Mercurial > hg > CbC > CbC_gcc
comparison gcc/tree-if-conv.c @ 131:84e7813d76e9
gcc-8.2
author | mir3636 |
---|---|
date | Thu, 25 Oct 2018 07:37:49 +0900 |
parents | 04ced10e8804 |
children | 1830386684a0 |
comparison
equal
deleted
inserted
replaced
111:04ced10e8804 | 131:84e7813d76e9 |
---|---|
1 /* If-conversion for vectorizer. | 1 /* If-conversion for vectorizer. |
2 Copyright (C) 2004-2017 Free Software Foundation, Inc. | 2 Copyright (C) 2004-2018 Free Software Foundation, Inc. |
3 Contributed by Devang Patel <dpatel@apple.com> | 3 Contributed by Devang Patel <dpatel@apple.com> |
4 | 4 |
5 This file is part of GCC. | 5 This file is part of GCC. |
6 | 6 |
7 GCC is free software; you can redistribute it and/or modify it under | 7 GCC is free software; you can redistribute it and/or modify it under |
114 #include "tree-hash-traits.h" | 114 #include "tree-hash-traits.h" |
115 #include "varasm.h" | 115 #include "varasm.h" |
116 #include "builtins.h" | 116 #include "builtins.h" |
117 #include "params.h" | 117 #include "params.h" |
118 #include "cfganal.h" | 118 #include "cfganal.h" |
119 #include "internal-fn.h" | |
120 #include "fold-const.h" | |
119 | 121 |
120 /* Only handle PHIs with no more arguments unless we are asked to by | 122 /* Only handle PHIs with no more arguments unless we are asked to by |
121 simd pragma. */ | 123 simd pragma. */ |
122 #define MAX_PHI_ARG_NUM \ | 124 #define MAX_PHI_ARG_NUM \ |
123 ((unsigned) PARAM_VALUE (PARAM_MAX_TREE_IF_CONVERSION_PHI_ARGS)) | 125 ((unsigned) PARAM_VALUE (PARAM_MAX_TREE_IF_CONVERSION_PHI_ARGS)) |
124 | 126 |
125 /* Indicate if new load/store that needs to be predicated is introduced | 127 /* True if we've converted a statement that was only executed when some |
126 during if conversion. */ | 128 condition C was true, and if for correctness we need to predicate the |
127 static bool any_pred_load_store; | 129 statement to ensure that it is a no-op when C is false. See |
130 predicate_statements for the kinds of predication we support. */ | |
131 static bool need_to_predicate; | |
128 | 132 |
129 /* Indicate if there are any complicated PHIs that need to be handled in | 133 /* Indicate if there are any complicated PHIs that need to be handled in |
130 if-conversion. Complicated PHI has more than two arguments and can't | 134 if-conversion. Complicated PHI has more than two arguments and can't |
131 be degenerated to two arguments PHI. See more information in comment | 135 be degenerated to two arguments PHI. See more information in comment |
132 before phi_convertible_by_degenerating_args. */ | 136 before phi_convertible_by_degenerating_args. */ |
191 data_reference_p> *innermost_DR_map; | 195 data_reference_p> *innermost_DR_map; |
192 | 196 |
193 /* Hash table to store <base reference, DR> pairs. */ | 197 /* Hash table to store <base reference, DR> pairs. */ |
194 static hash_map<tree_operand_hash, data_reference_p> *baseref_DR_map; | 198 static hash_map<tree_operand_hash, data_reference_p> *baseref_DR_map; |
195 | 199 |
200 /* List of redundant SSA names: the first should be replaced by the second. */ | |
201 static vec< std::pair<tree, tree> > redundant_ssa_names; | |
202 | |
196 /* Structure used to predicate basic blocks. This is attached to the | 203 /* Structure used to predicate basic blocks. This is attached to the |
197 ->aux field of the BBs in the loop to be if-converted. */ | 204 ->aux field of the BBs in the loop to be if-converted. */ |
198 struct bb_predicate { | 205 struct bb_predicate { |
199 | 206 |
200 /* The condition under which this basic block is executed. */ | 207 /* The condition under which this basic block is executed. */ |
255 of the predicate for basic block BB. */ | 262 of the predicate for basic block BB. */ |
256 | 263 |
257 static inline void | 264 static inline void |
258 add_bb_predicate_gimplified_stmts (basic_block bb, gimple_seq stmts) | 265 add_bb_predicate_gimplified_stmts (basic_block bb, gimple_seq stmts) |
259 { | 266 { |
267 /* We might have updated some stmts in STMTS via force_gimple_operand | |
268 calling fold_stmt and that producing multiple stmts. Delink immediate | |
269 uses so update_ssa after loop versioning doesn't get confused for | |
270 the not yet inserted predicates. | |
271 ??? This should go away once we reliably avoid updating stmts | |
272 not in any BB. */ | |
273 for (gimple_stmt_iterator gsi = gsi_start (stmts); | |
274 !gsi_end_p (gsi); gsi_next (&gsi)) | |
275 { | |
276 gimple *stmt = gsi_stmt (gsi); | |
277 delink_stmt_imm_use (stmt); | |
278 gimple_set_modified (stmt, true); | |
279 } | |
260 gimple_seq_add_seq_without_update | 280 gimple_seq_add_seq_without_update |
261 (&(((struct bb_predicate *) bb->aux)->predicate_gimplified_stmts), stmts); | 281 (&(((struct bb_predicate *) bb->aux)->predicate_gimplified_stmts), stmts); |
262 } | 282 } |
263 | 283 |
264 /* Initializes to TRUE the predicate of basic block BB. */ | 284 /* Initializes to TRUE the predicate of basic block BB. */ |
269 bb->aux = XNEW (struct bb_predicate); | 289 bb->aux = XNEW (struct bb_predicate); |
270 set_bb_predicate_gimplified_stmts (bb, NULL); | 290 set_bb_predicate_gimplified_stmts (bb, NULL); |
271 set_bb_predicate (bb, boolean_true_node); | 291 set_bb_predicate (bb, boolean_true_node); |
272 } | 292 } |
273 | 293 |
274 /* Release the SSA_NAMEs associated with the predicate of basic block BB, | 294 /* Release the SSA_NAMEs associated with the predicate of basic block BB. */ |
275 but don't actually free it. */ | |
276 | 295 |
277 static inline void | 296 static inline void |
278 release_bb_predicate (basic_block bb) | 297 release_bb_predicate (basic_block bb) |
279 { | 298 { |
280 gimple_seq stmts = bb_predicate_gimplified_stmts (bb); | 299 gimple_seq stmts = bb_predicate_gimplified_stmts (bb); |
281 if (stmts) | 300 if (stmts) |
282 { | 301 { |
302 /* Ensure that these stmts haven't yet been added to a bb. */ | |
283 if (flag_checking) | 303 if (flag_checking) |
284 for (gimple_stmt_iterator i = gsi_start (stmts); | 304 for (gimple_stmt_iterator i = gsi_start (stmts); |
285 !gsi_end_p (i); gsi_next (&i)) | 305 !gsi_end_p (i); gsi_next (&i)) |
286 gcc_assert (! gimple_use_ops (gsi_stmt (i))); | 306 gcc_assert (! gimple_bb (gsi_stmt (i))); |
287 | 307 |
308 /* Discard them. */ | |
309 gimple_seq_discard (stmts); | |
288 set_bb_predicate_gimplified_stmts (bb, NULL); | 310 set_bb_predicate_gimplified_stmts (bb, NULL); |
289 } | 311 } |
290 } | 312 } |
291 | 313 |
292 /* Free the predicate of basic block BB. */ | 314 /* Free the predicate of basic block BB. */ |
726 within array bound. Return false otherwise. */ | 748 within array bound. Return false otherwise. */ |
727 | 749 |
728 static bool | 750 static bool |
729 idx_within_array_bound (tree ref, tree *idx, void *dta) | 751 idx_within_array_bound (tree ref, tree *idx, void *dta) |
730 { | 752 { |
731 bool overflow; | 753 wi::overflow_type overflow; |
732 widest_int niter, valid_niter, delta, wi_step; | 754 widest_int niter, valid_niter, delta, wi_step; |
733 tree ev, init, step; | 755 tree ev, init, step; |
734 tree low, high; | 756 tree low, high; |
735 struct loop *loop = (struct loop*) dta; | 757 struct loop *loop = (struct loop*) dta; |
736 | 758 |
847 which is defined as read and write and is bound to the definition | 869 which is defined as read and write and is bound to the definition |
848 we are seeing. */ | 870 we are seeing. */ |
849 static bool | 871 static bool |
850 ifcvt_memrefs_wont_trap (gimple *stmt, vec<data_reference_p> drs) | 872 ifcvt_memrefs_wont_trap (gimple *stmt, vec<data_reference_p> drs) |
851 { | 873 { |
874 /* If DR didn't see a reference here we can't use it to tell | |
875 whether the ref traps or not. */ | |
876 if (gimple_uid (stmt) == 0) | |
877 return false; | |
878 | |
852 data_reference_p *master_dr, *base_master_dr; | 879 data_reference_p *master_dr, *base_master_dr; |
853 data_reference_p a = drs[gimple_uid (stmt) - 1]; | 880 data_reference_p a = drs[gimple_uid (stmt) - 1]; |
854 | 881 |
855 tree base = DR_BASE_OBJECT (a); | 882 tree base = DR_BASE_OBJECT (a); |
856 innermost_loop_behavior *innermost = &DR_INNERMOST (a); | 883 innermost_loop_behavior *innermost = &DR_INNERMOST (a); |
897 (conditional load or store based on a mask computed from bb predicate). */ | 924 (conditional load or store based on a mask computed from bb predicate). */ |
898 | 925 |
899 static bool | 926 static bool |
900 ifcvt_can_use_mask_load_store (gimple *stmt) | 927 ifcvt_can_use_mask_load_store (gimple *stmt) |
901 { | 928 { |
902 tree lhs, ref; | 929 /* Check whether this is a load or store. */ |
903 machine_mode mode; | 930 tree lhs = gimple_assign_lhs (stmt); |
904 basic_block bb = gimple_bb (stmt); | |
905 bool is_load; | 931 bool is_load; |
906 | 932 tree ref; |
907 if (!(flag_tree_loop_vectorize || bb->loop_father->force_vectorize) | |
908 || bb->loop_father->dont_vectorize | |
909 || !gimple_assign_single_p (stmt) | |
910 || gimple_has_volatile_ops (stmt)) | |
911 return false; | |
912 | |
913 /* Check whether this is a load or store. */ | |
914 lhs = gimple_assign_lhs (stmt); | |
915 if (gimple_store_p (stmt)) | 933 if (gimple_store_p (stmt)) |
916 { | 934 { |
917 if (!is_gimple_val (gimple_assign_rhs1 (stmt))) | 935 if (!is_gimple_val (gimple_assign_rhs1 (stmt))) |
918 return false; | 936 return false; |
919 is_load = false; | 937 is_load = false; |
930 if (may_be_nonaddressable_p (ref)) | 948 if (may_be_nonaddressable_p (ref)) |
931 return false; | 949 return false; |
932 | 950 |
933 /* Mask should be integer mode of the same size as the load/store | 951 /* Mask should be integer mode of the same size as the load/store |
934 mode. */ | 952 mode. */ |
935 mode = TYPE_MODE (TREE_TYPE (lhs)); | 953 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); |
936 if (!int_mode_for_mode (mode).exists () || VECTOR_MODE_P (mode)) | 954 if (!int_mode_for_mode (mode).exists () || VECTOR_MODE_P (mode)) |
937 return false; | 955 return false; |
938 | 956 |
939 if (can_vec_mask_load_store_p (mode, VOIDmode, is_load)) | 957 if (can_vec_mask_load_store_p (mode, VOIDmode, is_load)) |
940 return true; | 958 return true; |
941 | 959 |
942 return false; | 960 return false; |
961 } | |
962 | |
963 /* Return true if STMT could be converted from an operation that is | |
964 unconditional to one that is conditional on a bb predicate mask. */ | |
965 | |
966 static bool | |
967 ifcvt_can_predicate (gimple *stmt) | |
968 { | |
969 basic_block bb = gimple_bb (stmt); | |
970 | |
971 if (!(flag_tree_loop_vectorize || bb->loop_father->force_vectorize) | |
972 || bb->loop_father->dont_vectorize | |
973 || gimple_has_volatile_ops (stmt)) | |
974 return false; | |
975 | |
976 if (gimple_assign_single_p (stmt)) | |
977 return ifcvt_can_use_mask_load_store (stmt); | |
978 | |
979 tree_code code = gimple_assign_rhs_code (stmt); | |
980 tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt)); | |
981 tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (stmt)); | |
982 if (!types_compatible_p (lhs_type, rhs_type)) | |
983 return false; | |
984 internal_fn cond_fn = get_conditional_internal_fn (code); | |
985 return (cond_fn != IFN_LAST | |
986 && vectorized_internal_fn_supported_p (cond_fn, lhs_type)); | |
943 } | 987 } |
944 | 988 |
945 /* Return true when STMT is if-convertible. | 989 /* Return true when STMT is if-convertible. |
946 | 990 |
947 GIMPLE_ASSIGN statement is not if-convertible if, | 991 GIMPLE_ASSIGN statement is not if-convertible if, |
984 if ((! gimple_vuse (stmt) | 1028 if ((! gimple_vuse (stmt) |
985 || gimple_could_trap_p_1 (stmt, false, false) | 1029 || gimple_could_trap_p_1 (stmt, false, false) |
986 || ! ifcvt_memrefs_wont_trap (stmt, refs)) | 1030 || ! ifcvt_memrefs_wont_trap (stmt, refs)) |
987 && gimple_could_trap_p (stmt)) | 1031 && gimple_could_trap_p (stmt)) |
988 { | 1032 { |
989 if (ifcvt_can_use_mask_load_store (stmt)) | 1033 if (ifcvt_can_predicate (stmt)) |
990 { | 1034 { |
991 gimple_set_plf (stmt, GF_PLF_2, true); | 1035 gimple_set_plf (stmt, GF_PLF_2, true); |
992 any_pred_load_store = true; | 1036 need_to_predicate = true; |
993 return true; | 1037 return true; |
994 } | 1038 } |
995 if (dump_file && (dump_flags & TDF_DETAILS)) | 1039 if (dump_file && (dump_flags & TDF_DETAILS)) |
996 fprintf (dump_file, "tree could trap...\n"); | 1040 fprintf (dump_file, "tree could trap...\n"); |
997 return false; | 1041 return false; |
998 } | 1042 } |
999 | 1043 |
1000 /* When if-converting stores force versioning, likewise if we | 1044 /* When if-converting stores force versioning, likewise if we |
1001 ended up generating store data races. */ | 1045 ended up generating store data races. */ |
1002 if (gimple_vdef (stmt)) | 1046 if (gimple_vdef (stmt)) |
1003 any_pred_load_store = true; | 1047 need_to_predicate = true; |
1004 | 1048 |
1005 return true; | 1049 return true; |
1006 } | 1050 } |
1007 | 1051 |
1008 /* Return true when STMT is if-convertible. | 1052 /* Return true when STMT is if-convertible. |
1033 int flags = gimple_call_flags (stmt); | 1077 int flags = gimple_call_flags (stmt); |
1034 if ((flags & ECF_CONST) | 1078 if ((flags & ECF_CONST) |
1035 && !(flags & ECF_LOOPING_CONST_OR_PURE) | 1079 && !(flags & ECF_LOOPING_CONST_OR_PURE) |
1036 /* We can only vectorize some builtins at the moment, | 1080 /* We can only vectorize some builtins at the moment, |
1037 so restrict if-conversion to those. */ | 1081 so restrict if-conversion to those. */ |
1038 && DECL_BUILT_IN (fndecl)) | 1082 && fndecl_built_in_p (fndecl)) |
1039 return true; | 1083 return true; |
1040 } | 1084 } |
1041 return false; | 1085 return false; |
1042 } | 1086 } |
1043 | 1087 |
1066 | 1110 |
1067 FOR_EACH_EDGE (e, ei, bb->preds) | 1111 FOR_EACH_EDGE (e, ei, bb->preds) |
1068 if (EDGE_COUNT (e->src->succs) == 1) | 1112 if (EDGE_COUNT (e->src->succs) == 1) |
1069 return false; | 1113 return false; |
1070 return true; | 1114 return true; |
1071 } | |
1072 | |
1073 /* Returns true if at least one successor in on critical edge. */ | |
1074 static inline bool | |
1075 has_pred_critical_p (basic_block bb) | |
1076 { | |
1077 edge e; | |
1078 edge_iterator ei; | |
1079 | |
1080 FOR_EACH_EDGE (e, ei, bb->preds) | |
1081 if (EDGE_COUNT (e->src->succs) > 1) | |
1082 return true; | |
1083 return false; | |
1084 } | 1115 } |
1085 | 1116 |
1086 /* Return true when BB is if-convertible. This routine does not check | 1117 /* Return true when BB is if-convertible. This routine does not check |
1087 basic block's statements and phis. | 1118 basic block's statements and phis. |
1088 | 1119 |
2030 } | 2061 } |
2031 | 2062 |
2032 stmts = bb_predicate_gimplified_stmts (bb); | 2063 stmts = bb_predicate_gimplified_stmts (bb); |
2033 if (stmts) | 2064 if (stmts) |
2034 { | 2065 { |
2035 if (any_pred_load_store) | 2066 if (need_to_predicate) |
2036 { | 2067 { |
2037 /* Insert the predicate of the BB just after the label, | 2068 /* Insert the predicate of the BB just after the label, |
2038 as the if-conversion of memory writes will use this | 2069 as the if-conversion of memory writes will use this |
2039 predicate. */ | 2070 predicate. */ |
2040 gimple_stmt_iterator gsi = gsi_after_labels (bb); | 2071 gimple_stmt_iterator gsi = gsi_after_labels (bb); |
2058 set_bb_predicate_gimplified_stmts (bb, NULL); | 2089 set_bb_predicate_gimplified_stmts (bb, NULL); |
2059 } | 2090 } |
2060 } | 2091 } |
2061 } | 2092 } |
2062 | 2093 |
2063 /* Helper function for predicate_mem_writes. Returns index of existent | 2094 /* Helper function for predicate_statements. Returns index of existent |
2064 mask if it was created for given SIZE and -1 otherwise. */ | 2095 mask if it was created for given SIZE and -1 otherwise. */ |
2065 | 2096 |
2066 static int | 2097 static int |
2067 mask_exists (int size, vec<int> vec) | 2098 mask_exists (int size, vec<int> vec) |
2068 { | 2099 { |
2070 int v; | 2101 int v; |
2071 FOR_EACH_VEC_ELT (vec, ix, v) | 2102 FOR_EACH_VEC_ELT (vec, ix, v) |
2072 if (v == size) | 2103 if (v == size) |
2073 return (int) ix; | 2104 return (int) ix; |
2074 return -1; | 2105 return -1; |
2106 } | |
2107 | |
2108 /* Helper function for predicate_statements. STMT is a memory read or | |
2109 write and it needs to be predicated by MASK. Return a statement | |
2110 that does so. */ | |
2111 | |
2112 static gimple * | |
2113 predicate_load_or_store (gimple_stmt_iterator *gsi, gassign *stmt, tree mask) | |
2114 { | |
2115 gcall *new_stmt; | |
2116 | |
2117 tree lhs = gimple_assign_lhs (stmt); | |
2118 tree rhs = gimple_assign_rhs1 (stmt); | |
2119 tree ref = TREE_CODE (lhs) == SSA_NAME ? rhs : lhs; | |
2120 mark_addressable (ref); | |
2121 tree addr = force_gimple_operand_gsi (gsi, build_fold_addr_expr (ref), | |
2122 true, NULL_TREE, true, GSI_SAME_STMT); | |
2123 tree ptr = build_int_cst (reference_alias_ptr_type (ref), | |
2124 get_object_alignment (ref)); | |
2125 /* Copy points-to info if possible. */ | |
2126 if (TREE_CODE (addr) == SSA_NAME && !SSA_NAME_PTR_INFO (addr)) | |
2127 copy_ref_info (build2 (MEM_REF, TREE_TYPE (ref), addr, ptr), | |
2128 ref); | |
2129 if (TREE_CODE (lhs) == SSA_NAME) | |
2130 { | |
2131 new_stmt | |
2132 = gimple_build_call_internal (IFN_MASK_LOAD, 3, addr, | |
2133 ptr, mask); | |
2134 gimple_call_set_lhs (new_stmt, lhs); | |
2135 gimple_set_vuse (new_stmt, gimple_vuse (stmt)); | |
2136 } | |
2137 else | |
2138 { | |
2139 new_stmt | |
2140 = gimple_build_call_internal (IFN_MASK_STORE, 4, addr, ptr, | |
2141 mask, rhs); | |
2142 gimple_set_vuse (new_stmt, gimple_vuse (stmt)); | |
2143 gimple_set_vdef (new_stmt, gimple_vdef (stmt)); | |
2144 SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt; | |
2145 } | |
2146 gimple_call_set_nothrow (new_stmt, true); | |
2147 return new_stmt; | |
2148 } | |
2149 | |
2150 /* STMT uses OP_LHS. Check whether it is equivalent to: | |
2151 | |
2152 ... = OP_MASK ? OP_LHS : X; | |
2153 | |
2154 Return X if so, otherwise return null. OP_MASK is an SSA_NAME that is | |
2155 known to have value OP_COND. */ | |
2156 | |
2157 static tree | |
2158 check_redundant_cond_expr (gimple *stmt, tree op_mask, tree op_cond, | |
2159 tree op_lhs) | |
2160 { | |
2161 gassign *assign = dyn_cast <gassign *> (stmt); | |
2162 if (!assign || gimple_assign_rhs_code (assign) != COND_EXPR) | |
2163 return NULL_TREE; | |
2164 | |
2165 tree use_cond = gimple_assign_rhs1 (assign); | |
2166 tree if_true = gimple_assign_rhs2 (assign); | |
2167 tree if_false = gimple_assign_rhs3 (assign); | |
2168 | |
2169 if ((use_cond == op_mask || operand_equal_p (use_cond, op_cond, 0)) | |
2170 && if_true == op_lhs) | |
2171 return if_false; | |
2172 | |
2173 if (inverse_conditions_p (use_cond, op_cond) && if_false == op_lhs) | |
2174 return if_true; | |
2175 | |
2176 return NULL_TREE; | |
2177 } | |
2178 | |
2179 /* Return true if VALUE is available for use at STMT. SSA_NAMES is | |
2180 the set of SSA names defined earlier in STMT's block. */ | |
2181 | |
2182 static bool | |
2183 value_available_p (gimple *stmt, hash_set<tree_ssa_name_hash> *ssa_names, | |
2184 tree value) | |
2185 { | |
2186 if (is_gimple_min_invariant (value)) | |
2187 return true; | |
2188 | |
2189 if (TREE_CODE (value) == SSA_NAME) | |
2190 { | |
2191 if (SSA_NAME_IS_DEFAULT_DEF (value)) | |
2192 return true; | |
2193 | |
2194 basic_block def_bb = gimple_bb (SSA_NAME_DEF_STMT (value)); | |
2195 basic_block use_bb = gimple_bb (stmt); | |
2196 return (def_bb == use_bb | |
2197 ? ssa_names->contains (value) | |
2198 : dominated_by_p (CDI_DOMINATORS, use_bb, def_bb)); | |
2199 } | |
2200 | |
2201 return false; | |
2202 } | |
2203 | |
2204 /* Helper function for predicate_statements. STMT is a potentially-trapping | |
2205 arithmetic operation that needs to be predicated by MASK, an SSA_NAME that | |
2206 has value COND. Return a statement that does so. SSA_NAMES is the set of | |
2207 SSA names defined earlier in STMT's block. */ | |
2208 | |
2209 static gimple * | |
2210 predicate_rhs_code (gassign *stmt, tree mask, tree cond, | |
2211 hash_set<tree_ssa_name_hash> *ssa_names) | |
2212 { | |
2213 tree lhs = gimple_assign_lhs (stmt); | |
2214 tree_code code = gimple_assign_rhs_code (stmt); | |
2215 unsigned int nops = gimple_num_ops (stmt); | |
2216 internal_fn cond_fn = get_conditional_internal_fn (code); | |
2217 | |
2218 /* Construct the arguments to the conditional internal function. */ | |
2219 auto_vec<tree, 8> args; | |
2220 args.safe_grow (nops + 1); | |
2221 args[0] = mask; | |
2222 for (unsigned int i = 1; i < nops; ++i) | |
2223 args[i] = gimple_op (stmt, i); | |
2224 args[nops] = NULL_TREE; | |
2225 | |
2226 /* Look for uses of the result to see whether they are COND_EXPRs that can | |
2227 be folded into the conditional call. */ | |
2228 imm_use_iterator imm_iter; | |
2229 gimple *use_stmt; | |
2230 FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs) | |
2231 { | |
2232 tree new_else = check_redundant_cond_expr (use_stmt, mask, cond, lhs); | |
2233 if (new_else && value_available_p (stmt, ssa_names, new_else)) | |
2234 { | |
2235 if (!args[nops]) | |
2236 args[nops] = new_else; | |
2237 if (operand_equal_p (new_else, args[nops], 0)) | |
2238 { | |
2239 /* We have: | |
2240 | |
2241 LHS = IFN_COND (MASK, ..., ELSE); | |
2242 X = MASK ? LHS : ELSE; | |
2243 | |
2244 which makes X equivalent to LHS. */ | |
2245 tree use_lhs = gimple_assign_lhs (use_stmt); | |
2246 redundant_ssa_names.safe_push (std::make_pair (use_lhs, lhs)); | |
2247 } | |
2248 } | |
2249 } | |
2250 if (!args[nops]) | |
2251 args[nops] = targetm.preferred_else_value (cond_fn, TREE_TYPE (lhs), | |
2252 nops - 1, &args[1]); | |
2253 | |
2254 /* Create and insert the call. */ | |
2255 gcall *new_stmt = gimple_build_call_internal_vec (cond_fn, args); | |
2256 gimple_call_set_lhs (new_stmt, lhs); | |
2257 gimple_call_set_nothrow (new_stmt, true); | |
2258 | |
2259 return new_stmt; | |
2075 } | 2260 } |
2076 | 2261 |
2077 /* Predicate each write to memory in LOOP. | 2262 /* Predicate each write to memory in LOOP. |
2078 | 2263 |
2079 This function transforms control flow constructs containing memory | 2264 This function transforms control flow constructs containing memory |
2136 | | 2321 | |
2137 | bb_4 | 2322 | bb_4 |
2138 | goto bb_1 | 2323 | goto bb_1 |
2139 | end_bb_4 | 2324 | end_bb_4 |
2140 | 2325 |
2141 predicate_mem_writes is then predicating the memory write as follows: | 2326 predicate_statements is then predicating the memory write as follows: |
2142 | 2327 |
2143 | bb_0 | 2328 | bb_0 |
2144 | i = 0 | 2329 | i = 0 |
2145 | end_bb_0 | 2330 | end_bb_0 |
2146 | | 2331 | |
2180 | goto bb_1 | 2365 | goto bb_1 |
2181 | end_bb_4 | 2366 | end_bb_4 |
2182 */ | 2367 */ |
2183 | 2368 |
2184 static void | 2369 static void |
2185 predicate_mem_writes (loop_p loop) | 2370 predicate_statements (loop_p loop) |
2186 { | 2371 { |
2187 unsigned int i, orig_loop_num_nodes = loop->num_nodes; | 2372 unsigned int i, orig_loop_num_nodes = loop->num_nodes; |
2188 auto_vec<int, 1> vect_sizes; | 2373 auto_vec<int, 1> vect_sizes; |
2189 auto_vec<tree, 1> vect_masks; | 2374 auto_vec<tree, 1> vect_masks; |
2375 hash_set<tree_ssa_name_hash> ssa_names; | |
2190 | 2376 |
2191 for (i = 1; i < orig_loop_num_nodes; i++) | 2377 for (i = 1; i < orig_loop_num_nodes; i++) |
2192 { | 2378 { |
2193 gimple_stmt_iterator gsi; | 2379 gimple_stmt_iterator gsi; |
2194 basic_block bb = ifc_bbs[i]; | 2380 basic_block bb = ifc_bbs[i]; |
2195 tree cond = bb_predicate (bb); | 2381 tree cond = bb_predicate (bb); |
2196 bool swap; | 2382 bool swap; |
2197 gimple *stmt; | |
2198 int index; | 2383 int index; |
2199 | 2384 |
2200 if (is_true_predicate (cond)) | 2385 if (is_true_predicate (cond)) |
2201 continue; | 2386 continue; |
2202 | 2387 |
2210 vect_sizes.truncate (0); | 2395 vect_sizes.truncate (0); |
2211 vect_masks.truncate (0); | 2396 vect_masks.truncate (0); |
2212 | 2397 |
2213 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);) | 2398 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);) |
2214 { | 2399 { |
2215 if (!gimple_assign_single_p (stmt = gsi_stmt (gsi))) | 2400 gassign *stmt = dyn_cast <gassign *> (gsi_stmt (gsi)); |
2401 if (!stmt) | |
2216 ; | 2402 ; |
2217 else if (is_false_predicate (cond) | 2403 else if (is_false_predicate (cond) |
2218 && gimple_vdef (stmt)) | 2404 && gimple_vdef (stmt)) |
2219 { | 2405 { |
2220 unlink_stmt_vdef (stmt); | 2406 unlink_stmt_vdef (stmt); |
2223 continue; | 2409 continue; |
2224 } | 2410 } |
2225 else if (gimple_plf (stmt, GF_PLF_2)) | 2411 else if (gimple_plf (stmt, GF_PLF_2)) |
2226 { | 2412 { |
2227 tree lhs = gimple_assign_lhs (stmt); | 2413 tree lhs = gimple_assign_lhs (stmt); |
2228 tree rhs = gimple_assign_rhs1 (stmt); | 2414 tree mask; |
2229 tree ref, addr, ptr, mask; | 2415 gimple *new_stmt; |
2230 gcall *new_stmt; | |
2231 gimple_seq stmts = NULL; | 2416 gimple_seq stmts = NULL; |
2232 int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (lhs))); | 2417 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); |
2233 ref = TREE_CODE (lhs) == SSA_NAME ? rhs : lhs; | 2418 /* We checked before setting GF_PLF_2 that an equivalent |
2234 mark_addressable (ref); | 2419 integer mode exists. */ |
2235 addr = force_gimple_operand_gsi (&gsi, build_fold_addr_expr (ref), | 2420 int bitsize = GET_MODE_BITSIZE (mode).to_constant (); |
2236 true, NULL_TREE, true, | |
2237 GSI_SAME_STMT); | |
2238 if (!vect_sizes.is_empty () | 2421 if (!vect_sizes.is_empty () |
2239 && (index = mask_exists (bitsize, vect_sizes)) != -1) | 2422 && (index = mask_exists (bitsize, vect_sizes)) != -1) |
2240 /* Use created mask. */ | 2423 /* Use created mask. */ |
2241 mask = vect_masks[index]; | 2424 mask = vect_masks[index]; |
2242 else | 2425 else |
2245 mask = gimple_build (&stmts, TREE_CODE (cond), | 2428 mask = gimple_build (&stmts, TREE_CODE (cond), |
2246 boolean_type_node, | 2429 boolean_type_node, |
2247 TREE_OPERAND (cond, 0), | 2430 TREE_OPERAND (cond, 0), |
2248 TREE_OPERAND (cond, 1)); | 2431 TREE_OPERAND (cond, 1)); |
2249 else | 2432 else |
2250 { | 2433 mask = cond; |
2251 gcc_assert (TREE_CODE (cond) == SSA_NAME); | |
2252 mask = cond; | |
2253 } | |
2254 | 2434 |
2255 if (swap) | 2435 if (swap) |
2256 { | 2436 { |
2257 tree true_val | 2437 tree true_val |
2258 = constant_boolean_node (true, TREE_TYPE (mask)); | 2438 = constant_boolean_node (true, TREE_TYPE (mask)); |
2259 mask = gimple_build (&stmts, BIT_XOR_EXPR, | 2439 mask = gimple_build (&stmts, BIT_XOR_EXPR, |
2260 TREE_TYPE (mask), mask, true_val); | 2440 TREE_TYPE (mask), mask, true_val); |
2261 } | 2441 } |
2262 gsi_insert_seq_before (&gsi, stmts, GSI_SAME_STMT); | 2442 gsi_insert_seq_before (&gsi, stmts, GSI_SAME_STMT); |
2263 | 2443 |
2264 mask = ifc_temp_var (TREE_TYPE (mask), mask, &gsi); | |
2265 /* Save mask and its size for further use. */ | 2444 /* Save mask and its size for further use. */ |
2266 vect_sizes.safe_push (bitsize); | 2445 vect_sizes.safe_push (bitsize); |
2267 vect_masks.safe_push (mask); | 2446 vect_masks.safe_push (mask); |
2268 } | 2447 } |
2269 ptr = build_int_cst (reference_alias_ptr_type (ref), | 2448 if (gimple_assign_single_p (stmt)) |
2270 get_object_alignment (ref)); | 2449 new_stmt = predicate_load_or_store (&gsi, stmt, mask); |
2271 /* Copy points-to info if possible. */ | |
2272 if (TREE_CODE (addr) == SSA_NAME && !SSA_NAME_PTR_INFO (addr)) | |
2273 copy_ref_info (build2 (MEM_REF, TREE_TYPE (ref), addr, ptr), | |
2274 ref); | |
2275 if (TREE_CODE (lhs) == SSA_NAME) | |
2276 { | |
2277 new_stmt | |
2278 = gimple_build_call_internal (IFN_MASK_LOAD, 3, addr, | |
2279 ptr, mask); | |
2280 gimple_call_set_lhs (new_stmt, lhs); | |
2281 gimple_set_vuse (new_stmt, gimple_vuse (stmt)); | |
2282 } | |
2283 else | 2450 else |
2284 { | 2451 new_stmt = predicate_rhs_code (stmt, mask, cond, &ssa_names); |
2285 new_stmt | |
2286 = gimple_build_call_internal (IFN_MASK_STORE, 4, addr, ptr, | |
2287 mask, rhs); | |
2288 gimple_set_vuse (new_stmt, gimple_vuse (stmt)); | |
2289 gimple_set_vdef (new_stmt, gimple_vdef (stmt)); | |
2290 SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt; | |
2291 } | |
2292 gimple_call_set_nothrow (new_stmt, true); | |
2293 | 2452 |
2294 gsi_replace (&gsi, new_stmt, true); | 2453 gsi_replace (&gsi, new_stmt, true); |
2295 } | 2454 } |
2296 else if (gimple_vdef (stmt)) | 2455 else if (gimple_vdef (stmt)) |
2297 { | 2456 { |
2308 true, GSI_SAME_STMT); | 2467 true, GSI_SAME_STMT); |
2309 rhs = fold_build_cond_expr (type, unshare_expr (cond), rhs, lhs); | 2468 rhs = fold_build_cond_expr (type, unshare_expr (cond), rhs, lhs); |
2310 gimple_assign_set_rhs1 (stmt, ifc_temp_var (type, rhs, &gsi)); | 2469 gimple_assign_set_rhs1 (stmt, ifc_temp_var (type, rhs, &gsi)); |
2311 update_stmt (stmt); | 2470 update_stmt (stmt); |
2312 } | 2471 } |
2472 tree lhs = gimple_get_lhs (gsi_stmt (gsi)); | |
2473 if (lhs && TREE_CODE (lhs) == SSA_NAME) | |
2474 ssa_names.add (lhs); | |
2313 gsi_next (&gsi); | 2475 gsi_next (&gsi); |
2314 } | 2476 } |
2477 ssa_names.empty (); | |
2315 } | 2478 } |
2316 } | 2479 } |
2317 | 2480 |
2318 /* Remove all GIMPLE_CONDs and GIMPLE_LABELs of all the basic blocks | 2481 /* Remove all GIMPLE_CONDs and GIMPLE_LABELs of all the basic blocks |
2319 other than the exit and latch of the LOOP. Also resets the | 2482 other than the exit and latch of the LOOP. Also resets the |
2371 | 2534 |
2372 remove_conditions_and_labels (loop); | 2535 remove_conditions_and_labels (loop); |
2373 insert_gimplified_predicates (loop); | 2536 insert_gimplified_predicates (loop); |
2374 predicate_all_scalar_phis (loop); | 2537 predicate_all_scalar_phis (loop); |
2375 | 2538 |
2376 if (any_pred_load_store) | 2539 if (need_to_predicate) |
2377 predicate_mem_writes (loop); | 2540 predicate_statements (loop); |
2378 | 2541 |
2379 /* Merge basic blocks: first remove all the edges in the loop, | 2542 /* Merge basic blocks: first remove all the edges in the loop, |
2380 except for those from the exit block. */ | 2543 except for those from the exit block. */ |
2381 exit_bb = NULL; | 2544 exit_bb = NULL; |
2382 bool *predicated = XNEWVEC (bool, orig_loop_num_nodes); | 2545 bool *predicated = XNEWVEC (bool, orig_loop_num_nodes); |
2712 gimple_stmt_iterator gsi; | 2875 gimple_stmt_iterator gsi; |
2713 auto_vec<gimple *> worklist; | 2876 auto_vec<gimple *> worklist; |
2714 enum gimple_code code; | 2877 enum gimple_code code; |
2715 use_operand_p use_p; | 2878 use_operand_p use_p; |
2716 imm_use_iterator imm_iter; | 2879 imm_use_iterator imm_iter; |
2880 std::pair <tree, tree> *name_pair; | |
2881 unsigned int i; | |
2882 | |
2883 FOR_EACH_VEC_ELT (redundant_ssa_names, i, name_pair) | |
2884 replace_uses_by (name_pair->first, name_pair->second); | |
2885 redundant_ssa_names.release (); | |
2717 | 2886 |
2718 worklist.create (64); | 2887 worklist.create (64); |
2719 /* Consider all phi as live statements. */ | 2888 /* Consider all phi as live statements. */ |
2720 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi)) | 2889 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi)) |
2721 { | 2890 { |
2812 struct loop *rloop; | 2981 struct loop *rloop; |
2813 | 2982 |
2814 again: | 2983 again: |
2815 rloop = NULL; | 2984 rloop = NULL; |
2816 ifc_bbs = NULL; | 2985 ifc_bbs = NULL; |
2817 any_pred_load_store = false; | 2986 need_to_predicate = false; |
2818 any_complicated_phi = false; | 2987 any_complicated_phi = false; |
2819 | 2988 |
2820 /* Apply more aggressive if-conversion when loop or its outer loop were | 2989 /* Apply more aggressive if-conversion when loop or its outer loop were |
2821 marked with simd pragma. When that's the case, we try to if-convert | 2990 marked with simd pragma. When that's the case, we try to if-convert |
2822 loop containing PHIs with more than MAX_PHI_ARG_NUM arguments. */ | 2991 loop containing PHIs with more than MAX_PHI_ARG_NUM arguments. */ |
2833 | 3002 |
2834 if (!if_convertible_loop_p (loop) | 3003 if (!if_convertible_loop_p (loop) |
2835 || !dbg_cnt (if_conversion_tree)) | 3004 || !dbg_cnt (if_conversion_tree)) |
2836 goto cleanup; | 3005 goto cleanup; |
2837 | 3006 |
2838 if ((any_pred_load_store || any_complicated_phi) | 3007 if ((need_to_predicate || any_complicated_phi) |
2839 && ((!flag_tree_loop_vectorize && !loop->force_vectorize) | 3008 && ((!flag_tree_loop_vectorize && !loop->force_vectorize) |
2840 || loop->dont_vectorize)) | 3009 || loop->dont_vectorize)) |
2841 goto cleanup; | 3010 goto cleanup; |
2842 | 3011 |
2843 /* Since we have no cost model, always version loops unless the user | 3012 /* Since we have no cost model, always version loops unless the user |
2844 specified -ftree-loop-if-convert or unless versioning is required. | 3013 specified -ftree-loop-if-convert or unless versioning is required. |
2845 Either version this loop, or if the pattern is right for outer-loop | 3014 Either version this loop, or if the pattern is right for outer-loop |
2846 vectorization, version the outer loop. In the latter case we will | 3015 vectorization, version the outer loop. In the latter case we will |
2847 still if-convert the original inner loop. */ | 3016 still if-convert the original inner loop. */ |
2848 if (any_pred_load_store | 3017 if (need_to_predicate |
2849 || any_complicated_phi | 3018 || any_complicated_phi |
2850 || flag_tree_loop_if_convert != 1) | 3019 || flag_tree_loop_if_convert != 1) |
2851 { | 3020 { |
2852 struct loop *vloop | 3021 struct loop *vloop |
2853 = (versionable_outer_loop_p (loop_outer (loop)) | 3022 = (versionable_outer_loop_p (loop_outer (loop)) |
2960 if (flag_tree_loop_if_convert == 1 | 3129 if (flag_tree_loop_if_convert == 1 |
2961 || ((flag_tree_loop_vectorize || loop->force_vectorize) | 3130 || ((flag_tree_loop_vectorize || loop->force_vectorize) |
2962 && !loop->dont_vectorize)) | 3131 && !loop->dont_vectorize)) |
2963 todo |= tree_if_conversion (loop); | 3132 todo |= tree_if_conversion (loop); |
2964 | 3133 |
3134 if (todo) | |
3135 { | |
3136 free_numbers_of_iterations_estimates (fun); | |
3137 scev_reset (); | |
3138 } | |
3139 | |
2965 if (flag_checking) | 3140 if (flag_checking) |
2966 { | 3141 { |
2967 basic_block bb; | 3142 basic_block bb; |
2968 FOR_EACH_BB_FN (bb, fun) | 3143 FOR_EACH_BB_FN (bb, fun) |
2969 gcc_assert (!bb->aux); | 3144 gcc_assert (!bb->aux); |