From 8c25bf3b8c1dfa4da1cc583a182ea44d466bb8f9 Mon Sep 17 00:00:00 2001 From: rguenth Date: Tue, 19 Jun 2018 12:25:06 +0000 Subject: [PATCH] 2018-06-19 Richard Biener * tree-vectorizer.c (try_vectorize_loop_1): Split out of ... (vectorize_loops): ... here. Fix dbgcnt handling. (try_vectorize_loop): Wrap try_vectorize_loop_1. git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@261740 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 6 + gcc/tree-vectorizer.c | 312 ++++++++++++++++++++++++++------------------------ 2 files changed, 167 insertions(+), 151 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index f580b4eb196..7c4b9e6ae96 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,9 @@ +2018-06-19 Richard Biener + + * tree-vectorizer.c (try_vectorize_loop_1): Split out of ... + (vectorize_loops): ... here. Fix dbgcnt handling. + (try_vectorize_loop): Wrap try_vectorize_loop_1. + 2018-06-19 Segher Boessenkool PR target/86197 diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c index eb9129051b5..1c487e608ff 100644 --- a/gcc/tree-vectorizer.c +++ b/gcc/tree-vectorizer.c @@ -659,6 +659,158 @@ set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call) free (bbs); } +/* Try to vectorize LOOP. */ + +static unsigned +try_vectorize_loop_1 (hash_table *&simduid_to_vf_htab, + unsigned *num_vectorized_loops, + loop_p loop, loop_vec_info orig_loop_vinfo, + gimple *loop_vectorized_call, + gimple *loop_dist_alias_call) +{ + unsigned ret = 0; + vect_location = find_loop_location (loop); + if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOCATION + && dump_enabled_p ()) + dump_printf (MSG_NOTE, "\nAnalyzing loop at %s:%d\n", + LOCATION_FILE (vect_location), + LOCATION_LINE (vect_location)); + + loop_vec_info loop_vinfo = vect_analyze_loop (loop, orig_loop_vinfo); + loop->aux = loop_vinfo; + + if (!loop_vinfo || !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo)) + { + /* Free existing information if loop is analyzed with some + assumptions. */ + if (loop_constraint_set_p (loop, LOOP_C_FINITE)) + vect_free_loop_info_assumptions (loop); + + /* If we applied if-conversion then try to vectorize the + BB of innermost loops. + ??? Ideally BB vectorization would learn to vectorize + control flow by applying if-conversion on-the-fly, the + following retains the if-converted loop body even when + only non-if-converted parts took part in BB vectorization. */ + if (flag_tree_slp_vectorize != 0 + && loop_vectorized_call + && ! loop->inner) + { + basic_block bb = loop->header; + bool has_mask_load_store = false; + for (gimple_stmt_iterator gsi = gsi_start_bb (bb); + !gsi_end_p (gsi); gsi_next (&gsi)) + { + gimple *stmt = gsi_stmt (gsi); + if (is_gimple_call (stmt) + && gimple_call_internal_p (stmt) + && (gimple_call_internal_fn (stmt) == IFN_MASK_LOAD + || gimple_call_internal_fn (stmt) == IFN_MASK_STORE)) + { + has_mask_load_store = true; + break; + } + gimple_set_uid (stmt, -1); + gimple_set_visited (stmt, false); + } + if (! has_mask_load_store && vect_slp_bb (bb)) + { + dump_printf_loc (MSG_NOTE, vect_location, + "basic block vectorized\n"); + fold_loop_internal_call (loop_vectorized_call, + boolean_true_node); + loop_vectorized_call = NULL; + ret |= TODO_cleanup_cfg; + } + } + /* If outer loop vectorization fails for LOOP_VECTORIZED guarded + loop, don't vectorize its inner loop; we'll attempt to + vectorize LOOP_VECTORIZED guarded inner loop of the scalar + loop version. */ + if (loop_vectorized_call && loop->inner) + loop->inner->dont_vectorize = true; + return ret; + } + + if (!dbg_cnt (vect_loop)) + { + /* Free existing information if loop is analyzed with some + assumptions. */ + if (loop_constraint_set_p (loop, LOOP_C_FINITE)) + vect_free_loop_info_assumptions (loop); + return ret; + } + + if (loop_vectorized_call) + set_uid_loop_bbs (loop_vinfo, loop_vectorized_call); + + unsigned HOST_WIDE_INT bytes; + if (current_vector_size.is_constant (&bytes)) + dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location, + "loop vectorized vectorized using " + HOST_WIDE_INT_PRINT_UNSIGNED " byte " + "vectors\n", bytes); + else + dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location, + "loop vectorized using variable length vectors\n"); + + loop_p new_loop = vect_transform_loop (loop_vinfo); + (*num_vectorized_loops)++; + /* Now that the loop has been vectorized, allow it to be unrolled + etc. */ + loop->force_vectorize = false; + + if (loop->simduid) + { + simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf); + if (!simduid_to_vf_htab) + simduid_to_vf_htab = new hash_table (15); + simduid_to_vf_data->simduid = DECL_UID (loop->simduid); + simduid_to_vf_data->vf = loop_vinfo->vectorization_factor; + *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT) + = simduid_to_vf_data; + } + + if (loop_vectorized_call) + { + fold_loop_internal_call (loop_vectorized_call, boolean_true_node); + loop_vectorized_call = NULL; + ret |= TODO_cleanup_cfg; + } + if (loop_dist_alias_call) + { + tree value = gimple_call_arg (loop_dist_alias_call, 1); + fold_loop_internal_call (loop_dist_alias_call, value); + loop_dist_alias_call = NULL; + ret |= TODO_cleanup_cfg; + } + + /* Epilogue of vectorized loop must be vectorized too. */ + if (new_loop) + ret |= try_vectorize_loop_1 (simduid_to_vf_htab, num_vectorized_loops, + new_loop, loop_vinfo, NULL, NULL); + + return ret; +} + +/* Try to vectorize LOOP. */ + +static unsigned +try_vectorize_loop (hash_table *&simduid_to_vf_htab, + unsigned *num_vectorized_loops, loop_p loop) +{ + if (!((flag_tree_loop_vectorize + && optimize_loop_nest_for_speed_p (loop)) + || loop->force_vectorize)) + return 0; + + return try_vectorize_loop_1 (simduid_to_vf_htab, num_vectorized_loops, + loop, NULL, + vect_loop_vectorized_call (loop), + vect_loop_dist_alias_call (loop)); +} + + /* Function vectorize_loops. Entry point to loop vectorization phase. */ @@ -674,7 +826,6 @@ vectorize_loops (void) hash_table *simd_array_to_simduid_htab = NULL; bool any_ifcvt_loops = false; unsigned ret = 0; - struct loop *new_loop; vect_loops_num = number_of_loops (cfun); @@ -727,157 +878,18 @@ vectorize_loops (void) = get_loop (cfun, tree_to_shwi (arg)); if (vector_loop && vector_loop != loop) { - loop = vector_loop; /* Make sure we don't vectorize it twice. */ - loop->dont_vectorize = true; - goto try_vectorize; + vector_loop->dont_vectorize = true; + ret |= try_vectorize_loop (simduid_to_vf_htab, + &num_vectorized_loops, + vector_loop); } } } } else - { - loop_vec_info loop_vinfo, orig_loop_vinfo; - gimple *loop_vectorized_call, *loop_dist_alias_call; - try_vectorize: - if (!((flag_tree_loop_vectorize - && optimize_loop_nest_for_speed_p (loop)) - || loop->force_vectorize)) - continue; - orig_loop_vinfo = NULL; - loop_vectorized_call = vect_loop_vectorized_call (loop); - loop_dist_alias_call = vect_loop_dist_alias_call (loop); - vectorize_epilogue: - vect_location = find_loop_location (loop); - if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOCATION - && dump_enabled_p ()) - dump_printf (MSG_NOTE, "\nAnalyzing loop at %s:%d\n", - LOCATION_FILE (vect_location), - LOCATION_LINE (vect_location)); - - loop_vinfo = vect_analyze_loop (loop, orig_loop_vinfo); - loop->aux = loop_vinfo; - - if (!loop_vinfo || !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo)) - { - /* Free existing information if loop is analyzed with some - assumptions. */ - if (loop_constraint_set_p (loop, LOOP_C_FINITE)) - vect_free_loop_info_assumptions (loop); - - /* If we applied if-conversion then try to vectorize the - BB of innermost loops. - ??? Ideally BB vectorization would learn to vectorize - control flow by applying if-conversion on-the-fly, the - following retains the if-converted loop body even when - only non-if-converted parts took part in BB vectorization. */ - if (flag_tree_slp_vectorize != 0 - && loop_vectorized_call - && ! loop->inner) - { - basic_block bb = loop->header; - bool has_mask_load_store = false; - for (gimple_stmt_iterator gsi = gsi_start_bb (bb); - !gsi_end_p (gsi); gsi_next (&gsi)) - { - gimple *stmt = gsi_stmt (gsi); - if (is_gimple_call (stmt) - && gimple_call_internal_p (stmt) - && (gimple_call_internal_fn (stmt) == IFN_MASK_LOAD - || gimple_call_internal_fn (stmt) == IFN_MASK_STORE)) - { - has_mask_load_store = true; - break; - } - gimple_set_uid (stmt, -1); - gimple_set_visited (stmt, false); - } - if (! has_mask_load_store && vect_slp_bb (bb)) - { - dump_printf_loc (MSG_NOTE, vect_location, - "basic block vectorized\n"); - fold_loop_internal_call (loop_vectorized_call, - boolean_true_node); - loop_vectorized_call = NULL; - ret |= TODO_cleanup_cfg; - } - } - /* If outer loop vectorization fails for LOOP_VECTORIZED guarded - loop, don't vectorize its inner loop; we'll attempt to - vectorize LOOP_VECTORIZED guarded inner loop of the scalar - loop version. */ - if (loop_vectorized_call && loop->inner) - loop->inner->dont_vectorize = true; - continue; - } - - if (!dbg_cnt (vect_loop)) - { - /* We may miss some if-converted loops due to - debug counter. Set any_ifcvt_loops to visit - them at finalization. */ - any_ifcvt_loops = true; - /* Free existing information if loop is analyzed with some - assumptions. */ - if (loop_constraint_set_p (loop, LOOP_C_FINITE)) - vect_free_loop_info_assumptions (loop); - - break; - } - - if (loop_vectorized_call) - set_uid_loop_bbs (loop_vinfo, loop_vectorized_call); - - unsigned HOST_WIDE_INT bytes; - if (current_vector_size.is_constant (&bytes)) - dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location, - "loop vectorized vectorized using " - HOST_WIDE_INT_PRINT_UNSIGNED " byte " - "vectors\n", bytes); - else - dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location, - "loop vectorized using variable length vectors\n"); - - new_loop = vect_transform_loop (loop_vinfo); - num_vectorized_loops++; - /* Now that the loop has been vectorized, allow it to be unrolled - etc. */ - loop->force_vectorize = false; - - if (loop->simduid) - { - simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf); - if (!simduid_to_vf_htab) - simduid_to_vf_htab = new hash_table (15); - simduid_to_vf_data->simduid = DECL_UID (loop->simduid); - simduid_to_vf_data->vf = loop_vinfo->vectorization_factor; - *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT) - = simduid_to_vf_data; - } - - if (loop_vectorized_call) - { - fold_loop_internal_call (loop_vectorized_call, boolean_true_node); - loop_vectorized_call = NULL; - ret |= TODO_cleanup_cfg; - } - if (loop_dist_alias_call) - { - tree value = gimple_call_arg (loop_dist_alias_call, 1); - fold_loop_internal_call (loop_dist_alias_call, value); - loop_dist_alias_call = NULL; - ret |= TODO_cleanup_cfg; - } - - if (new_loop) - { - /* Epilogue of vectorized loop must be vectorized too. */ - vect_loops_num = number_of_loops (cfun); - loop = new_loop; - orig_loop_vinfo = loop_vinfo; /* To pass vect_analyze_loop. */ - goto vectorize_epilogue; - } - } + ret |= try_vectorize_loop (simduid_to_vf_htab, &num_vectorized_loops, + loop); vect_location = UNKNOWN_LOCATION; @@ -914,18 +926,16 @@ vectorize_loops (void) } } - for (i = 1; i < vect_loops_num; i++) + for (i = 1; i < number_of_loops (cfun); i++) { loop_vec_info loop_vinfo; bool has_mask_store; loop = get_loop (cfun, i); - if (!loop) + if (!loop || !loop->aux) continue; loop_vinfo = (loop_vec_info) loop->aux; - has_mask_store = false; - if (loop_vinfo) - has_mask_store = LOOP_VINFO_HAS_MASK_STORE (loop_vinfo); + has_mask_store = LOOP_VINFO_HAS_MASK_STORE (loop_vinfo); delete loop_vinfo; if (has_mask_store && targetm.vectorize.empty_mask_is_expensive (IFN_MASK_STORE)) -- 2.11.4.GIT