From 470da3b27e6dbeb3286b09dcb1c1b810ac75b276 Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Fri, 25 Aug 2023 11:43:36 +0200 Subject: [PATCH] Apply some TLC to vect_slp_analyze_instance_dependence This refactors things, separating load and store handing, adjusting comments to reflect reality and removing some dead code. * tree-vect-data-refs.cc (vect_slp_analyze_store_dependences): Split out from vect_slp_analyze_node_dependences, remove dead code. (vect_slp_analyze_load_dependences): Split out from vect_slp_analyze_node_dependences, adjust comments. Process queued stores before any disambiguation. (vect_slp_analyze_node_dependences): Remove. (vect_slp_analyze_instance_dependence): Adjust. --- gcc/tree-vect-data-refs.cc | 238 ++++++++++++++++++++------------------------- 1 file changed, 108 insertions(+), 130 deletions(-) diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc index a2caf6cb1c7..0295e256a44 100644 --- a/gcc/tree-vect-data-refs.cc +++ b/gcc/tree-vect-data-refs.cc @@ -670,160 +670,138 @@ vect_slp_analyze_data_ref_dependence (vec_info *vinfo, } -/* Analyze dependences involved in the transform of SLP NODE. STORES - contain the vector of scalar stores of this instance if we are - disambiguating the loads. */ +/* Analyze dependences involved in the transform of a store SLP NODE. */ static bool -vect_slp_analyze_node_dependences (vec_info *vinfo, slp_tree node, - vec stores, - stmt_vec_info last_store_info) +vect_slp_analyze_store_dependences (vec_info *vinfo, slp_tree node) { - /* This walks over all stmts involved in the SLP load/store done + /* This walks over all stmts involved in the SLP store done in NODE verifying we can sink them up to the last stmt in the group. */ - if (DR_IS_WRITE (STMT_VINFO_DATA_REF (SLP_TREE_REPRESENTATIVE (node)))) + stmt_vec_info last_access_info = vect_find_last_scalar_stmt_in_slp (node); + gcc_assert (DR_IS_WRITE (STMT_VINFO_DATA_REF (last_access_info))); + + for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (node).length (); ++k) { - stmt_vec_info last_access_info = vect_find_last_scalar_stmt_in_slp (node); - for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (node).length (); ++k) + stmt_vec_info access_info + = vect_orig_stmt (SLP_TREE_SCALAR_STMTS (node)[k]); + if (access_info == last_access_info) + continue; + data_reference *dr_a = STMT_VINFO_DATA_REF (access_info); + ao_ref ref; + bool ref_initialized_p = false; + for (gimple_stmt_iterator gsi = gsi_for_stmt (access_info->stmt); + gsi_stmt (gsi) != last_access_info->stmt; gsi_next (&gsi)) { - stmt_vec_info access_info - = vect_orig_stmt (SLP_TREE_SCALAR_STMTS (node)[k]); - if (access_info == last_access_info) + gimple *stmt = gsi_stmt (gsi); + if (! gimple_vuse (stmt)) continue; - data_reference *dr_a = STMT_VINFO_DATA_REF (access_info); - ao_ref ref; - bool ref_initialized_p = false; - for (gimple_stmt_iterator gsi = gsi_for_stmt (access_info->stmt); - gsi_stmt (gsi) != last_access_info->stmt; gsi_next (&gsi)) - { - gimple *stmt = gsi_stmt (gsi); - if (! gimple_vuse (stmt)) - continue; - - /* If we couldn't record a (single) data reference for this - stmt we have to resort to the alias oracle. */ - stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt); - data_reference *dr_b = STMT_VINFO_DATA_REF (stmt_info); - if (!dr_b) - { - /* We are moving a store - this means - we cannot use TBAA for disambiguation. */ - if (!ref_initialized_p) - ao_ref_init (&ref, DR_REF (dr_a)); - if (stmt_may_clobber_ref_p_1 (stmt, &ref, false) - || ref_maybe_used_by_stmt_p (stmt, &ref, false)) - return false; - continue; - } - - bool dependent = false; - /* If we run into a store of this same instance (we've just - marked those) then delay dependence checking until we run - into the last store because this is where it will have - been sunk to (and we verify if we can do that as well). */ - if (gimple_visited_p (stmt)) - { - if (stmt_info != last_store_info) - continue; - for (stmt_vec_info &store_info : stores) - { - data_reference *store_dr - = STMT_VINFO_DATA_REF (store_info); - ddr_p ddr = initialize_data_dependence_relation - (dr_a, store_dr, vNULL); - dependent - = vect_slp_analyze_data_ref_dependence (vinfo, ddr); - free_dependence_relation (ddr); - if (dependent) - break; - } - } - else - { - ddr_p ddr = initialize_data_dependence_relation (dr_a, - dr_b, vNULL); - dependent = vect_slp_analyze_data_ref_dependence (vinfo, ddr); - free_dependence_relation (ddr); - } - if (dependent) + /* If we couldn't record a (single) data reference for this + stmt we have to resort to the alias oracle. */ + stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt); + data_reference *dr_b = STMT_VINFO_DATA_REF (stmt_info); + if (!dr_b) + { + /* We are moving a store - this means + we cannot use TBAA for disambiguation. */ + if (!ref_initialized_p) + ao_ref_init (&ref, DR_REF (dr_a)); + if (stmt_may_clobber_ref_p_1 (stmt, &ref, false) + || ref_maybe_used_by_stmt_p (stmt, &ref, false)) return false; + continue; } + + gcc_assert (!gimple_visited_p (stmt)); + + ddr_p ddr = initialize_data_dependence_relation (dr_a, + dr_b, vNULL); + bool dependent = vect_slp_analyze_data_ref_dependence (vinfo, ddr); + free_dependence_relation (ddr); + if (dependent) + return false; } } - else /* DR_IS_READ */ + return true; +} + +/* Analyze dependences involved in the transform of a load SLP NODE. STORES + contain the vector of scalar stores of this instance if we are + disambiguating the loads. */ + +static bool +vect_slp_analyze_load_dependences (vec_info *vinfo, slp_tree node, + vec stores, + stmt_vec_info last_store_info) +{ + /* This walks over all stmts involved in the SLP load done + in NODE verifying we can hoist them up to the first stmt in the + group. */ + stmt_vec_info first_access_info = vect_find_first_scalar_stmt_in_slp (node); + gcc_assert (DR_IS_READ (STMT_VINFO_DATA_REF (first_access_info))); + + for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (node).length (); ++k) { - stmt_vec_info first_access_info - = vect_find_first_scalar_stmt_in_slp (node); - for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (node).length (); ++k) + stmt_vec_info access_info + = vect_orig_stmt (SLP_TREE_SCALAR_STMTS (node)[k]); + if (access_info == first_access_info) + continue; + data_reference *dr_a = STMT_VINFO_DATA_REF (access_info); + ao_ref ref; + bool ref_initialized_p = false; + for (gimple_stmt_iterator gsi = gsi_for_stmt (access_info->stmt); + gsi_stmt (gsi) != first_access_info->stmt; gsi_prev (&gsi)) { - stmt_vec_info access_info - = vect_orig_stmt (SLP_TREE_SCALAR_STMTS (node)[k]); - if (access_info == first_access_info) + gimple *stmt = gsi_stmt (gsi); + if (! gimple_vdef (stmt)) continue; - data_reference *dr_a = STMT_VINFO_DATA_REF (access_info); - ao_ref ref; - bool ref_initialized_p = false; - for (gimple_stmt_iterator gsi = gsi_for_stmt (access_info->stmt); - gsi_stmt (gsi) != first_access_info->stmt; gsi_prev (&gsi)) - { - gimple *stmt = gsi_stmt (gsi); - if (! gimple_vdef (stmt)) - continue; - /* If we couldn't record a (single) data reference for this - stmt we have to resort to the alias oracle. */ - stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt); - data_reference *dr_b = STMT_VINFO_DATA_REF (stmt_info); + stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt); - /* We are hoisting a load - this means we can use - TBAA for disambiguation. */ - if (!ref_initialized_p) - ao_ref_init (&ref, DR_REF (dr_a)); - if (stmt_may_clobber_ref_p_1 (stmt, &ref, true)) - { - if (!dr_b) - return false; - /* Resort to dependence checking below. */ - } - else - /* No dependence. */ + /* If we run into a store of this same instance (we've just + marked those) then delay dependence checking until we run + into the last store because this is where it will have + been sunk to (and we verified that we can do that already). */ + if (gimple_visited_p (stmt)) + { + if (stmt_info != last_store_info) continue; - bool dependent = false; - /* If we run into a store of this same instance (we've just - marked those) then delay dependence checking until we run - into the last store because this is where it will have - been sunk to (and we verify if we can do that as well). */ - if (gimple_visited_p (stmt)) - { - if (stmt_info != last_store_info) - continue; - - for (stmt_vec_info &store_info : stores) - { - data_reference *store_dr - = STMT_VINFO_DATA_REF (store_info); - ddr_p ddr = initialize_data_dependence_relation - (dr_a, store_dr, vNULL); - dependent - = vect_slp_analyze_data_ref_dependence (vinfo, ddr); - free_dependence_relation (ddr); - if (dependent) - break; - } - } - else + for (stmt_vec_info &store_info : stores) { - ddr_p ddr = initialize_data_dependence_relation (dr_a, - dr_b, vNULL); - dependent = vect_slp_analyze_data_ref_dependence (vinfo, ddr); + data_reference *store_dr = STMT_VINFO_DATA_REF (store_info); + ddr_p ddr = initialize_data_dependence_relation + (dr_a, store_dr, vNULL); + bool dependent + = vect_slp_analyze_data_ref_dependence (vinfo, ddr); free_dependence_relation (ddr); + if (dependent) + return false; } + continue; + } + + /* We are hoisting a load - this means we can use TBAA for + disambiguation. */ + if (!ref_initialized_p) + ao_ref_init (&ref, DR_REF (dr_a)); + if (stmt_may_clobber_ref_p_1 (stmt, &ref, true)) + { + /* If we couldn't record a (single) data reference for this + stmt we have to give up now. */ + data_reference *dr_b = STMT_VINFO_DATA_REF (stmt_info); + if (!dr_b) + return false; + ddr_p ddr = initialize_data_dependence_relation (dr_a, + dr_b, vNULL); + bool dependent + = vect_slp_analyze_data_ref_dependence (vinfo, ddr); + free_dependence_relation (ddr); if (dependent) return false; } + /* No dependence. */ } } return true; @@ -850,7 +828,7 @@ vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance) stmt_vec_info last_store_info = NULL; if (store) { - if (! vect_slp_analyze_node_dependences (vinfo, store, vNULL, NULL)) + if (! vect_slp_analyze_store_dependences (vinfo, store)) return false; /* Mark stores in this instance and remember the last one. */ @@ -864,7 +842,7 @@ vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance) /* Verify we can sink loads to the vectorized stmt insert location, special-casing stores of this instance. */ for (slp_tree &load : SLP_INSTANCE_LOADS (instance)) - if (! vect_slp_analyze_node_dependences (vinfo, load, + if (! vect_slp_analyze_load_dependences (vinfo, load, store ? SLP_TREE_SCALAR_STMTS (store) : vNULL, last_store_info)) -- 2.11.4.GIT