1 /* FMA steering optimization pass for Cortex-A57.
2 Copyright (C) 2015-2018 Free Software Foundation, Inc.
3 Contributed by ARM Ltd.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #define IN_TARGET_CODE 1
26 #include "coretypes.h"
31 #include "insn-config.h"
37 #include "insn-attr.h"
39 #include "tree-pass.h"
40 #include "regrename.h"
41 #include "aarch64-protos.h"
43 /* For better performance, the destination of FMADD/FMSUB instructions should
44 have the same parity as their accumulator register if the accumulator
45 contains the result of a previous FMUL or FMADD/FMSUB instruction if
46 targetting Cortex-A57 processors. Performance is also increased by
47 otherwise keeping a good balance in the parity of the destination register
48 of FMUL or FMADD/FMSUB.
50 This pass ensure that registers are renamed so that these conditions hold.
51 We reuse the existing register renaming facility from regrename.c to build
52 dependency chains and expose candidate registers for renaming.
55 The algorithm has three steps:
57 First, the functions of the register renaming pass are called. These
58 analyze the instructions and produce a list of def/use chains of
61 Next, this information is used to build trees of multiply and
62 multiply-accumulate instructions. The roots of these trees are any
63 multiply, or any multiply-accumulate whose accumulator is not dependent on
64 a multiply or multiply-accumulate instruction. A child is added to the
65 tree where a dependency chain exists between the result of the parent
66 instruction and the accumulator operand of the child, as in the diagram
71 fmadd s0, s1, s1, s2 fmadd s4, s1, s1 s2
75 Trees made of a single instruction are permitted.
77 Finally, renaming is performed. The parity of the destination register at
78 the root of a tree is checked against the current balance of multiply and
79 multiply-accumulate on each pipeline. If necessary, the root of a tree is
80 renamed, in which case the rest of the tree is then renamed to keep the same
81 parity in the destination registers of all instructions in the tree. */
85 /* Forward declarations. */
88 class func_fma_steering
;
90 /* Dependencies between FMUL or FMADD/FMSUB instructions and subsequent
91 FMADD/FMSUB instructions form a graph. This is because alternatives can
92 make a register be set by several FMUL or FMADD/FMSUB instructions in
93 different basic blocks and because of loops. For ease of browsing, the
94 connected components of this graph are broken up into forests of trees.
95 Forests are represented by fma_forest objects, contained in the fma_forests
96 list. Using a separate object for the forests allows for a better use of
97 memory as there is some information that is global to each forest, such as
98 the number of FMSUB and FMADD/FMSUB instructions currently scheduled on each
99 floating-point execution pipelines. */
104 fma_forest (func_fma_steering
*, fma_root_node
*, int);
108 std::list
<fma_root_node
*> *get_roots ();
109 func_fma_steering
*get_globals ();
110 int get_target_parity ();
111 void fma_node_created (fma_node
*);
112 void merge_forest (fma_forest
*);
117 /* The list of roots that form this forest. */
118 std::list
<fma_root_node
*> *m_roots
;
120 /* Target parity the destination register of all FMUL and FMADD/FMSUB
121 instructions in this forest should have. */
124 /* Link to the instance of func_fma_steering holding data related to the
125 FMA steering of the current function (cfun). */
126 func_fma_steering
*m_globals
;
128 /* Identifier for the forest (used for dumps). */
131 /* Total number of nodes in the forest (for statistics). */
138 fma_node (fma_node
*parent
, du_chain
*chain
);
142 fma_forest
*get_forest ();
143 std::list
<fma_node
*> *get_children ();
144 rtx_insn
*get_insn ();
145 void add_child (fma_node
*);
147 void set_head (du_head
*);
148 void rename (fma_forest
*);
149 void dump_info (fma_forest
*);
152 /* Root node that lead to this node. */
153 fma_root_node
*m_root
;
155 /* The parent node of this node. If the node belong to a chain with several
156 parent nodes, the first one encountered in a depth-first search is chosen
157 as canonical parent. */
160 /* The list of child nodes. If a chain contains several parent nodes, one is
161 chosen as canonical parent and the others will have no children. */
162 std::list
<fma_node
*> *m_children
;
164 /* The associated DU_HEAD chain that the insn represented by this object
165 is (one of) the root of. When a chain contains several roots, the non
166 canonical ones have this field set to NULL. */
167 struct du_head
*m_head
;
169 /* The FMUL or FMADD/FMSUB instruction this object corresponds to. */
173 class fma_root_node
: public fma_node
176 fma_root_node (func_fma_steering
*, du_chain
*, int);
178 fma_forest
*get_forest ();
179 void set_forest (fma_forest
*);
180 void dump_info (fma_forest
*);
183 /* The forest this node belonged to when it was created. */
184 fma_forest
*m_forest
;
187 /* Class holding all data and methods relative to the FMA steering of a given
188 function. The FMA steering pass could then run in parallel for different
191 class func_fma_steering
194 func_fma_steering ();
195 ~func_fma_steering ();
197 int get_fpu_balance ();
198 void remove_forest (fma_forest
*);
199 bool put_node (fma_node
*);
200 void update_balance (int);
201 fma_node
*get_fma_node (rtx_insn
*);
202 void analyze_fma_fmul_insn (fma_forest
*, du_chain
*, du_head_p
);
203 void execute_fma_steering ();
206 void dfs (void (*) (fma_forest
*), void (*) (fma_forest
*, fma_root_node
*),
207 void (*) (fma_forest
*, fma_node
*), bool);
209 void rename_fma_trees ();
211 /* Mapping between FMUL or FMADD/FMSUB instructions and the associated
212 fma_node object. Used when analyzing an instruction that is a root of
213 a chain to find if such an object was created because this instruction
214 is also a use in another chain. */
215 hash_map
<rtx_insn
*, fma_node
*> *m_insn_fma_head_map
;
217 /* A list of all the forests in a given function. */
218 std::list
<fma_forest
*> m_fma_forests
;
220 /* Balance of FMUL and FMADD/FMSUB instructions between the two FPU
222 < 0: more instruction dispatched to the first pipeline
223 == 0: perfect balance
224 > 0: more instruction dispatched to the second pipeline. */
227 /* Identifier for the next forest created. */
228 int m_next_forest_id
;
231 /* Rename the register HEAD->regno in all the insns in the chain HEAD to any
232 register not in the set UNAVAILABLE. Adapted from rename_chains in
236 rename_single_chain (du_head_p head
, HARD_REG_SET
*unavailable
)
240 struct du_chain
*tmp
;
241 int reg
= head
->regno
;
242 enum reg_class super_class
= NO_REGS
;
244 if (head
->cannot_rename
)
247 if (fixed_regs
[reg
] || global_regs
[reg
]
248 || (frame_pointer_needed
&& reg
== HARD_FRAME_POINTER_REGNUM
))
251 /* Iterate over elements in the chain in order to:
252 1. Count number of uses, and narrow the set of registers we can
254 2. Compute the superunion of register classes in this chain. */
255 for (tmp
= head
->first
; tmp
; tmp
= tmp
->next_use
)
257 if (DEBUG_INSN_P (tmp
->insn
))
260 IOR_COMPL_HARD_REG_SET (*unavailable
, reg_class_contents
[tmp
->cl
]);
261 super_class
= reg_class_superunion
[(int) super_class
][(int) tmp
->cl
];
267 best_new_reg
= find_rename_reg (head
, super_class
, unavailable
, reg
,
272 fprintf (dump_file
, "Register %s in insn %d", reg_names
[reg
],
273 INSN_UID (head
->first
->insn
));
274 if (head
->need_caller_save_reg
)
275 fprintf (dump_file
, " crosses a call");
278 if (best_new_reg
== reg
)
281 fprintf (dump_file
, "; no available better choice\n");
285 if (regrename_do_replace (head
, best_new_reg
))
288 fprintf (dump_file
, ", renamed as %s\n", reg_names
[best_new_reg
]);
289 df_set_regs_ever_live (best_new_reg
, true);
294 fprintf (dump_file
, ", renaming as %s failed\n",
295 reg_names
[best_new_reg
]);
301 /* Return whether T is the attribute of a FMADD/FMSUB-like instruction. */
304 is_fmac_op (enum attr_type t
)
306 return (t
== TYPE_FMACS
) || (t
== TYPE_FMACD
) || (t
== TYPE_NEON_FP_MLA_S
);
309 /* Return whether T is the attribute of a FMUL instruction. */
312 is_fmul_op (enum attr_type t
)
314 return (t
== TYPE_FMULS
) || (t
== TYPE_FMULD
) || (t
== TYPE_NEON_FP_MUL_S
);
317 /* Return whether INSN is an FMUL (if FMUL_OK is true) or FMADD/FMSUB
321 is_fmul_fmac_insn (rtx_insn
*insn
, bool fmul_ok
)
325 if (!NONDEBUG_INSN_P (insn
))
328 if (recog_memoized (insn
) < 0)
331 /* Only consider chain(s) this instruction is a root of if this is an FMUL or
332 FMADD/FMSUB instruction. This allows to avoid browsing chains of all
333 instructions for FMUL or FMADD/FMSUB in them. */
334 t
= get_attr_type (insn
);
335 return is_fmac_op (t
) || (fmul_ok
&& is_fmul_op (t
));
340 * Class fma_forest method definitions.
343 fma_forest::fma_forest (func_fma_steering
*fma_steer
, fma_root_node
*fma_root
,
346 memset (this, 0, sizeof (*this));
347 this->m_globals
= fma_steer
;
348 this->m_roots
= new std::list
<fma_root_node
*>;
349 this->m_roots
->push_back (fma_root
);
353 fma_forest::~fma_forest ()
355 delete this->m_roots
;
359 fma_forest::get_id ()
364 std::list
<fma_root_node
*> *
365 fma_forest::get_roots ()
367 return this->m_roots
;
371 fma_forest::get_globals ()
373 return this->m_globals
;
377 fma_forest::get_target_parity ()
379 return this->m_target_parity
;
382 /* Act on the creation of NODE by updating statistics in FOREST and adding an
383 entry for it in the func_fma_steering hashmap. */
385 void fma_forest::fma_node_created (fma_node
*node
)
387 bool created
= !this->m_globals
->put_node (node
);
389 gcc_assert (created
);
393 /* Merge REF_FOREST and OTHER_FOREST together, making REF_FOREST the canonical
394 fma_forest object to represent both. */
397 fma_forest::merge_forest (fma_forest
*other_forest
)
399 std::list
<fma_root_node
*> *other_roots
;
400 std::list
<fma_root_node
*>::iterator other_root_iter
;
402 if (this == other_forest
)
405 other_roots
= other_forest
->m_roots
;
407 /* Update root nodes' pointer to forest. */
408 for (other_root_iter
= other_roots
->begin ();
409 other_root_iter
!= other_roots
->end (); ++other_root_iter
)
410 (*other_root_iter
)->set_forest (this);
412 /* Remove other_forest from the list of forests and move its tree roots in
413 the list of tree roots of ref_forest. */
414 this->m_globals
->remove_forest (other_forest
);
415 this->m_roots
->splice (this->m_roots
->begin (), *other_roots
);
416 this->m_nb_nodes
+= other_forest
->m_nb_nodes
;
421 /* Dump information about the forest FOREST. */
424 fma_forest::dump_info ()
426 gcc_assert (dump_file
);
428 fprintf (dump_file
, "Forest #%d has %d nodes\n", this->m_id
,
432 /* Wrapper around fma_forest::dump_info for use as parameter of function
433 pointer type in func_fma_steering::dfs. */
436 dump_forest_info (fma_forest
*forest
)
438 forest
->dump_info ();
441 /* Dispatch forest to the least utilized pipeline. */
444 fma_forest::dispatch ()
446 this->m_target_parity
= this->m_roots
->front ()->get_parity ();
447 int fpu_balance
= this->m_globals
->get_fpu_balance ();
448 if (fpu_balance
!= 0)
449 this->m_target_parity
= (fpu_balance
< 0);
452 fprintf (dump_file
, "Target parity for forest #%d: %s\n", this->m_id
,
453 this->m_target_parity
? "odd" : "even");
456 /* Wrapper around fma_forest::dispatch for use as parameter of function pointer
457 type in func_fma_steering::dfs. */
460 dispatch_forest (fma_forest
*forest
)
465 fma_node::fma_node (fma_node
*parent
, du_chain
*chain
)
467 memset (this, 0, sizeof (*this));
468 this->m_parent
= parent
;
469 this->m_children
= new std::list
<fma_node
*>;
470 this->m_insn
= chain
->insn
;
471 /* root_p () cannot be used to check for root before root is set. */
472 if (this->m_parent
== this)
473 this->m_root
= static_cast<fma_root_node
*> (parent
);
476 this->m_root
= parent
->m_root
;
477 this->get_forest ()->fma_node_created (this);
481 fma_node::~fma_node ()
483 delete this->m_children
;
486 std::list
<fma_node
*> *
487 fma_node::get_children ()
489 return this->m_children
;
493 fma_node::get_insn ()
499 fma_node::set_head (du_head
*head
)
501 gcc_assert (!this->m_head
);
505 /* Add a child to this node in the list of children. */
508 fma_node::add_child (fma_node
*child
)
510 this->m_children
->push_back (child
);
513 /* Return the parity of the destination register of the instruction represented
517 fma_node::get_parity ()
519 return this->m_head
->regno
% 2;
522 /* Get the actual forest associated with a non root node as the one the node
523 points to might have been merged into another one. In that case the pointer
524 in the root nodes are updated so we return the forest pointer of a root node
525 pointed to by the initial forest. Despite being a oneliner, this method is
526 defined here as it references a method from fma_root_node. */
529 fma_node::get_forest ()
531 return this->m_root
->get_forest ();
534 /* Return whether a node is a root node. */
539 return this->m_root
== this;
542 /* Dump information about the children of node FMA_NODE in forest FOREST. */
545 fma_node::dump_info (ATTRIBUTE_UNUSED fma_forest
*forest
)
547 struct du_chain
*chain
;
548 std::list
<fma_node
*>::iterator fma_child
;
550 gcc_assert (dump_file
);
552 if (this->get_children ()->empty ())
555 fprintf (dump_file
, "Instruction(s)");
556 for (chain
= this->m_head
->first
; chain
; chain
= chain
->next_use
)
558 if (!is_fmul_fmac_insn (chain
->insn
, true))
561 if (chain
->loc
!= &SET_DEST (PATTERN (chain
->insn
)))
564 fprintf (dump_file
, " %d", INSN_UID (chain
->insn
));
567 fprintf (dump_file
, " is(are) accumulator dependency of instructions");
568 for (fma_child
= this->get_children ()->begin ();
569 fma_child
!= this->get_children ()->end (); fma_child
++)
570 fprintf (dump_file
, " %d", INSN_UID ((*fma_child
)->m_insn
));
571 fprintf (dump_file
, "\n");
574 /* Wrapper around fma_node::dump_info for use as parameter of function pointer
575 type in func_fma_steering::dfs. */
578 dump_tree_node_info (fma_forest
*forest
, fma_node
*node
)
580 node
->dump_info (forest
);
583 /* Rename the destination register of a single FMUL or FMADD/FMSUB instruction
584 represented by FMA_NODE to a register that respect the target parity for
585 FOREST or with same parity of the instruction represented by its parent node
589 fma_node::rename (fma_forest
*forest
)
591 int cur_parity
, target_parity
;
593 /* This is alternate root of a chain and thus has no children. It will be
594 renamed when processing the canonical root for that chain. */
598 target_parity
= forest
->get_target_parity ();
600 target_parity
= this->m_parent
->get_parity ();
601 cur_parity
= this->get_parity ();
603 /* Rename if parity differs. */
604 if (cur_parity
!= target_parity
)
606 rtx_insn
*insn
= this->m_insn
;
607 HARD_REG_SET unavailable
;
613 unsigned cur_dest_reg
= this->m_head
->regno
;
615 fprintf (dump_file
, "FMA or FMUL at insn %d but destination "
616 "register (%s) has different parity from expected to "
617 "maximize FPU pipeline utilization\n", INSN_UID (insn
),
618 reg_names
[cur_dest_reg
]);
621 /* Don't clobber traceback for noreturn functions. */
622 CLEAR_HARD_REG_SET (unavailable
);
623 if (frame_pointer_needed
)
625 add_to_hard_reg_set (&unavailable
, Pmode
, FRAME_POINTER_REGNUM
);
626 add_to_hard_reg_set (&unavailable
, Pmode
, HARD_FRAME_POINTER_REGNUM
);
629 /* Exclude registers with wrong parity. */
630 mode
= GET_MODE (SET_DEST (PATTERN (insn
)));
631 for (reg
= cur_parity
; reg
< FIRST_PSEUDO_REGISTER
; reg
+= 2)
632 add_to_hard_reg_set (&unavailable
, mode
, reg
);
634 if (!rename_single_chain (this->m_head
, &unavailable
))
637 fprintf (dump_file
, "Destination register of insn %d could not be "
638 "renamed. Dependent FMA insns will use this parity from "
639 "there on.\n", INSN_UID (insn
));
642 cur_parity
= target_parity
;
645 forest
->get_globals ()->update_balance (cur_parity
);
648 /* Wrapper around fma_node::dump_info for use as parameter of function pointer
649 type in func_fma_steering::dfs. */
652 rename_fma_node (fma_forest
*forest
, fma_node
*node
)
654 node
->rename (forest
);
657 fma_root_node::fma_root_node (func_fma_steering
*globals
, du_chain
*chain
,
658 int id
) : fma_node (this, chain
)
660 this->m_forest
= new fma_forest (globals
, this, id
);
661 this->m_forest
->fma_node_created (this);
665 fma_root_node::get_forest ()
667 return this->m_forest
;
671 fma_root_node::set_forest (fma_forest
*ref_forest
)
673 this->m_forest
= ref_forest
;
676 /* Dump information about the roots of forest FOREST. */
679 fma_root_node::dump_info (fma_forest
*forest
)
681 gcc_assert (dump_file
);
683 if (this == forest
->get_roots ()->front ())
684 fprintf (dump_file
, "Instruction(s) at root of forest #%d:",
686 fprintf (dump_file
, " %d", INSN_UID (this->m_insn
));
687 if (this == forest
->get_roots ()->back ())
688 fprintf (dump_file
, "\n");
691 /* Wrapper around fma_root_node::dump_info for use as parameter of function
692 pointer type in func_fma_steering::dfs. */
695 dump_tree_root_info (fma_forest
*forest
, fma_root_node
*node
)
697 node
->dump_info (forest
);
700 func_fma_steering::func_fma_steering () : m_fpu_balance (0)
702 this->m_insn_fma_head_map
= new hash_map
<rtx_insn
*, fma_node
*>;
703 this->m_fma_forests
.clear ();
704 this->m_next_forest_id
= 0;
707 func_fma_steering::~func_fma_steering ()
709 delete this->m_insn_fma_head_map
;
713 func_fma_steering::get_fpu_balance ()
715 return this->m_fpu_balance
;
719 func_fma_steering::remove_forest (fma_forest
*forest
)
721 this->m_fma_forests
.remove (forest
);
724 /* Memorize the mapping of this instruction to its fma_node object and return
725 whether such a mapping existed. */
728 func_fma_steering::put_node (fma_node
*node
)
730 return this->m_insn_fma_head_map
->put (node
->get_insn (), node
);
733 /* Update the current balance considering a node with the given PARITY. */
736 func_fma_steering::update_balance (int parity
)
738 this->m_fpu_balance
= parity
? this->m_fpu_balance
+ 1
739 : this->m_fpu_balance
- 1;
742 /* Return whether an fma_node object exists for instruction INSN and, if not,
743 allocate one in *RET. */
746 func_fma_steering::get_fma_node (rtx_insn
*insn
)
750 fma_slot
= this->m_insn_fma_head_map
->get (insn
);
756 /* Allocate and initialize fma_node objects for the FMUL or FMADD/FMSUB
757 instruction in CHAIN->insn and its dependent FMADD/FMSUB instructions, all
758 part of FOREST. For the children, the associated head is left untouched
759 (and thus null) as this function will be called again when considering the
760 chain where they are def. For the parent, the chain is given in HEAD. */
763 func_fma_steering::analyze_fma_fmul_insn (fma_forest
*ref_forest
,
764 du_chain
*chain
, du_head_p head
)
767 fma_node
*node
= this->get_fma_node (chain
->insn
);
769 /* This is a root node. */
772 fma_root_node
*root_node
;
774 root_node
= new fma_root_node (this, chain
, this->m_next_forest_id
++);
775 forest
= root_node
->get_forest ();
778 /* Until proved otherwise, assume this root is not part of an existing
779 forest and thus add its forest to the list of forests. */
780 this->m_fma_forests
.push_back (forest
);
783 forest
= node
->get_forest ();
785 node
->set_head (head
);
787 /* fma_node is part of a chain with several defs, one of them having already
788 been processed. The root of that already processed def is the canonical
789 one and the root of fma_node is added to its forest. No need to process
790 the children nodes as they were already processed when the other def was
794 ref_forest
->merge_forest (forest
);
798 for (chain
= head
->first
; chain
; chain
= chain
->next_use
)
801 rtx fma_rtx
, *accum_rtx_p
;
803 if (!is_fmul_fmac_insn (chain
->insn
, false))
807 fma_rtx
= SET_SRC (PATTERN (chain
->insn
));
808 /* FMA is negated. */
809 if (GET_CODE (fma_rtx
) == NEG
)
810 fma_rtx
= XEXP (fma_rtx
, 0);
811 /* Get accumulator rtx. */
812 accum_rtx_p
= &XEXP (fma_rtx
, 2);
813 /* Accumulator is negated. */
814 if (!REG_P (*accum_rtx_p
))
815 accum_rtx_p
= &XEXP (*accum_rtx_p
, 0);
817 /* This du_chain structure is not for the accumulator register. */
818 if (accum_rtx_p
!= chain
->loc
)
821 /* If object already created, this is a loop carried dependency. We
822 don't include this object in the children as we want trees for
823 rename_fma_trees to not be an infinite loop. */
824 if (this->get_fma_node (chain
->insn
))
827 child_fma
= new fma_node (node
, chain
);
829 /* Memorize the mapping of this instruction to its fma_node object
830 as it will be processed for the chain starting at its destination
833 /* Link to siblings. */
834 node
->add_child (child_fma
);
838 /* Perform a depth-first search of the forests of fma_node in
839 THIS->m_fma_forests, calling PROCESS_FOREST () on each fma_forest object in
840 THIS->m_fma_forests list, PROCESS_ROOT () on each tree root and
841 PROCESS_NODE () on each node. If FREE is true, free all std::list in the
845 func_fma_steering::dfs (void (*process_forest
) (fma_forest
*),
846 void (*process_root
) (fma_forest
*, fma_root_node
*),
847 void (*process_node
) (fma_forest
*, fma_node
*),
850 auto_vec
<fma_node
*> to_process
;
851 auto_vec
<fma_node
*> to_free
;
852 std::list
<fma_forest
*>::iterator forest_iter
;
854 /* For each forest. */
855 for (forest_iter
= this->m_fma_forests
.begin ();
856 forest_iter
!= this->m_fma_forests
.end (); ++forest_iter
)
858 std::list
<fma_root_node
*>::iterator root_iter
;
861 process_forest (*forest_iter
);
863 /* For each tree root in this forest. */
864 for (root_iter
= (*forest_iter
)->get_roots ()->begin ();
865 root_iter
!= (*forest_iter
)->get_roots ()->end (); ++root_iter
)
868 process_root (*forest_iter
, *root_iter
);
869 to_process
.safe_push (*root_iter
);
872 /* For each tree node in this forest. */
873 while (!to_process
.is_empty ())
876 std::list
<fma_node
*>::iterator child_iter
;
878 node
= to_process
.pop ();
881 process_node (*forest_iter
, node
);
883 for (child_iter
= node
->get_children ()->begin ();
884 child_iter
!= node
->get_children ()->end (); ++child_iter
)
885 to_process
.safe_push (*child_iter
);
887 /* Defer freeing so that the process_node callback can access the
888 parent and children of the node being processed. */
890 to_free
.safe_push (node
);
897 while (!to_free
.is_empty ())
899 fma_node
*node
= to_free
.pop ();
901 delete static_cast<fma_root_node
*> (node
);
909 /* Build the dependency trees of FMUL and FMADD/FMSUB instructions. */
912 func_fma_steering::analyze ()
914 int i
, n_blocks
, *bb_dfs_preorder
;
918 bb_dfs_preorder
= XNEWVEC (int, last_basic_block_for_fn (cfun
));
919 n_blocks
= pre_and_rev_post_order_compute (bb_dfs_preorder
, NULL
, false);
921 /* Browse the graph of basic blocks looking for FMUL or FMADD/FMSUB
923 for (i
= 0; i
< n_blocks
; i
++)
925 bb
= BASIC_BLOCK_FOR_FN (cfun
, bb_dfs_preorder
[i
]);
926 FOR_BB_INSNS (bb
, insn
)
928 operand_rr_info
*dest_op_info
;
929 struct du_chain
*chain
= NULL
;
931 fma_forest
*forest
= NULL
;
932 du_head_p head
= NULL
;
935 if (!is_fmul_fmac_insn (insn
, true))
938 /* Search the chain where this instruction is (one of) the root. */
939 dest_op_info
= insn_rr
[INSN_UID (insn
)].op_info
;
940 dest_regno
= REGNO (SET_DEST (PATTERN (insn
)));
941 for (i
= 0; i
< dest_op_info
->n_chains
; i
++)
943 /* The register tracked by this chain does not match the
944 destination register of insn. */
945 if (dest_op_info
->heads
[i
]->regno
!= dest_regno
)
948 head
= dest_op_info
->heads
[i
];
949 /* The chain was merged in another, find the new head. */
951 head
= regrename_chain_from_id (head
->id
);
953 /* Search the chain element for this instruction and, if another
954 FMUL or FMADD/FMSUB instruction was already processed, note
955 the forest of its tree. */
957 for (chain
= head
->first
; chain
; chain
= chain
->next_use
)
961 if (!is_fmul_fmac_insn (chain
->insn
, true))
964 /* This is a use, continue. */
965 if (chain
->loc
!= &SET_DEST (PATTERN (chain
->insn
)))
968 if (chain
->insn
== insn
)
971 fma_slot
= this->m_insn_fma_head_map
->get (chain
->insn
);
972 if (fma_slot
&& (*fma_slot
)->get_children ())
973 forest
= (*fma_slot
)->get_forest ();
979 /* Due to implementation of regrename, dest register can slip away
980 from regrename's analysis. As a result, there is no chain for
981 the destination register of insn. We simply skip the insn even
982 it is a fmul/fmac instruction. This can happen when the dest
983 register is also a source register of insn and one of the below
984 conditions is satisfied:
985 1) the source reg is setup in larger mode than this insn;
986 2) the source reg is uninitialized;
987 3) the source reg is passed in as parameter. */
988 if (i
< dest_op_info
->n_chains
)
989 this->analyze_fma_fmul_insn (forest
, chain
, head
);
992 free (bb_dfs_preorder
);
995 this->dfs (dump_forest_info
, dump_tree_root_info
, dump_tree_node_info
,
999 /* Perform the renaming of all chains with FMUL or FMADD/FMSUB involved with
1000 the objective of keeping FPU pipeline balanced in term of instructions and
1001 having FMADD/FMSUB with dependencies on previous FMUL or FMADD/FMSUB be
1002 scheduled on the same pipeline. */
1005 func_fma_steering::rename_fma_trees ()
1007 this->dfs (dispatch_forest
, NULL
, rename_fma_node
, true);
1009 if (dump_file
&& !this->m_fma_forests
.empty ())
1011 fprintf (dump_file
, "Function %s has ", current_function_name ());
1012 if (this->m_fpu_balance
== 0)
1013 fprintf (dump_file
, "perfect balance of FMUL/FMA chains between the "
1014 "two FPU pipelines\n");
1015 else if (this->m_fpu_balance
> 0)
1016 fprintf (dump_file
, "%d more FMUL/FMA chains scheduled on the second "
1017 "FPU pipeline\n", this->m_fpu_balance
);
1018 else /* this->m_fpu_balance < 0 */
1019 fprintf (dump_file
, "%d more FMUL/FMA chains scheduled on the first "
1020 "FPU pipeline\n", - this->m_fpu_balance
);
1024 /* Execute FMA steering pass. */
1027 func_fma_steering::execute_fma_steering ()
1029 df_set_flags (DF_LR_RUN_DCE
);
1030 df_note_add_problem ();
1032 df_set_flags (DF_DEFER_INSN_RESCAN
);
1034 regrename_init (true);
1035 regrename_analyze (NULL
);
1037 this->rename_fma_trees ();
1038 regrename_finish ();
1041 const pass_data pass_data_fma_steering
=
1043 RTL_PASS
, /* type */
1044 "fma_steering", /* name */
1045 OPTGROUP_NONE
, /* optinfo_flags */
1046 TV_NONE
, /* tv_id */
1047 0, /* properties_required */
1048 0, /* properties_provided */
1049 0, /* properties_destroyed */
1050 0, /* todo_flags_start */
1051 TODO_df_finish
, /* todo_flags_finish */
1054 class pass_fma_steering
: public rtl_opt_pass
1057 pass_fma_steering (gcc::context
*ctxt
)
1058 : rtl_opt_pass (pass_data_fma_steering
, ctxt
)
1061 /* opt_pass methods: */
1062 virtual bool gate (function
*)
1064 return (aarch64_tune_params
.extra_tuning_flags
1065 & AARCH64_EXTRA_TUNE_RENAME_FMA_REGS
)
1069 virtual unsigned int execute (function
*)
1071 func_fma_steering
*fma_steering
= new func_fma_steering
;
1072 fma_steering
->execute_fma_steering ();
1073 delete fma_steering
;
1077 }; // class pass_fma_steering
1079 /* Create a new fma steering pass instance. */
1082 make_pass_fma_steering (gcc::context
*ctxt
)
1084 return new pass_fma_steering (ctxt
);