gcc/lra-spills.c

   1 /* Change pseudos by memory.
   2    Copyright (C) 2010-2015 Free Software Foundation, Inc.
   3    Contributed by Vladimir Makarov <vmakarov@redhat.com>.
   4
   5 This file is part of GCC.
   6
   7 GCC is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 3, or (at your option) any later
  10 version.
  11
  12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with GCC; see the file COPYING3.  If not see
  19 <http://www.gnu.org/licenses/>.  */
  20
  21
  22 /* This file contains code for a pass to change spilled pseudos into
  23    memory.
  24
  25    The pass creates necessary stack slots and assigns spilled pseudos
  26    to the stack slots in following way:
  27
  28    for all spilled pseudos P most frequently used first do
  29      for all stack slots S do
  30        if P doesn't conflict with pseudos assigned to S then
  31          assign S to P and goto to the next pseudo process
  32        end
  33      end
  34      create new stack slot S and assign P to S
  35    end
  36
  37    The actual algorithm is bit more complicated because of different
  38    pseudo sizes.
  39
  40    After that the code changes spilled pseudos (except ones created
  41    from scratches) by corresponding stack slot memory in RTL.
  42
  43    If at least one stack slot was created, we need to run more passes
  44    because we have new addresses which should be checked and because
  45    the old address displacements might change and address constraints
  46    (or insn memory constraints) might not be satisfied any more.
  47
  48    For some targets, the pass can spill some pseudos into hard
  49    registers of different class (usually into vector registers)
  50    instead of spilling them into memory if it is possible and
  51    profitable.  Spilling GENERAL_REGS pseudo into SSE registers for
  52    Intel Corei7 is an example of such optimization.  And this is
  53    actually recommended by Intel optimization guide.
  54
  55    The file also contains code for final change of pseudos on hard
  56    regs correspondingly assigned to them.  */
  57
  58 #include "config.h"
  59 #include "system.h"
  60 #include "coretypes.h"
  61 #include "tm.h"
  62 #include "rtl.h"
  63 #include "tm_p.h"
  64 #include "insn-config.h"
  65 #include "recog.h"
  66 #include "output.h"
  67 #include "regs.h"
  68 #include "hard-reg-set.h"
  69 #include "flags.h"
  70 #include "hashtab.h"
  71 #include "hash-set.h"
  72 #include "vec.h"
  73 #include "machmode.h"
  74 #include "input.h"
  75 #include "function.h"
  76 #include "symtab.h"
  77 #include "statistics.h"
  78 #include "double-int.h"
  79 #include "real.h"
  80 #include "fixed-value.h"
  81 #include "alias.h"
  82 #include "wide-int.h"
  83 #include "inchash.h"
  84 #include "tree.h"
  85 #include "expmed.h"
  86 #include "dojump.h"
  87 #include "explow.h"
  88 #include "calls.h"
  89 #include "emit-rtl.h"
  90 #include "varasm.h"
  91 #include "stmt.h"
  92 #include "expr.h"
  93 #include "predict.h"
  94 #include "dominance.h"
  95 #include "cfg.h"
  96 #include "cfgrtl.h"
  97 #include "basic-block.h"
  98 #include "except.h"
  99 #include "timevar.h"
 100 #include "target.h"
 101 #include "lra-int.h"
 102 #include "ira.h"
 103 #include "df.h"
 104
 105
 106 /* Max regno at the start of the pass.  */
 107 static int regs_num;
 108
 109 /* Map spilled regno -> hard regno used instead of memory for
 110    spilling.  */
 111 static rtx *spill_hard_reg;
 112
 113 /* The structure describes stack slot of a spilled pseudo.  */
 114 struct pseudo_slot
 115 {
 116   /* Number (0, 1, ...) of the stack slot to which given pseudo
 117      belongs.  */
 118   int slot_num;
 119   /* First or next slot with the same slot number.  */
 120   struct pseudo_slot *next, *first;
 121   /* Memory representing the spilled pseudo.  */
 122   rtx mem;
 123 };
 124
 125 /* The stack slots for each spilled pseudo.  Indexed by regnos.  */
 126 static struct pseudo_slot *pseudo_slots;
 127
 128 /* The structure describes a register or a stack slot which can be
 129    used for several spilled pseudos.  */
 130 struct slot
 131 {
 132   /* First pseudo with given stack slot.  */
 133   int regno;
 134   /* Hard reg into which the slot pseudos are spilled.  The value is
 135      negative for pseudos spilled into memory.  */
 136   int hard_regno;
 137   /* Memory representing the all stack slot.  It can be different from
 138      memory representing a pseudo belonging to give stack slot because
 139      pseudo can be placed in a part of the corresponding stack slot.
 140      The value is NULL for pseudos spilled into a hard reg.  */
 141   rtx mem;
 142   /* Combined live ranges of all pseudos belonging to given slot.  It
 143      is used to figure out that a new spilled pseudo can use given
 144      stack slot.  */
 145   lra_live_range_t live_ranges;
 146 };
 147
 148 /* Array containing info about the stack slots.  The array element is
 149    indexed by the stack slot number in the range [0..slots_num).  */
 150 static struct slot *slots;
 151 /* The number of the stack slots currently existing.  */
 152 static int slots_num;
 153
 154 /* Set up memory of the spilled pseudo I.  The function can allocate
 155    the corresponding stack slot if it is not done yet.  */
 156 static void
 157 assign_mem_slot (int i)
 158 {
 159   rtx x = NULL_RTX;
 160   machine_mode mode = GET_MODE (regno_reg_rtx[i]);
 161   unsigned int inherent_size = PSEUDO_REGNO_BYTES (i);
 162   unsigned int inherent_align = GET_MODE_ALIGNMENT (mode);
 163   unsigned int max_ref_width = GET_MODE_SIZE (lra_reg_info[i].biggest_mode);
 164   unsigned int total_size = MAX (inherent_size, max_ref_width);
 165   unsigned int min_align = max_ref_width * BITS_PER_UNIT;
 166   int adjust = 0;
 167
 168   lra_assert (regno_reg_rtx[i] != NULL_RTX && REG_P (regno_reg_rtx[i])
 169               && lra_reg_info[i].nrefs != 0 && reg_renumber[i] < 0);
 170
 171   x = slots[pseudo_slots[i].slot_num].mem;
 172
 173   /* We can use a slot already allocated because it is guaranteed the
 174      slot provides both enough inherent space and enough total
 175      space.  */
 176   if (x)
 177     ;
 178   /* Each pseudo has an inherent size which comes from its own mode,
 179      and a total size which provides room for paradoxical subregs
 180      which refer to the pseudo reg in wider modes.  We allocate a new
 181      slot, making sure that it has enough inherent space and total
 182      space.  */
 183   else
 184     {
 185       rtx stack_slot;
 186
 187       /* No known place to spill from => no slot to reuse.  */
 188       x = assign_stack_local (mode, total_size,
 189                               min_align > inherent_align
 190                               || total_size > inherent_size ? -1 : 0);
 191       stack_slot = x;
 192       /* Cancel the big-endian correction done in assign_stack_local.
 193          Get the address of the beginning of the slot.  This is so we
 194          can do a big-endian correction unconditionally below.  */
 195       if (BYTES_BIG_ENDIAN)
 196         {
 197           adjust = inherent_size - total_size;
 198           if (adjust)
 199             stack_slot
 200               = adjust_address_nv (x,
 201                                    mode_for_size (total_size * BITS_PER_UNIT,
 202                                                   MODE_INT, 1),
 203                                    adjust);
 204         }
 205       slots[pseudo_slots[i].slot_num].mem = stack_slot;
 206     }
 207
 208   /* On a big endian machine, the "address" of the slot is the address
 209      of the low part that fits its inherent mode.  */
 210   if (BYTES_BIG_ENDIAN && inherent_size < total_size)
 211     adjust += (total_size - inherent_size);
 212
 213   x = adjust_address_nv (x, GET_MODE (regno_reg_rtx[i]), adjust);
 214
 215   /* Set all of the memory attributes as appropriate for a spill.  */
 216   set_mem_attrs_for_spill (x);
 217   pseudo_slots[i].mem = x;
 218 }
 219
 220 /* Sort pseudos according their usage frequencies.  */
 221 static int
 222 regno_freq_compare (const void *v1p, const void *v2p)
 223 {
 224   const int regno1 = *(const int *) v1p;
 225   const int regno2 = *(const int *) v2p;
 226   int diff;
 227
 228   if ((diff = lra_reg_info[regno2].freq - lra_reg_info[regno1].freq) != 0)
 229     return diff;
 230   return regno1 - regno2;
 231 }
 232
 233 /* Sort pseudos according to their slots, putting the slots in the order
 234    that they should be allocated.  Slots with lower numbers have the highest
 235    priority and should get the smallest displacement from the stack or
 236    frame pointer (whichever is being used).
 237
 238    The first allocated slot is always closest to the frame pointer,
 239    so prefer lower slot numbers when frame_pointer_needed.  If the stack
 240    and frame grow in the same direction, then the first allocated slot is
 241    always closest to the initial stack pointer and furthest away from the
 242    final stack pointer, so allocate higher numbers first when using the
 243    stack pointer in that case.  The reverse is true if the stack and
 244    frame grow in opposite directions.  */
 245 static int
 246 pseudo_reg_slot_compare (const void *v1p, const void *v2p)
 247 {
 248   const int regno1 = *(const int *) v1p;
 249   const int regno2 = *(const int *) v2p;
 250   int diff, slot_num1, slot_num2;
 251   int total_size1, total_size2;
 252
 253   slot_num1 = pseudo_slots[regno1].slot_num;
 254   slot_num2 = pseudo_slots[regno2].slot_num;
 255   if ((diff = slot_num1 - slot_num2) != 0)
 256     return (frame_pointer_needed
 257             || (!FRAME_GROWS_DOWNWARD) == STACK_GROWS_DOWNWARD ? diff : -diff);
 258   total_size1 = GET_MODE_SIZE (lra_reg_info[regno1].biggest_mode);
 259   total_size2 = GET_MODE_SIZE (lra_reg_info[regno2].biggest_mode);
 260   if ((diff = total_size2 - total_size1) != 0)
 261     return diff;
 262   return regno1 - regno2;
 263 }
 264
 265 /* Assign spill hard registers to N pseudos in PSEUDO_REGNOS which is
 266    sorted in order of highest frequency first.  Put the pseudos which
 267    did not get a spill hard register at the beginning of array
 268    PSEUDO_REGNOS.  Return the number of such pseudos.  */
 269 static int
 270 assign_spill_hard_regs (int *pseudo_regnos, int n)
 271 {
 272   int i, k, p, regno, res, spill_class_size, hard_regno, nr;
 273   enum reg_class rclass, spill_class;
 274   machine_mode mode;
 275   lra_live_range_t r;
 276   rtx_insn *insn;
 277   rtx set;
 278   basic_block bb;
 279   HARD_REG_SET conflict_hard_regs;
 280   bitmap_head ok_insn_bitmap;
 281   bitmap setjump_crosses = regstat_get_setjmp_crosses ();
 282   /* Hard registers which can not be used for any purpose at given
 283      program point because they are unallocatable or already allocated
 284      for other pseudos.  */
 285   HARD_REG_SET *reserved_hard_regs;
 286
 287   if (! lra_reg_spill_p)
 288     return n;
 289   /* Set up reserved hard regs for every program point.  */
 290   reserved_hard_regs = XNEWVEC (HARD_REG_SET, lra_live_max_point);
 291   for (p = 0; p < lra_live_max_point; p++)
 292     COPY_HARD_REG_SET (reserved_hard_regs[p], lra_no_alloc_regs);
 293   for (i = FIRST_PSEUDO_REGISTER; i < regs_num; i++)
 294     if (lra_reg_info[i].nrefs != 0
 295         && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
 296       for (r = lra_reg_info[i].live_ranges; r != NULL; r = r->next)
 297         for (p = r->start; p <= r->finish; p++)
 298           add_to_hard_reg_set (&reserved_hard_regs[p],
 299                                lra_reg_info[i].biggest_mode, hard_regno);
 300   bitmap_initialize (&ok_insn_bitmap, &reg_obstack);
 301   FOR_EACH_BB_FN (bb, cfun)
 302     FOR_BB_INSNS (bb, insn)
 303       if (DEBUG_INSN_P (insn)
 304           || ((set = single_set (insn)) != NULL_RTX
 305               && REG_P (SET_SRC (set)) && REG_P (SET_DEST (set))))
 306         bitmap_set_bit (&ok_insn_bitmap, INSN_UID (insn));
 307   for (res = i = 0; i < n; i++)
 308     {
 309       regno = pseudo_regnos[i];
 310       rclass = lra_get_allocno_class (regno);
 311       if (bitmap_bit_p (setjump_crosses, regno)
 312           || (spill_class
 313               = ((enum reg_class)
 314                  targetm.spill_class ((reg_class_t) rclass,
 315                                       PSEUDO_REGNO_MODE (regno)))) == NO_REGS
 316           || bitmap_intersect_compl_p (&lra_reg_info[regno].insn_bitmap,
 317                                        &ok_insn_bitmap))
 318         {
 319           pseudo_regnos[res++] = regno;
 320           continue;
 321         }
 322       lra_assert (spill_class != NO_REGS);
 323       COPY_HARD_REG_SET (conflict_hard_regs,
 324                          lra_reg_info[regno].conflict_hard_regs);
 325       for (r = lra_reg_info[regno].live_ranges; r != NULL; r = r->next)
 326         for (p = r->start; p <= r->finish; p++)
 327           IOR_HARD_REG_SET (conflict_hard_regs, reserved_hard_regs[p]);
 328       spill_class_size = ira_class_hard_regs_num[spill_class];
 329       mode = lra_reg_info[regno].biggest_mode;
 330       for (k = 0; k < spill_class_size; k++)
 331         {
 332           hard_regno = ira_class_hard_regs[spill_class][k];
 333           if (! overlaps_hard_reg_set_p (conflict_hard_regs, mode, hard_regno))
 334             break;
 335         }
 336       if (k >= spill_class_size)
 337         {
 338            /* There is no available regs -- assign memory later.  */
 339           pseudo_regnos[res++] = regno;
 340           continue;
 341         }
 342       if (lra_dump_file != NULL)
 343         fprintf (lra_dump_file, "  Spill r%d into hr%d\n", regno, hard_regno);
 344       /* Update reserved_hard_regs.  */
 345       for (r = lra_reg_info[regno].live_ranges; r != NULL; r = r->next)
 346         for (p = r->start; p <= r->finish; p++)
 347           add_to_hard_reg_set (&reserved_hard_regs[p],
 348                                lra_reg_info[regno].biggest_mode, hard_regno);
 349       spill_hard_reg[regno]
 350         = gen_raw_REG (PSEUDO_REGNO_MODE (regno), hard_regno);
 351       for (nr = 0;
 352            nr < hard_regno_nregs[hard_regno][lra_reg_info[regno].biggest_mode];
 353            nr++)
 354         /* Just loop.  */
 355         df_set_regs_ever_live (hard_regno + nr, true);
 356     }
 357   bitmap_clear (&ok_insn_bitmap);
 358   free (reserved_hard_regs);
 359   return res;
 360 }
 361
 362 /* Add pseudo REGNO to slot SLOT_NUM.  */
 363 static void
 364 add_pseudo_to_slot (int regno, int slot_num)
 365 {
 366   struct pseudo_slot *first;
 367
 368   if (slots[slot_num].regno < 0)
 369     {
 370       /* It is the first pseudo in the slot.  */
 371       slots[slot_num].regno = regno;
 372       pseudo_slots[regno].first = &pseudo_slots[regno];
 373       pseudo_slots[regno].next = NULL;
 374     }
 375   else
 376     {
 377       first = pseudo_slots[regno].first = &pseudo_slots[slots[slot_num].regno];
 378       pseudo_slots[regno].next = first->next;
 379       first->next = &pseudo_slots[regno];
 380     }
 381   pseudo_slots[regno].mem = NULL_RTX;
 382   pseudo_slots[regno].slot_num = slot_num;
 383   slots[slot_num].live_ranges
 384     = lra_merge_live_ranges (slots[slot_num].live_ranges,
 385                              lra_copy_live_range_list
 386                              (lra_reg_info[regno].live_ranges));
 387 }
 388
 389 /* Assign stack slot numbers to pseudos in array PSEUDO_REGNOS of
 390    length N.  Sort pseudos in PSEUDO_REGNOS for subsequent assigning
 391    memory stack slots.  */
 392 static void
 393 assign_stack_slot_num_and_sort_pseudos (int *pseudo_regnos, int n)
 394 {
 395   int i, j, regno;
 396
 397   slots_num = 0;
 398   /* Assign stack slot numbers to spilled pseudos, use smaller numbers
 399      for most frequently used pseudos.  */
 400   for (i = 0; i < n; i++)
 401     {
 402       regno = pseudo_regnos[i];
 403       if (! flag_ira_share_spill_slots)
 404         j = slots_num;
 405       else
 406         {
 407           for (j = 0; j < slots_num; j++)
 408             if (slots[j].hard_regno < 0
 409                 && ! (lra_intersected_live_ranges_p
 410                       (slots[j].live_ranges,
 411                        lra_reg_info[regno].live_ranges)))
 412               break;
 413         }
 414       if (j >= slots_num)
 415         {
 416           /* New slot.  */
 417           slots[j].live_ranges = NULL;
 418           slots[j].regno = slots[j].hard_regno = -1;
 419           slots[j].mem = NULL_RTX;
 420           slots_num++;
 421         }
 422       add_pseudo_to_slot (regno, j);
 423     }
 424   /* Sort regnos according to their slot numbers.  */
 425   qsort (pseudo_regnos, n, sizeof (int), pseudo_reg_slot_compare);
 426 }
 427
 428 /* Recursively process LOC in INSN and change spilled pseudos to the
 429    corresponding memory or spilled hard reg.  Ignore spilled pseudos
 430    created from the scratches.  */
 431 static void
 432 remove_pseudos (rtx *loc, rtx_insn *insn)
 433 {
 434   int i;
 435   rtx hard_reg;
 436   const char *fmt;
 437   enum rtx_code code;
 438
 439   if (*loc == NULL_RTX)
 440     return;
 441   code = GET_CODE (*loc);
 442   if (code == REG && (i = REGNO (*loc)) >= FIRST_PSEUDO_REGISTER
 443       && lra_get_regno_hard_regno (i) < 0
 444       /* We do not want to assign memory for former scratches because
 445          it might result in an address reload for some targets.  In
 446          any case we transform such pseudos not getting hard registers
 447          into scratches back.  */
 448       && ! lra_former_scratch_p (i))
 449     {
 450       if ((hard_reg = spill_hard_reg[i]) != NULL_RTX)
 451         *loc = copy_rtx (hard_reg);
 452       else
 453         {
 454           rtx x = lra_eliminate_regs_1 (insn, pseudo_slots[i].mem,
 455                                         GET_MODE (pseudo_slots[i].mem),
 456                                         false, false, 0, true);
 457           *loc = x != pseudo_slots[i].mem ? x : copy_rtx (x);
 458         }
 459       return;
 460     }
 461
 462   fmt = GET_RTX_FORMAT (code);
 463   for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
 464     {
 465       if (fmt[i] == 'e')
 466         remove_pseudos (&XEXP (*loc, i), insn);
 467       else if (fmt[i] == 'E')
 468         {
 469           int j;
 470
 471           for (j = XVECLEN (*loc, i) - 1; j >= 0; j--)
 472             remove_pseudos (&XVECEXP (*loc, i, j), insn);
 473         }
 474     }
 475 }
 476
 477 /* Convert spilled pseudos into their stack slots or spill hard regs,
 478    put insns to process on the constraint stack (that is all insns in
 479    which pseudos were changed to memory or spill hard regs).   */
 480 static void
 481 spill_pseudos (void)
 482 {
 483   basic_block bb;
 484   rtx_insn *insn;
 485   int i;
 486   bitmap_head spilled_pseudos, changed_insns;
 487
 488   bitmap_initialize (&spilled_pseudos, &reg_obstack);
 489   bitmap_initialize (&changed_insns, &reg_obstack);
 490   for (i = FIRST_PSEUDO_REGISTER; i < regs_num; i++)
 491     {
 492       if (lra_reg_info[i].nrefs != 0 && lra_get_regno_hard_regno (i) < 0
 493           && ! lra_former_scratch_p (i))
 494         {
 495           bitmap_set_bit (&spilled_pseudos, i);
 496           bitmap_ior_into (&changed_insns, &lra_reg_info[i].insn_bitmap);
 497         }
 498     }
 499   FOR_EACH_BB_FN (bb, cfun)
 500     {
 501       FOR_BB_INSNS (bb, insn)
 502         if (bitmap_bit_p (&changed_insns, INSN_UID (insn)))
 503           {
 504             rtx *link_loc, link;
 505             remove_pseudos (&PATTERN (insn), insn);
 506             if (CALL_P (insn))
 507               remove_pseudos (&CALL_INSN_FUNCTION_USAGE (insn), insn);
 508             for (link_loc = &REG_NOTES (insn);
 509                  (link = *link_loc) != NULL_RTX;
 510                  link_loc = &XEXP (link, 1))
 511               {
 512                 switch (REG_NOTE_KIND (link))
 513                   {
 514                   case REG_FRAME_RELATED_EXPR:
 515                   case REG_CFA_DEF_CFA:
 516                   case REG_CFA_ADJUST_CFA:
 517                   case REG_CFA_OFFSET:
 518                   case REG_CFA_REGISTER:
 519                   case REG_CFA_EXPRESSION:
 520                   case REG_CFA_RESTORE:
 521                   case REG_CFA_SET_VDRAP:
 522                     remove_pseudos (&XEXP (link, 0), insn);
 523                     break;
 524                   default:
 525                     break;
 526                   }
 527               }
 528             if (lra_dump_file != NULL)
 529               fprintf (lra_dump_file,
 530                        "Changing spilled pseudos to memory in insn #%u\n",
 531                        INSN_UID (insn));
 532             lra_push_insn (insn);
 533             if (lra_reg_spill_p || targetm.different_addr_displacement_p ())
 534               lra_set_used_insn_alternative (insn, -1);
 535           }
 536         else if (CALL_P (insn))
 537           /* Presence of any pseudo in CALL_INSN_FUNCTION_USAGE does
 538              not affect value of insn_bitmap of the corresponding
 539              lra_reg_info.  That is because we don't need to reload
 540              pseudos in CALL_INSN_FUNCTION_USAGEs.  So if we process
 541              only insns in the insn_bitmap of given pseudo here, we
 542              can miss the pseudo in some
 543              CALL_INSN_FUNCTION_USAGEs.  */
 544           remove_pseudos (&CALL_INSN_FUNCTION_USAGE (insn), insn);
 545       bitmap_and_compl_into (df_get_live_in (bb), &spilled_pseudos);
 546       bitmap_and_compl_into (df_get_live_out (bb), &spilled_pseudos);
 547     }
 548   bitmap_clear (&spilled_pseudos);
 549   bitmap_clear (&changed_insns);
 550 }
 551
 552 /* Return true if we need to change some pseudos into memory.  */
 553 bool
 554 lra_need_for_spills_p (void)
 555 {
 556   int i; max_regno = max_reg_num ();
 557
 558   for (i = FIRST_PSEUDO_REGISTER; i < max_regno; i++)
 559     if (lra_reg_info[i].nrefs != 0 && lra_get_regno_hard_regno (i) < 0
 560         && ! lra_former_scratch_p (i))
 561       return true;
 562   return false;
 563 }
 564
 565 /* Change spilled pseudos into memory or spill hard regs.  Put changed
 566    insns on the constraint stack (these insns will be considered on
 567    the next constraint pass).  The changed insns are all insns in
 568    which pseudos were changed.  */
 569 void
 570 lra_spill (void)
 571 {
 572   int i, n, curr_regno;
 573   int *pseudo_regnos;
 574
 575   regs_num = max_reg_num ();
 576   spill_hard_reg = XNEWVEC (rtx, regs_num);
 577   pseudo_regnos = XNEWVEC (int, regs_num);
 578   for (n = 0, i = FIRST_PSEUDO_REGISTER; i < regs_num; i++)
 579     if (lra_reg_info[i].nrefs != 0 && lra_get_regno_hard_regno (i) < 0
 580         /* We do not want to assign memory for former scratches.  */
 581         && ! lra_former_scratch_p (i))
 582       {
 583         spill_hard_reg[i] = NULL_RTX;
 584         pseudo_regnos[n++] = i;
 585       }
 586   lra_assert (n > 0);
 587   pseudo_slots = XNEWVEC (struct pseudo_slot, regs_num);
 588   slots = XNEWVEC (struct slot, regs_num);
 589   /* Sort regnos according their usage frequencies.  */
 590   qsort (pseudo_regnos, n, sizeof (int), regno_freq_compare);
 591   n = assign_spill_hard_regs (pseudo_regnos, n);
 592   assign_stack_slot_num_and_sort_pseudos (pseudo_regnos, n);
 593   for (i = 0; i < n; i++)
 594     if (pseudo_slots[pseudo_regnos[i]].mem == NULL_RTX)
 595       assign_mem_slot (pseudo_regnos[i]);
 596   if (n > 0 && crtl->stack_alignment_needed)
 597     /* If we have a stack frame, we must align it now.  The stack size
 598        may be a part of the offset computation for register
 599        elimination.  */
 600     assign_stack_local (BLKmode, 0, crtl->stack_alignment_needed);
 601   if (lra_dump_file != NULL)
 602     {
 603       for (i = 0; i < slots_num; i++)
 604         {
 605           fprintf (lra_dump_file, "  Slot %d regnos (width = %d):", i,
 606                    GET_MODE_SIZE (GET_MODE (slots[i].mem)));
 607           for (curr_regno = slots[i].regno;;
 608                curr_regno = pseudo_slots[curr_regno].next - pseudo_slots)
 609             {
 610               fprintf (lra_dump_file, "  %d", curr_regno);
 611               if (pseudo_slots[curr_regno].next == NULL)
 612                 break;
 613             }
 614           fprintf (lra_dump_file, "\n");
 615         }
 616     }
 617   spill_pseudos ();
 618   free (slots);
 619   free (pseudo_slots);
 620   free (pseudo_regnos);
 621   free (spill_hard_reg);
 622 }
 623
 624 /* Apply alter_subreg for subregs of regs in *LOC.  Use FINAL_P for
 625    alter_subreg calls. Return true if any subreg of reg is
 626    processed.  */
 627 static bool
 628 alter_subregs (rtx *loc, bool final_p)
 629 {
 630   int i;
 631   rtx x = *loc;
 632   bool res;
 633   const char *fmt;
 634   enum rtx_code code;
 635
 636   if (x == NULL_RTX)
 637     return false;
 638   code = GET_CODE (x);
 639   if (code == SUBREG && REG_P (SUBREG_REG (x)))
 640     {
 641       lra_assert (REGNO (SUBREG_REG (x)) < FIRST_PSEUDO_REGISTER);
 642       alter_subreg (loc, final_p);
 643       return true;
 644     }
 645   fmt = GET_RTX_FORMAT (code);
 646   res = false;
 647   for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
 648     {
 649       if (fmt[i] == 'e')
 650         {
 651           if (alter_subregs (&XEXP (x, i), final_p))
 652             res = true;
 653         }
 654       else if (fmt[i] == 'E')
 655         {
 656           int j;
 657
 658           for (j = XVECLEN (x, i) - 1; j >= 0; j--)
 659             if (alter_subregs (&XVECEXP (x, i, j), final_p))
 660               res = true;
 661         }
 662     }
 663   return res;
 664 }
 665
 666 /* Return true if REGNO is used for return in the current
 667    function.  */
 668 static bool
 669 return_regno_p (unsigned int regno)
 670 {
 671   rtx outgoing = crtl->return_rtx;
 672
 673   if (! outgoing)
 674     return false;
 675
 676   if (REG_P (outgoing))
 677     return REGNO (outgoing) == regno;
 678   else if (GET_CODE (outgoing) == PARALLEL)
 679     {
 680       int i;
 681
 682       for (i = 0; i < XVECLEN (outgoing, 0); i++)
 683         {
 684           rtx x = XEXP (XVECEXP (outgoing, 0, i), 0);
 685
 686           if (REG_P (x) && REGNO (x) == regno)
 687             return true;
 688         }
 689     }
 690   return false;
 691 }
 692
 693 /* Final change of pseudos got hard registers into the corresponding
 694    hard registers and removing temporary clobbers.  */
 695 void
 696 lra_final_code_change (void)
 697 {
 698   int i, hard_regno;
 699   basic_block bb;
 700   rtx_insn *insn, *curr;
 701   int max_regno = max_reg_num ();
 702
 703   for (i = FIRST_PSEUDO_REGISTER; i < max_regno; i++)
 704     if (lra_reg_info[i].nrefs != 0
 705         && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
 706       SET_REGNO (regno_reg_rtx[i], hard_regno);
 707   FOR_EACH_BB_FN (bb, cfun)
 708     FOR_BB_INSNS_SAFE (bb, insn, curr)
 709       if (INSN_P (insn))
 710         {
 711           rtx pat = PATTERN (insn);
 712
 713           if (GET_CODE (pat) == CLOBBER && LRA_TEMP_CLOBBER_P (pat))
 714             {
 715               /* Remove clobbers temporarily created in LRA.  We don't
 716                  need them anymore and don't want to waste compiler
 717                  time processing them in a few subsequent passes.  */
 718               lra_invalidate_insn_data (insn);
 719               delete_insn (insn);
 720               continue;
 721             }
 722
 723           /* IRA can generate move insns involving pseudos.  It is
 724              better remove them earlier to speed up compiler a bit.
 725              It is also better to do it here as they might not pass
 726              final RTL check in LRA, (e.g. insn moving a control
 727              register into itself).  So remove an useless move insn
 728              unless next insn is USE marking the return reg (we should
 729              save this as some subsequent optimizations assume that
 730              such original insns are saved).  */
 731           if (NONJUMP_INSN_P (insn) && GET_CODE (pat) == SET
 732               && REG_P (SET_SRC (pat)) && REG_P (SET_DEST (pat))
 733               && REGNO (SET_SRC (pat)) == REGNO (SET_DEST (pat))
 734               && ! return_regno_p (REGNO (SET_SRC (pat))))
 735             {
 736               lra_invalidate_insn_data (insn);
 737               delete_insn (insn);
 738               continue;
 739             }
 740
 741           lra_insn_recog_data_t id = lra_get_insn_recog_data (insn);
 742           struct lra_static_insn_data *static_id = id->insn_static_data;
 743           bool insn_change_p = false;
 744
 745           for (i = id->insn_static_data->n_operands - 1; i >= 0; i--)
 746             if ((DEBUG_INSN_P (insn) || ! static_id->operand[i].is_operator)
 747                 && alter_subregs (id->operand_loc[i], ! DEBUG_INSN_P (insn)))
 748               {
 749                 lra_update_dup (id, i);
 750                 insn_change_p = true;
 751               }
 752           if (insn_change_p)
 753             lra_update_operator_dups (id);
 754         }
 755 }