gcc/lra-spills.c

   1 /* Change pseudos by memory.
   2    Copyright (C) 2010-2015 Free Software Foundation, Inc.
   3    Contributed by Vladimir Makarov <vmakarov@redhat.com>.
   4
   5 This file is part of GCC.
   6
   7 GCC is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 3, or (at your option) any later
  10 version.
  11
  12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with GCC; see the file COPYING3.  If not see
  19 <http://www.gnu.org/licenses/>.  */
  20
  21
  22 /* This file contains code for a pass to change spilled pseudos into
  23    memory.
  24
  25    The pass creates necessary stack slots and assigns spilled pseudos
  26    to the stack slots in following way:
  27
  28    for all spilled pseudos P most frequently used first do
  29      for all stack slots S do
  30        if P doesn't conflict with pseudos assigned to S then
  31          assign S to P and goto to the next pseudo process
  32        end
  33      end
  34      create new stack slot S and assign P to S
  35    end
  36
  37    The actual algorithm is bit more complicated because of different
  38    pseudo sizes.
  39
  40    After that the code changes spilled pseudos (except ones created
  41    from scratches) by corresponding stack slot memory in RTL.
  42
  43    If at least one stack slot was created, we need to run more passes
  44    because we have new addresses which should be checked and because
  45    the old address displacements might change and address constraints
  46    (or insn memory constraints) might not be satisfied any more.
  47
  48    For some targets, the pass can spill some pseudos into hard
  49    registers of different class (usually into vector registers)
  50    instead of spilling them into memory if it is possible and
  51    profitable.  Spilling GENERAL_REGS pseudo into SSE registers for
  52    Intel Corei7 is an example of such optimization.  And this is
  53    actually recommended by Intel optimization guide.
  54
  55    The file also contains code for final change of pseudos on hard
  56    regs correspondingly assigned to them.  */
  57
  58 #include "config.h"
  59 #include "system.h"
  60 #include "coretypes.h"
  61 #include "tm.h"
  62 #include "rtl.h"
  63 #include "tm_p.h"
  64 #include "insn-config.h"
  65 #include "recog.h"
  66 #include "output.h"
  67 #include "regs.h"
  68 #include "hard-reg-set.h"
  69 #include "flags.h"
  70 #include "hashtab.h"
  71 #include "hash-set.h"
  72 #include "vec.h"
  73 #include "machmode.h"
  74 #include "input.h"
  75 #include "function.h"
  76 #include "symtab.h"
  77 #include "statistics.h"
  78 #include "double-int.h"
  79 #include "real.h"
  80 #include "fixed-value.h"
  81 #include "alias.h"
  82 #include "wide-int.h"
  83 #include "inchash.h"
  84 #include "tree.h"
  85 #include "expmed.h"
  86 #include "dojump.h"
  87 #include "explow.h"
  88 #include "calls.h"
  89 #include "emit-rtl.h"
  90 #include "varasm.h"
  91 #include "stmt.h"
  92 #include "expr.h"
  93 #include "predict.h"
  94 #include "dominance.h"
  95 #include "cfg.h"
  96 #include "cfgrtl.h"
  97 #include "basic-block.h"
  98 #include "except.h"
  99 #include "timevar.h"
 100 #include "target.h"
 101 #include "lra-int.h"
 102 #include "ira.h"
 103 #include "df.h"
 104
 105
 106 /* Max regno at the start of the pass.  */
 107 static int regs_num;
 108
 109 /* Map spilled regno -> hard regno used instead of memory for
 110    spilling.  */
 111 static rtx *spill_hard_reg;
 112
 113 /* The structure describes stack slot of a spilled pseudo.  */
 114 struct pseudo_slot
 115 {
 116   /* Number (0, 1, ...) of the stack slot to which given pseudo
 117      belongs.  */
 118   int slot_num;
 119   /* First or next slot with the same slot number.  */
 120   struct pseudo_slot *next, *first;
 121   /* Memory representing the spilled pseudo.  */
 122   rtx mem;
 123 };
 124
 125 /* The stack slots for each spilled pseudo.  Indexed by regnos.  */
 126 static struct pseudo_slot *pseudo_slots;
 127
 128 /* The structure describes a register or a stack slot which can be
 129    used for several spilled pseudos.  */
 130 struct slot
 131 {
 132   /* First pseudo with given stack slot.  */
 133   int regno;
 134   /* Hard reg into which the slot pseudos are spilled.  The value is
 135      negative for pseudos spilled into memory.  */
 136   int hard_regno;
 137   /* Memory representing the all stack slot.  It can be different from
 138      memory representing a pseudo belonging to give stack slot because
 139      pseudo can be placed in a part of the corresponding stack slot.
 140      The value is NULL for pseudos spilled into a hard reg.  */
 141   rtx mem;
 142   /* Combined live ranges of all pseudos belonging to given slot.  It
 143      is used to figure out that a new spilled pseudo can use given
 144      stack slot.  */
 145   lra_live_range_t live_ranges;
 146 };
 147
 148 /* Array containing info about the stack slots.  The array element is
 149    indexed by the stack slot number in the range [0..slots_num).  */
 150 static struct slot *slots;
 151 /* The number of the stack slots currently existing.  */
 152 static int slots_num;
 153
 154 /* Set up memory of the spilled pseudo I.  The function can allocate
 155    the corresponding stack slot if it is not done yet.  */
 156 static void
 157 assign_mem_slot (int i)
 158 {
 159   rtx x = NULL_RTX;
 160   machine_mode mode = GET_MODE (regno_reg_rtx[i]);
 161   unsigned int inherent_size = PSEUDO_REGNO_BYTES (i);
 162   unsigned int inherent_align = GET_MODE_ALIGNMENT (mode);
 163   unsigned int max_ref_width = GET_MODE_SIZE (lra_reg_info[i].biggest_mode);
 164   unsigned int total_size = MAX (inherent_size, max_ref_width);
 165   unsigned int min_align = max_ref_width * BITS_PER_UNIT;
 166   int adjust = 0;
 167
 168   lra_assert (regno_reg_rtx[i] != NULL_RTX && REG_P (regno_reg_rtx[i])
 169               && lra_reg_info[i].nrefs != 0 && reg_renumber[i] < 0);
 170
 171   x = slots[pseudo_slots[i].slot_num].mem;
 172
 173   /* We can use a slot already allocated because it is guaranteed the
 174      slot provides both enough inherent space and enough total
 175      space.  */
 176   if (x)
 177     ;
 178   /* Each pseudo has an inherent size which comes from its own mode,
 179      and a total size which provides room for paradoxical subregs
 180      which refer to the pseudo reg in wider modes.  We allocate a new
 181      slot, making sure that it has enough inherent space and total
 182      space.  */
 183   else
 184     {
 185       rtx stack_slot;
 186
 187       /* No known place to spill from => no slot to reuse.  */
 188       x = assign_stack_local (mode, total_size,
 189                               min_align > inherent_align
 190                               || total_size > inherent_size ? -1 : 0);
 191       stack_slot = x;
 192       /* Cancel the big-endian correction done in assign_stack_local.
 193          Get the address of the beginning of the slot.  This is so we
 194          can do a big-endian correction unconditionally below.  */
 195       if (BYTES_BIG_ENDIAN)
 196         {
 197           adjust = inherent_size - total_size;
 198           if (adjust)
 199             stack_slot
 200               = adjust_address_nv (x,
 201                                    mode_for_size (total_size * BITS_PER_UNIT,
 202                                                   MODE_INT, 1),
 203                                    adjust);
 204         }
 205       slots[pseudo_slots[i].slot_num].mem = stack_slot;
 206     }
 207
 208   /* On a big endian machine, the "address" of the slot is the address
 209      of the low part that fits its inherent mode.  */
 210   if (BYTES_BIG_ENDIAN && inherent_size < total_size)
 211     adjust += (total_size - inherent_size);
 212
 213   x = adjust_address_nv (x, GET_MODE (regno_reg_rtx[i]), adjust);
 214
 215   /* Set all of the memory attributes as appropriate for a spill.  */
 216   set_mem_attrs_for_spill (x);
 217   pseudo_slots[i].mem = x;
 218 }
 219
 220 /* Sort pseudos according their usage frequencies.  */
 221 static int
 222 regno_freq_compare (const void *v1p, const void *v2p)
 223 {
 224   const int regno1 = *(const int *) v1p;
 225   const int regno2 = *(const int *) v2p;
 226   int diff;
 227
 228   if ((diff = lra_reg_info[regno2].freq - lra_reg_info[regno1].freq) != 0)
 229     return diff;
 230   return regno1 - regno2;
 231 }
 232
 233 /* Redefine STACK_GROWS_DOWNWARD in terms of 0 or 1.  */
 234 #ifdef STACK_GROWS_DOWNWARD
 235 # undef STACK_GROWS_DOWNWARD
 236 # define STACK_GROWS_DOWNWARD 1
 237 #else
 238 # define STACK_GROWS_DOWNWARD 0
 239 #endif
 240
 241 /* Sort pseudos according to their slots, putting the slots in the order
 242    that they should be allocated.  Slots with lower numbers have the highest
 243    priority and should get the smallest displacement from the stack or
 244    frame pointer (whichever is being used).
 245
 246    The first allocated slot is always closest to the frame pointer,
 247    so prefer lower slot numbers when frame_pointer_needed.  If the stack
 248    and frame grow in the same direction, then the first allocated slot is
 249    always closest to the initial stack pointer and furthest away from the
 250    final stack pointer, so allocate higher numbers first when using the
 251    stack pointer in that case.  The reverse is true if the stack and
 252    frame grow in opposite directions.  */
 253 static int
 254 pseudo_reg_slot_compare (const void *v1p, const void *v2p)
 255 {
 256   const int regno1 = *(const int *) v1p;
 257   const int regno2 = *(const int *) v2p;
 258   int diff, slot_num1, slot_num2;
 259   int total_size1, total_size2;
 260
 261   slot_num1 = pseudo_slots[regno1].slot_num;
 262   slot_num2 = pseudo_slots[regno2].slot_num;
 263   if ((diff = slot_num1 - slot_num2) != 0)
 264     return (frame_pointer_needed
 265             || (!FRAME_GROWS_DOWNWARD) == STACK_GROWS_DOWNWARD ? diff : -diff);
 266   total_size1 = GET_MODE_SIZE (lra_reg_info[regno1].biggest_mode);
 267   total_size2 = GET_MODE_SIZE (lra_reg_info[regno2].biggest_mode);
 268   if ((diff = total_size2 - total_size1) != 0)
 269     return diff;
 270   return regno1 - regno2;
 271 }
 272
 273 /* Assign spill hard registers to N pseudos in PSEUDO_REGNOS which is
 274    sorted in order of highest frequency first.  Put the pseudos which
 275    did not get a spill hard register at the beginning of array
 276    PSEUDO_REGNOS.  Return the number of such pseudos.  */
 277 static int
 278 assign_spill_hard_regs (int *pseudo_regnos, int n)
 279 {
 280   int i, k, p, regno, res, spill_class_size, hard_regno, nr;
 281   enum reg_class rclass, spill_class;
 282   machine_mode mode;
 283   lra_live_range_t r;
 284   rtx_insn *insn;
 285   rtx set;
 286   basic_block bb;
 287   HARD_REG_SET conflict_hard_regs;
 288   bitmap_head ok_insn_bitmap;
 289   bitmap setjump_crosses = regstat_get_setjmp_crosses ();
 290   /* Hard registers which can not be used for any purpose at given
 291      program point because they are unallocatable or already allocated
 292      for other pseudos.  */
 293   HARD_REG_SET *reserved_hard_regs;
 294
 295   if (! lra_reg_spill_p)
 296     return n;
 297   /* Set up reserved hard regs for every program point.  */
 298   reserved_hard_regs = XNEWVEC (HARD_REG_SET, lra_live_max_point);
 299   for (p = 0; p < lra_live_max_point; p++)
 300     COPY_HARD_REG_SET (reserved_hard_regs[p], lra_no_alloc_regs);
 301   for (i = FIRST_PSEUDO_REGISTER; i < regs_num; i++)
 302     if (lra_reg_info[i].nrefs != 0
 303         && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
 304       for (r = lra_reg_info[i].live_ranges; r != NULL; r = r->next)
 305         for (p = r->start; p <= r->finish; p++)
 306           add_to_hard_reg_set (&reserved_hard_regs[p],
 307                                lra_reg_info[i].biggest_mode, hard_regno);
 308   bitmap_initialize (&ok_insn_bitmap, &reg_obstack);
 309   FOR_EACH_BB_FN (bb, cfun)
 310     FOR_BB_INSNS (bb, insn)
 311       if (DEBUG_INSN_P (insn)
 312           || ((set = single_set (insn)) != NULL_RTX
 313               && REG_P (SET_SRC (set)) && REG_P (SET_DEST (set))))
 314         bitmap_set_bit (&ok_insn_bitmap, INSN_UID (insn));
 315   for (res = i = 0; i < n; i++)
 316     {
 317       regno = pseudo_regnos[i];
 318       rclass = lra_get_allocno_class (regno);
 319       if (bitmap_bit_p (setjump_crosses, regno)
 320           || (spill_class
 321               = ((enum reg_class)
 322                  targetm.spill_class ((reg_class_t) rclass,
 323                                       PSEUDO_REGNO_MODE (regno)))) == NO_REGS
 324           || bitmap_intersect_compl_p (&lra_reg_info[regno].insn_bitmap,
 325                                        &ok_insn_bitmap))
 326         {
 327           pseudo_regnos[res++] = regno;
 328           continue;
 329         }
 330       lra_assert (spill_class != NO_REGS);
 331       COPY_HARD_REG_SET (conflict_hard_regs,
 332                          lra_reg_info[regno].conflict_hard_regs);
 333       for (r = lra_reg_info[regno].live_ranges; r != NULL; r = r->next)
 334         for (p = r->start; p <= r->finish; p++)
 335           IOR_HARD_REG_SET (conflict_hard_regs, reserved_hard_regs[p]);
 336       spill_class_size = ira_class_hard_regs_num[spill_class];
 337       mode = lra_reg_info[regno].biggest_mode;
 338       for (k = 0; k < spill_class_size; k++)
 339         {
 340           hard_regno = ira_class_hard_regs[spill_class][k];
 341           if (! overlaps_hard_reg_set_p (conflict_hard_regs, mode, hard_regno))
 342             break;
 343         }
 344       if (k >= spill_class_size)
 345         {
 346            /* There is no available regs -- assign memory later.  */
 347           pseudo_regnos[res++] = regno;
 348           continue;
 349         }
 350       if (lra_dump_file != NULL)
 351         fprintf (lra_dump_file, "  Spill r%d into hr%d\n", regno, hard_regno);
 352       /* Update reserved_hard_regs.  */
 353       for (r = lra_reg_info[regno].live_ranges; r != NULL; r = r->next)
 354         for (p = r->start; p <= r->finish; p++)
 355           add_to_hard_reg_set (&reserved_hard_regs[p],
 356                                lra_reg_info[regno].biggest_mode, hard_regno);
 357       spill_hard_reg[regno]
 358         = gen_raw_REG (PSEUDO_REGNO_MODE (regno), hard_regno);
 359       for (nr = 0;
 360            nr < hard_regno_nregs[hard_regno][lra_reg_info[regno].biggest_mode];
 361            nr++)
 362         /* Just loop.  */
 363         df_set_regs_ever_live (hard_regno + nr, true);
 364     }
 365   bitmap_clear (&ok_insn_bitmap);
 366   free (reserved_hard_regs);
 367   return res;
 368 }
 369
 370 /* Add pseudo REGNO to slot SLOT_NUM.  */
 371 static void
 372 add_pseudo_to_slot (int regno, int slot_num)
 373 {
 374   struct pseudo_slot *first;
 375
 376   if (slots[slot_num].regno < 0)
 377     {
 378       /* It is the first pseudo in the slot.  */
 379       slots[slot_num].regno = regno;
 380       pseudo_slots[regno].first = &pseudo_slots[regno];
 381       pseudo_slots[regno].next = NULL;
 382     }
 383   else
 384     {
 385       first = pseudo_slots[regno].first = &pseudo_slots[slots[slot_num].regno];
 386       pseudo_slots[regno].next = first->next;
 387       first->next = &pseudo_slots[regno];
 388     }
 389   pseudo_slots[regno].mem = NULL_RTX;
 390   pseudo_slots[regno].slot_num = slot_num;
 391   slots[slot_num].live_ranges
 392     = lra_merge_live_ranges (slots[slot_num].live_ranges,
 393                              lra_copy_live_range_list
 394                              (lra_reg_info[regno].live_ranges));
 395 }
 396
 397 /* Assign stack slot numbers to pseudos in array PSEUDO_REGNOS of
 398    length N.  Sort pseudos in PSEUDO_REGNOS for subsequent assigning
 399    memory stack slots.  */
 400 static void
 401 assign_stack_slot_num_and_sort_pseudos (int *pseudo_regnos, int n)
 402 {
 403   int i, j, regno;
 404
 405   slots_num = 0;
 406   /* Assign stack slot numbers to spilled pseudos, use smaller numbers
 407      for most frequently used pseudos.  */
 408   for (i = 0; i < n; i++)
 409     {
 410       regno = pseudo_regnos[i];
 411       if (! flag_ira_share_spill_slots)
 412         j = slots_num;
 413       else
 414         {
 415           for (j = 0; j < slots_num; j++)
 416             if (slots[j].hard_regno < 0
 417                 && ! (lra_intersected_live_ranges_p
 418                       (slots[j].live_ranges,
 419                        lra_reg_info[regno].live_ranges)))
 420               break;
 421         }
 422       if (j >= slots_num)
 423         {
 424           /* New slot.  */
 425           slots[j].live_ranges = NULL;
 426           slots[j].regno = slots[j].hard_regno = -1;
 427           slots[j].mem = NULL_RTX;
 428           slots_num++;
 429         }
 430       add_pseudo_to_slot (regno, j);
 431     }
 432   /* Sort regnos according to their slot numbers.  */
 433   qsort (pseudo_regnos, n, sizeof (int), pseudo_reg_slot_compare);
 434 }
 435
 436 /* Recursively process LOC in INSN and change spilled pseudos to the
 437    corresponding memory or spilled hard reg.  Ignore spilled pseudos
 438    created from the scratches.  */
 439 static void
 440 remove_pseudos (rtx *loc, rtx_insn *insn)
 441 {
 442   int i;
 443   rtx hard_reg;
 444   const char *fmt;
 445   enum rtx_code code;
 446
 447   if (*loc == NULL_RTX)
 448     return;
 449   code = GET_CODE (*loc);
 450   if (code == REG && (i = REGNO (*loc)) >= FIRST_PSEUDO_REGISTER
 451       && lra_get_regno_hard_regno (i) < 0
 452       /* We do not want to assign memory for former scratches because
 453          it might result in an address reload for some targets.  In
 454          any case we transform such pseudos not getting hard registers
 455          into scratches back.  */
 456       && ! lra_former_scratch_p (i))
 457     {
 458       if ((hard_reg = spill_hard_reg[i]) != NULL_RTX)
 459         *loc = copy_rtx (hard_reg);
 460       else
 461         {
 462           rtx x = lra_eliminate_regs_1 (insn, pseudo_slots[i].mem,
 463                                         GET_MODE (pseudo_slots[i].mem),
 464                                         0, false, false, true);
 465           *loc = x != pseudo_slots[i].mem ? x : copy_rtx (x);
 466         }
 467       return;
 468     }
 469
 470   fmt = GET_RTX_FORMAT (code);
 471   for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
 472     {
 473       if (fmt[i] == 'e')
 474         remove_pseudos (&XEXP (*loc, i), insn);
 475       else if (fmt[i] == 'E')
 476         {
 477           int j;
 478
 479           for (j = XVECLEN (*loc, i) - 1; j >= 0; j--)
 480             remove_pseudos (&XVECEXP (*loc, i, j), insn);
 481         }
 482     }
 483 }
 484
 485 /* Convert spilled pseudos into their stack slots or spill hard regs,
 486    put insns to process on the constraint stack (that is all insns in
 487    which pseudos were changed to memory or spill hard regs).   */
 488 static void
 489 spill_pseudos (void)
 490 {
 491   basic_block bb;
 492   rtx_insn *insn;
 493   int i;
 494   bitmap_head spilled_pseudos, changed_insns;
 495
 496   bitmap_initialize (&spilled_pseudos, &reg_obstack);
 497   bitmap_initialize (&changed_insns, &reg_obstack);
 498   for (i = FIRST_PSEUDO_REGISTER; i < regs_num; i++)
 499     {
 500       if (lra_reg_info[i].nrefs != 0 && lra_get_regno_hard_regno (i) < 0
 501           && ! lra_former_scratch_p (i))
 502         {
 503           bitmap_set_bit (&spilled_pseudos, i);
 504           bitmap_ior_into (&changed_insns, &lra_reg_info[i].insn_bitmap);
 505         }
 506     }
 507   FOR_EACH_BB_FN (bb, cfun)
 508     {
 509       FOR_BB_INSNS (bb, insn)
 510         if (bitmap_bit_p (&changed_insns, INSN_UID (insn)))
 511           {
 512             rtx *link_loc, link;
 513             remove_pseudos (&PATTERN (insn), insn);
 514             if (CALL_P (insn))
 515               remove_pseudos (&CALL_INSN_FUNCTION_USAGE (insn), insn);
 516             for (link_loc = &REG_NOTES (insn);
 517                  (link = *link_loc) != NULL_RTX;
 518                  link_loc = &XEXP (link, 1))
 519               {
 520                 switch (REG_NOTE_KIND (link))
 521                   {
 522                   case REG_FRAME_RELATED_EXPR:
 523                   case REG_CFA_DEF_CFA:
 524                   case REG_CFA_ADJUST_CFA:
 525                   case REG_CFA_OFFSET:
 526                   case REG_CFA_REGISTER:
 527                   case REG_CFA_EXPRESSION:
 528                   case REG_CFA_RESTORE:
 529                   case REG_CFA_SET_VDRAP:
 530                     remove_pseudos (&XEXP (link, 0), insn);
 531                     break;
 532                   default:
 533                     break;
 534                   }
 535               }
 536             if (lra_dump_file != NULL)
 537               fprintf (lra_dump_file,
 538                        "Changing spilled pseudos to memory in insn #%u\n",
 539                        INSN_UID (insn));
 540             lra_push_insn (insn);
 541             if (lra_reg_spill_p || targetm.different_addr_displacement_p ())
 542               lra_set_used_insn_alternative (insn, -1);
 543           }
 544         else if (CALL_P (insn))
 545           /* Presence of any pseudo in CALL_INSN_FUNCTION_USAGE does
 546              not affect value of insn_bitmap of the corresponding
 547              lra_reg_info.  That is because we don't need to reload
 548              pseudos in CALL_INSN_FUNCTION_USAGEs.  So if we process
 549              only insns in the insn_bitmap of given pseudo here, we
 550              can miss the pseudo in some
 551              CALL_INSN_FUNCTION_USAGEs.  */
 552           remove_pseudos (&CALL_INSN_FUNCTION_USAGE (insn), insn);
 553       bitmap_and_compl_into (df_get_live_in (bb), &spilled_pseudos);
 554       bitmap_and_compl_into (df_get_live_out (bb), &spilled_pseudos);
 555     }
 556   bitmap_clear (&spilled_pseudos);
 557   bitmap_clear (&changed_insns);
 558 }
 559
 560 /* Return true if we need to change some pseudos into memory.  */
 561 bool
 562 lra_need_for_spills_p (void)
 563 {
 564   int i; max_regno = max_reg_num ();
 565
 566   for (i = FIRST_PSEUDO_REGISTER; i < max_regno; i++)
 567     if (lra_reg_info[i].nrefs != 0 && lra_get_regno_hard_regno (i) < 0
 568         && ! lra_former_scratch_p (i))
 569       return true;
 570   return false;
 571 }
 572
 573 /* Change spilled pseudos into memory or spill hard regs.  Put changed
 574    insns on the constraint stack (these insns will be considered on
 575    the next constraint pass).  The changed insns are all insns in
 576    which pseudos were changed.  */
 577 void
 578 lra_spill (void)
 579 {
 580   int i, n, curr_regno;
 581   int *pseudo_regnos;
 582
 583   regs_num = max_reg_num ();
 584   spill_hard_reg = XNEWVEC (rtx, regs_num);
 585   pseudo_regnos = XNEWVEC (int, regs_num);
 586   for (n = 0, i = FIRST_PSEUDO_REGISTER; i < regs_num; i++)
 587     if (lra_reg_info[i].nrefs != 0 && lra_get_regno_hard_regno (i) < 0
 588         /* We do not want to assign memory for former scratches.  */
 589         && ! lra_former_scratch_p (i))
 590       {
 591         spill_hard_reg[i] = NULL_RTX;
 592         pseudo_regnos[n++] = i;
 593       }
 594   lra_assert (n > 0);
 595   pseudo_slots = XNEWVEC (struct pseudo_slot, regs_num);
 596   slots = XNEWVEC (struct slot, regs_num);
 597   /* Sort regnos according their usage frequencies.  */
 598   qsort (pseudo_regnos, n, sizeof (int), regno_freq_compare);
 599   n = assign_spill_hard_regs (pseudo_regnos, n);
 600   assign_stack_slot_num_and_sort_pseudos (pseudo_regnos, n);
 601   for (i = 0; i < n; i++)
 602     if (pseudo_slots[pseudo_regnos[i]].mem == NULL_RTX)
 603       assign_mem_slot (pseudo_regnos[i]);
 604   if (n > 0 && crtl->stack_alignment_needed)
 605     /* If we have a stack frame, we must align it now.  The stack size
 606        may be a part of the offset computation for register
 607        elimination.  */
 608     assign_stack_local (BLKmode, 0, crtl->stack_alignment_needed);
 609   if (lra_dump_file != NULL)
 610     {
 611       for (i = 0; i < slots_num; i++)
 612         {
 613           fprintf (lra_dump_file, "  Slot %d regnos (width = %d):", i,
 614                    GET_MODE_SIZE (GET_MODE (slots[i].mem)));
 615           for (curr_regno = slots[i].regno;;
 616                curr_regno = pseudo_slots[curr_regno].next - pseudo_slots)
 617             {
 618               fprintf (lra_dump_file, "  %d", curr_regno);
 619               if (pseudo_slots[curr_regno].next == NULL)
 620                 break;
 621             }
 622           fprintf (lra_dump_file, "\n");
 623         }
 624     }
 625   spill_pseudos ();
 626   free (slots);
 627   free (pseudo_slots);
 628   free (pseudo_regnos);
 629   free (spill_hard_reg);
 630 }
 631
 632 /* Apply alter_subreg for subregs of regs in *LOC.  Use FINAL_P for
 633    alter_subreg calls. Return true if any subreg of reg is
 634    processed.  */
 635 static bool
 636 alter_subregs (rtx *loc, bool final_p)
 637 {
 638   int i;
 639   rtx x = *loc;
 640   bool res;
 641   const char *fmt;
 642   enum rtx_code code;
 643
 644   if (x == NULL_RTX)
 645     return false;
 646   code = GET_CODE (x);
 647   if (code == SUBREG && REG_P (SUBREG_REG (x)))
 648     {
 649       lra_assert (REGNO (SUBREG_REG (x)) < FIRST_PSEUDO_REGISTER);
 650       alter_subreg (loc, final_p);
 651       return true;
 652     }
 653   fmt = GET_RTX_FORMAT (code);
 654   res = false;
 655   for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
 656     {
 657       if (fmt[i] == 'e')
 658         {
 659           if (alter_subregs (&XEXP (x, i), final_p))
 660             res = true;
 661         }
 662       else if (fmt[i] == 'E')
 663         {
 664           int j;
 665
 666           for (j = XVECLEN (x, i) - 1; j >= 0; j--)
 667             if (alter_subregs (&XVECEXP (x, i, j), final_p))
 668               res = true;
 669         }
 670     }
 671   return res;
 672 }
 673
 674 /* Return true if REGNO is used for return in the current
 675    function.  */
 676 static bool
 677 return_regno_p (unsigned int regno)
 678 {
 679   rtx outgoing = crtl->return_rtx;
 680
 681   if (! outgoing)
 682     return false;
 683
 684   if (REG_P (outgoing))
 685     return REGNO (outgoing) == regno;
 686   else if (GET_CODE (outgoing) == PARALLEL)
 687     {
 688       int i;
 689
 690       for (i = 0; i < XVECLEN (outgoing, 0); i++)
 691         {
 692           rtx x = XEXP (XVECEXP (outgoing, 0, i), 0);
 693
 694           if (REG_P (x) && REGNO (x) == regno)
 695             return true;
 696         }
 697     }
 698   return false;
 699 }
 700
 701 /* Final change of pseudos got hard registers into the corresponding
 702    hard registers and removing temporary clobbers.  */
 703 void
 704 lra_final_code_change (void)
 705 {
 706   int i, hard_regno;
 707   basic_block bb;
 708   rtx_insn *insn, *curr;
 709   int max_regno = max_reg_num ();
 710
 711   for (i = FIRST_PSEUDO_REGISTER; i < max_regno; i++)
 712     if (lra_reg_info[i].nrefs != 0
 713         && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
 714       SET_REGNO (regno_reg_rtx[i], hard_regno);
 715   FOR_EACH_BB_FN (bb, cfun)
 716     FOR_BB_INSNS_SAFE (bb, insn, curr)
 717       if (INSN_P (insn))
 718         {
 719           rtx pat = PATTERN (insn);
 720
 721           if (GET_CODE (pat) == CLOBBER && LRA_TEMP_CLOBBER_P (pat))
 722             {
 723               /* Remove clobbers temporarily created in LRA.  We don't
 724                  need them anymore and don't want to waste compiler
 725                  time processing them in a few subsequent passes.  */
 726               lra_invalidate_insn_data (insn);
 727               delete_insn (insn);
 728               continue;
 729             }
 730
 731           /* IRA can generate move insns involving pseudos.  It is
 732              better remove them earlier to speed up compiler a bit.
 733              It is also better to do it here as they might not pass
 734              final RTL check in LRA, (e.g. insn moving a control
 735              register into itself).  So remove an useless move insn
 736              unless next insn is USE marking the return reg (we should
 737              save this as some subsequent optimizations assume that
 738              such original insns are saved).  */
 739           if (NONJUMP_INSN_P (insn) && GET_CODE (pat) == SET
 740               && REG_P (SET_SRC (pat)) && REG_P (SET_DEST (pat))
 741               && REGNO (SET_SRC (pat)) == REGNO (SET_DEST (pat))
 742               && ! return_regno_p (REGNO (SET_SRC (pat))))
 743             {
 744               lra_invalidate_insn_data (insn);
 745               delete_insn (insn);
 746               continue;
 747             }
 748
 749           lra_insn_recog_data_t id = lra_get_insn_recog_data (insn);
 750           struct lra_static_insn_data *static_id = id->insn_static_data;
 751           bool insn_change_p = false;
 752
 753           for (i = id->insn_static_data->n_operands - 1; i >= 0; i--)
 754             if ((DEBUG_INSN_P (insn) || ! static_id->operand[i].is_operator)
 755                 && alter_subregs (id->operand_loc[i], ! DEBUG_INSN_P (insn)))
 756               {
 757                 lra_update_dup (id, i);
 758                 insn_change_p = true;
 759               }
 760           if (insn_change_p)
 761             lra_update_operator_dups (id);
 762         }
 763 }