src/northbridge/intel/gm45/raminit_read_write_training.c

   1 /*
   2  * This file is part of the coreboot project.
   3  *
   4  * Copyright (C) 2012 secunet Security Networks AG
   5  * (Written by Nico Huber <nico.huber@secunet.com> for secunet)
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; version 2 of the License.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  */
  16
  17 #include <stdint.h>
  18 #include <arch/io.h>
  19 #include <pc80/mc146818rtc.h>
  20 #include <console/console.h>
  21 #include "gm45.h"
  22
  23 typedef struct {
  24         u32 addr[RANKS_PER_CHANNEL];
  25         unsigned count;
  26 } address_bunch_t;
  27
  28 /* Read Training. */
  29 #define CxRDTy_MCHBAR(ch, bl)   (0x14b0 + (ch * 0x0100) + ((7 - bl) * 4))
  30 #define CxRDTy_T_SHIFT          20
  31 #define CxRDTy_T_MASK           (0xf << CxRDTy_T_SHIFT)
  32 #define CxRDTy_T(t)             ((t << CxRDTy_T_SHIFT) & CxRDTy_T_MASK)
  33 #define CxRDTy_P_SHIFT          16
  34 #define CxRDTy_P_MASK           (0x7 << CxRDTy_P_SHIFT)
  35 #define CxRDTy_P(p)             ((p << CxRDTy_P_SHIFT) & CxRDTy_P_MASK)
  36 static const u32 read_training_schedule[] = {
  37         0xfefefefe, 0x7f7f7f7f, 0xbebebebe, 0xdfdfdfdf,
  38         0xeeeeeeee, 0xf7f7f7f7, 0xfafafafa, 0xfdfdfdfd,
  39         0x00000000, 0x81818181, 0x40404040, 0x21212121,
  40         0x10101010, 0x09090909, 0x04040404, 0x03030303,
  41         0x10101010, 0x11111111, 0xeeeeeeee, 0xefefefef,
  42         0x10101010, 0x11111111, 0xeeeeeeee, 0xefefefef,
  43         0x10101010, 0xefefefef, 0x10101010, 0xefefefef,
  44         0x10101010, 0xefefefef, 0x10101010, 0xefefefef,
  45         0x00000000, 0xffffffff, 0x00000000, 0xffffffff,
  46         0x00000000, 0xffffffff, 0x00000000, 0x00000000,
  47 };
  48 #define READ_TIMING_P_SHIFT     3
  49 #define READ_TIMING_P_BOUND     (1 << READ_TIMING_P_SHIFT)
  50 #define READ_TIMING_T_BOUND     14
  51 typedef struct {
  52         int t;
  53         int p;
  54 } read_timing_t;
  55 static void print_read_timing(const int msg_lvl, const char *const msg,
  56                               const int lane, const int channel,
  57                               const read_timing_t *const timing)
  58 {
  59         printk(msg_lvl, "%s for byte lane %d on channel %d: %d.%d\n",
  60                msg, lane, channel, timing->t, timing->p);
  61 }
  62
  63 static int normalize_read_timing(read_timing_t *const timing)
  64 {
  65         while (timing->p >= READ_TIMING_P_BOUND) {
  66                 timing->t++;
  67                 timing->p -= READ_TIMING_P_BOUND;
  68         }
  69         while (timing->p < 0) {
  70                 timing->t--;
  71                 timing->p += READ_TIMING_P_BOUND;
  72         }
  73         if (timing->t < 0) {
  74                 printk(BIOS_WARNING,
  75                        "Timing underflow during read training.\n");
  76                 timing->t = 0;
  77                 timing->p = 0;
  78                 return -1;
  79         } else if (timing->t >= READ_TIMING_T_BOUND) {
  80                 printk(BIOS_WARNING,
  81                        "Timing overflow during read training.\n");
  82                 timing->t = READ_TIMING_T_BOUND - 1;
  83                 timing->p = READ_TIMING_P_BOUND - 1;
  84                 return -1;
  85         }
  86         return 0;
  87 }
  88 static int program_read_timing(const int ch, const int lane,
  89                                read_timing_t *const timing)
  90 {
  91         if (normalize_read_timing(timing) < 0)
  92                 return -1;
  93
  94         u32 reg = MCHBAR32(CxRDTy_MCHBAR(ch, lane));
  95         reg &= ~(CxRDTy_T_MASK | CxRDTy_P_MASK);
  96         reg |= CxRDTy_T(timing->t) | CxRDTy_P(timing->p);
  97         MCHBAR32(CxRDTy_MCHBAR(ch, lane)) = reg;
  98
  99         return 0;
 100 }
 101 /* Returns 1 on success, 0 on failure. */
 102 static int read_training_test(const int channel, const int lane,
 103                               const address_bunch_t *const addresses)
 104 {
 105         int i;
 106
 107         const int lane_offset = lane & 4;
 108         const int lane_mask = 0xff << ((lane & ~4) << 3);
 109
 110         for (i = 0; i < addresses->count; ++i) {
 111                 unsigned int offset;
 112                 for (offset = lane_offset; offset < 320; offset += 8) {
 113                         const u32 read = read32((u32 *)(addresses->addr[i] + offset));
 114                         const u32 good = read_training_schedule[offset >> 3];
 115                         if ((read & lane_mask) != (good & lane_mask))
 116                                 return 0;
 117                 }
 118         }
 119         return 1;
 120 }
 121 static int read_training_find_lower(const int channel, const int lane,
 122                                     const address_bunch_t *const addresses,
 123                                     read_timing_t *const lower)
 124 {
 125         /* Coarse search for good t. */
 126         program_read_timing(channel, lane, lower);
 127         while (!read_training_test(channel, lane, addresses)) {
 128                 ++lower->t;
 129                 if (program_read_timing(channel, lane, lower) < 0)
 130                         return -1;
 131         }
 132
 133         /* Step back, then fine search for good p. */
 134         if (lower->t <= 0)
 135                 /* Can't step back, zero is good. */
 136                 return 0;
 137
 138         --lower->t;
 139         program_read_timing(channel, lane, lower);
 140         while (!read_training_test(channel, lane, addresses)) {
 141                 ++lower->p;
 142                 if (program_read_timing(channel, lane, lower) < 0)
 143                         return -1;
 144         }
 145
 146         return 0;
 147 }
 148 static int read_training_find_upper(const int channel, const int lane,
 149                                     const address_bunch_t *const addresses,
 150                                     read_timing_t *const upper)
 151 {
 152         if (program_read_timing(channel, lane, upper) < 0)
 153                 return -1;
 154         if (!read_training_test(channel, lane, addresses)) {
 155                 printk(BIOS_WARNING,
 156                        "Read training failure: limits too narrow.\n");
 157                 return -1;
 158         }
 159         /* Coarse search for bad t. */
 160         do {
 161                 ++upper->t;
 162                 if (program_read_timing(channel, lane, upper) < 0)
 163                         return -1;
 164         } while (read_training_test(channel, lane, addresses));
 165         /* Fine search for bad p. */
 166         --upper->t;
 167         program_read_timing(channel, lane, upper);
 168         while (read_training_test(channel, lane, addresses)) {
 169                 ++upper->p;
 170                 if (program_read_timing(channel, lane, upper) < 0)
 171                         return -1;
 172         }
 173
 174         return 0;
 175 }
 176 static void read_training_per_lane(const int channel, const int lane,
 177                                    const address_bunch_t *const addresses)
 178 {
 179         read_timing_t lower, upper;
 180
 181         MCHBAR32(CxRDTy_MCHBAR(channel, lane)) |= 3 << 25;
 182
 183         /*** Search lower bound. ***/
 184
 185         /* Start at zero. */
 186         lower.t = 0;
 187         lower.p = 0;
 188         if (read_training_find_lower(channel, lane, addresses, &lower) < 0)
 189                 die("Read training failure: lower bound.\n");
 190         print_read_timing(BIOS_SPEW, "Lower bound", lane, channel, &lower);
 191
 192         /*** Search upper bound. ***/
 193
 194         /* Start at lower + 1t. */
 195         upper.t = lower.t + 1;
 196         upper.p = lower.p;
 197         if (read_training_find_upper(channel, lane, addresses, &upper) < 0)
 198                 /* Overflow on upper edge is not fatal. */
 199                 printk(BIOS_WARNING, "Read training failure: upper bound.\n");
 200         print_read_timing(BIOS_SPEW, "Upper bound", lane, channel, &upper);
 201
 202         /*** Calculate and program mean value. ***/
 203
 204         lower.p += lower.t << READ_TIMING_P_SHIFT;
 205         upper.p += upper.t << READ_TIMING_P_SHIFT;
 206         const int mean_p = (lower.p + upper.p) >> 1;
 207         /* lower becomes the mean value. */
 208         lower.t = mean_p >> READ_TIMING_P_SHIFT;
 209         lower.p = mean_p & (READ_TIMING_P_BOUND - 1);
 210         program_read_timing(channel, lane, &lower);
 211         print_read_timing(BIOS_DEBUG, "Final timings", lane, channel, &lower);
 212 }
 213 static void perform_read_training(const dimminfo_t *const dimms)
 214 {
 215         int ch, i;
 216
 217         FOR_EACH_POPULATED_CHANNEL(dimms, ch) {
 218                 address_bunch_t addresses = { { 0, }, 0 };
 219                 FOR_EACH_POPULATED_RANK_IN_CHANNEL(dimms, ch, i)
 220                         addresses.addr[addresses.count++] =
 221                                 raminit_get_rank_addr(ch, i);
 222
 223                 for (i = 0; i < addresses.count; ++i) {
 224                         /* Write test pattern. */
 225                         unsigned int offset;
 226                         for (offset = 0; offset < 320; offset += 4)
 227                                 write32((u32 *)(addresses.addr[i] + offset),
 228                                         read_training_schedule[offset >> 3]);
 229                 }
 230
 231                 for (i = 0; i < 8; ++i)
 232                         read_training_per_lane(ch, i, &addresses);
 233         }
 234 }
 235 static void read_training_store_results(void)
 236 {
 237         u8 bytes[TOTAL_CHANNELS * 8];
 238         int ch, i;
 239
 240         /* Store one timing pair in one byte each. */
 241         FOR_EACH_CHANNEL(ch) {
 242                 for (i = 0; i < 8; ++i) {
 243                         const u32 bl_reg = MCHBAR32(CxRDTy_MCHBAR(ch, i));
 244                         bytes[(ch * 8) + i] =
 245                                 (((bl_reg & CxRDTy_T_MASK) >> CxRDTy_T_SHIFT)
 246                                  << 4) |
 247                                 ((bl_reg & CxRDTy_P_MASK) >> CxRDTy_P_SHIFT);
 248                 }
 249         }
 250
 251         /* Store everything in CMOS above 128 bytes. */
 252         for (i = 0; i < (TOTAL_CHANNELS * 8); ++i)
 253                 cmos_write(bytes[i], CMOS_READ_TRAINING + i);
 254 }
 255 static void read_training_restore_results(void)
 256 {
 257         u8 bytes[TOTAL_CHANNELS * 8];
 258         int ch, i;
 259
 260         /* Read from CMOS. */
 261         for (i = 0; i < (TOTAL_CHANNELS * 8); ++i)
 262                 bytes[i] = cmos_read(CMOS_READ_TRAINING + i);
 263
 264         /* Program restored results. */
 265         FOR_EACH_CHANNEL(ch) {
 266                 for (i = 0; i < 8; ++i) {
 267                         const int t = bytes[(ch * 8) + i] >> 4;
 268                         const int p = bytes[(ch * 8) + i] & 7;
 269                         u32 bl_reg = MCHBAR32(CxRDTy_MCHBAR(ch, i));
 270                         bl_reg &= ~(CxRDTy_T_MASK | CxRDTy_P_MASK);
 271                         bl_reg |= (3 << 25) | CxRDTy_T(t) | CxRDTy_P(p);
 272                         MCHBAR32(CxRDTy_MCHBAR(ch, i)) = bl_reg;
 273                         printk(BIOS_DEBUG, "Restored timings for byte lane "
 274                                "%d on channel %d: %d.%d\n", i, ch, t, p);
 275                 }
 276         }
 277 }
 278 void raminit_read_training(const dimminfo_t *const dimms, const int s3resume)
 279 {
 280         if (!s3resume) {
 281                 perform_read_training(dimms);
 282                 read_training_store_results();
 283         } else {
 284                 read_training_restore_results();
 285         }
 286         raminit_reset_readwrite_pointers();
 287 }
 288
 289 /* Write Training. */
 290 #define CxWRTy_T_SHIFT          28
 291 #define CxWRTy_T_MASK           (0xf << CxWRTy_T_SHIFT)
 292 #define CxWRTy_T(t)             ((t << CxWRTy_T_SHIFT) & CxWRTy_T_MASK)
 293 #define CxWRTy_P_SHIFT          24
 294 #define CxWRTy_P_MASK           (0x7 << CxWRTy_P_SHIFT)
 295 #define CxWRTy_P(p)             ((p << CxWRTy_P_SHIFT) & CxWRTy_P_MASK)
 296 #define CxWRTy_F_SHIFT          18
 297 #define CxWRTy_F_MASK           (0x3 << CxWRTy_F_SHIFT)
 298 #define CxWRTy_F(f)             ((f << CxWRTy_F_SHIFT) & CxWRTy_F_MASK)
 299 #define CxWRTy_D_SHIFT          16
 300 #define CxWRTy_D_MASK           (0x3 << CxWRTy_D_SHIFT)
 301 #define CxWRTy_BELOW_D          (0x3 << CxWRTy_D_SHIFT)
 302 #define CxWRTy_ABOVE_D          (0x1 << CxWRTy_D_SHIFT)
 303 static const u32 write_training_schedule[] = {
 304         0xffffffff, 0x00000000, 0xffffffff, 0x00000000,
 305         0xffffffff, 0x00000000, 0xffffffff, 0x00000000,
 306         0xffffffff, 0x00000000, 0xffffffff, 0x00000000,
 307         0xffffffff, 0x00000000, 0xffffffff, 0x00000000,
 308         0xefefefef, 0x10101010, 0xefefefef, 0x10101010,
 309         0xefefefef, 0x10101010, 0xefefefef, 0x10101010,
 310         0xefefefef, 0x10101010, 0xefefefef, 0x10101010,
 311         0xefefefef, 0x10101010, 0xefefefef, 0x10101010,
 312         0xefefefef, 0xeeeeeeee, 0x11111111, 0x10101010,
 313         0xefefefef, 0xeeeeeeee, 0x11111111, 0x10101010,
 314         0xefefefef, 0xeeeeeeee, 0x11111111, 0x10101010,
 315         0xefefefef, 0xeeeeeeee, 0x11111111, 0x10101010,
 316         0x03030303, 0x04040404, 0x09090909, 0x10101010,
 317         0x21212121, 0x40404040, 0x81818181, 0x00000000,
 318         0x03030303, 0x04040404, 0x09090909, 0x10101010,
 319         0x21212121, 0x40404040, 0x81818181, 0x00000000,
 320         0xfdfdfdfd, 0xfafafafa, 0xf7f7f7f7, 0xeeeeeeee,
 321         0xdfdfdfdf, 0xbebebebe, 0x7f7f7f7f, 0xfefefefe,
 322         0xfdfdfdfd, 0xfafafafa, 0xf7f7f7f7, 0xeeeeeeee,
 323         0xdfdfdfdf, 0xbebebebe, 0x7f7f7f7f, 0xfefefefe,
 324 };
 325 /* for raw card types A, B and C: MEM_CLOCK_1067MT? X group X lower/upper */
 326 static const u32 write_training_bytelane_masks_abc[2][4][2] = {
 327         { /* clock < MEM_CLOCK_1067MT */
 328                 { 0xffffffff, 0x00000000 }, { 0x00000000, 0x00000000 },
 329                 { 0x00000000, 0xffffffff }, { 0x00000000, 0x00000000 },
 330         },
 331         { /* clock == MEM_CLOCK_1067MT */
 332                 { 0x0000ffff, 0x00000000 }, { 0xffff0000, 0x00000000 },
 333                 { 0x00000000, 0x0000ffff }, { 0x00000000, 0xffff0000 },
 334         },
 335 };
 336 /* for raw card type F: group X lower/upper */
 337 static const u32 write_training_bytelane_masks_f[4][2] = {
 338         { 0xff00ff00, 0x00000000 }, { 0x00ff00ff, 0x00000000 },
 339         { 0x00000000, 0xff00ff00 }, { 0x00000000, 0x00ff00ff },
 340 };
 341 #define WRITE_TIMING_P_SHIFT    3
 342 #define WRITE_TIMING_P_BOUND    (1 << WRITE_TIMING_P_SHIFT)
 343 #define WRITE_TIMING_F_BOUND    4
 344 typedef struct {
 345         int f;
 346         int t;
 347         const int t_bound;
 348         int p;
 349 } write_timing_t;
 350 static void print_write_timing(const int msg_lvl, const char *const msg,
 351                                const int group, const int channel,
 352                                const write_timing_t *const timing)
 353 {
 354         printk(msg_lvl, "%s for group %d on channel %d: %d.%d.%d\n",
 355                msg, group, channel, timing->f, timing->t, timing->p);
 356 }
 357
 358 static int normalize_write_timing(write_timing_t *const timing)
 359 {
 360         while (timing->p >= WRITE_TIMING_P_BOUND) {
 361                 timing->t++;
 362                 timing->p -= WRITE_TIMING_P_BOUND;
 363         }
 364         while (timing->p < 0) {
 365                 timing->t--;
 366                 timing->p += WRITE_TIMING_P_BOUND;
 367         }
 368         while (timing->t >= timing->t_bound) {
 369                 timing->f++;
 370                 timing->t -= timing->t_bound;
 371         }
 372         while (timing->t < 0) {
 373                 timing->f--;
 374                 timing->t += timing->t_bound;
 375         }
 376         if (timing->f < 0) {
 377                 printk(BIOS_WARNING,
 378                        "Timing underflow during write training.\n");
 379                 timing->f = 0;
 380                 timing->t = 0;
 381                 timing->p = 0;
 382                 return -1;
 383         } else if (timing->f >= WRITE_TIMING_F_BOUND) {
 384                 printk(BIOS_WARNING,
 385                        "Timing overflow during write training.\n");
 386                 timing->f = WRITE_TIMING_F_BOUND - 1;
 387                 timing->t = timing->t_bound - 1;
 388                 timing->p = WRITE_TIMING_P_BOUND - 1;
 389                 return -1;
 390         }
 391         return 0;
 392 }
 393 static int program_write_timing(const int ch, const int group,
 394                                 write_timing_t *const timing, int memclk1067)
 395 {
 396         /* MEM_CLOCK_1067MT? X lower/upper */
 397         const u32 d_bounds[2][2] = { { 1, 6 }, { 2, 9 } };
 398
 399         if (normalize_write_timing(timing) < 0)
 400                 return -1;
 401
 402         const int f = timing->f;
 403         const int t = timing->t;
 404         const int p = (memclk1067 && (((t ==  9) && (timing->p >= 4)) ||
 405                                        ((t == 10) && (timing->p < 4))))
 406                 ? 4 : timing->p;
 407         const int d =
 408                 (t <= d_bounds[memclk1067][0]) ? CxWRTy_BELOW_D :
 409                 ((t >  d_bounds[memclk1067][1]) ? CxWRTy_ABOVE_D : 0);
 410
 411         u32 reg = MCHBAR32(CxWRTy_MCHBAR(ch, group));
 412         reg &= ~(CxWRTy_T_MASK | CxWRTy_P_MASK | CxWRTy_F_MASK);
 413         reg &= ~CxWRTy_D_MASK;
 414         reg |= CxWRTy_T(t) | CxWRTy_P(p) | CxWRTy_F(f) | d;
 415         MCHBAR32(CxWRTy_MCHBAR(ch, group)) = reg;
 416
 417         return 0;
 418 }
 419 /* Returns 1 on success, 0 on failure. */
 420 static int write_training_test(const address_bunch_t *const addresses,
 421                                const u32 *const masks)
 422 {
 423         int i, ret = 0;
 424
 425         const u32 mmarb0 = MCHBAR32(0x0220);
 426         const u8  wrcctl = MCHBAR8(0x0218);
 427         MCHBAR32(0x0220) |= 0xf << 28;
 428         MCHBAR8(0x0218)  |= 0x1 <<  4;
 429
 430         for (i = 0; i < addresses->count; ++i) {
 431                 const unsigned int addr = addresses->addr[i];
 432                 unsigned int off;
 433                 for (off = 0; off < 640; off += 8) {
 434                         const u32 pattern = write_training_schedule[off >> 3];
 435                         write32((u32 *)(addr + off), pattern);
 436                         write32((u32 *)(addr + off + 4), pattern);
 437                 }
 438
 439                 MCHBAR8(0x78) |= 1;
 440
 441                 for (off = 0; off < 640; off += 8) {
 442                         const u32 good = write_training_schedule[off >> 3];
 443                         const u32 read1 = read32((u32 *)(addr + off));
 444                         if ((read1 & masks[0]) != (good & masks[0]))
 445                                 goto _bad_timing_out;
 446                         const u32 read2 = read32((u32 *)(addr + off + 4));
 447                         if ((read2 & masks[1]) != (good & masks[1]))
 448                                 goto _bad_timing_out;
 449                 }
 450         }
 451         ret = 1;
 452
 453 _bad_timing_out:
 454         MCHBAR32(0x0220) = mmarb0;
 455         MCHBAR8(0x0218)  = wrcctl;
 456
 457         return ret;
 458 }
 459 static int write_training_find_lower(const int ch, const int group,
 460                                      const address_bunch_t *const addresses,
 461                                      const u32 masks[][2], const int memclk1067,
 462                                      write_timing_t *const lower)
 463 {
 464         program_write_timing(ch, group, lower, memclk1067);
 465         /* Coarse search for good t. */
 466         while (!write_training_test(addresses, masks[group])) {
 467                 ++lower->t;
 468                 if (program_write_timing(ch, group, lower, memclk1067) < 0)
 469                         return -1;
 470         }
 471         /* Step back, then fine search for good p. */
 472         if ((lower->f <= 0) && (lower->t <= 0))
 473                 /* Can't step back, zero is good. */
 474                 return 0;
 475
 476         --lower->t;
 477         program_write_timing(ch, group, lower, memclk1067);
 478         while (!write_training_test(addresses, masks[group])) {
 479                 ++lower->p;
 480                 if (program_write_timing(ch, group, lower, memclk1067) < 0)
 481                         return -1;
 482         }
 483
 484         return 0;
 485 }
 486 static int write_training_find_upper(const int ch, const int group,
 487                                      const address_bunch_t *const addresses,
 488                                      const u32 masks[][2], const int memclk1067,
 489                                      write_timing_t *const upper)
 490 {
 491         if (program_write_timing(ch, group, upper, memclk1067) < 0)
 492                 return -1;
 493         if (!write_training_test(addresses, masks[group])) {
 494                 printk(BIOS_WARNING,
 495                        "Write training failure; limits too narrow.\n");
 496                 return -1;
 497         }
 498         /* Coarse search for bad t. */
 499         while (write_training_test(addresses, masks[group])) {
 500                 ++upper->t;
 501                 if (program_write_timing(ch, group, upper, memclk1067) < 0)
 502                         return -1;
 503         }
 504         /* Fine search for bad p. */
 505         --upper->t;
 506         program_write_timing(ch, group, upper, memclk1067);
 507         while (write_training_test(addresses, masks[group])) {
 508                 ++upper->p;
 509                 if (program_write_timing(ch, group, upper, memclk1067) < 0)
 510                         return -1;
 511         }
 512
 513         return 0;
 514 }
 515 static void write_training_per_group(const int ch, const int group,
 516                                      const address_bunch_t *const addresses,
 517                                      const u32 masks[][2], const int memclk1067)
 518 {
 519         const int t_bound = memclk1067 ? 12 : 11;
 520         write_timing_t lower = { 0, 0, t_bound, 0 },
 521                        upper = { 0, 0, t_bound, 0 };
 522
 523         /*** Search lower bound. ***/
 524
 525         /* Start at -1f from current values. */
 526         const u32 reg = MCHBAR32(CxWRTy_MCHBAR(ch, group));
 527         lower.t =  (reg >> 12) & 0xf;
 528         lower.p =  (reg >>  8) & 0x7;
 529         lower.f = ((reg >>  2) & 0x3) - 1;
 530
 531         if (write_training_find_lower(ch, group, addresses,
 532                                       masks, memclk1067, &lower) < 0)
 533                 die("Write training failure: lower bound.\n");
 534         print_write_timing(BIOS_SPEW, "Lower bound", group, ch, &lower);
 535
 536         /*** Search upper bound. ***/
 537
 538         /* Start at lower + 3t. */
 539         upper.t = lower.t + 3;
 540         upper.p = lower.p;
 541         upper.f = lower.f;
 542
 543         if (write_training_find_upper(ch, group, addresses,
 544                                       masks, memclk1067, &upper) < 0)
 545                 printk(BIOS_WARNING, "Write training failure: upper bound.\n");
 546         print_write_timing(BIOS_SPEW, "Upper bound", group, ch, &upper);
 547
 548         /*** Calculate and program mean value. ***/
 549
 550         lower.t += lower.f * lower.t_bound;
 551         lower.p += lower.t << WRITE_TIMING_P_SHIFT;
 552         upper.t += upper.f * upper.t_bound;
 553         upper.p += upper.t << WRITE_TIMING_P_SHIFT;
 554         /* lower becomes the mean value. */
 555         const int mean_p = (lower.p + upper.p) >> 1;
 556         lower.f = mean_p / (lower.t_bound << WRITE_TIMING_P_SHIFT);
 557         lower.t = (mean_p >> WRITE_TIMING_P_SHIFT) % lower.t_bound;
 558         lower.p = mean_p & (WRITE_TIMING_P_BOUND - 1);
 559         program_write_timing(ch, group, &lower, memclk1067);
 560         print_write_timing(BIOS_DEBUG, "Final timings", group, ch, &lower);
 561 }
 562 static void perform_write_training(const int memclk1067,
 563                                    const dimminfo_t *const dimms)
 564 {
 565         const int cardF[] = { dimms[0].card_type == 0xf,
 566                               dimms[1].card_type == 0xf };
 567         int ch, r, group;
 568
 569         address_bunch_t addr[2] = { { { 0, }, 0 }, { { 0, }, 0 }, };
 570         /* Add check if channel A is populated, i.e. if cardF[0] is valid.
 571          * Otherwise we would write channel A registers when DIMM in channel B
 572          * is of raw card type A, B or C (cardF[1] == 0) even if channel A is
 573          * not populated.
 574          * Needs raw card type A, B or C for testing. */
 575         if ((dimms[0].card_type != 0) && (cardF[0] == cardF[1])) {
 576                 /* Common path for both channels. */
 577                 FOR_EACH_POPULATED_RANK(dimms, ch, r)
 578                         addr[0].addr[addr[0].count++] =
 579                                 raminit_get_rank_addr(ch, r);
 580         } else {
 581                 FOR_EACH_POPULATED_RANK(dimms, ch, r)
 582                         addr[ch].addr[addr[ch].count++] =
 583                                 raminit_get_rank_addr(ch, r);
 584         }
 585
 586         FOR_EACH_CHANNEL(ch) if (addr[ch].count > 0) {
 587                 const u32 (*const masks)[2] = (!cardF[ch])
 588                         ? write_training_bytelane_masks_abc[memclk1067]
 589                         : write_training_bytelane_masks_f;
 590                 for (group = 0; group < 4; ++group) {
 591                         if (!masks[group][0] && !masks[group][1])
 592                                 continue;
 593                         write_training_per_group(
 594                                 ch, group, &addr[ch], masks, memclk1067);
 595                 }
 596         }
 597 }
 598 static void write_training_store_results(void)
 599 {
 600         u8 bytes[TOTAL_CHANNELS * 4 * 2]; /* two bytes per group */
 601         int ch, i;
 602
 603         /* Store one T/P pair in one, F in the other byte. */
 604         /* We could save six bytes by putting all F values in two bytes. */
 605         FOR_EACH_CHANNEL(ch) {
 606                 for (i = 0; i < 4; ++i) {
 607                         const u32 reg = MCHBAR32(CxWRTy_MCHBAR(ch, i));
 608                         bytes[(ch * 8) + (i * 2)] =
 609                                 (((reg & CxWRTy_T_MASK)
 610                                   >> CxWRTy_T_SHIFT) << 4) |
 611                                 ((reg & CxWRTy_P_MASK) >> CxWRTy_P_SHIFT);
 612                         bytes[(ch * 8) + (i * 2) + 1] =
 613                                 ((reg & CxWRTy_F_MASK) >> CxWRTy_F_SHIFT);
 614                 }
 615         }
 616
 617         /* Store everything in CMOS above 128 bytes. */
 618         for (i = 0; i < (TOTAL_CHANNELS * 4 * 2); ++i)
 619                 cmos_write(bytes[i], CMOS_WRITE_TRAINING + i);
 620 }
 621 static void write_training_restore_results(const int memclk1067)
 622 {
 623         const int t_bound = memclk1067 ? 12 : 11;
 624
 625         u8 bytes[TOTAL_CHANNELS * 4 * 2]; /* two bytes per group */
 626         int ch, i;
 627
 628         /* Read from CMOS. */
 629         for (i = 0; i < (TOTAL_CHANNELS * 4 * 2); ++i)
 630                 bytes[i] = cmos_read(CMOS_WRITE_TRAINING + i);
 631
 632         /* Program with original program_write_timing(). */
 633         FOR_EACH_CHANNEL(ch) {
 634                 for (i = 0; i < 4; ++i) {
 635                         write_timing_t timing = { 0, 0, t_bound, 0 };
 636                         timing.f = bytes[(ch * 8) + (i * 2) + 1] & 3;
 637                         timing.t = bytes[(ch * 8) + (i * 2)] >> 4;
 638                         timing.p = bytes[(ch * 8) + (i * 2)] & 7;
 639                         program_write_timing(ch, i, &timing, memclk1067);
 640                         printk(BIOS_DEBUG, "Restored timings for group %d "
 641                                            "on channel %d: %d.%d.%d\n",
 642                                i, ch, timing.f, timing.t, timing.p);
 643                 }
 644         }
 645 }
 646 void raminit_write_training(const mem_clock_t ddr3clock,
 647                             const dimminfo_t *const dimms,
 648                             const int s3resume)
 649 {
 650         const int memclk1067 = ddr3clock == MEM_CLOCK_1067MT;
 651
 652         if (!s3resume) {
 653                 perform_write_training(memclk1067, dimms);
 654                 write_training_store_results();
 655         } else {
 656                 write_training_restore_results(memclk1067);
 657         }
 658         raminit_reset_readwrite_pointers();
 659 }