?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
tree: drop last paragraph of GPL copyright header
[coreboot.git] / src / northbridge / intel / nehalem / raminit.c
blob4bc95ba27d87f11e8760c989fc6d2fa227a80034
1 /*
2 * This file is part of the coreboot project.
3 *
4 * Copyright (C) 2013 Vladimir Serbinenko.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 */
16
17 /* Please don't remove this. It's needed it to do debugging
18 and reverse engineering to support in futur more nehalem variants. */
19 #ifndef REAL
20 #define REAL 1
21 #endif
22
23 #if REAL
24 #include <stdlib.h>
25 #include <console/console.h>
26 #include <string.h>
27 #include <arch/io.h>
28 #include <cpu/x86/msr.h>
29 #include <cbmem.h>
30 #include <arch/cbfs.h>
31 #include <cbfs.h>
32 #include <ip_checksum.h>
33 #include <pc80/mc146818rtc.h>
34 #include <device/pci_def.h>
35 #include <arch/cpu.h>
36 #include <halt.h>
37 #include <spd.h>
38 #include "raminit.h"
39 #include <timestamp.h>
40 #include <cpu/x86/mtrr.h>
41 #include <cpu/intel/speedstep.h>
42 #include <cpu/intel/turbo.h>
43 #endif
44
45 #if !REAL
46 typedef unsigned char u8;
47 typedef unsigned short u16;
48 typedef unsigned int u32;
49 typedef u32 device_t;
50 #endif
51
52 #include "nehalem.h"
53
54 #include "southbridge/intel/ibexpeak/me.h"
55
56 #if REAL
57 #include <delay.h>
58 #endif
59
60 #define NORTHBRIDGE PCI_DEV(0, 0, 0)
61 #define SOUTHBRIDGE PCI_DEV(0, 0x1f, 0)
62 #define GMA PCI_DEV (0, 0x2, 0x0)
63 #define HECIDEV PCI_DEV(0, 0x16, 0)
64 #define HECIBAR 0x10
65
66 #define FOR_ALL_RANKS \
67 for (channel = 0; channel < NUM_CHANNELS; channel++) \
68 for (slot = 0; slot < NUM_SLOTS; slot++) \
69 for (rank = 0; rank < NUM_RANKS; rank++)
70
71 #define FOR_POPULATED_RANKS \
72 for (channel = 0; channel < NUM_CHANNELS; channel++) \
73 for (slot = 0; slot < NUM_SLOTS; slot++) \
74 for (rank = 0; rank < NUM_RANKS; rank++) \
75 if (info->populated_ranks[channel][slot][rank])
76
77 #define FOR_POPULATED_RANKS_BACKWARDS \
78 for (channel = NUM_CHANNELS - 1; channel >= 0; channel--) \
79 for (slot = 0; slot < NUM_SLOTS; slot++) \
80 for (rank = 0; rank < NUM_RANKS; rank++) \
81 if (info->populated_ranks[channel][slot][rank])
82
83 /* [REG_178][CHANNEL][2 * SLOT + RANK][LANE] */
84 typedef struct {
85 u8 smallest;
86 u8 largest;
87 } timing_bounds_t[2][2][2][9];
88
89 struct ram_training {
90 /* [TM][CHANNEL][SLOT][RANK][LANE] */
91 u16 lane_timings[4][2][2][2][9];
92 u16 reg_178;
93 u16 reg_10b;
94
95 u8 reg178_center;
96 u8 reg178_smallest;
97 u8 reg178_largest;
98 timing_bounds_t timing_bounds[2];
99 u16 timing_offset[2][2][2][9];
100 u16 timing2_offset[2][2][2][9];
101 u16 timing2_bounds[2][2][2][9][2];
102 u8 reg274265[2][3]; /* [CHANNEL][REGISTER] */
103 u8 reg2ca9_bit0;
104 u32 reg_6dc;
105 u32 reg_6e8;
106 };
107
108 #if !REAL
109 #include "raminit_fake.c"
110 #else
111
112 #include <lib.h> /* Prototypes */
113
114 static inline void write_mchbar32(u32 addr, u32 val)
115 {
116 MCHBAR32(addr) = val;
117 }
118
119 static inline void write_mchbar16(u32 addr, u16 val)
120 {
121 MCHBAR16(addr) = val;
122 }
123
124 static inline void write_mchbar8(u32 addr, u8 val)
125 {
126 MCHBAR8(addr) = val;
127 }
128
129
130 static inline u32 read_mchbar32(u32 addr)
131 {
132 return MCHBAR32(addr);
133 }
134
135 static inline u16 read_mchbar16(u32 addr)
136 {
137 return MCHBAR16(addr);
138 }
139
140 static inline u8 read_mchbar8(u32 addr)
141 {
142 return MCHBAR8(addr);
143 }
144
145 static void clflush(u32 addr)
146 {
147 asm volatile ("clflush (%0)"::"r" (addr));
148 }
149
150 typedef struct _u128 {
151 u64 lo;
152 u64 hi;
153 } u128;
154
155 static void read128(u32 addr, u64 * out)
156 {
157 u128 ret;
158 u128 stor;
159 asm volatile ("movdqu %%xmm0, %0\n"
160 "movdqa (%2), %%xmm0\n"
161 "movdqu %%xmm0, %1\n"
162 "movdqu %0, %%xmm0":"+m" (stor), "=m"(ret):"r"(addr));
163 out[0] = ret.lo;
164 out[1] = ret.hi;
165 }
166
167 #endif
168
169 /* OK */
170 static void write_1d0(u32 val, u16 addr, int bits, int flag)
171 {
172 write_mchbar32(0x1d0, 0);
173 while (read_mchbar32(0x1d0) & 0x800000) ;
174 write_mchbar32(0x1d4,
175 (val & ((1 << bits) - 1)) | (2 << bits) | (flag <<
176 bits));
177 write_mchbar32(0x1d0, 0x40000000 | addr);
178 while (read_mchbar32(0x1d0) & 0x800000) ;
179 }
180
181 /* OK */
182 static u16 read_1d0(u16 addr, int split)
183 {
184 u32 val;
185 write_mchbar32(0x1d0, 0);
186 while (read_mchbar32(0x1d0) & 0x800000) ;
187 write_mchbar32(0x1d0,
188 0x80000000 | (((read_mchbar8(0x246) >> 2) & 3) +
189 0x361 - addr));
190 while (read_mchbar32(0x1d0) & 0x800000) ;
191 val = read_mchbar32(0x1d8);
192 write_1d0(0, 0x33d, 0, 0);
193 write_1d0(0, 0x33d, 0, 0);
194 val &= ((1 << split) - 1);
195 // printk (BIOS_ERR, "R1D0C [%x] => %x\n", addr, val);
196 return val;
197 }
198
199 static void write32p(uintptr_t addr, uint32_t val)
200 {
201 write32((void *)addr, val);
202 }
203
204 static uint32_t read32p(uintptr_t addr)
205 {
206 return read32((void *)addr);
207 }
208
209 static void sfence(void)
210 {
211 #if REAL
212 asm volatile ("sfence");
213 #endif
214 }
215
216 static inline u16 get_lane_offset(int slot, int rank, int lane)
217 {
218 return 0x124 * lane + ((lane & 4) ? 0x23e : 0) + 11 * rank + 22 * slot -
219 0x452 * (lane == 8);
220 }
221
222 static inline u16 get_timing_register_addr(int lane, int tm, int slot, int rank)
223 {
224 const u16 offs[] = { 0x1d, 0xa8, 0xe6, 0x5c };
225 return get_lane_offset(slot, rank, lane) + offs[(tm + 3) % 4];
226 }
227
228 #if REAL
229 static u32 gav_real(int line, u32 in)
230 {
231 // printk (BIOS_DEBUG, "%d: GAV: %x\n", line, in);
232 return in;
233 }
234
235 #define gav(x) gav_real (__LINE__, (x))
236 #endif
237 struct raminfo {
238 u16 clock_speed_index; /* clock_speed (REAL, not DDR) / 133.(3) - 3 */
239 u16 fsb_frequency; /* in 1.(1)/2 MHz. */
240 u8 is_x16_module[2][2]; /* [CHANNEL][SLOT] */
241 u8 density[2][2]; /* [CHANNEL][SLOT] */
242 u8 populated_ranks[2][2][2]; /* [CHANNEL][SLOT][RANK] */
243 int rank_start[2][2][2];
244 u8 cas_latency;
245 u8 board_lane_delay[9];
246 u8 use_ecc;
247 u8 revision;
248 u8 max_supported_clock_speed_index;
249 u8 uma_enabled;
250 u8 spd[2][2][151]; /* [CHANNEL][SLOT][BYTE] */
251 u8 silicon_revision;
252 u8 populated_ranks_mask[2];
253 u8 max_slots_used_in_channel;
254 u8 mode4030[2];
255 u16 avg4044[2];
256 u16 max4048[2];
257 unsigned total_memory_mb;
258 unsigned interleaved_part_mb;
259 unsigned non_interleaved_part_mb;
260
261 u32 heci_bar;
262 u64 heci_uma_addr;
263 unsigned memory_reserved_for_heci_mb;
264
265 struct ram_training training;
266 u32 last_500_command[2];
267
268 u32 delay46_ps[2];
269 u32 delay54_ps[2];
270 u8 revision_flag_1;
271 u8 some_delay_1_cycle_floor;
272 u8 some_delay_2_halfcycles_ceil;
273 u8 some_delay_3_ps_rounded;
274
275 const struct ram_training *cached_training;
276 };
277
278 static void
279 write_500(struct raminfo *info, int channel, u32 val, u16 addr, int bits,
280 int flag);
281
282 /* OK */
283 static u16
284 read_500(struct raminfo *info, int channel, u16 addr, int split)
285 {
286 u32 val;
287 info->last_500_command[channel] = 0x80000000;
288 write_mchbar32(0x500 + (channel << 10), 0);
289 while (read_mchbar32(0x500 + (channel << 10)) & 0x800000) ;
290 write_mchbar32(0x500 + (channel << 10),
291 0x80000000 |
292 (((read_mchbar8(0x246 + (channel << 10)) >> 2) &
293 3) + 0xb88 - addr));
294 while (read_mchbar32(0x500 + (channel << 10)) & 0x800000) ;
295 val = read_mchbar32(0x508 + (channel << 10));
296 return val & ((1 << split) - 1);
297 }
298
299 /* OK */
300 static void
301 write_500(struct raminfo *info, int channel, u32 val, u16 addr, int bits,
302 int flag)
303 {
304 if (info->last_500_command[channel] == 0x80000000) {
305 info->last_500_command[channel] = 0x40000000;
306 write_500(info, channel, 0, 0xb61, 0, 0);
307 }
308 write_mchbar32(0x500 + (channel << 10), 0);
309 while (read_mchbar32(0x500 + (channel << 10)) & 0x800000) ;
310 write_mchbar32(0x504 + (channel << 10),
311 (val & ((1 << bits) - 1)) | (2 << bits) | (flag <<
312 bits));
313 write_mchbar32(0x500 + (channel << 10), 0x40000000 | addr);
314 while (read_mchbar32(0x500 + (channel << 10)) & 0x800000) ;
315 }
316
317 static int rw_test(int rank)
318 {
319 const u32 mask = 0xf00fc33c;
320 int ok = 0xff;
321 int i;
322 for (i = 0; i < 64; i++)
323 write32p((rank << 28) | (i << 2), 0);
324 sfence();
325 for (i = 0; i < 64; i++)
326 gav(read32p((rank << 28) | (i << 2)));
327 sfence();
328 for (i = 0; i < 32; i++) {
329 u32 pat = (((mask >> i) & 1) ? 0xffffffff : 0);
330 write32p((rank << 28) | (i << 3), pat);
331 write32p((rank << 28) | (i << 3) | 4, pat);
332 }
333 sfence();
334 for (i = 0; i < 32; i++) {
335 u8 pat = (((mask >> i) & 1) ? 0xff : 0);
336 int j;
337 u32 val;
338 gav(val = read32p((rank << 28) | (i << 3)));
339 for (j = 0; j < 4; j++)
340 if (((val >> (j * 8)) & 0xff) != pat)
341 ok &= ~(1 << j);
342 gav(val = read32p((rank << 28) | (i << 3) | 4));
343 for (j = 0; j < 4; j++)
344 if (((val >> (j * 8)) & 0xff) != pat)
345 ok &= ~(16 << j);
346 }
347 sfence();
348 for (i = 0; i < 64; i++)
349 write32p((rank << 28) | (i << 2), 0);
350 sfence();
351 for (i = 0; i < 64; i++)
352 gav(read32p((rank << 28) | (i << 2)));
353
354 return ok;
355 }
356
357 static void
358 program_timings(struct raminfo *info, u16 base, int channel, int slot, int rank)
359 {
360 int lane;
361 for (lane = 0; lane < 8; lane++) {
362 write_500(info, channel,
363 base +
364 info->training.
365 lane_timings[2][channel][slot][rank][lane],
366 get_timing_register_addr(lane, 2, slot, rank), 9, 0);
367 write_500(info, channel,
368 base +
369 info->training.
370 lane_timings[3][channel][slot][rank][lane],
371 get_timing_register_addr(lane, 3, slot, rank), 9, 0);
372 }
373 }
374
375 static void write_26c(int channel, u16 si)
376 {
377 write_mchbar32(0x26c + (channel << 10), 0x03243f35);
378 write_mchbar32(0x268 + (channel << 10), 0xcfc00000 | (si << 9));
379 write_mchbar16(0x2b9 + (channel << 10), si);
380 }
381
382 static u32 get_580(int channel, u8 addr)
383 {
384 u32 ret;
385 gav(read_1d0(0x142, 3));
386 write_mchbar8(0x5ff, 0x0); /* OK */
387 write_mchbar8(0x5ff, 0x80); /* OK */
388 write_mchbar32(0x580 + (channel << 10), 0x8493c012 | addr);
389 write_mchbar8(0x580 + (channel << 10),
390 read_mchbar8(0x580 + (channel << 10)) | 1);
391 while (!((ret = read_mchbar32(0x580 + (channel << 10))) & 0x10000)) ;
392 write_mchbar8(0x580 + (channel << 10),
393 read_mchbar8(0x580 + (channel << 10)) & ~1);
394 return ret;
395 }
396
397 const int cached_config = 0;
398
399 #define NUM_CHANNELS 2
400 #define NUM_SLOTS 2
401 #define NUM_RANKS 2
402 #define RANK_SHIFT 28
403 #define CHANNEL_SHIFT 10
404
405 #include "raminit_tables.c"
406
407 static void seq9(struct raminfo *info, int channel, int slot, int rank)
408 {
409 int i, lane;
410
411 for (i = 0; i < 2; i++)
412 for (lane = 0; lane < 8; lane++)
413 write_500(info, channel,
414 info->training.lane_timings[i +
415 1][channel][slot]
416 [rank][lane], get_timing_register_addr(lane,
417 i + 1,
418 slot,
419 rank),
420 9, 0);
421
422 write_1d0(1, 0x103, 6, 1);
423 for (lane = 0; lane < 8; lane++)
424 write_500(info, channel,
425 info->training.
426 lane_timings[0][channel][slot][rank][lane],
427 get_timing_register_addr(lane, 0, slot, rank), 9, 0);
428
429 for (i = 0; i < 2; i++) {
430 for (lane = 0; lane < 8; lane++)
431 write_500(info, channel,
432 info->training.lane_timings[i +
433 1][channel][slot]
434 [rank][lane], get_timing_register_addr(lane,
435 i + 1,
436 slot,
437 rank),
438 9, 0);
439 gav(get_580(channel, ((i + 1) << 2) | (rank << 5)));
440 }
441
442 gav(read_1d0(0x142, 3)); // = 0x10408118
443 write_mchbar8(0x5ff, 0x0); /* OK */
444 write_mchbar8(0x5ff, 0x80); /* OK */
445 write_1d0(0x2, 0x142, 3, 1);
446 for (lane = 0; lane < 8; lane++) {
447 // printk (BIOS_ERR, "before: %x\n", info->training.lane_timings[2][channel][slot][rank][lane]);
448 info->training.lane_timings[2][channel][slot][rank][lane] =
449 read_500(info, channel,
450 get_timing_register_addr(lane, 2, slot, rank), 9);
451 //printk (BIOS_ERR, "after: %x\n", info->training.lane_timings[2][channel][slot][rank][lane]);
452 info->training.lane_timings[3][channel][slot][rank][lane] =
453 info->training.lane_timings[2][channel][slot][rank][lane] +
454 0x20;
455 }
456 }
457
458 static int count_ranks_in_channel(struct raminfo *info, int channel)
459 {
460 int slot, rank;
461 int res = 0;
462 for (slot = 0; slot < NUM_SLOTS; slot++)
463 for (rank = 0; rank < NUM_SLOTS; rank++)
464 res += info->populated_ranks[channel][slot][rank];
465 return res;
466 }
467
468 static void
469 config_rank(struct raminfo *info, int s3resume, int channel, int slot, int rank)
470 {
471 int add;
472
473 write_1d0(0, 0x178, 7, 1);
474 seq9(info, channel, slot, rank);
475 program_timings(info, 0x80, channel, slot, rank);
476
477 if (channel == 0)
478 add = count_ranks_in_channel(info, 1);
479 else
480 add = 0;
481 if (!s3resume)
482 gav(rw_test(rank + add));
483 program_timings(info, 0x00, channel, slot, rank);
484 if (!s3resume)
485 gav(rw_test(rank + add));
486 if (!s3resume)
487 gav(rw_test(rank + add));
488 write_1d0(0, 0x142, 3, 1);
489 write_1d0(0, 0x103, 6, 1);
490
491 gav(get_580(channel, 0xc | (rank << 5)));
492 gav(read_1d0(0x142, 3));
493
494 write_mchbar8(0x5ff, 0x0); /* OK */
495 write_mchbar8(0x5ff, 0x80); /* OK */
496 }
497
498 static void set_4cf(struct raminfo *info, int channel, u8 val)
499 {
500 gav(read_500(info, channel, 0x4cf, 4)); // = 0xc2300cf9
501 write_500(info, channel, val, 0x4cf, 4, 1);
502 gav(read_500(info, channel, 0x659, 4)); // = 0x80300839
503 write_500(info, channel, val, 0x659, 4, 1);
504 gav(read_500(info, channel, 0x697, 4)); // = 0x80300839
505 write_500(info, channel, val, 0x697, 4, 1);
506 }
507
508 static void set_334(int zero)
509 {
510 int j, k, channel;
511 const u32 val3[] = { 0x2a2b2a2b, 0x26272627, 0x2e2f2e2f, 0x2a2b };
512 u32 vd8[2][16];
513
514 for (channel = 0; channel < NUM_CHANNELS; channel++) {
515 for (j = 0; j < 4; j++) {
516 u32 a = (j == 1) ? 0x29292929 : 0x31313131;
517 u32 lmask = (j == 3) ? 0xffff : 0xffffffff;
518 u16 c;
519 if ((j == 0 || j == 3) && zero)
520 c = 0;
521 else if (j == 3)
522 c = 0x5f;
523 else
524 c = 0x5f5f;
525
526 for (k = 0; k < 2; k++) {
527 write_mchbar32(0x138 + 8 * k,
528 (channel << 26) | (j << 24));
529 gav(vd8[1][(channel << 3) | (j << 1) | k] =
530 read_mchbar32(0x138 + 8 * k));
531 gav(vd8[0][(channel << 3) | (j << 1) | k] =
532 read_mchbar32(0x13c + 8 * k));
533 }
534
535 write_mchbar32(0x334 + (channel << 10) + (j * 0x44),
536 zero ? 0 : val3[j]);
537 write_mchbar32(0x32c + (channel << 10) + (j * 0x44),
538 zero ? 0 : (0x18191819 & lmask));
539 write_mchbar16(0x34a + (channel << 10) + (j * 0x44), c);
540 write_mchbar32(0x33c + (channel << 10) + (j * 0x44),
541 zero ? 0 : (a & lmask));
542 write_mchbar32(0x344 + (channel << 10) + (j * 0x44),
543 zero ? 0 : (a & lmask));
544 }
545 }
546
547 write_mchbar32(0x130, read_mchbar32(0x130) | 1); /* OK */
548 while (read_mchbar8(0x130) & 1) ; /* OK */
549 }
550
551 static void rmw_1d0(u16 addr, u32 and, u32 or, int split, int flag)
552 {
553 u32 v;
554 v = read_1d0(addr, split);
555 write_1d0((v & and) | or, addr, split, flag);
556 }
557
558 static int find_highest_bit_set(u16 val)
559 {
560 int i;
561 for (i = 15; i >= 0; i--)
562 if (val & (1 << i))
563 return i;
564 return -1;
565 }
566
567 static int find_lowest_bit_set32(u32 val)
568 {
569 int i;
570 for (i = 0; i < 32; i++)
571 if (val & (1 << i))
572 return i;
573 return -1;
574 }
575
576 enum {
577 DEVICE_TYPE = 2,
578 MODULE_TYPE = 3,
579 DENSITY = 4,
580 RANKS_AND_DQ = 7,
581 MEMORY_BUS_WIDTH = 8,
582 TIMEBASE_DIVIDEND = 10,
583 TIMEBASE_DIVISOR = 11,
584 CYCLETIME = 12,
585
586 CAS_LATENCIES_LSB = 14,
587 CAS_LATENCIES_MSB = 15,
588 CAS_LATENCY_TIME = 16,
589 THERMAL_AND_REFRESH = 31,
590 REFERENCE_RAW_CARD_USED = 62,
591 RANK1_ADDRESS_MAPPING = 63
592 };
593
594 static void calculate_timings(struct raminfo *info)
595 {
596 unsigned cycletime;
597 unsigned cas_latency_time;
598 unsigned supported_cas_latencies;
599 unsigned channel, slot;
600 unsigned clock_speed_index;
601 unsigned min_cas_latency;
602 unsigned cas_latency;
603 unsigned max_clock_index;
604
605 /* Find common CAS latency */
606 supported_cas_latencies = 0x3fe;
607 for (channel = 0; channel < NUM_CHANNELS; channel++)
608 for (slot = 0; slot < NUM_SLOTS; slot++)
609 if (info->populated_ranks[channel][slot][0])
610 supported_cas_latencies &=
611 2 *
612 (info->
613 spd[channel][slot][CAS_LATENCIES_LSB] |
614 (info->
615 spd[channel][slot][CAS_LATENCIES_MSB] <<
616 8));
617
618 max_clock_index = min(3, info->max_supported_clock_speed_index);
619
620 cycletime = min_cycletime[max_clock_index];
621 cas_latency_time = min_cas_latency_time[max_clock_index];
622
623 for (channel = 0; channel < NUM_CHANNELS; channel++)
624 for (slot = 0; slot < NUM_SLOTS; slot++)
625 if (info->populated_ranks[channel][slot][0]) {
626 unsigned timebase;
627 timebase =
628 1000 *
629 info->
630 spd[channel][slot][TIMEBASE_DIVIDEND] /
631 info->spd[channel][slot][TIMEBASE_DIVISOR];
632 cycletime =
633 max(cycletime,
634 timebase *
635 info->spd[channel][slot][CYCLETIME]);
636 cas_latency_time =
637 max(cas_latency_time,
638 timebase *
639 info->
640 spd[channel][slot][CAS_LATENCY_TIME]);
641 }
642 for (clock_speed_index = 0; clock_speed_index < 3; clock_speed_index++) {
643 if (cycletime == min_cycletime[clock_speed_index])
644 break;
645 if (cycletime > min_cycletime[clock_speed_index]) {
646 clock_speed_index--;
647 cycletime = min_cycletime[clock_speed_index];
648 break;
649 }
650 }
651 min_cas_latency = CEIL_DIV(cas_latency_time, cycletime);
652 cas_latency = 0;
653 while (supported_cas_latencies) {
654 cas_latency = find_highest_bit_set(supported_cas_latencies) + 3;
655 if (cas_latency <= min_cas_latency)
656 break;
657 supported_cas_latencies &=
658 ~(1 << find_highest_bit_set(supported_cas_latencies));
659 }
660
661 if (cas_latency != min_cas_latency && clock_speed_index)
662 clock_speed_index--;
663
664 if (cas_latency * min_cycletime[clock_speed_index] > 20000)
665 die("Couldn't configure DRAM");
666 info->clock_speed_index = clock_speed_index;
667 info->cas_latency = cas_latency;
668 }
669
670 static void program_base_timings(struct raminfo *info)
671 {
672 unsigned channel;
673 unsigned slot, rank, lane;
674 unsigned extended_silicon_revision;
675 int i;
676
677 extended_silicon_revision = info->silicon_revision;
678 if (info->silicon_revision == 0)
679 for (channel = 0; channel < NUM_CHANNELS; channel++)
680 for (slot = 0; slot < NUM_SLOTS; slot++)
681 if ((info->
682 spd[channel][slot][MODULE_TYPE] & 0xF) ==
683 3)
684 extended_silicon_revision = 4;
685
686 for (channel = 0; channel < NUM_CHANNELS; channel++) {
687 for (slot = 0; slot < NUM_SLOTS; slot++)
688 for (rank = 0; rank < NUM_SLOTS; rank++) {
689 int card_timing_2;
690 if (!info->populated_ranks[channel][slot][rank])
691 continue;
692
693 for (lane = 0; lane < 9; lane++) {
694 int tm_reg;
695 int card_timing;
696
697 card_timing = 0;
698 if ((info->
699 spd[channel][slot][MODULE_TYPE] &
700 0xF) == 3) {
701 int reference_card;
702 reference_card =
703 info->
704 spd[channel][slot]
705 [REFERENCE_RAW_CARD_USED] &
706 0x1f;
707 if (reference_card == 3)
708 card_timing =
709 u16_ffd1188[0][lane]
710 [info->
711 clock_speed_index];
712 if (reference_card == 5)
713 card_timing =
714 u16_ffd1188[1][lane]
715 [info->
716 clock_speed_index];
717 }
718
719 info->training.
720 lane_timings[0][channel][slot][rank]
721 [lane] =
722 u8_FFFD1218[info->
723 clock_speed_index];
724 info->training.
725 lane_timings[1][channel][slot][rank]
726 [lane] = 256;
727
728 for (tm_reg = 2; tm_reg < 4; tm_reg++)
729 info->training.
730 lane_timings[tm_reg]
731 [channel][slot][rank][lane]
732 =
733 u8_FFFD1240[channel]
734 [extended_silicon_revision]
735 [lane][2 * slot +
736 rank][info->
737 clock_speed_index]
738 + info->max4048[channel]
739 +
740 u8_FFFD0C78[channel]
741 [extended_silicon_revision]
742 [info->
743 mode4030[channel]][slot]
744 [rank][info->
745 clock_speed_index]
746 + card_timing;
747 for (tm_reg = 0; tm_reg < 4; tm_reg++)
748 write_500(info, channel,
749 info->training.
750 lane_timings[tm_reg]
751 [channel][slot][rank]
752 [lane],
753 get_timing_register_addr
754 (lane, tm_reg, slot,
755 rank), 9, 0);
756 }
757
758 card_timing_2 = 0;
759 if (!(extended_silicon_revision != 4
760 || (info->
761 populated_ranks_mask[channel] & 5) ==
762 5)) {
763 if ((info->
764 spd[channel][slot]
765 [REFERENCE_RAW_CARD_USED] & 0x1F)
766 == 3)
767 card_timing_2 =
768 u16_FFFE0EB8[0][info->
769 clock_speed_index];
770 if ((info->
771 spd[channel][slot]
772 [REFERENCE_RAW_CARD_USED] & 0x1F)
773 == 5)
774 card_timing_2 =
775 u16_FFFE0EB8[1][info->
776 clock_speed_index];
777 }
778
779 for (i = 0; i < 3; i++)
780 write_500(info, channel,
781 (card_timing_2 +
782 info->max4048[channel]
783 +
784 u8_FFFD0EF8[channel]
785 [extended_silicon_revision]
786 [info->
787 mode4030[channel]][info->
788 clock_speed_index]),
789 u16_fffd0c50[i][slot][rank],
790 8, 1);
791 write_500(info, channel,
792 (info->max4048[channel] +
793 u8_FFFD0C78[channel]
794 [extended_silicon_revision][info->
795 mode4030
796 [channel]]
797 [slot][rank][info->
798 clock_speed_index]),
799 u16_fffd0c70[slot][rank], 7, 1);
800 }
801 if (!info->populated_ranks_mask[channel])
802 continue;
803 for (i = 0; i < 3; i++)
804 write_500(info, channel,
805 (info->max4048[channel] +
806 info->avg4044[channel]
807 +
808 u8_FFFD17E0[channel]
809 [extended_silicon_revision][info->
810 mode4030
811 [channel]][info->
812 clock_speed_index]),
813 u16_fffd0c68[i], 8, 1);
814 }
815 }
816
817 static unsigned int fsbcycle_ps(struct raminfo *info)
818 {
819 return 900000 / info->fsb_frequency;
820 }
821
822 /* The time of DDR transfer in ps. */
823 static unsigned int halfcycle_ps(struct raminfo *info)
824 {
825 return 3750 / (info->clock_speed_index + 3);
826 }
827
828 /* The time of clock cycle in ps. */
829 static unsigned int cycle_ps(struct raminfo *info)
830 {
831 return 2 * halfcycle_ps(info);
832 }
833
834 /* Frequency in 1.(1)=10/9 MHz units. */
835 static unsigned frequency_11(struct raminfo *info)
836 {
837 return (info->clock_speed_index + 3) * 120;
838 }
839
840 /* Frequency in 0.1 MHz units. */
841 static unsigned frequency_01(struct raminfo *info)
842 {
843 return 100 * frequency_11(info) / 9;
844 }
845
846 static unsigned ps_to_halfcycles(struct raminfo *info, unsigned int ps)
847 {
848 return (frequency_11(info) * 2) * ps / 900000;
849 }
850
851 static unsigned ns_to_cycles(struct raminfo *info, unsigned int ns)
852 {
853 return (frequency_11(info)) * ns / 900;
854 }
855
856 static void compute_derived_timings(struct raminfo *info)
857 {
858 unsigned channel, slot, rank;
859 int extended_silicon_revision;
860 int some_delay_1_ps;
861 int some_delay_2_ps;
862 int some_delay_2_halfcycles_ceil;
863 int some_delay_2_halfcycles_floor;
864 int some_delay_3_ps;
865 int some_delay_3_halfcycles;
866 int some_delay_3_ps_rounded;
867 int some_delay_1_cycle_ceil;
868 int some_delay_1_cycle_floor;
869
870 some_delay_3_halfcycles = 0;
871 some_delay_3_ps_rounded = 0;
872 extended_silicon_revision = info->silicon_revision;
873 if (!info->silicon_revision)
874 for (channel = 0; channel < NUM_CHANNELS; channel++)
875 for (slot = 0; slot < NUM_SLOTS; slot++)
876 if ((info->
877 spd[channel][slot][MODULE_TYPE] & 0xF) ==
878 3)
879 extended_silicon_revision = 4;
880 if (info->board_lane_delay[7] < 5)
881 info->board_lane_delay[7] = 5;
882 info->revision_flag_1 = 2;
883 if (info->silicon_revision == 2 || info->silicon_revision == 3)
884 info->revision_flag_1 = 0;
885 if (info->revision < 16)
886 info->revision_flag_1 = 0;
887
888 if (info->revision < 8)
889 info->revision_flag_1 = 0;
890 if (info->revision >= 8 && (info->silicon_revision == 0
891 || info->silicon_revision == 1))
892 some_delay_2_ps = 735;
893 else
894 some_delay_2_ps = 750;
895
896 if (info->revision >= 0x10 && (info->silicon_revision == 0
897 || info->silicon_revision == 1))
898 some_delay_1_ps = 3929;
899 else
900 some_delay_1_ps = 3490;
901
902 some_delay_1_cycle_floor = some_delay_1_ps / cycle_ps(info);
903 some_delay_1_cycle_ceil = some_delay_1_ps / cycle_ps(info);
904 if (some_delay_1_ps % cycle_ps(info))
905 some_delay_1_cycle_ceil++;
906 else
907 some_delay_1_cycle_floor--;
908 info->some_delay_1_cycle_floor = some_delay_1_cycle_floor;
909 if (info->revision_flag_1)
910 some_delay_2_ps = halfcycle_ps(info) >> 6;
911 some_delay_2_ps +=
912 max(some_delay_1_ps - 30,
913 2 * halfcycle_ps(info) * (some_delay_1_cycle_ceil - 1) + 1000) +
914 375;
915 some_delay_3_ps =
916 halfcycle_ps(info) - some_delay_2_ps % halfcycle_ps(info);
917 if (info->revision_flag_1) {
918 if (some_delay_3_ps < 150)
919 some_delay_3_halfcycles = 0;
920 else
921 some_delay_3_halfcycles =
922 (some_delay_3_ps << 6) / halfcycle_ps(info);
923 some_delay_3_ps_rounded =
924 halfcycle_ps(info) * some_delay_3_halfcycles >> 6;
925 }
926 some_delay_2_halfcycles_ceil =
927 (some_delay_2_ps + halfcycle_ps(info) - 1) / halfcycle_ps(info) -
928 2 * (some_delay_1_cycle_ceil - 1);
929 if (info->revision_flag_1 && some_delay_3_ps < 150)
930 some_delay_2_halfcycles_ceil++;
931 some_delay_2_halfcycles_floor = some_delay_2_halfcycles_ceil;
932 if (info->revision < 0x10)
933 some_delay_2_halfcycles_floor =
934 some_delay_2_halfcycles_ceil - 1;
935 if (!info->revision_flag_1)
936 some_delay_2_halfcycles_floor++;
937 info->some_delay_2_halfcycles_ceil = some_delay_2_halfcycles_ceil;
938 info->some_delay_3_ps_rounded = some_delay_3_ps_rounded;
939 if ((info->populated_ranks[0][0][0] && info->populated_ranks[0][1][0])
940 || (info->populated_ranks[1][0][0]
941 && info->populated_ranks[1][1][0]))
942 info->max_slots_used_in_channel = 2;
943 else
944 info->max_slots_used_in_channel = 1;
945 for (channel = 0; channel < 2; channel++)
946 write_mchbar32(0x244 + (channel << 10),
947 ((info->revision < 8) ? 1 : 0x200)
948 | ((2 - info->max_slots_used_in_channel) << 17) |
949 (channel << 21) | (info->
950 some_delay_1_cycle_floor <<
951 18) | 0x9510);
952 if (info->max_slots_used_in_channel == 1) {
953 info->mode4030[0] = (count_ranks_in_channel(info, 0) == 2);
954 info->mode4030[1] = (count_ranks_in_channel(info, 1) == 2);
955 } else {
956 info->mode4030[0] = ((count_ranks_in_channel(info, 0) == 1) || (count_ranks_in_channel(info, 0) == 2)) ? 2 : 3; /* 2 if 1 or 2 ranks */
957 info->mode4030[1] = ((count_ranks_in_channel(info, 1) == 1)
958 || (count_ranks_in_channel(info, 1) ==
959 2)) ? 2 : 3;
960 }
961 for (channel = 0; channel < NUM_CHANNELS; channel++) {
962 int max_of_unk;
963 int min_of_unk_2;
964
965 int i, count;
966 int sum;
967
968 if (!info->populated_ranks_mask[channel])
969 continue;
970
971 max_of_unk = 0;
972 min_of_unk_2 = 32767;
973
974 sum = 0;
975 count = 0;
976 for (i = 0; i < 3; i++) {
977 int unk1;
978 if (info->revision < 8)
979 unk1 =
980 u8_FFFD1891[0][channel][info->
981 clock_speed_index]
982 [i];
983 else if (!
984 (info->revision >= 0x10
985 || info->revision_flag_1))
986 unk1 =
987 u8_FFFD1891[1][channel][info->
988 clock_speed_index]
989 [i];
990 else
991 unk1 = 0;
992 for (slot = 0; slot < NUM_SLOTS; slot++)
993 for (rank = 0; rank < NUM_RANKS; rank++) {
994 int a = 0;
995 int b = 0;
996
997 if (!info->
998 populated_ranks[channel][slot]
999 [rank])
1000 continue;
1001 if (extended_silicon_revision == 4
1002 && (info->
1003 populated_ranks_mask[channel] &
1004 5) != 5) {
1005 if ((info->
1006 spd[channel][slot]
1007 [REFERENCE_RAW_CARD_USED] &
1008 0x1F) == 3) {
1009 a = u16_ffd1178[0]
1010 [info->
1011 clock_speed_index];
1012 b = u16_fe0eb8[0][info->
1013 clock_speed_index];
1014 } else
1015 if ((info->
1016 spd[channel][slot]
1017 [REFERENCE_RAW_CARD_USED]
1018 & 0x1F) == 5) {
1019 a = u16_ffd1178[1]
1020 [info->
1021 clock_speed_index];
1022 b = u16_fe0eb8[1][info->
1023 clock_speed_index];
1024 }
1025 }
1026 min_of_unk_2 = min(min_of_unk_2, a);
1027 min_of_unk_2 = min(min_of_unk_2, b);
1028 if (rank == 0) {
1029 sum += a;
1030 count++;
1031 }
1032 {
1033 int t;
1034 t = b +
1035 u8_FFFD0EF8[channel]
1036 [extended_silicon_revision]
1037 [info->
1038 mode4030[channel]][info->
1039 clock_speed_index];
1040 if (unk1 >= t)
1041 max_of_unk =
1042 max(max_of_unk,
1043 unk1 - t);
1044 }
1045 }
1046 {
1047 int t =
1048 u8_FFFD17E0[channel]
1049 [extended_silicon_revision][info->
1050 mode4030
1051 [channel]]
1052 [info->clock_speed_index] + min_of_unk_2;
1053 if (unk1 >= t)
1054 max_of_unk = max(max_of_unk, unk1 - t);
1055 }
1056 }
1057
1058 info->avg4044[channel] = sum / count;
1059 info->max4048[channel] = max_of_unk;
1060 }
1061 }
1062
1063 static void jedec_read(struct raminfo *info,
1064 int channel, int slot, int rank,
1065 int total_rank, u8 addr3, unsigned int value)
1066 {
1067 /* Handle mirrored mapping. */
1068 if ((rank & 1) && (info->spd[channel][slot][RANK1_ADDRESS_MAPPING] & 1))
1069 addr3 =
1070 (addr3 & 0xCF) | ((addr3 & 0x10) << 1) | ((addr3 >> 1) &
1071 0x10);
1072 write_mchbar8(0x271, addr3 | (read_mchbar8(0x271) & 0xC1));
1073 write_mchbar8(0x671, addr3 | (read_mchbar8(0x671) & 0xC1));
1074
1075 /* Handle mirrored mapping. */
1076 if ((rank & 1) && (info->spd[channel][slot][RANK1_ADDRESS_MAPPING] & 1))
1077 value =
1078 (value & ~0x1f8) | ((value >> 1) & 0xa8) | ((value & 0xa8)
1079 << 1);
1080
1081 read32p((value << 3) | (total_rank << 28));
1082
1083 write_mchbar8(0x271, (read_mchbar8(0x271) & 0xC3) | 2);
1084 write_mchbar8(0x671, (read_mchbar8(0x671) & 0xC3) | 2);
1085
1086 read32p(total_rank << 28);
1087 }
1088
1089 enum {
1090 MR1_RZQ12 = 512,
1091 MR1_RZQ2 = 64,
1092 MR1_RZQ4 = 4,
1093 MR1_ODS34OHM = 2
1094 };
1095
1096 enum {
1097 MR0_BT_INTERLEAVED = 8,
1098 MR0_DLL_RESET_ON = 256
1099 };
1100
1101 enum {
1102 MR2_RTT_WR_DISABLED = 0,
1103 MR2_RZQ2 = 1 << 10
1104 };
1105
1106 static void jedec_init(struct raminfo *info)
1107 {
1108 int write_recovery;
1109 int channel, slot, rank;
1110 int total_rank;
1111 int dll_on;
1112 int self_refresh_temperature;
1113 int auto_self_refresh;
1114
1115 auto_self_refresh = 1;
1116 self_refresh_temperature = 1;
1117 if (info->board_lane_delay[3] <= 10) {
1118 if (info->board_lane_delay[3] <= 8)
1119 write_recovery = info->board_lane_delay[3] - 4;
1120 else
1121 write_recovery = 5;
1122 } else {
1123 write_recovery = 6;
1124 }
1125 FOR_POPULATED_RANKS {
1126 auto_self_refresh &=
1127 (info->spd[channel][slot][THERMAL_AND_REFRESH] >> 2) & 1;
1128 self_refresh_temperature &=
1129 info->spd[channel][slot][THERMAL_AND_REFRESH] & 1;
1130 }
1131 if (auto_self_refresh == 1)
1132 self_refresh_temperature = 0;
1133
1134 dll_on = ((info->silicon_revision != 2 && info->silicon_revision != 3)
1135 || (info->populated_ranks[0][0][0]
1136 && info->populated_ranks[0][1][0])
1137 || (info->populated_ranks[1][0][0]
1138 && info->populated_ranks[1][1][0]));
1139
1140 total_rank = 0;
1141
1142 for (channel = NUM_CHANNELS - 1; channel >= 0; channel--) {
1143 int rtt, rtt_wr = MR2_RTT_WR_DISABLED;
1144 int rzq_reg58e;
1145
1146 if (info->silicon_revision == 2 || info->silicon_revision == 3) {
1147 rzq_reg58e = 64;
1148 rtt = MR1_RZQ2;
1149 if (info->clock_speed_index != 0) {
1150 rzq_reg58e = 4;
1151 if (info->populated_ranks_mask[channel] == 3)
1152 rtt = MR1_RZQ4;
1153 }
1154 } else {
1155 if ((info->populated_ranks_mask[channel] & 5) == 5) {
1156 rtt = MR1_RZQ12;
1157 rzq_reg58e = 64;
1158 rtt_wr = MR2_RZQ2;
1159 } else {
1160 rzq_reg58e = 4;
1161 rtt = MR1_RZQ4;
1162 }
1163 }
1164
1165 write_mchbar16(0x588 + (channel << 10), 0x0);
1166 write_mchbar16(0x58a + (channel << 10), 0x4);
1167 write_mchbar16(0x58c + (channel << 10), rtt | MR1_ODS34OHM);
1168 write_mchbar16(0x58e + (channel << 10), rzq_reg58e | 0x82);
1169 write_mchbar16(0x590 + (channel << 10), 0x1282);
1170
1171 for (slot = 0; slot < NUM_SLOTS; slot++)
1172 for (rank = 0; rank < NUM_RANKS; rank++)
1173 if (info->populated_ranks[channel][slot][rank]) {
1174 jedec_read(info, channel, slot, rank,
1175 total_rank, 0x28,
1176 rtt_wr | (info->
1177 clock_speed_index
1178 << 3)
1179 | (auto_self_refresh << 6) |
1180 (self_refresh_temperature <<
1181 7));
1182 jedec_read(info, channel, slot, rank,
1183 total_rank, 0x38, 0);
1184 jedec_read(info, channel, slot, rank,
1185 total_rank, 0x18,
1186 rtt | MR1_ODS34OHM);
1187 jedec_read(info, channel, slot, rank,
1188 total_rank, 6,
1189 (dll_on << 12) |
1190 (write_recovery << 9)
1191 | ((info->cas_latency - 4) <<
1192 4) | MR0_BT_INTERLEAVED |
1193 MR0_DLL_RESET_ON);
1194 total_rank++;
1195 }
1196 }
1197 }
1198
1199 static void program_modules_memory_map(struct raminfo *info, int pre_jedec)
1200 {
1201 unsigned channel, slot, rank;
1202 unsigned int total_mb[2] = { 0, 0 }; /* total memory per channel in MB */
1203 unsigned int channel_0_non_interleaved;
1204
1205 FOR_ALL_RANKS {
1206 if (info->populated_ranks[channel][slot][rank]) {
1207 total_mb[channel] +=
1208 pre_jedec ? 256 : (256 << info->
1209 density[channel][slot] >> info->
1210 is_x16_module[channel][slot]);
1211 write_mchbar8(0x208 + rank + 2 * slot + (channel << 10),
1212 (pre_jedec ? (1 | ((1 + 1) << 1))
1213 : (info->
1214 is_x16_module[channel][slot] |
1215 ((info->density[channel][slot] +
1216 1) << 1))) | 0x80);
1217 }
1218 write_mchbar16(0x200 + (channel << 10) + 4 * slot + 2 * rank,
1219 total_mb[channel] >> 6);
1220 }
1221
1222 info->total_memory_mb = total_mb[0] + total_mb[1];
1223
1224 info->interleaved_part_mb =
1225 pre_jedec ? 0 : 2 * min(total_mb[0], total_mb[1]);
1226 info->non_interleaved_part_mb =
1227 total_mb[0] + total_mb[1] - info->interleaved_part_mb;
1228 channel_0_non_interleaved = total_mb[0] - info->interleaved_part_mb / 2;
1229 write_mchbar32(0x100,
1230 channel_0_non_interleaved | (info->
1231 non_interleaved_part_mb <<
1232 16));
1233 if (!pre_jedec)
1234 write_mchbar16(0x104, info->interleaved_part_mb);
1235 }
1236
1237 static void program_board_delay(struct raminfo *info)
1238 {
1239 int cas_latency_shift;
1240 int some_delay_ns;
1241 int some_delay_3_half_cycles;
1242
1243 unsigned channel, i;
1244 int high_multiplier;
1245 int lane_3_delay;
1246 int cas_latency_derived;
1247
1248 high_multiplier = 0;
1249 some_delay_ns = 200;
1250 some_delay_3_half_cycles = 4;
1251 cas_latency_shift = info->silicon_revision == 0
1252 || info->silicon_revision == 1 ? 1 : 0;
1253 if (info->revision < 8) {
1254 some_delay_ns = 600;
1255 cas_latency_shift = 0;
1256 }
1257 {
1258 int speed_bit;
1259 speed_bit =
1260 ((info->clock_speed_index > 1
1261 || (info->silicon_revision != 2
1262 && info->silicon_revision != 3))) ^ (info->revision >=
1263 0x10);
1264 write_500(info, 0, speed_bit | ((!info->use_ecc) << 1), 0x60e,
1265 3, 1);
1266 write_500(info, 1, speed_bit | ((!info->use_ecc) << 1), 0x60e,
1267 3, 1);
1268 if (info->revision >= 0x10 && info->clock_speed_index <= 1
1269 && (info->silicon_revision == 2
1270 || info->silicon_revision == 3))
1271 rmw_1d0(0x116, 5, 2, 4, 1);
1272 }
1273 write_mchbar32(0x120,
1274 (1 << (info->max_slots_used_in_channel + 28)) |
1275 0x188e7f9f);
1276
1277 write_mchbar8(0x124,
1278 info->board_lane_delay[4] +
1279 ((frequency_01(info) + 999) / 1000));
1280 write_mchbar16(0x125, 0x1360);
1281 write_mchbar8(0x127, 0x40);
1282 if (info->fsb_frequency < frequency_11(info) / 2) {
1283 unsigned some_delay_2_half_cycles;
1284 high_multiplier = 1;
1285 some_delay_2_half_cycles = ps_to_halfcycles(info,
1286 ((3 *
1287 fsbcycle_ps(info))
1288 >> 1) +
1289 (halfcycle_ps(info)
1290 *
1291 reg178_min[info->
1292 clock_speed_index]
1293 >> 6)
1294 +
1295 4 *
1296 halfcycle_ps(info)
1297 + 2230);
1298 some_delay_3_half_cycles =
1299 min((some_delay_2_half_cycles +
1300 (frequency_11(info) * 2) * (28 -
1301 some_delay_2_half_cycles) /
1302 (frequency_11(info) * 2 -
1303 4 * (info->fsb_frequency))) >> 3, 7);
1304 }
1305 if (read_mchbar8(0x2ca9) & 1)
1306 some_delay_3_half_cycles = 3;
1307 for (channel = 0; channel < NUM_CHANNELS; channel++) {
1308 write_mchbar32(0x220 + (channel << 10),
1309 read_mchbar32(0x220 +
1310 (channel << 10)) | 0x18001117);
1311 write_mchbar32(0x224 + (channel << 10),
1312 (info->max_slots_used_in_channel - 1)
1313 |
1314 ((info->cas_latency - 5 -
1315 info->clock_speed_index) << 21)
1316 |
1317 ((info->max_slots_used_in_channel +
1318 info->cas_latency - cas_latency_shift -
1319 4) << 16)
1320 | ((info->cas_latency - cas_latency_shift - 4) <<
1321 26)
1322 |
1323 ((info->cas_latency - info->clock_speed_index +
1324 info->max_slots_used_in_channel - 6) << 8));
1325 write_mchbar32(0x228 + (channel << 10),
1326 info->max_slots_used_in_channel);
1327 write_mchbar8(0x239 + (channel << 10), 32);
1328 write_mchbar32(0x248 + (channel << 10),
1329 (high_multiplier << 24) |
1330 (some_delay_3_half_cycles << 25) | 0x840000);
1331 write_mchbar32(0x278 + (channel << 10), 0xc362042);
1332 write_mchbar32(0x27c + (channel << 10), 0x8b000062);
1333 write_mchbar32(0x24c + (channel << 10),
1334 ((! !info->
1335 clock_speed_index) << 17) | (((2 +
1336 info->
1337 clock_speed_index
1338 -
1339 (! !info->
1340 clock_speed_index)))
1341 << 12) | 0x10200);
1342
1343 write_mchbar8(0x267 + (channel << 10), 0x4);
1344 write_mchbar16(0x272 + (channel << 10), 0x155);
1345 write_mchbar32(0x2bc + (channel << 10),
1346 (read_mchbar32(0x2bc + (channel << 10)) &
1347 0xFF000000)
1348 | 0x707070);
1349
1350 write_500(info, channel,
1351 ((!info->populated_ranks[channel][1][1])
1352 | (!info->populated_ranks[channel][1][0] << 1)
1353 | (!info->populated_ranks[channel][0][1] << 2)
1354 | (!info->populated_ranks[channel][0][0] << 3)),
1355 0x4c9, 4, 1);
1356 }
1357
1358 write_mchbar8(0x2c4, ((1 + (info->clock_speed_index != 0)) << 6) | 0xC);
1359 {
1360 u8 freq_divisor = 2;
1361 if (info->fsb_frequency == frequency_11(info))
1362 freq_divisor = 3;
1363 else if (2 * info->fsb_frequency < 3 * (frequency_11(info) / 2))
1364 freq_divisor = 1;
1365 else
1366 freq_divisor = 2;
1367 write_mchbar32(0x2c0, (freq_divisor << 11) | 0x6009c400);
1368 }
1369
1370 if (info->board_lane_delay[3] <= 10) {
1371 if (info->board_lane_delay[3] <= 8)
1372 lane_3_delay = info->board_lane_delay[3];
1373 else
1374 lane_3_delay = 10;
1375 } else {
1376 lane_3_delay = 12;
1377 }
1378 cas_latency_derived = info->cas_latency - info->clock_speed_index + 2;
1379 if (info->clock_speed_index > 1)
1380 cas_latency_derived++;
1381 for (channel = 0; channel < NUM_CHANNELS; channel++) {
1382 write_mchbar32(0x240 + (channel << 10),
1383 ((info->clock_speed_index ==
1384 0) * 0x11000) | 0x1002100 | ((2 +
1385 info->
1386 clock_speed_index)
1387 << 4) | (info->
1388 cas_latency
1389 - 3));
1390 write_500(info, channel, (info->clock_speed_index << 1) | 1,
1391 0x609, 6, 1);
1392 write_500(info, channel,
1393 info->clock_speed_index + 2 * info->cas_latency - 7,
1394 0x601, 6, 1);
1395
1396 write_mchbar32(0x250 + (channel << 10),
1397 ((lane_3_delay + info->clock_speed_index +
1398 9) << 6)
1399 | (info->board_lane_delay[7] << 2) | (info->
1400 board_lane_delay
1401 [4] << 16)
1402 | (info->board_lane_delay[1] << 25) | (info->
1403 board_lane_delay
1404 [1] << 29)
1405 | 1);
1406 write_mchbar32(0x254 + (channel << 10),
1407 (info->
1408 board_lane_delay[1] >> 3) | ((info->
1409 board_lane_delay
1410 [8] +
1411 4 *
1412 info->
1413 use_ecc) << 6) |
1414 0x80 | (info->board_lane_delay[6] << 1) | (info->
1415 board_lane_delay
1416 [2] <<
1417 28) |
1418 (cas_latency_derived << 16) | 0x4700000);
1419 write_mchbar32(0x258 + (channel << 10),
1420 ((info->board_lane_delay[5] +
1421 info->clock_speed_index +
1422 9) << 12) | ((info->clock_speed_index -
1423 info->cas_latency + 12) << 8)
1424 | (info->board_lane_delay[2] << 17) | (info->
1425 board_lane_delay
1426 [4] << 24)
1427 | 0x47);
1428 write_mchbar32(0x25c + (channel << 10),
1429 (info->board_lane_delay[1] << 1) | (info->
1430 board_lane_delay
1431 [0] << 8) |
1432 0x1da50000);
1433 write_mchbar8(0x264 + (channel << 10), 0xff);
1434 write_mchbar8(0x5f8 + (channel << 10),
1435 (cas_latency_shift << 3) | info->use_ecc);
1436 }
1437
1438 program_modules_memory_map(info, 1);
1439
1440 write_mchbar16(0x610,
1441 (min(ns_to_cycles(info, some_delay_ns) / 2, 127) << 9)
1442 | (read_mchbar16(0x610) & 0x1C3) | 0x3C);
1443 write_mchbar16(0x612, read_mchbar16(0x612) | 0x100);
1444 write_mchbar16(0x214, read_mchbar16(0x214) | 0x3E00);
1445 for (i = 0; i < 8; i++) {
1446 pcie_write_config32(PCI_DEV (QUICKPATH_BUS, 0, 1), 0x80 + 4 * i,
1447 (info->total_memory_mb - 64) | !i | 2);
1448 pcie_write_config32(PCI_DEV (QUICKPATH_BUS, 0, 1), 0xc0 + 4 * i, 0);
1449 }
1450 }
1451
1452 #define BETTER_MEMORY_MAP 0
1453
1454 static void program_total_memory_map(struct raminfo *info)
1455 {
1456 unsigned int TOM, TOLUD, TOUUD;
1457 unsigned int quickpath_reserved;
1458 unsigned int REMAPbase;
1459 unsigned int uma_base_igd;
1460 unsigned int uma_base_gtt;
1461 int memory_remap;
1462 unsigned int memory_map[8];
1463 int i;
1464 unsigned int current_limit;
1465 unsigned int tseg_base;
1466 int uma_size_igd = 0, uma_size_gtt = 0;
1467
1468 memset(memory_map, 0, sizeof(memory_map));
1469
1470 #if REAL
1471 if (info->uma_enabled) {
1472 u16 t = pcie_read_config16(NORTHBRIDGE, D0F0_GGC);
1473 gav(t);
1474 const int uma_sizes_gtt[16] =
1475 { 0, 1, 0, 2, 0, 0, 0, 0, 0, 2, 3, 4, 42, 42, 42, 42 };
1476 /* Igd memory */
1477 const int uma_sizes_igd[16] = {
1478 0, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352,
1479 256, 512
1480 };
1481
1482 uma_size_igd = uma_sizes_igd[(t >> 4) & 0xF];
1483 uma_size_gtt = uma_sizes_gtt[(t >> 8) & 0xF];
1484 }
1485 #endif
1486
1487 TOM = info->total_memory_mb;
1488 if (TOM == 4096)
1489 TOM = 4032;
1490 TOUUD = ALIGN_DOWN(TOM - info->memory_reserved_for_heci_mb, 64);
1491 TOLUD = ALIGN_DOWN(min(3072 + ALIGN_UP(uma_size_igd + uma_size_gtt, 64)
1492 , TOUUD), 64);
1493 memory_remap = 0;
1494 if (TOUUD - TOLUD > 64) {
1495 memory_remap = 1;
1496 REMAPbase = max(4096, TOUUD);
1497 TOUUD = TOUUD - TOLUD + 4096;
1498 }
1499 if (TOUUD > 4096)
1500 memory_map[2] = TOUUD | 1;
1501 quickpath_reserved = 0;
1502
1503 {
1504 u32 t;
1505
1506 gav(t = pcie_read_config32(PCI_DEV(QUICKPATH_BUS, 0, 1), 0x68));
1507 if (t & 0x800)
1508 quickpath_reserved =
1509 (1 << find_lowest_bit_set32(t >> 20));
1510 }
1511 if (memory_remap)
1512 TOUUD -= quickpath_reserved;
1513
1514 #if !REAL
1515 if (info->uma_enabled) {
1516 u16 t = pcie_read_config16(NORTHBRIDGE, D0F0_GGC);
1517 gav(t);
1518 const int uma_sizes_gtt[16] =
1519 { 0, 1, 0, 2, 0, 0, 0, 0, 0, 2, 3, 4, 42, 42, 42, 42 };
1520 /* Igd memory */
1521 const int uma_sizes_igd[16] = {
1522 0, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352,
1523 256, 512
1524 };
1525
1526 uma_size_igd = uma_sizes_igd[(t >> 4) & 0xF];
1527 uma_size_gtt = uma_sizes_gtt[(t >> 8) & 0xF];
1528 }
1529 #endif
1530
1531 uma_base_igd = TOLUD - uma_size_igd;
1532 uma_base_gtt = uma_base_igd - uma_size_gtt;
1533 tseg_base = ALIGN_DOWN(uma_base_gtt, 64) - (CONFIG_SMM_TSEG_SIZE >> 20);
1534 if (!memory_remap)
1535 tseg_base -= quickpath_reserved;
1536 tseg_base = ALIGN_DOWN(tseg_base, 8);
1537
1538 pcie_write_config16(NORTHBRIDGE, D0F0_TOLUD, TOLUD << 4);
1539 pcie_write_config16(NORTHBRIDGE, D0F0_TOM, TOM >> 6);
1540 if (memory_remap) {
1541 pcie_write_config16(NORTHBRIDGE, D0F0_REMAPBASE, REMAPbase >> 6);
1542 pcie_write_config16(NORTHBRIDGE, D0F0_REMAPLIMIT, (TOUUD - 64) >> 6);
1543 }
1544 pcie_write_config16(NORTHBRIDGE, D0F0_TOUUD, TOUUD);
1545
1546 if (info->uma_enabled) {
1547 pcie_write_config32(NORTHBRIDGE, D0F0_IGD_BASE, uma_base_igd << 20);
1548 pcie_write_config32(NORTHBRIDGE, D0F0_GTT_BASE, uma_base_gtt << 20);
1549 }
1550 pcie_write_config32(NORTHBRIDGE, TSEG, tseg_base << 20);
1551
1552 current_limit = 0;
1553 memory_map[0] = ALIGN_DOWN(uma_base_gtt, 64) | 1;
1554 memory_map[1] = 4096;
1555 for (i = 0; i < ARRAY_SIZE(memory_map); i++) {
1556 current_limit = max(current_limit, memory_map[i] & ~1);
1557 pcie_write_config32(PCI_DEV(QUICKPATH_BUS, 0, 1), 4 * i + 0x80,
1558 (memory_map[i] & 1) | ALIGN_DOWN(current_limit -
1559 1, 64) | 2);
1560 pcie_write_config32(PCI_DEV(QUICKPATH_BUS, 0, 1), 4 * i + 0xc0, 0);
1561 }
1562 }
1563
1564 static void collect_system_info(struct raminfo *info)
1565 {
1566 u32 capid0[3];
1567 int i;
1568 unsigned channel;
1569
1570 /* Wait for some bit, maybe TXT clear. */
1571 while (!(read8((u8 *)0xfed40000) & (1 << 7))) ;
1572
1573 if (!info->heci_bar)
1574 gav(info->heci_bar =
1575 pcie_read_config32(HECIDEV, HECIBAR) & 0xFFFFFFF8);
1576 if (!info->memory_reserved_for_heci_mb) {
1577 /* Wait for ME to be ready */
1578 intel_early_me_init();
1579 info->memory_reserved_for_heci_mb = intel_early_me_uma_size();
1580 }
1581
1582 for (i = 0; i < 3; i++)
1583 gav(capid0[i] =
1584 pcie_read_config32(NORTHBRIDGE, D0F0_CAPID0 | (i << 2)));
1585 gav(info->revision = pcie_read_config8(NORTHBRIDGE, PCI_REVISION_ID));
1586 info->max_supported_clock_speed_index = (~capid0[1] & 7);
1587
1588 if ((capid0[1] >> 11) & 1)
1589 info->uma_enabled = 0;
1590 else
1591 gav(info->uma_enabled =
1592 pcie_read_config8(NORTHBRIDGE, D0F0_DEVEN) & 8);
1593 /* Unrecognised: [0000:fffd3d2d] 37f81.37f82 ! CPUID: eax: 00000001; ecx: 00000e00 => 00020655.00010800.029ae3ff.bfebfbff */
1594 info->silicon_revision = 0;
1595
1596 if (capid0[2] & 2) {
1597 info->silicon_revision = 0;
1598 info->max_supported_clock_speed_index = 2;
1599 for (channel = 0; channel < NUM_CHANNELS; channel++)
1600 if (info->populated_ranks[channel][0][0]
1601 && (info->spd[channel][0][MODULE_TYPE] & 0xf) ==
1602 3) {
1603 info->silicon_revision = 2;
1604 info->max_supported_clock_speed_index = 1;
1605 }
1606 } else {
1607 switch (((capid0[2] >> 18) & 1) + 2 * ((capid0[1] >> 3) & 1)) {
1608 case 1:
1609 case 2:
1610 info->silicon_revision = 3;
1611 break;
1612 case 3:
1613 info->silicon_revision = 0;
1614 break;
1615 case 0:
1616 info->silicon_revision = 2;
1617 break;
1618 }
1619 switch (pcie_read_config16(NORTHBRIDGE, PCI_DEVICE_ID)) {
1620 case 0x40:
1621 info->silicon_revision = 0;
1622 break;
1623 case 0x48:
1624 info->silicon_revision = 1;
1625 break;
1626 }
1627 }
1628 }
1629
1630 static void write_training_data(struct raminfo *info)
1631 {
1632 int tm, channel, slot, rank, lane;
1633 if (info->revision < 8)
1634 return;
1635
1636 for (tm = 0; tm < 4; tm++)
1637 for (channel = 0; channel < NUM_CHANNELS; channel++)
1638 for (slot = 0; slot < NUM_SLOTS; slot++)
1639 for (rank = 0; rank < NUM_RANKS; rank++)
1640 for (lane = 0; lane < 9; lane++)
1641 write_500(info, channel,
1642 info->
1643 cached_training->
1644 lane_timings[tm]
1645 [channel][slot][rank]
1646 [lane],
1647 get_timing_register_addr
1648 (lane, tm, slot,
1649 rank), 9, 0);
1650 write_1d0(info->cached_training->reg_178, 0x178, 7, 1);
1651 write_1d0(info->cached_training->reg_10b, 0x10b, 6, 1);
1652 }
1653
1654 static void dump_timings(struct raminfo *info)
1655 {
1656 #if REAL
1657 int channel, slot, rank, lane, i;
1658 printk(BIOS_DEBUG, "Timings:\n");
1659 FOR_POPULATED_RANKS {
1660 printk(BIOS_DEBUG, "channel %d, slot %d, rank %d\n", channel,
1661 slot, rank);
1662 for (lane = 0; lane < 9; lane++) {
1663 printk(BIOS_DEBUG, "lane %d: ", lane);
1664 for (i = 0; i < 4; i++) {
1665 printk(BIOS_DEBUG, "%x (%x) ",
1666 read_500(info, channel,
1667 get_timing_register_addr
1668 (lane, i, slot, rank),
1669 9),
1670 info->training.
1671 lane_timings[i][channel][slot][rank]
1672 [lane]);
1673 }
1674 printk(BIOS_DEBUG, "\n");
1675 }
1676 }
1677 printk(BIOS_DEBUG, "[178] = %x (%x)\n", read_1d0(0x178, 7),
1678 info->training.reg_178);
1679 printk(BIOS_DEBUG, "[10b] = %x (%x)\n", read_1d0(0x10b, 6),
1680 info->training.reg_10b);
1681 #endif
1682 }
1683
1684 /* Read timings and other registers that need to be restored verbatim and
1685 put them to CBMEM.
1686 */
1687 static void save_timings(struct raminfo *info)
1688 {
1689 struct ram_training train;
1690 struct mrc_data_container *mrcdata;
1691 int output_len = ALIGN(sizeof(train), 16);
1692 int channel, slot, rank, lane, i;
1693
1694 train = info->training;
1695 FOR_POPULATED_RANKS for (lane = 0; lane < 9; lane++)
1696 for (i = 0; i < 4; i++)
1697 train.lane_timings[i][channel][slot][rank][lane] =
1698 read_500(info, channel,
1699 get_timing_register_addr(lane, i, slot,
1700 rank), 9);
1701 train.reg_178 = read_1d0(0x178, 7);
1702 train.reg_10b = read_1d0(0x10b, 6);
1703
1704 for (channel = 0; channel < NUM_CHANNELS; channel++) {
1705 u32 reg32;
1706 reg32 = read_mchbar32 ((channel << 10) + 0x274);
1707 train.reg274265[channel][0] = reg32 >> 16;
1708 train.reg274265[channel][1] = reg32 & 0xffff;
1709 train.reg274265[channel][2] = read_mchbar16 ((channel << 10) + 0x265) >> 8;
1710 }
1711 train.reg2ca9_bit0 = read_mchbar8(0x2ca9) & 1;
1712 train.reg_6dc = read_mchbar32 (0x6dc);
1713 train.reg_6e8 = read_mchbar32 (0x6e8);
1714
1715 printk (BIOS_SPEW, "[6dc] = %x\n", train.reg_6dc);
1716 printk (BIOS_SPEW, "[6e8] = %x\n", train.reg_6e8);
1717
1718 /* Save the MRC S3 restore data to cbmem */
1719 mrcdata = cbmem_add
1720 (CBMEM_ID_MRCDATA, output_len + sizeof(struct mrc_data_container));
1721
1722 if (mrcdata != NULL) {
1723 printk(BIOS_DEBUG, "Relocate MRC DATA from %p to %p (%u bytes)\n",
1724 &train, mrcdata, output_len);
1725
1726 mrcdata->mrc_signature = MRC_DATA_SIGNATURE;
1727 mrcdata->mrc_data_size = output_len;
1728 mrcdata->reserved = 0;
1729 memcpy(mrcdata->mrc_data, &train, sizeof(train));
1730
1731 /* Zero the unused space in aligned buffer. */
1732 if (output_len > sizeof(train))
1733 memset(mrcdata->mrc_data + sizeof(train), 0,
1734 output_len - sizeof(train));
1735
1736 mrcdata->mrc_checksum = compute_ip_checksum(mrcdata->mrc_data,
1737 mrcdata->mrc_data_size);
1738 }
1739 }
1740
1741 #if REAL
1742 static const struct ram_training *get_cached_training(void)
1743 {
1744 struct mrc_data_container *cont;
1745 cont = find_current_mrc_cache();
1746 if (!cont)
1747 return 0;
1748 return (void *)cont->mrc_data;
1749 }
1750 #endif
1751
1752 /* FIXME: add timeout. */
1753 static void wait_heci_ready(void)
1754 {
1755 while (!(read32(DEFAULT_HECIBAR + 0xc) & 8)) ; // = 0x8000000c
1756 write32((DEFAULT_HECIBAR + 0x4),
1757 (read32(DEFAULT_HECIBAR + 0x4) & ~0x10) | 0xc);
1758 }
1759
1760 /* FIXME: add timeout. */
1761 static void wait_heci_cb_avail(int len)
1762 {
1763 union {
1764 struct mei_csr csr;
1765 u32 raw;
1766 } csr;
1767
1768 while (!(read32(DEFAULT_HECIBAR + 0xc) & 8)) ;
1769
1770 do
1771 csr.raw = read32(DEFAULT_HECIBAR + 0x4);
1772 while (len >
1773 csr.csr.buffer_depth - (csr.csr.buffer_write_ptr -
1774 csr.csr.buffer_read_ptr));
1775 }
1776
1777 static void send_heci_packet(struct mei_header *head, u32 * payload)
1778 {
1779 int len = (head->length + 3) / 4;
1780 int i;
1781
1782 wait_heci_cb_avail(len + 1);
1783
1784 /* FIXME: handle leftovers correctly. */
1785 write32(DEFAULT_HECIBAR + 0, *(u32 *) head);
1786 for (i = 0; i < len - 1; i++)
1787 write32(DEFAULT_HECIBAR + 0, payload[i]);
1788
1789 write32(DEFAULT_HECIBAR + 0, payload[i] & ((1 << (8 * len)) - 1));
1790 write32(DEFAULT_HECIBAR + 0x4, read32(DEFAULT_HECIBAR + 0x4) | 0x4);
1791 }
1792
1793 static void
1794 send_heci_message(u8 * msg, int len, u8 hostaddress, u8 clientaddress)
1795 {
1796 struct mei_header head;
1797 int maxlen;
1798
1799 wait_heci_ready();
1800 maxlen = (read32(DEFAULT_HECIBAR + 0x4) >> 24) * 4 - 4;
1801
1802 while (len) {
1803 int cur = len;
1804 if (cur > maxlen) {
1805 cur = maxlen;
1806 head.is_complete = 0;
1807 } else
1808 head.is_complete = 1;
1809 head.length = cur;
1810 head.reserved = 0;
1811 head.client_address = clientaddress;
1812 head.host_address = hostaddress;
1813 send_heci_packet(&head, (u32 *) msg);
1814 len -= cur;
1815 msg += cur;
1816 }
1817 }
1818
1819 /* FIXME: Add timeout. */
1820 static int
1821 recv_heci_packet(struct raminfo *info, struct mei_header *head, u32 * packet,
1822 u32 * packet_size)
1823 {
1824 union {
1825 struct mei_csr csr;
1826 u32 raw;
1827 } csr;
1828 int i = 0;
1829
1830 write32(DEFAULT_HECIBAR + 0x4, read32(DEFAULT_HECIBAR + 0x4) | 2);
1831 do {
1832 csr.raw = read32(DEFAULT_HECIBAR + 0xc);
1833 #if !REAL
1834 if (i++ > 346)
1835 return -1;
1836 #endif
1837 }
1838 while (csr.csr.buffer_write_ptr == csr.csr.buffer_read_ptr);
1839 *(u32 *) head = read32(DEFAULT_HECIBAR + 0x8);
1840 if (!head->length) {
1841 write32(DEFAULT_HECIBAR + 0x4,
1842 read32(DEFAULT_HECIBAR + 0x4) | 2);
1843 *packet_size = 0;
1844 return 0;
1845 }
1846 if (head->length + 4 > 4 * csr.csr.buffer_depth
1847 || head->length > *packet_size) {
1848 *packet_size = 0;
1849 return -1;
1850 }
1851
1852 do
1853 csr.raw = read32(DEFAULT_HECIBAR + 0xc);
1854 while ((head->length + 3) >> 2 >
1855 csr.csr.buffer_write_ptr - csr.csr.buffer_read_ptr);
1856
1857 for (i = 0; i < (head->length + 3) >> 2; i++)
1858 packet[i++] = read32(DEFAULT_HECIBAR + 0x8);
1859 *packet_size = head->length;
1860 if (!csr.csr.ready)
1861 *packet_size = 0;
1862 write32(DEFAULT_HECIBAR + 0x4, read32(DEFAULT_HECIBAR + 0x4) | 4);
1863 return 0;
1864 }
1865
1866 /* FIXME: Add timeout. */
1867 static int
1868 recv_heci_message(struct raminfo *info, u32 * message, u32 * message_size)
1869 {
1870 struct mei_header head;
1871 int current_position;
1872
1873 current_position = 0;
1874 while (1) {
1875 u32 current_size;
1876 current_size = *message_size - current_position;
1877 if (recv_heci_packet
1878 (info, &head, message + (current_position >> 2),
1879 &current_size) == -1)
1880 break;
1881 if (!current_size)
1882 break;
1883 current_position += current_size;
1884 if (head.is_complete) {
1885 *message_size = current_position;
1886 return 0;
1887 }
1888
1889 if (current_position >= *message_size)
1890 break;
1891 }
1892 *message_size = 0;
1893 return -1;
1894 }
1895
1896 static void send_heci_uma_message(struct raminfo *info)
1897 {
1898 struct uma_reply {
1899 u8 group_id;
1900 u8 command;
1901 u8 reserved;
1902 u8 result;
1903 u8 field2;
1904 u8 unk3[0x48 - 4 - 1];
1905 } __attribute__ ((packed)) reply;
1906 struct uma_message {
1907 u8 group_id;
1908 u8 cmd;
1909 u8 reserved;
1910 u8 result;
1911 u32 c2;
1912 u64 heci_uma_addr;
1913 u32 memory_reserved_for_heci_mb;
1914 u16 c3;
1915 } __attribute__ ((packed)) msg = {
1916 0, MKHI_SET_UMA, 0, 0,
1917 0x82,
1918 info->heci_uma_addr, info->memory_reserved_for_heci_mb, 0};
1919 u32 reply_size;
1920
1921 send_heci_message((u8 *) & msg, sizeof(msg), 0, 7);
1922
1923 reply_size = sizeof(reply);
1924 if (recv_heci_message(info, (u32 *) & reply, &reply_size) == -1)
1925 return;
1926
1927 if (reply.command != (MKHI_SET_UMA | (1 << 7)))
1928 die("HECI init failed\n");
1929 }
1930
1931 static void setup_heci_uma(struct raminfo *info)
1932 {
1933 u32 reg44;
1934
1935 reg44 = pcie_read_config32(HECIDEV, 0x44); // = 0x80010020
1936 info->memory_reserved_for_heci_mb = 0;
1937 info->heci_uma_addr = 0;
1938 if (!((reg44 & 0x10000) && !(pcie_read_config32(HECIDEV, 0x40) & 0x20)))
1939 return;
1940
1941 info->heci_bar = pcie_read_config32(HECIDEV, 0x10) & 0xFFFFFFF0;
1942 info->memory_reserved_for_heci_mb = reg44 & 0x3f;
1943 info->heci_uma_addr =
1944 ((u64)
1945 ((((u64) pcie_read_config16(NORTHBRIDGE, D0F0_TOM)) << 6) -
1946 info->memory_reserved_for_heci_mb)) << 20;
1947
1948 pcie_read_config32(NORTHBRIDGE, DMIBAR);
1949 if (info->memory_reserved_for_heci_mb) {
1950 write32(DEFAULT_DMIBAR + 0x14,
1951 read32(DEFAULT_DMIBAR + 0x14) & ~0x80);
1952 write32(DEFAULT_RCBA + 0x14,
1953 read32(DEFAULT_RCBA + 0x14) & ~0x80);
1954 write32(DEFAULT_DMIBAR + 0x20,
1955 read32(DEFAULT_DMIBAR + 0x20) & ~0x80);
1956 write32(DEFAULT_RCBA + 0x20,
1957 read32(DEFAULT_RCBA + 0x20) & ~0x80);
1958 write32(DEFAULT_DMIBAR + 0x2c,
1959 read32(DEFAULT_DMIBAR + 0x2c) & ~0x80);
1960 write32(DEFAULT_RCBA + 0x30,
1961 read32(DEFAULT_RCBA + 0x30) & ~0x80);
1962 write32(DEFAULT_DMIBAR + 0x38,
1963 read32(DEFAULT_DMIBAR + 0x38) & ~0x80);
1964 write32(DEFAULT_RCBA + 0x40,
1965 read32(DEFAULT_RCBA + 0x40) & ~0x80);
1966
1967 write32(DEFAULT_RCBA + 0x40, 0x87000080); // OK
1968 write32(DEFAULT_DMIBAR + 0x38, 0x87000080); // OK
1969 while (read16(DEFAULT_RCBA + 0x46) & 2
1970 && read16(DEFAULT_DMIBAR + 0x3e) & 2) ;
1971 }
1972
1973 write_mchbar32(0x24, 0x10000 + info->memory_reserved_for_heci_mb);
1974
1975 send_heci_uma_message(info);
1976
1977 pcie_write_config32(HECIDEV, 0x10, 0x0);
1978 pcie_write_config8(HECIDEV, 0x4, 0x0);
1979
1980 }
1981
1982 static int have_match_ranks(struct raminfo *info, int channel, int ranks)
1983 {
1984 int ranks_in_channel;
1985 ranks_in_channel = info->populated_ranks[channel][0][0]
1986 + info->populated_ranks[channel][0][1]
1987 + info->populated_ranks[channel][1][0]
1988 + info->populated_ranks[channel][1][1];
1989
1990 /* empty channel */
1991 if (ranks_in_channel == 0)
1992 return 1;
1993
1994 if (ranks_in_channel != ranks)
1995 return 0;
1996 /* single slot */
1997 if (info->populated_ranks[channel][0][0] !=
1998 info->populated_ranks[channel][1][0])
1999 return 1;
2000 if (info->populated_ranks[channel][0][1] !=
2001 info->populated_ranks[channel][1][1])
2002 return 1;
2003 if (info->is_x16_module[channel][0] != info->is_x16_module[channel][1])
2004 return 0;
2005 if (info->density[channel][0] != info->density[channel][1])
2006 return 0;
2007 return 1;
2008 }
2009
2010 static void read_4090(struct raminfo *info)
2011 {
2012 int i, channel, slot, rank, lane;
2013 for (i = 0; i < 2; i++)
2014 for (slot = 0; slot < NUM_SLOTS; slot++)
2015 for (rank = 0; rank < NUM_RANKS; rank++)
2016 for (lane = 0; lane < 9; lane++)
2017 info->training.
2018 lane_timings[0][i][slot][rank][lane]
2019 = 32;
2020
2021 for (i = 1; i < 4; i++)
2022 for (channel = 0; channel < NUM_CHANNELS; channel++)
2023 for (slot = 0; slot < NUM_SLOTS; slot++)
2024 for (rank = 0; rank < NUM_RANKS; rank++)
2025 for (lane = 0; lane < 9; lane++) {
2026 info->training.
2027 lane_timings[i][channel]
2028 [slot][rank][lane] =
2029 read_500(info, channel,
2030 get_timing_register_addr
2031 (lane, i, slot,
2032 rank), 9)
2033 + (i == 1) * 11; // !!!!
2034 }
2035
2036 }
2037
2038 static u32 get_etalon2(int flip, u32 addr)
2039 {
2040 const u16 invmask[] = {
2041 0xaaaa, 0x6db6, 0x4924, 0xeeee, 0xcccc, 0x8888, 0x7bde, 0x739c,
2042 0x6318, 0x4210, 0xefbe, 0xcf3c, 0x8e38, 0x0c30, 0x0820
2043 };
2044 u32 ret;
2045 u32 comp4 = addr / 480;
2046 addr %= 480;
2047 u32 comp1 = addr & 0xf;
2048 u32 comp2 = (addr >> 4) & 1;
2049 u32 comp3 = addr >> 5;
2050
2051 if (comp4)
2052 ret = 0x1010101 << (comp4 - 1);
2053 else
2054 ret = 0;
2055 if (flip ^ (((invmask[comp3] >> comp1) ^ comp2) & 1))
2056 ret = ~ret;
2057
2058 return ret;
2059 }
2060
2061 static void disable_cache(void)
2062 {
2063 msr_t msr = {.lo = 0, .hi = 0 };
2064
2065 wrmsr(MTRR_PHYS_BASE(3), msr);
2066 wrmsr(MTRR_PHYS_MASK(3), msr);
2067 }
2068
2069 static void enable_cache(unsigned int base, unsigned int size)
2070 {
2071 msr_t msr;
2072 msr.lo = base | MTRR_TYPE_WRPROT;
2073 msr.hi = 0;
2074 wrmsr(MTRR_PHYS_BASE(3), msr);
2075 msr.lo = ((~(ALIGN_DOWN(size + 4096, 4096) - 1) | MTRR_DEF_TYPE_EN)
2076 & 0xffffffff);
2077 msr.hi = 0x0000000f;
2078 wrmsr(MTRR_PHYS_MASK(3), msr);
2079 }
2080
2081 static void flush_cache(u32 start, u32 size)
2082 {
2083 u32 end;
2084 u32 addr;
2085
2086 end = start + (ALIGN_DOWN(size + 4096, 4096));
2087 for (addr = start; addr < end; addr += 64)
2088 clflush(addr);
2089 }
2090
2091 static void clear_errors(void)
2092 {
2093 pcie_write_config8(NORTHBRIDGE, 0xc0, 0x01);
2094 }
2095
2096 static void write_testing(struct raminfo *info, int totalrank, int flip)
2097 {
2098 int nwrites = 0;
2099 /* in 8-byte units. */
2100 u32 offset;
2101 u8 *base;
2102
2103 base = (u8 *)(totalrank << 28);
2104 for (offset = 0; offset < 9 * 480; offset += 2) {
2105 write32(base + offset * 8, get_etalon2(flip, offset));
2106 write32(base + offset * 8 + 4, get_etalon2(flip, offset));
2107 write32(base + offset * 8 + 8, get_etalon2(flip, offset + 1));
2108 write32(base + offset * 8 + 12, get_etalon2(flip, offset + 1));
2109 nwrites += 4;
2110 if (nwrites >= 320) {
2111 clear_errors();
2112 nwrites = 0;
2113 }
2114 }
2115 }
2116
2117 static u8 check_testing(struct raminfo *info, u8 total_rank, int flip)
2118 {
2119 u8 failmask = 0;
2120 int i;
2121 int comp1, comp2, comp3;
2122 u32 failxor[2] = { 0, 0 };
2123
2124 enable_cache((total_rank << 28), 1728 * 5 * 4);
2125
2126 for (comp3 = 0; comp3 < 9 && failmask != 0xff; comp3++) {
2127 for (comp1 = 0; comp1 < 4; comp1++)
2128 for (comp2 = 0; comp2 < 60; comp2++) {
2129 u32 re[4];
2130 u32 curroffset =
2131 comp3 * 8 * 60 + 2 * comp1 + 8 * comp2;
2132 read128((total_rank << 28) | (curroffset << 3),
2133 (u64 *) re);
2134 failxor[0] |=
2135 get_etalon2(flip, curroffset) ^ re[0];
2136 failxor[1] |=
2137 get_etalon2(flip, curroffset) ^ re[1];
2138 failxor[0] |=
2139 get_etalon2(flip, curroffset | 1) ^ re[2];
2140 failxor[1] |=
2141 get_etalon2(flip, curroffset | 1) ^ re[3];
2142 }
2143 for (i = 0; i < 8; i++)
2144 if ((0xff << (8 * (i % 4))) & failxor[i / 4])
2145 failmask |= 1 << i;
2146 }
2147 disable_cache();
2148 flush_cache((total_rank << 28), 1728 * 5 * 4);
2149 return failmask;
2150 }
2151
2152 const u32 seed1[0x18] = {
2153 0x3a9d5ab5, 0x576cb65b, 0x555773b6, 0x2ab772ee,
2154 0x555556ee, 0x3a9d5ab5, 0x576cb65b, 0x555773b6,
2155 0x2ab772ee, 0x555556ee, 0x5155a555, 0x5155a555,
2156 0x5155a555, 0x5155a555, 0x3a9d5ab5, 0x576cb65b,
2157 0x555773b6, 0x2ab772ee, 0x555556ee, 0x55d6b4a5,
2158 0x366d6b3a, 0x2ae5ddbb, 0x3b9ddbb7, 0x55d6b4a5,
2159 };
2160
2161 static u32 get_seed2(int a, int b)
2162 {
2163 const u32 seed2[5] = {
2164 0x55555555, 0x33333333, 0x2e555a55, 0x55555555,
2165 0x5b6db6db,
2166 };
2167 u32 r;
2168 r = seed2[(a + (a >= 10)) / 5];
2169 return b ? ~r : r;
2170 }
2171
2172 static int make_shift(int comp2, int comp5, int x)
2173 {
2174 const u8 seed3[32] = {
2175 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2176 0x00, 0x00, 0x38, 0x1c, 0x3c, 0x18, 0x38, 0x38,
2177 0x38, 0x38, 0x38, 0x38, 0x0f, 0x0f, 0x0f, 0x0f,
2178 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
2179 };
2180
2181 return (comp2 - ((seed3[comp5] >> (x & 7)) & 1)) & 0x1f;
2182 }
2183
2184 static u32 get_etalon(int flip, u32 addr)
2185 {
2186 u32 mask_byte = 0;
2187 int comp1 = (addr >> 1) & 1;
2188 int comp2 = (addr >> 3) & 0x1f;
2189 int comp3 = (addr >> 8) & 0xf;
2190 int comp4 = (addr >> 12) & 0xf;
2191 int comp5 = (addr >> 16) & 0x1f;
2192 u32 mask_bit = ~(0x10001 << comp3);
2193 u32 part1;
2194 u32 part2;
2195 int byte;
2196
2197 part2 =
2198 ((seed1[comp5] >>
2199 make_shift(comp2, comp5,
2200 (comp3 >> 3) | (comp1 << 2) | 2)) & 1) ^ flip;
2201 part1 =
2202 ((seed1[comp5] >>
2203 make_shift(comp2, comp5,
2204 (comp3 >> 3) | (comp1 << 2) | 0)) & 1) ^ flip;
2205
2206 for (byte = 0; byte < 4; byte++)
2207 if ((get_seed2(comp5, comp4) >>
2208 make_shift(comp2, comp5, (byte | (comp1 << 2)))) & 1)
2209 mask_byte |= 0xff << (8 * byte);
2210
2211 return (mask_bit & mask_byte) | (part1 << comp3) | (part2 <<
2212 (comp3 + 16));
2213 }
2214
2215 static void
2216 write_testing_type2(struct raminfo *info, u8 totalrank, u8 region, u8 block,
2217 char flip)
2218 {
2219 int i;
2220 for (i = 0; i < 2048; i++)
2221 write32p((totalrank << 28) | (region << 25) | (block << 16) |
2222 (i << 2), get_etalon(flip, (block << 16) | (i << 2)));
2223 }
2224
2225 static u8
2226 check_testing_type2(struct raminfo *info, u8 totalrank, u8 region, u8 block,
2227 char flip)
2228 {
2229 u8 failmask = 0;
2230 u32 failxor[2];
2231 int i;
2232 int comp1, comp2, comp3;
2233
2234 failxor[0] = 0;
2235 failxor[1] = 0;
2236
2237 enable_cache(totalrank << 28, 134217728);
2238 for (comp3 = 0; comp3 < 2 && failmask != 0xff; comp3++) {
2239 for (comp1 = 0; comp1 < 16; comp1++)
2240 for (comp2 = 0; comp2 < 64; comp2++) {
2241 u32 addr =
2242 (totalrank << 28) | (region << 25) | (block
2243 << 16)
2244 | (comp3 << 12) | (comp2 << 6) | (comp1 <<
2245 2);
2246 failxor[comp1 & 1] |=
2247 read32p(addr) ^ get_etalon(flip, addr);
2248 }
2249 for (i = 0; i < 8; i++)
2250 if ((0xff << (8 * (i % 4))) & failxor[i / 4])
2251 failmask |= 1 << i;
2252 }
2253 disable_cache();
2254 flush_cache((totalrank << 28) | (region << 25) | (block << 16), 16384);
2255 return failmask;
2256 }
2257
2258 static int check_bounded(unsigned short *vals, u16 bound)
2259 {
2260 int i;
2261
2262 for (i = 0; i < 8; i++)
2263 if (vals[i] < bound)
2264 return 0;
2265 return 1;
2266 }
2267
2268 enum state {
2269 BEFORE_USABLE = 0, AT_USABLE = 1, AT_MARGIN = 2, COMPLETE = 3
2270 };
2271
2272 static int validate_state(enum state *in)
2273 {
2274 int i;
2275 for (i = 0; i < 8; i++)
2276 if (in[i] != COMPLETE)
2277 return 0;
2278 return 1;
2279 }
2280
2281 static void
2282 do_fsm(enum state *state, u16 * counter,
2283 u8 fail_mask, int margin, int uplimit,
2284 u8 * res_low, u8 * res_high, u8 val)
2285 {
2286 int lane;
2287
2288 for (lane = 0; lane < 8; lane++) {
2289 int is_fail = (fail_mask >> lane) & 1;
2290 switch (state[lane]) {
2291 case BEFORE_USABLE:
2292 if (!is_fail) {
2293 counter[lane] = 1;
2294 state[lane] = AT_USABLE;
2295 break;
2296 }
2297 counter[lane] = 0;
2298 state[lane] = BEFORE_USABLE;
2299 break;
2300 case AT_USABLE:
2301 if (!is_fail) {
2302 ++counter[lane];
2303 if (counter[lane] >= margin) {
2304 state[lane] = AT_MARGIN;
2305 res_low[lane] = val - margin + 1;
2306 break;
2307 }
2308 state[lane] = 1;
2309 break;
2310 }
2311 counter[lane] = 0;
2312 state[lane] = BEFORE_USABLE;
2313 break;
2314 case AT_MARGIN:
2315 if (is_fail) {
2316 state[lane] = COMPLETE;
2317 res_high[lane] = val - 1;
2318 } else {
2319 counter[lane]++;
2320 state[lane] = AT_MARGIN;
2321 if (val == uplimit) {
2322 state[lane] = COMPLETE;
2323 res_high[lane] = uplimit;
2324 }
2325 }
2326 break;
2327 case COMPLETE:
2328 break;
2329 }
2330 }
2331 }
2332
2333 static void
2334 train_ram_at_178(struct raminfo *info, u8 channel, int slot, int rank,
2335 u8 total_rank, u8 reg_178, int first_run, int niter,
2336 timing_bounds_t * timings)
2337 {
2338 int lane;
2339 enum state state[8];
2340 u16 count[8];
2341 u8 lower_usable[8];
2342 u8 upper_usable[8];
2343 unsigned short num_sucessfully_checked[8];
2344 u8 secondary_total_rank;
2345 u8 reg1b3;
2346
2347 if (info->populated_ranks_mask[1]) {
2348 if (channel == 1)
2349 secondary_total_rank =
2350 info->populated_ranks[1][0][0] +
2351 info->populated_ranks[1][0][1]
2352 + info->populated_ranks[1][1][0] +
2353 info->populated_ranks[1][1][1];
2354 else
2355 secondary_total_rank = 0;
2356 } else
2357 secondary_total_rank = total_rank;
2358
2359 {
2360 int i;
2361 for (i = 0; i < 8; i++)
2362 state[i] = BEFORE_USABLE;
2363 }
2364
2365 if (!first_run) {
2366 int is_all_ok = 1;
2367 for (lane = 0; lane < 8; lane++)
2368 if (timings[reg_178][channel][slot][rank][lane].
2369 smallest ==
2370 timings[reg_178][channel][slot][rank][lane].
2371 largest) {
2372 timings[reg_178][channel][slot][rank][lane].
2373 smallest = 0;
2374 timings[reg_178][channel][slot][rank][lane].
2375 largest = 0;
2376 is_all_ok = 0;
2377 }
2378 if (is_all_ok) {
2379 int i;
2380 for (i = 0; i < 8; i++)
2381 state[i] = COMPLETE;
2382 }
2383 }
2384
2385 for (reg1b3 = 0; reg1b3 < 0x30 && !validate_state(state); reg1b3++) {
2386 u8 failmask = 0;
2387 write_1d0(reg1b3 ^ 32, 0x1b3, 6, 1);
2388 write_1d0(reg1b3 ^ 32, 0x1a3, 6, 1);
2389 failmask = check_testing(info, total_rank, 0);
2390 write_mchbar32(0xfb0, read_mchbar32(0xfb0) | 0x00030000);
2391 do_fsm(state, count, failmask, 5, 47, lower_usable,
2392 upper_usable, reg1b3);
2393 }
2394
2395 if (reg1b3) {
2396 write_1d0(0, 0x1b3, 6, 1);
2397 write_1d0(0, 0x1a3, 6, 1);
2398 for (lane = 0; lane < 8; lane++) {
2399 if (state[lane] == COMPLETE) {
2400 timings[reg_178][channel][slot][rank][lane].
2401 smallest =
2402 lower_usable[lane] +
2403 (info->training.
2404 lane_timings[0][channel][slot][rank][lane]
2405 & 0x3F) - 32;
2406 timings[reg_178][channel][slot][rank][lane].
2407 largest =
2408 upper_usable[lane] +
2409 (info->training.
2410 lane_timings[0][channel][slot][rank][lane]
2411 & 0x3F) - 32;
2412 }
2413 }
2414 }
2415
2416 if (!first_run) {
2417 for (lane = 0; lane < 8; lane++)
2418 if (state[lane] == COMPLETE) {
2419 write_500(info, channel,
2420 timings[reg_178][channel][slot][rank]
2421 [lane].smallest,
2422 get_timing_register_addr(lane, 0,
2423 slot, rank),
2424 9, 1);
2425 write_500(info, channel,
2426 timings[reg_178][channel][slot][rank]
2427 [lane].smallest +
2428 info->training.
2429 lane_timings[1][channel][slot][rank]
2430 [lane]
2431 -
2432 info->training.
2433 lane_timings[0][channel][slot][rank]
2434 [lane], get_timing_register_addr(lane,
2435 1,
2436 slot,
2437 rank),
2438 9, 1);
2439 num_sucessfully_checked[lane] = 0;
2440 } else
2441 num_sucessfully_checked[lane] = -1;
2442
2443 do {
2444 u8 failmask = 0;
2445 int i;
2446 for (i = 0; i < niter; i++) {
2447 if (failmask == 0xFF)
2448 break;
2449 failmask |=
2450 check_testing_type2(info, total_rank, 2, i,
2451 0);
2452 failmask |=
2453 check_testing_type2(info, total_rank, 3, i,
2454 1);
2455 }
2456 write_mchbar32(0xfb0,
2457 read_mchbar32(0xfb0) | 0x00030000);
2458 for (lane = 0; lane < 8; lane++)
2459 if (num_sucessfully_checked[lane] != 0xffff) {
2460 if ((1 << lane) & failmask) {
2461 if (timings[reg_178][channel]
2462 [slot][rank][lane].
2463 largest <=
2464 timings[reg_178][channel]
2465 [slot][rank][lane].smallest)
2466 num_sucessfully_checked
2467 [lane] = -1;
2468 else {
2469 num_sucessfully_checked
2470 [lane] = 0;
2471 timings[reg_178]
2472 [channel][slot]
2473 [rank][lane].
2474 smallest++;
2475 write_500(info, channel,
2476 timings
2477 [reg_178]
2478 [channel]
2479 [slot][rank]
2480 [lane].
2481 smallest,
2482 get_timing_register_addr
2483 (lane, 0,
2484 slot, rank),
2485 9, 1);
2486 write_500(info, channel,
2487 timings
2488 [reg_178]
2489 [channel]
2490 [slot][rank]
2491 [lane].
2492 smallest +
2493 info->
2494 training.
2495 lane_timings
2496 [1][channel]
2497 [slot][rank]
2498 [lane]
2499 -
2500 info->
2501 training.
2502 lane_timings
2503 [0][channel]
2504 [slot][rank]
2505 [lane],
2506 get_timing_register_addr
2507 (lane, 1,
2508 slot, rank),
2509 9, 1);
2510 }
2511 } else
2512 num_sucessfully_checked[lane]++;
2513 }
2514 }
2515 while (!check_bounded(num_sucessfully_checked, 2));
2516
2517 for (lane = 0; lane < 8; lane++)
2518 if (state[lane] == COMPLETE) {
2519 write_500(info, channel,
2520 timings[reg_178][channel][slot][rank]
2521 [lane].largest,
2522 get_timing_register_addr(lane, 0,
2523 slot, rank),
2524 9, 1);
2525 write_500(info, channel,
2526 timings[reg_178][channel][slot][rank]
2527 [lane].largest +
2528 info->training.
2529 lane_timings[1][channel][slot][rank]
2530 [lane]
2531 -
2532 info->training.
2533 lane_timings[0][channel][slot][rank]
2534 [lane], get_timing_register_addr(lane,
2535 1,
2536 slot,
2537 rank),
2538 9, 1);
2539 num_sucessfully_checked[lane] = 0;
2540 } else
2541 num_sucessfully_checked[lane] = -1;
2542
2543 do {
2544 int failmask = 0;
2545 int i;
2546 for (i = 0; i < niter; i++) {
2547 if (failmask == 0xFF)
2548 break;
2549 failmask |=
2550 check_testing_type2(info, total_rank, 2, i,
2551 0);
2552 failmask |=
2553 check_testing_type2(info, total_rank, 3, i,
2554 1);
2555 }
2556
2557 write_mchbar32(0xfb0,
2558 read_mchbar32(0xfb0) | 0x00030000);
2559 for (lane = 0; lane < 8; lane++) {
2560 if (num_sucessfully_checked[lane] != 0xffff) {
2561 if ((1 << lane) & failmask) {
2562 if (timings[reg_178][channel]
2563 [slot][rank][lane].
2564 largest <=
2565 timings[reg_178][channel]
2566 [slot][rank][lane].
2567 smallest) {
2568 num_sucessfully_checked
2569 [lane] = -1;
2570 } else {
2571 num_sucessfully_checked
2572 [lane] = 0;
2573 timings[reg_178]
2574 [channel][slot]
2575 [rank][lane].
2576 largest--;
2577 write_500(info, channel,
2578 timings
2579 [reg_178]
2580 [channel]
2581 [slot][rank]
2582 [lane].
2583 largest,
2584 get_timing_register_addr
2585 (lane, 0,
2586 slot, rank),
2587 9, 1);
2588 write_500(info, channel,
2589 timings
2590 [reg_178]
2591 [channel]
2592 [slot][rank]
2593 [lane].
2594 largest +
2595 info->
2596 training.
2597 lane_timings
2598 [1][channel]
2599 [slot][rank]
2600 [lane]
2601 -
2602 info->
2603 training.
2604 lane_timings
2605 [0][channel]
2606 [slot][rank]
2607 [lane],
2608 get_timing_register_addr
2609 (lane, 1,
2610 slot, rank),
2611 9, 1);
2612 }
2613 } else
2614 num_sucessfully_checked[lane]++;
2615 }
2616 }
2617 }
2618 while (!check_bounded(num_sucessfully_checked, 3));
2619
2620 for (lane = 0; lane < 8; lane++) {
2621 write_500(info, channel,
2622 info->training.
2623 lane_timings[0][channel][slot][rank][lane],
2624 get_timing_register_addr(lane, 0, slot, rank),
2625 9, 1);
2626 write_500(info, channel,
2627 info->training.
2628 lane_timings[1][channel][slot][rank][lane],
2629 get_timing_register_addr(lane, 1, slot, rank),
2630 9, 1);
2631 if (timings[reg_178][channel][slot][rank][lane].
2632 largest <=
2633 timings[reg_178][channel][slot][rank][lane].
2634 smallest) {
2635 timings[reg_178][channel][slot][rank][lane].
2636 largest = 0;
2637 timings[reg_178][channel][slot][rank][lane].
2638 smallest = 0;
2639 }
2640 }
2641 }
2642 }
2643
2644 static void set_10b(struct raminfo *info, u8 val)
2645 {
2646 int channel;
2647 int slot, rank;
2648 int lane;
2649
2650 if (read_1d0(0x10b, 6) == val)
2651 return;
2652
2653 write_1d0(val, 0x10b, 6, 1);
2654
2655 FOR_POPULATED_RANKS_BACKWARDS for (lane = 0; lane < 9; lane++) {
2656 u16 reg_500;
2657 reg_500 = read_500(info, channel,
2658 get_timing_register_addr(lane, 0, slot,
2659 rank), 9);
2660 if (val == 1) {
2661 if (lut16[info->clock_speed_index] <= reg_500)
2662 reg_500 -= lut16[info->clock_speed_index];
2663 else
2664 reg_500 = 0;
2665 } else {
2666 reg_500 += lut16[info->clock_speed_index];
2667 }
2668 write_500(info, channel, reg_500,
2669 get_timing_register_addr(lane, 0, slot, rank), 9, 1);
2670 }
2671 }
2672
2673 static void set_ecc(int onoff)
2674 {
2675 int channel;
2676 for (channel = 0; channel < NUM_CHANNELS; channel++) {
2677 u8 t;
2678 t = read_mchbar8((channel << 10) + 0x5f8);
2679 if (onoff)
2680 t |= 1;
2681 else
2682 t &= ~1;
2683 write_mchbar8((channel << 10) + 0x5f8, t);
2684 }
2685 }
2686
2687 static void set_178(u8 val)
2688 {
2689 if (val >= 31)
2690 val = val - 31;
2691 else
2692 val = 63 - val;
2693
2694 write_1d0(2 * val, 0x178, 7, 1);
2695 }
2696
2697 static void
2698 write_500_timings_type(struct raminfo *info, int channel, int slot, int rank,
2699 int type)
2700 {
2701 int lane;
2702
2703 for (lane = 0; lane < 8; lane++)
2704 write_500(info, channel,
2705 info->training.
2706 lane_timings[type][channel][slot][rank][lane],
2707 get_timing_register_addr(lane, type, slot, rank), 9,
2708 0);
2709 }
2710
2711 static void
2712 try_timing_offsets(struct raminfo *info, int channel,
2713 int slot, int rank, int totalrank)
2714 {
2715 u16 count[8];
2716 enum state state[8];
2717 u8 lower_usable[8], upper_usable[8];
2718 int lane;
2719 int i;
2720 int flip = 1;
2721 int timing_offset;
2722
2723 for (i = 0; i < 8; i++)
2724 state[i] = BEFORE_USABLE;
2725
2726 memset(count, 0, sizeof(count));
2727
2728 for (lane = 0; lane < 8; lane++)
2729 write_500(info, channel,
2730 info->training.
2731 lane_timings[2][channel][slot][rank][lane] + 32,
2732 get_timing_register_addr(lane, 3, slot, rank), 9, 1);
2733
2734 for (timing_offset = 0; !validate_state(state) && timing_offset < 64;
2735 timing_offset++) {
2736 u8 failmask;
2737 write_1d0(timing_offset ^ 32, 0x1bb, 6, 1);
2738 failmask = 0;
2739 for (i = 0; i < 2 && failmask != 0xff; i++) {
2740 flip = !flip;
2741 write_testing(info, totalrank, flip);
2742 failmask |= check_testing(info, totalrank, flip);
2743 }
2744 do_fsm(state, count, failmask, 10, 63, lower_usable,
2745 upper_usable, timing_offset);
2746 }
2747 write_1d0(0, 0x1bb, 6, 1);
2748 dump_timings(info);
2749 if (!validate_state(state))
2750 die("Couldn't discover DRAM timings (1)\n");
2751
2752 for (lane = 0; lane < 8; lane++) {
2753 u8 bias = 0;
2754
2755 if (info->silicon_revision) {
2756 int usable_length;
2757
2758 usable_length = upper_usable[lane] - lower_usable[lane];
2759 if (usable_length >= 20) {
2760 bias = usable_length / 2 - 10;
2761 if (bias >= 2)
2762 bias = 2;
2763 }
2764 }
2765 write_500(info, channel,
2766 info->training.
2767 lane_timings[2][channel][slot][rank][lane] +
2768 (upper_usable[lane] + lower_usable[lane]) / 2 - bias,
2769 get_timing_register_addr(lane, 3, slot, rank), 9, 1);
2770 info->training.timing2_bounds[channel][slot][rank][lane][0] =
2771 info->training.lane_timings[2][channel][slot][rank][lane] +
2772 lower_usable[lane];
2773 info->training.timing2_bounds[channel][slot][rank][lane][1] =
2774 info->training.lane_timings[2][channel][slot][rank][lane] +
2775 upper_usable[lane];
2776 info->training.timing2_offset[channel][slot][rank][lane] =
2777 info->training.lane_timings[2][channel][slot][rank][lane];
2778 }
2779 }
2780
2781 static u8
2782 choose_training(struct raminfo *info, int channel, int slot, int rank,
2783 int lane, timing_bounds_t * timings, u8 center_178)
2784 {
2785 u16 central_weight;
2786 u16 side_weight;
2787 unsigned int sum = 0, count = 0;
2788 u8 span;
2789 u8 lower_margin, upper_margin;
2790 u8 reg_178;
2791 u8 result;
2792
2793 span = 12;
2794 central_weight = 20;
2795 side_weight = 20;
2796 if (info->silicon_revision == 1 && channel == 1) {
2797 central_weight = 5;
2798 side_weight = 20;
2799 if ((info->
2800 populated_ranks_mask[1] ^ (info->
2801 populated_ranks_mask[1] >> 2)) &
2802 1)
2803 span = 18;
2804 }
2805 if ((info->populated_ranks_mask[0] & 5) == 5) {
2806 central_weight = 20;
2807 side_weight = 20;
2808 }
2809 if (info->clock_speed_index >= 2
2810 && (info->populated_ranks_mask[0] & 5) == 5 && slot == 1) {
2811 if (info->silicon_revision == 1) {
2812 switch (channel) {
2813 case 0:
2814 if (lane == 1) {
2815 central_weight = 10;
2816 side_weight = 20;
2817 }
2818 break;
2819 case 1:
2820 if (lane == 6) {
2821 side_weight = 5;
2822 central_weight = 20;
2823 }
2824 break;
2825 }
2826 }
2827 if (info->silicon_revision == 0 && channel == 0 && lane == 0) {
2828 side_weight = 5;
2829 central_weight = 20;
2830 }
2831 }
2832 for (reg_178 = center_178 - span; reg_178 <= center_178 + span;
2833 reg_178 += span) {
2834 u8 smallest;
2835 u8 largest;
2836 largest = timings[reg_178][channel][slot][rank][lane].largest;
2837 smallest = timings[reg_178][channel][slot][rank][lane].smallest;
2838 if (largest - smallest + 1 >= 5) {
2839 unsigned int weight;
2840 if (reg_178 == center_178)
2841 weight = central_weight;
2842 else
2843 weight = side_weight;
2844 sum += weight * (largest + smallest);
2845 count += weight;
2846 }
2847 }
2848 dump_timings(info);
2849 if (count == 0)
2850 die("Couldn't discover DRAM timings (2)\n");
2851 result = sum / (2 * count);
2852 lower_margin =
2853 result - timings[center_178][channel][slot][rank][lane].smallest;
2854 upper_margin =
2855 timings[center_178][channel][slot][rank][lane].largest - result;
2856 if (upper_margin < 10 && lower_margin > 10)
2857 result -= min(lower_margin - 10, 10 - upper_margin);
2858 if (upper_margin > 10 && lower_margin < 10)
2859 result += min(upper_margin - 10, 10 - lower_margin);
2860 return result;
2861 }
2862
2863 #define STANDARD_MIN_MARGIN 5
2864
2865 static u8 choose_reg178(struct raminfo *info, timing_bounds_t * timings)
2866 {
2867 u16 margin[64];
2868 int lane, rank, slot, channel;
2869 u8 reg178;
2870 int count = 0, sum = 0;
2871
2872 for (reg178 = reg178_min[info->clock_speed_index];
2873 reg178 < reg178_max[info->clock_speed_index];
2874 reg178 += reg178_step[info->clock_speed_index]) {
2875 margin[reg178] = -1;
2876 FOR_POPULATED_RANKS_BACKWARDS for (lane = 0; lane < 8; lane++) {
2877 int curmargin =
2878 timings[reg178][channel][slot][rank][lane].largest -
2879 timings[reg178][channel][slot][rank][lane].
2880 smallest + 1;
2881 if (curmargin < margin[reg178])
2882 margin[reg178] = curmargin;
2883 }
2884 if (margin[reg178] >= STANDARD_MIN_MARGIN) {
2885 u16 weight;
2886 weight = margin[reg178] - STANDARD_MIN_MARGIN;
2887 sum += weight * reg178;
2888 count += weight;
2889 }
2890 }
2891 dump_timings(info);
2892 if (count == 0)
2893 die("Couldn't discover DRAM timings (3)\n");
2894
2895 u8 threshold;
2896
2897 for (threshold = 30; threshold >= 5; threshold--) {
2898 int usable_length = 0;
2899 int smallest_fount = 0;
2900 for (reg178 = reg178_min[info->clock_speed_index];
2901 reg178 < reg178_max[info->clock_speed_index];
2902 reg178 += reg178_step[info->clock_speed_index])
2903 if (margin[reg178] >= threshold) {
2904 usable_length +=
2905 reg178_step[info->clock_speed_index];
2906 info->training.reg178_largest =
2907 reg178 -
2908 2 * reg178_step[info->clock_speed_index];
2909
2910 if (!smallest_fount) {
2911 smallest_fount = 1;
2912 info->training.reg178_smallest =
2913 reg178 +
2914 reg178_step[info->
2915 clock_speed_index];
2916 }
2917 }
2918 if (usable_length >= 0x21)
2919 break;
2920 }
2921
2922 return sum / count;
2923 }
2924
2925 static int check_cached_sanity(struct raminfo *info)
2926 {
2927 int lane;
2928 int slot, rank;
2929 int channel;
2930
2931 if (!info->cached_training)
2932 return 0;
2933
2934 for (channel = 0; channel < NUM_CHANNELS; channel++)
2935 for (slot = 0; slot < NUM_SLOTS; slot++)
2936 for (rank = 0; rank < NUM_RANKS; rank++)
2937 for (lane = 0; lane < 8 + info->use_ecc; lane++) {
2938 u16 cached_value, estimation_value;
2939 cached_value =
2940 info->cached_training->
2941 lane_timings[1][channel][slot][rank]
2942 [lane];
2943 if (cached_value >= 0x18
2944 && cached_value <= 0x1E7) {
2945 estimation_value =
2946 info->training.
2947 lane_timings[1][channel]
2948 [slot][rank][lane];
2949 if (estimation_value <
2950 cached_value - 24)
2951 return 0;
2952 if (estimation_value >
2953 cached_value + 24)
2954 return 0;
2955 }
2956 }
2957 return 1;
2958 }
2959
2960 static int try_cached_training(struct raminfo *info)
2961 {
2962 u8 saved_243[2];
2963 u8 tm;
2964
2965 int channel, slot, rank, lane;
2966 int flip = 1;
2967 int i, j;
2968
2969 if (!check_cached_sanity(info))
2970 return 0;
2971
2972 info->training.reg178_center = info->cached_training->reg178_center;
2973 info->training.reg178_smallest = info->cached_training->reg178_smallest;
2974 info->training.reg178_largest = info->cached_training->reg178_largest;
2975 memcpy(&info->training.timing_bounds,
2976 &info->cached_training->timing_bounds,
2977 sizeof(info->training.timing_bounds));
2978 memcpy(&info->training.timing_offset,
2979 &info->cached_training->timing_offset,
2980 sizeof(info->training.timing_offset));
2981
2982 write_1d0(2, 0x142, 3, 1);
2983 saved_243[0] = read_mchbar8(0x243);
2984 saved_243[1] = read_mchbar8(0x643);
2985 write_mchbar8(0x243, saved_243[0] | 2);
2986 write_mchbar8(0x643, saved_243[1] | 2);
2987 set_ecc(0);
2988 pcie_write_config16(NORTHBRIDGE, 0xc8, 3);
2989 if (read_1d0(0x10b, 6) & 1)
2990 set_10b(info, 0);
2991 for (tm = 0; tm < 2; tm++) {
2992 int totalrank;
2993
2994 set_178(tm ? info->cached_training->reg178_largest : info->
2995 cached_training->reg178_smallest);
2996
2997 totalrank = 0;
2998 /* Check timing ranges. With i == 0 we check smallest one and with
2999 i == 1 the largest bound. With j == 0 we check that on the bound
3000 it still works whereas with j == 1 we check that just outside of
3001 bound we fail.
3002 */
3003 FOR_POPULATED_RANKS_BACKWARDS {
3004 for (i = 0; i < 2; i++) {
3005 for (lane = 0; lane < 8; lane++) {
3006 write_500(info, channel,
3007 info->cached_training->
3008 timing2_bounds[channel][slot]
3009 [rank][lane][i],
3010 get_timing_register_addr(lane,
3011 3,
3012 slot,
3013 rank),
3014 9, 1);
3015
3016 if (!i)
3017 write_500(info, channel,
3018 info->
3019 cached_training->
3020 timing2_offset
3021 [channel][slot][rank]
3022 [lane],
3023 get_timing_register_addr
3024 (lane, 2, slot, rank),
3025 9, 1);
3026 write_500(info, channel,
3027 i ? info->cached_training->
3028 timing_bounds[tm][channel]
3029 [slot][rank][lane].
3030 largest : info->
3031 cached_training->
3032 timing_bounds[tm][channel]
3033 [slot][rank][lane].smallest,
3034 get_timing_register_addr(lane,
3035 0,
3036 slot,
3037 rank),
3038 9, 1);
3039 write_500(info, channel,
3040 info->cached_training->
3041 timing_offset[channel][slot]
3042 [rank][lane] +
3043 (i ? info->cached_training->
3044 timing_bounds[tm][channel]
3045 [slot][rank][lane].
3046 largest : info->
3047 cached_training->
3048 timing_bounds[tm][channel]
3049 [slot][rank][lane].
3050 smallest) - 64,
3051 get_timing_register_addr(lane,
3052 1,
3053 slot,
3054 rank),
3055 9, 1);
3056 }
3057 for (j = 0; j < 2; j++) {
3058 u8 failmask;
3059 u8 expected_failmask;
3060 char reg1b3;
3061
3062 reg1b3 = (j == 1) + 4;
3063 reg1b3 =
3064 j == i ? reg1b3 : (-reg1b3) & 0x3f;
3065 write_1d0(reg1b3, 0x1bb, 6, 1);
3066 write_1d0(reg1b3, 0x1b3, 6, 1);
3067 write_1d0(reg1b3, 0x1a3, 6, 1);
3068
3069 flip = !flip;
3070 write_testing(info, totalrank, flip);
3071 failmask =
3072 check_testing(info, totalrank,
3073 flip);
3074 expected_failmask =
3075 j == 0 ? 0x00 : 0xff;
3076 if (failmask != expected_failmask)
3077 goto fail;
3078 }
3079 }
3080 totalrank++;
3081 }
3082 }
3083
3084 set_178(info->cached_training->reg178_center);
3085 if (info->use_ecc)
3086 set_ecc(1);
3087 write_training_data(info);
3088 write_1d0(0, 322, 3, 1);
3089 info->training = *info->cached_training;
3090
3091 write_1d0(0, 0x1bb, 6, 1);
3092 write_1d0(0, 0x1b3, 6, 1);
3093 write_1d0(0, 0x1a3, 6, 1);
3094 write_mchbar8(0x243, saved_243[0]);
3095 write_mchbar8(0x643, saved_243[1]);
3096
3097 return 1;
3098
3099 fail:
3100 FOR_POPULATED_RANKS {
3101 write_500_timings_type(info, channel, slot, rank, 1);
3102 write_500_timings_type(info, channel, slot, rank, 2);
3103 write_500_timings_type(info, channel, slot, rank, 3);
3104 }
3105
3106 write_1d0(0, 0x1bb, 6, 1);
3107 write_1d0(0, 0x1b3, 6, 1);
3108 write_1d0(0, 0x1a3, 6, 1);
3109 write_mchbar8(0x243, saved_243[0]);
3110 write_mchbar8(0x643, saved_243[1]);
3111
3112 return 0;
3113 }
3114
3115 static void do_ram_training(struct raminfo *info)
3116 {
3117 u8 saved_243[2];
3118 int totalrank = 0;
3119 u8 reg_178;
3120 int niter;
3121
3122 timing_bounds_t timings[64];
3123 int lane, rank, slot, channel;
3124 u8 reg178_center;
3125
3126 write_1d0(2, 0x142, 3, 1);
3127 saved_243[0] = read_mchbar8(0x243);
3128 saved_243[1] = read_mchbar8(0x643);
3129 write_mchbar8(0x243, saved_243[0] | 2);
3130 write_mchbar8(0x643, saved_243[1] | 2);
3131 switch (info->clock_speed_index) {
3132 case 0:
3133 niter = 5;
3134 break;
3135 case 1:
3136 niter = 10;
3137 break;
3138 default:
3139 niter = 19;
3140 break;
3141 }
3142 set_ecc(0);
3143
3144 FOR_POPULATED_RANKS_BACKWARDS {
3145 int i;
3146
3147 write_500_timings_type(info, channel, slot, rank, 0);
3148
3149 write_testing(info, totalrank, 0);
3150 for (i = 0; i < niter; i++) {
3151 write_testing_type2(info, totalrank, 2, i, 0);
3152 write_testing_type2(info, totalrank, 3, i, 1);
3153 }
3154 pcie_write_config8(NORTHBRIDGE, 0xc0, 0x01);
3155 totalrank++;
3156 }
3157
3158 if (reg178_min[info->clock_speed_index] <
3159 reg178_max[info->clock_speed_index])
3160 memset(timings[reg178_min[info->clock_speed_index]], 0,
3161 sizeof(timings[0]) *
3162 (reg178_max[info->clock_speed_index] -
3163 reg178_min[info->clock_speed_index]));
3164 for (reg_178 = reg178_min[info->clock_speed_index];
3165 reg_178 < reg178_max[info->clock_speed_index];
3166 reg_178 += reg178_step[info->clock_speed_index]) {
3167 totalrank = 0;
3168 set_178(reg_178);
3169 for (channel = NUM_CHANNELS - 1; channel >= 0; channel--)
3170 for (slot = 0; slot < NUM_SLOTS; slot++)
3171 for (rank = 0; rank < NUM_RANKS; rank++) {
3172 memset(&timings[reg_178][channel][slot]
3173 [rank][0].smallest, 0, 16);
3174 if (info->
3175 populated_ranks[channel][slot]
3176 [rank]) {
3177 train_ram_at_178(info, channel,
3178 slot, rank,
3179 totalrank,
3180 reg_178, 1,
3181 niter,
3182 timings);
3183 totalrank++;
3184 }
3185 }
3186 }
3187
3188 reg178_center = choose_reg178(info, timings);
3189
3190 FOR_POPULATED_RANKS_BACKWARDS for (lane = 0; lane < 8; lane++) {
3191 info->training.timing_bounds[0][channel][slot][rank][lane].
3192 smallest =
3193 timings[info->training.
3194 reg178_smallest][channel][slot][rank][lane].
3195 smallest;
3196 info->training.timing_bounds[0][channel][slot][rank][lane].
3197 largest =
3198 timings[info->training.
3199 reg178_smallest][channel][slot][rank][lane].largest;
3200 info->training.timing_bounds[1][channel][slot][rank][lane].
3201 smallest =
3202 timings[info->training.
3203 reg178_largest][channel][slot][rank][lane].smallest;
3204 info->training.timing_bounds[1][channel][slot][rank][lane].
3205 largest =
3206 timings[info->training.
3207 reg178_largest][channel][slot][rank][lane].largest;
3208 info->training.timing_offset[channel][slot][rank][lane] =
3209 info->training.lane_timings[1][channel][slot][rank][lane]
3210 -
3211 info->training.lane_timings[0][channel][slot][rank][lane] +
3212 64;
3213 }
3214
3215 if (info->silicon_revision == 1
3216 && (info->
3217 populated_ranks_mask[1] ^ (info->
3218 populated_ranks_mask[1] >> 2)) & 1) {
3219 int ranks_after_channel1;
3220
3221 totalrank = 0;
3222 for (reg_178 = reg178_center - 18;
3223 reg_178 <= reg178_center + 18; reg_178 += 18) {
3224 totalrank = 0;
3225 set_178(reg_178);
3226 for (slot = 0; slot < NUM_SLOTS; slot++)
3227 for (rank = 0; rank < NUM_RANKS; rank++) {
3228 if (info->
3229 populated_ranks[1][slot][rank]) {
3230 train_ram_at_178(info, 1, slot,
3231 rank,
3232 totalrank,
3233 reg_178, 0,
3234 niter,
3235 timings);
3236 totalrank++;
3237 }
3238 }
3239 }
3240 ranks_after_channel1 = totalrank;
3241
3242 for (reg_178 = reg178_center - 12;
3243 reg_178 <= reg178_center + 12; reg_178 += 12) {
3244 totalrank = ranks_after_channel1;
3245 set_178(reg_178);
3246 for (slot = 0; slot < NUM_SLOTS; slot++)
3247 for (rank = 0; rank < NUM_RANKS; rank++)
3248 if (info->
3249 populated_ranks[0][slot][rank]) {
3250 train_ram_at_178(info, 0, slot,
3251 rank,
3252 totalrank,
3253 reg_178, 0,
3254 niter,
3255 timings);
3256 totalrank++;
3257 }
3258
3259 }
3260 } else {
3261 for (reg_178 = reg178_center - 12;
3262 reg_178 <= reg178_center + 12; reg_178 += 12) {
3263 totalrank = 0;
3264 set_178(reg_178);
3265 FOR_POPULATED_RANKS_BACKWARDS {
3266 train_ram_at_178(info, channel, slot, rank,
3267 totalrank, reg_178, 0, niter,
3268 timings);
3269 totalrank++;
3270 }
3271 }
3272 }
3273
3274 set_178(reg178_center);
3275 FOR_POPULATED_RANKS_BACKWARDS for (lane = 0; lane < 8; lane++) {
3276 u16 tm0;
3277
3278 tm0 =
3279 choose_training(info, channel, slot, rank, lane, timings,
3280 reg178_center);
3281 write_500(info, channel, tm0,
3282 get_timing_register_addr(lane, 0, slot, rank), 9, 1);
3283 write_500(info, channel,
3284 tm0 +
3285 info->training.
3286 lane_timings[1][channel][slot][rank][lane] -
3287 info->training.
3288 lane_timings[0][channel][slot][rank][lane],
3289 get_timing_register_addr(lane, 1, slot, rank), 9, 1);
3290 }
3291
3292 totalrank = 0;
3293 FOR_POPULATED_RANKS_BACKWARDS {
3294 try_timing_offsets(info, channel, slot, rank, totalrank);
3295 totalrank++;
3296 }
3297 write_mchbar8(0x243, saved_243[0]);
3298 write_mchbar8(0x643, saved_243[1]);
3299 write_1d0(0, 0x142, 3, 1);
3300 info->training.reg178_center = reg178_center;
3301 }
3302
3303 static void ram_training(struct raminfo *info)
3304 {
3305 u16 saved_fc4;
3306
3307 saved_fc4 = read_mchbar16(0xfc4);
3308 write_mchbar16(0xfc4, 0xffff);
3309
3310 if (info->revision >= 8)
3311 read_4090(info);
3312
3313 if (!try_cached_training(info))
3314 do_ram_training(info);
3315 if ((info->silicon_revision == 2 || info->silicon_revision == 3)
3316 && info->clock_speed_index < 2)
3317 set_10b(info, 1);
3318 write_mchbar16(0xfc4, saved_fc4);
3319 }
3320
3321 static unsigned gcd(unsigned a, unsigned b)
3322 {
3323 unsigned t;
3324 if (a > b) {
3325 t = a;
3326 a = b;
3327 b = t;
3328 }
3329 /* invariant a < b. */
3330 while (a) {
3331 t = b % a;
3332 b = a;
3333 a = t;
3334 }
3335 return b;
3336 }
3337
3338 static inline int div_roundup(int a, int b)
3339 {
3340 return CEIL_DIV(a, b);
3341 }
3342
3343 static unsigned lcm(unsigned a, unsigned b)
3344 {
3345 return (a * b) / gcd(a, b);
3346 }
3347
3348 struct stru1 {
3349 u8 freqs_reversed;
3350 u8 freq_diff_reduced;
3351 u8 freq_min_reduced;
3352 u8 divisor_f4_to_fmax;
3353 u8 divisor_f3_to_fmax;
3354 u8 freq4_to_max_remainder;
3355 u8 freq3_to_2_remainder;
3356 u8 freq3_to_2_remaindera;
3357 u8 freq4_to_2_remainder;
3358 int divisor_f3_to_f1, divisor_f4_to_f2;
3359 int common_time_unit_ps;
3360 int freq_max_reduced;
3361 };
3362
3363 static void
3364 compute_frequence_ratios(struct raminfo *info, u16 freq1, u16 freq2,
3365 int num_cycles_2, int num_cycles_1, int round_it,
3366 int add_freqs, struct stru1 *result)
3367 {
3368 int g;
3369 int common_time_unit_ps;
3370 int freq1_reduced, freq2_reduced;
3371 int freq_min_reduced;
3372 int freq_max_reduced;
3373 int freq3, freq4;
3374
3375 g = gcd(freq1, freq2);
3376 freq1_reduced = freq1 / g;
3377 freq2_reduced = freq2 / g;
3378 freq_min_reduced = min(freq1_reduced, freq2_reduced);
3379 freq_max_reduced = max(freq1_reduced, freq2_reduced);
3380
3381 common_time_unit_ps = div_roundup(900000, lcm(freq1, freq2));
3382 freq3 = div_roundup(num_cycles_2, common_time_unit_ps) - 1;
3383 freq4 = div_roundup(num_cycles_1, common_time_unit_ps) - 1;
3384 if (add_freqs) {
3385 freq3 += freq2_reduced;
3386 freq4 += freq1_reduced;
3387 }
3388
3389 if (round_it) {
3390 result->freq3_to_2_remainder = 0;
3391 result->freq3_to_2_remaindera = 0;
3392 result->freq4_to_max_remainder = 0;
3393 result->divisor_f4_to_f2 = 0;
3394 result->divisor_f3_to_f1 = 0;
3395 } else {
3396 if (freq2_reduced < freq1_reduced) {
3397 result->freq3_to_2_remainder =
3398 result->freq3_to_2_remaindera =
3399 freq3 % freq1_reduced - freq1_reduced + 1;
3400 result->freq4_to_max_remainder =
3401 -(freq4 % freq1_reduced);
3402 result->divisor_f3_to_f1 = freq3 / freq1_reduced;
3403 result->divisor_f4_to_f2 =
3404 (freq4 -
3405 (freq1_reduced - freq2_reduced)) / freq2_reduced;
3406 result->freq4_to_2_remainder =
3407 -(char)((freq1_reduced - freq2_reduced) +
3408 ((u8) freq4 -
3409 (freq1_reduced -
3410 freq2_reduced)) % (u8) freq2_reduced);
3411 } else {
3412 if (freq2_reduced > freq1_reduced) {
3413 result->freq4_to_max_remainder =
3414 (freq4 % freq2_reduced) - freq2_reduced + 1;
3415 result->freq4_to_2_remainder =
3416 freq4 % freq_max_reduced -
3417 freq_max_reduced + 1;
3418 } else {
3419 result->freq4_to_max_remainder =
3420 -(freq4 % freq2_reduced);
3421 result->freq4_to_2_remainder =
3422 -(char)(freq4 % freq_max_reduced);
3423 }
3424 result->divisor_f4_to_f2 = freq4 / freq2_reduced;
3425 result->divisor_f3_to_f1 =
3426 (freq3 -
3427 (freq2_reduced - freq1_reduced)) / freq1_reduced;
3428 result->freq3_to_2_remainder = -(freq3 % freq2_reduced);
3429 result->freq3_to_2_remaindera =
3430 -(char)((freq_max_reduced - freq_min_reduced) +
3431 (freq3 -
3432 (freq_max_reduced -
3433 freq_min_reduced)) % freq1_reduced);
3434 }
3435 }
3436 result->divisor_f3_to_fmax = freq3 / freq_max_reduced;
3437 result->divisor_f4_to_fmax = freq4 / freq_max_reduced;
3438 if (round_it) {
3439 if (freq2_reduced > freq1_reduced) {
3440 if (freq3 % freq_max_reduced)
3441 result->divisor_f3_to_fmax++;
3442 }
3443 if (freq2_reduced < freq1_reduced) {
3444 if (freq4 % freq_max_reduced)
3445 result->divisor_f4_to_fmax++;
3446 }
3447 }
3448 result->freqs_reversed = (freq2_reduced < freq1_reduced);
3449 result->freq_diff_reduced = freq_max_reduced - freq_min_reduced;
3450 result->freq_min_reduced = freq_min_reduced;
3451 result->common_time_unit_ps = common_time_unit_ps;
3452 result->freq_max_reduced = freq_max_reduced;
3453 }
3454
3455 static void
3456 set_2d5x_reg(struct raminfo *info, u16 reg, u16 freq1, u16 freq2,
3457 int num_cycles_2, int num_cycles_1, int num_cycles_3,
3458 int num_cycles_4, int reverse)
3459 {
3460 struct stru1 vv;
3461 char multiplier;
3462
3463 compute_frequence_ratios(info, freq1, freq2, num_cycles_2, num_cycles_1,
3464 0, 1, &vv);
3465
3466 multiplier =
3467 div_roundup(max
3468 (div_roundup(num_cycles_2, vv.common_time_unit_ps) +
3469 div_roundup(num_cycles_3, vv.common_time_unit_ps),
3470 div_roundup(num_cycles_1,
3471 vv.common_time_unit_ps) +
3472 div_roundup(num_cycles_4, vv.common_time_unit_ps))
3473 + vv.freq_min_reduced - 1, vv.freq_max_reduced) - 1;
3474
3475 u32 y =
3476 (u8) ((vv.freq_max_reduced - vv.freq_min_reduced) +
3477 vv.freq_max_reduced * multiplier)
3478 | (vv.
3479 freqs_reversed << 8) | ((u8) (vv.freq_min_reduced *
3480 multiplier) << 16) | ((u8) (vv.
3481 freq_min_reduced
3482 *
3483 multiplier)
3484 << 24);
3485 u32 x =
3486 vv.freq3_to_2_remaindera | (vv.freq4_to_2_remainder << 8) | (vv.
3487 divisor_f3_to_f1
3488 << 16)
3489 | (vv.divisor_f4_to_f2 << 20) | (vv.freq_min_reduced << 24);
3490 if (reverse) {
3491 write_mchbar32(reg, y);
3492 write_mchbar32(reg + 4, x);
3493 } else {
3494 write_mchbar32(reg + 4, y);
3495 write_mchbar32(reg, x);
3496 }
3497 }
3498
3499 static void
3500 set_6d_reg(struct raminfo *info, u16 reg, u16 freq1, u16 freq2,
3501 int num_cycles_1, int num_cycles_2, int num_cycles_3,
3502 int num_cycles_4)
3503 {
3504 struct stru1 ratios1;
3505 struct stru1 ratios2;
3506
3507 compute_frequence_ratios(info, freq1, freq2, num_cycles_1, num_cycles_2,
3508 0, 1, &ratios2);
3509 compute_frequence_ratios(info, freq1, freq2, num_cycles_3, num_cycles_4,
3510 0, 1, &ratios1);
3511 printk (BIOS_SPEW, "[%x] <= %x\n", reg,
3512 ratios1.freq4_to_max_remainder | (ratios2.
3513 freq4_to_max_remainder
3514 << 8)
3515 | (ratios1.divisor_f4_to_fmax << 16) | (ratios2.
3516 divisor_f4_to_fmax
3517 << 20));
3518 write_mchbar32(reg,
3519 ratios1.freq4_to_max_remainder | (ratios2.
3520 freq4_to_max_remainder
3521 << 8)
3522 | (ratios1.divisor_f4_to_fmax << 16) | (ratios2.
3523 divisor_f4_to_fmax
3524 << 20));
3525 }
3526
3527 static void
3528 set_2dx8_reg(struct raminfo *info, u16 reg, u8 mode, u16 freq1, u16 freq2,
3529 int num_cycles_2, int num_cycles_1, int round_it, int add_freqs)
3530 {
3531 struct stru1 ratios;
3532
3533 compute_frequence_ratios(info, freq1, freq2, num_cycles_2, num_cycles_1,
3534 round_it, add_freqs, &ratios);
3535 switch (mode) {
3536 case 0:
3537 write_mchbar32(reg + 4,
3538 ratios.freq_diff_reduced | (ratios.
3539 freqs_reversed <<
3540 8));
3541 write_mchbar32(reg,
3542 ratios.freq3_to_2_remainder | (ratios.
3543 freq4_to_max_remainder
3544 << 8)
3545 | (ratios.divisor_f3_to_fmax << 16) | (ratios.
3546 divisor_f4_to_fmax
3547 << 20) |
3548 (ratios.freq_min_reduced << 24));
3549 break;
3550
3551 case 1:
3552 write_mchbar32(reg,
3553 ratios.freq3_to_2_remainder | (ratios.
3554 divisor_f3_to_fmax
3555 << 16));
3556 break;
3557
3558 case 2:
3559 write_mchbar32(reg,
3560 ratios.freq3_to_2_remainder | (ratios.
3561 freq4_to_max_remainder
3562 << 8) | (ratios.
3563 divisor_f3_to_fmax
3564 << 16) |
3565 (ratios.divisor_f4_to_fmax << 20));
3566 break;
3567
3568 case 4:
3569 write_mchbar32(reg, (ratios.divisor_f3_to_fmax << 4)
3570 | (ratios.divisor_f4_to_fmax << 8) | (ratios.
3571 freqs_reversed
3572 << 12) |
3573 (ratios.freq_min_reduced << 16) | (ratios.
3574 freq_diff_reduced
3575 << 24));
3576 break;
3577 }
3578 }
3579
3580 static void set_2dxx_series(struct raminfo *info, int s3resume)
3581 {
3582 set_2dx8_reg(info, 0x2d00, 0, 0x78, frequency_11(info) / 2, 1359, 1005,
3583 0, 1);
3584 set_2dx8_reg(info, 0x2d08, 0, 0x78, 0x78, 3273, 5033, 1, 1);
3585 set_2dx8_reg(info, 0x2d10, 0, 0x78, info->fsb_frequency, 1475, 1131, 0,
3586 1);
3587 set_2dx8_reg(info, 0x2d18, 0, 2 * info->fsb_frequency,
3588 frequency_11(info), 1231, 1524, 0, 1);
3589 set_2dx8_reg(info, 0x2d20, 0, 2 * info->fsb_frequency,
3590 frequency_11(info) / 2, 1278, 2008, 0, 1);
3591 set_2dx8_reg(info, 0x2d28, 0, info->fsb_frequency, frequency_11(info),
3592 1167, 1539, 0, 1);
3593 set_2dx8_reg(info, 0x2d30, 0, info->fsb_frequency,
3594 frequency_11(info) / 2, 1403, 1318, 0, 1);
3595 set_2dx8_reg(info, 0x2d38, 0, info->fsb_frequency, 0x78, 3460, 5363, 1,
3596 1);
3597 set_2dx8_reg(info, 0x2d40, 0, info->fsb_frequency, 0x3c, 2792, 5178, 1,
3598 1);
3599 set_2dx8_reg(info, 0x2d48, 0, 2 * info->fsb_frequency, 0x78, 2738, 4610,
3600 1, 1);
3601 set_2dx8_reg(info, 0x2d50, 0, info->fsb_frequency, 0x78, 2819, 5932, 1,
3602 1);
3603 set_2dx8_reg(info, 0x6d4, 1, info->fsb_frequency,
3604 frequency_11(info) / 2, 4000, 0, 0, 0);
3605 set_2dx8_reg(info, 0x6d8, 2, info->fsb_frequency,
3606 frequency_11(info) / 2, 4000, 4000, 0, 0);
3607
3608 if (s3resume) {
3609 printk (BIOS_SPEW, "[6dc] <= %x\n", info->cached_training->reg_6dc);
3610 write_mchbar32(0x6dc, info->cached_training->reg_6dc);
3611 } else
3612 set_6d_reg(info, 0x6dc, 2 * info->fsb_frequency, frequency_11(info), 0,
3613 info->delay46_ps[0], 0,
3614 info->delay54_ps[0]);
3615 set_2dx8_reg(info, 0x6e0, 1, 2 * info->fsb_frequency,
3616 frequency_11(info), 2500, 0, 0, 0);
3617 set_2dx8_reg(info, 0x6e4, 1, 2 * info->fsb_frequency,
3618 frequency_11(info) / 2, 3500, 0, 0, 0);
3619 if (s3resume) {
3620 printk (BIOS_SPEW, "[6e8] <= %x\n", info->cached_training->reg_6e8);
3621 write_mchbar32(0x6e8, info->cached_training->reg_6e8);
3622 } else
3623 set_6d_reg(info, 0x6e8, 2 * info->fsb_frequency, frequency_11(info), 0,
3624 info->delay46_ps[1], 0,
3625 info->delay54_ps[1]);
3626 set_2d5x_reg(info, 0x2d58, 0x78, 0x78, 864, 1195, 762, 786, 0);
3627 set_2d5x_reg(info, 0x2d60, 0x195, info->fsb_frequency, 1352, 725, 455,
3628 470, 0);
3629 set_2d5x_reg(info, 0x2d68, 0x195, 0x3c, 2707, 5632, 3277, 2207, 0);
3630 set_2d5x_reg(info, 0x2d70, 0x195, frequency_11(info) / 2, 1276, 758,
3631 454, 459, 0);
3632 set_2d5x_reg(info, 0x2d78, 0x195, 0x78, 1021, 799, 510, 513, 0);
3633 set_2d5x_reg(info, 0x2d80, info->fsb_frequency, 0xe1, 0, 2862, 2579,
3634 2588, 0);
3635 set_2d5x_reg(info, 0x2d88, info->fsb_frequency, 0xe1, 0, 2690, 2405,
3636 2405, 0);
3637 set_2d5x_reg(info, 0x2da0, 0x78, 0xe1, 0, 2560, 2264, 2251, 0);
3638 set_2d5x_reg(info, 0x2da8, 0x195, frequency_11(info), 1060, 775, 484,
3639 480, 0);
3640 set_2d5x_reg(info, 0x2db0, 0x195, 0x78, 4183, 6023, 2217, 2048, 0);
3641 write_mchbar32(0x2dbc, ((frequency_11(info) / 2) - 1) | 0xe00000);
3642 write_mchbar32(0x2db8, ((info->fsb_frequency - 1) << 16) | 0x77);
3643 }
3644
3645 static u16 get_max_timing(struct raminfo *info, int channel)
3646 {
3647 int slot, rank, lane;
3648 u16 ret = 0;
3649
3650 if ((read_mchbar8(0x2ca8) >> 2) < 1)
3651 return 384;
3652
3653 if (info->revision < 8)
3654 return 256;
3655
3656 for (slot = 0; slot < NUM_SLOTS; slot++)
3657 for (rank = 0; rank < NUM_RANKS; rank++)
3658 if (info->populated_ranks[channel][slot][rank])
3659 for (lane = 0; lane < 8 + info->use_ecc; lane++)
3660 ret = max(ret, read_500(info, channel,
3661 get_timing_register_addr
3662 (lane, 0, slot,
3663 rank), 9));
3664 return ret;
3665 }
3666
3667 static void set_274265(struct raminfo *info)
3668 {
3669 int delay_a_ps, delay_b_ps, delay_c_ps, delay_d_ps;
3670 int delay_e_ps, delay_e_cycles, delay_f_cycles;
3671 int delay_e_over_cycle_ps;
3672 int cycletime_ps;
3673 int channel;
3674
3675 delay_a_ps = 4 * halfcycle_ps(info) + 6 * fsbcycle_ps(info);
3676 info->training.reg2ca9_bit0 = 0;
3677 for (channel = 0; channel < NUM_CHANNELS; channel++) {
3678 cycletime_ps =
3679 900000 / lcm(2 * info->fsb_frequency, frequency_11(info));
3680 delay_d_ps =
3681 (halfcycle_ps(info) * get_max_timing(info, channel) >> 6)
3682 - info->some_delay_3_ps_rounded + 200;
3683 if (!
3684 ((info->silicon_revision == 0
3685 || info->silicon_revision == 1)
3686 && (info->revision >= 8)))
3687 delay_d_ps += halfcycle_ps(info) * 2;
3688 delay_d_ps +=
3689 halfcycle_ps(info) * (!info->revision_flag_1 +
3690 info->some_delay_2_halfcycles_ceil +
3691 2 * info->some_delay_1_cycle_floor +
3692 info->clock_speed_index +
3693 2 * info->cas_latency - 7 + 11);
3694 delay_d_ps += info->revision >= 8 ? 2758 : 4428;
3695
3696 write_mchbar32(0x140,
3697 (read_mchbar32(0x140) & 0xfaffffff) | 0x2000000);
3698 write_mchbar32(0x138,
3699 (read_mchbar32(0x138) & 0xfaffffff) | 0x2000000);
3700 if ((read_mchbar8(0x144) & 0x1f) > 0x13)
3701 delay_d_ps += 650;
3702 delay_c_ps = delay_d_ps + 1800;
3703 if (delay_c_ps <= delay_a_ps)
3704 delay_e_ps = 0;
3705 else
3706 delay_e_ps =
3707 cycletime_ps * div_roundup(delay_c_ps - delay_a_ps,
3708 cycletime_ps);
3709
3710 delay_e_over_cycle_ps = delay_e_ps % (2 * halfcycle_ps(info));
3711 delay_e_cycles = delay_e_ps / (2 * halfcycle_ps(info));
3712 delay_f_cycles =
3713 div_roundup(2500 - delay_e_over_cycle_ps,
3714 2 * halfcycle_ps(info));
3715 if (delay_f_cycles > delay_e_cycles) {
3716 info->delay46_ps[channel] = delay_e_ps;
3717 delay_e_cycles = 0;
3718 } else {
3719 info->delay46_ps[channel] =
3720 delay_e_over_cycle_ps +
3721 2 * halfcycle_ps(info) * delay_f_cycles;
3722 delay_e_cycles -= delay_f_cycles;
3723 }
3724
3725 if (info->delay46_ps[channel] < 2500) {
3726 info->delay46_ps[channel] = 2500;
3727 info->training.reg2ca9_bit0 = 1;
3728 }
3729 delay_b_ps = halfcycle_ps(info) + delay_c_ps;
3730 if (delay_b_ps <= delay_a_ps)
3731 delay_b_ps = 0;
3732 else
3733 delay_b_ps -= delay_a_ps;
3734 info->delay54_ps[channel] =
3735 cycletime_ps * div_roundup(delay_b_ps,
3736 cycletime_ps) -
3737 2 * halfcycle_ps(info) * delay_e_cycles;
3738 if (info->delay54_ps[channel] < 2500)
3739 info->delay54_ps[channel] = 2500;
3740 info->training.reg274265[channel][0] = delay_e_cycles;
3741 if (delay_d_ps + 7 * halfcycle_ps(info) <=
3742 24 * halfcycle_ps(info))
3743 info->training.reg274265[channel][1] = 0;
3744 else
3745 info->training.reg274265[channel][1] =
3746 div_roundup(delay_d_ps + 7 * halfcycle_ps(info),
3747 4 * halfcycle_ps(info)) - 6;
3748 write_mchbar32((channel << 10) + 0x274,
3749 info->training.reg274265[channel][1]
3750 | (info->training.reg274265[channel][0] << 16));
3751 info->training.reg274265[channel][2] =
3752 div_roundup(delay_c_ps + 3 * fsbcycle_ps(info),
3753 4 * halfcycle_ps(info)) + 1;
3754 write_mchbar16((channel << 10) + 0x265,
3755 info->training.reg274265[channel][2] << 8);
3756 }
3757 if (info->training.reg2ca9_bit0)
3758 write_mchbar8(0x2ca9, read_mchbar8(0x2ca9) | 1);
3759 else
3760 write_mchbar8(0x2ca9, read_mchbar8(0x2ca9) & ~1);
3761 }
3762
3763 static void restore_274265(struct raminfo *info)
3764 {
3765 int channel;
3766
3767 for (channel = 0; channel < NUM_CHANNELS; channel++) {
3768 write_mchbar32((channel << 10) + 0x274,
3769 (info->cached_training->reg274265[channel][0] << 16)
3770 | info->cached_training->reg274265[channel][1]);
3771 write_mchbar16((channel << 10) + 0x265,
3772 info->cached_training->reg274265[channel][2] << 8);
3773 }
3774 if (info->cached_training->reg2ca9_bit0)
3775 write_mchbar8(0x2ca9, read_mchbar8(0x2ca9) | 1);
3776 else
3777 write_mchbar8(0x2ca9, read_mchbar8(0x2ca9) & ~1);
3778 }
3779
3780 #if REAL
3781 static void dmi_setup(void)
3782 {
3783 gav(read8(DEFAULT_DMIBAR + 0x254));
3784 write8(DEFAULT_DMIBAR + 0x254, 0x1);
3785 write16(DEFAULT_DMIBAR + 0x1b8, 0x18f2);
3786 read_mchbar16(0x48);
3787 write_mchbar16(0x48, 0x2);
3788
3789 write32(DEFAULT_DMIBAR + 0xd68, read32(DEFAULT_DMIBAR + 0xd68) | 0x08000000);
3790
3791 outl((gav(inl(DEFAULT_GPIOBASE | 0x38)) & ~0x140000) | 0x400000,
3792 DEFAULT_GPIOBASE | 0x38);
3793 gav(inb(DEFAULT_GPIOBASE | 0xe)); // = 0xfdcaff6e
3794 }
3795 #endif
3796
3797 void chipset_init(const int s3resume)
3798 {
3799 u8 x2ca8;
3800 u16 ggc;
3801 u8 gfxsize;
3802
3803 x2ca8 = read_mchbar8(0x2ca8);
3804 if ((x2ca8 & 1) || (x2ca8 == 8 && !s3resume)) {
3805 printk(BIOS_DEBUG, "soft reset detected, rebooting properly\n");
3806 write_mchbar8(0x2ca8, 0);
3807 outb(0x6, 0xcf9);
3808 #if REAL
3809 halt();
3810 #else
3811 printf("CP5\n");
3812 exit(0);
3813 #endif
3814 }
3815 #if !REAL
3816 if (!s3resume) {
3817 pre_raminit_3(x2ca8);
3818 }
3819 pre_raminit_4a(x2ca8);
3820 #endif
3821
3822 dmi_setup();
3823
3824 write_mchbar16(0x1170, 0xa880);
3825 write_mchbar8(0x11c1, 0x1);
3826 write_mchbar16(0x1170, 0xb880);
3827 read_mchbar8(0x1210);
3828 write_mchbar8(0x1210, 0x84);
3829
3830 if (get_option(&gfxsize, "gfx_uma_size") != CB_SUCCESS) {
3831 /* 0 for 32MB */
3832 gfxsize = 0;
3833 }
3834
3835 ggc = 0xb00 | ((gfxsize + 5) << 4);
3836
3837 pcie_write_config16(NORTHBRIDGE, D0F0_GGC, ggc | 2);
3838
3839 u16 deven;
3840 deven = pcie_read_config16(NORTHBRIDGE, D0F0_DEVEN); // = 0x3
3841
3842 if (deven & 8) {
3843 write_mchbar8(0x2c30, 0x20);
3844 pcie_read_config8(NORTHBRIDGE, 0x8); // = 0x18
3845 write_mchbar16(0x2c30, read_mchbar16(0x2c30) | 0x200);
3846 write_mchbar16(0x2c32, 0x434);
3847 read_mchbar32(0x2c44);
3848 write_mchbar32(0x2c44, 0x1053687);
3849 pcie_read_config8(GMA, 0x62); // = 0x2
3850 pcie_write_config8(GMA, 0x62, 0x2);
3851 read8(DEFAULT_RCBA + 0x2318);
3852 write8(DEFAULT_RCBA + 0x2318, 0x47);
3853 read8(DEFAULT_RCBA + 0x2320);
3854 write8(DEFAULT_RCBA + 0x2320, 0xfc);
3855 }
3856
3857 read_mchbar32(0x30);
3858 write_mchbar32(0x30, 0x40);
3859
3860 pcie_write_config16(NORTHBRIDGE, D0F0_GGC, ggc);
3861 gav(read32(DEFAULT_RCBA + 0x3428));
3862 write32(DEFAULT_RCBA + 0x3428, 0x1d);
3863 }
3864
3865 void raminit(const int s3resume, const u8 *spd_addrmap)
3866 {
3867 unsigned channel, slot, lane, rank;
3868 int i;
3869 struct raminfo info;
3870 u8 x2ca8;
3871 u16 deven;
3872 int cbmem_wasnot_inited;
3873
3874 x2ca8 = read_mchbar8(0x2ca8);
3875 deven = pcie_read_config16(NORTHBRIDGE, D0F0_DEVEN);
3876
3877 memset(&info, 0x5a, sizeof(info));
3878
3879 info.last_500_command[0] = 0;
3880 info.last_500_command[1] = 0;
3881
3882 info.fsb_frequency = 135 * 2;
3883 info.board_lane_delay[0] = 0x14;
3884 info.board_lane_delay[1] = 0x07;
3885 info.board_lane_delay[2] = 0x07;
3886 info.board_lane_delay[3] = 0x08;
3887 info.board_lane_delay[4] = 0x56;
3888 info.board_lane_delay[5] = 0x04;
3889 info.board_lane_delay[6] = 0x04;
3890 info.board_lane_delay[7] = 0x05;
3891 info.board_lane_delay[8] = 0x10;
3892
3893 info.training.reg_178 = 0;
3894 info.training.reg_10b = 0;
3895
3896 info.heci_bar = 0;
3897 info.memory_reserved_for_heci_mb = 0;
3898
3899 /* before SPD */
3900 timestamp_add_now(101);
3901
3902 if (!s3resume || REAL) {
3903 pcie_read_config8(SOUTHBRIDGE, GEN_PMCON_2); // = 0x80
3904
3905 collect_system_info(&info);
3906
3907 #if REAL
3908 /* Enable SMBUS. */
3909 enable_smbus();
3910 #endif
3911
3912 memset(&info.populated_ranks, 0, sizeof(info.populated_ranks));
3913
3914 info.use_ecc = 1;
3915 for (channel = 0; channel < NUM_CHANNELS; channel++)
3916 for (slot = 0; slot < NUM_SLOTS; slot++) {
3917 int v;
3918 int try;
3919 int addr;
3920 const u8 useful_addresses[] = {
3921 DEVICE_TYPE,
3922 MODULE_TYPE,
3923 DENSITY,
3924 RANKS_AND_DQ,
3925 MEMORY_BUS_WIDTH,
3926 TIMEBASE_DIVIDEND,
3927 TIMEBASE_DIVISOR,
3928 CYCLETIME,
3929 CAS_LATENCIES_LSB,
3930 CAS_LATENCIES_MSB,
3931 CAS_LATENCY_TIME,
3932 0x11, 0x12, 0x13, 0x14, 0x15,
3933 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b,
3934 0x1c, 0x1d,
3935 THERMAL_AND_REFRESH,
3936 0x20,
3937 REFERENCE_RAW_CARD_USED,
3938 RANK1_ADDRESS_MAPPING,
3939 0x75, 0x76, 0x77, 0x78,
3940 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e,
3941 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84,
3942 0x85, 0x86, 0x87, 0x88,
3943 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e,
3944 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94,
3945 0x95
3946 };
3947 if (!spd_addrmap[2 * channel + slot])
3948 continue;
3949 for (try = 0; try < 5; try++) {
3950 v = smbus_read_byte(spd_addrmap[2 * channel + slot],
3951 DEVICE_TYPE);
3952 if (v >= 0)
3953 break;
3954 }
3955 if (v < 0)
3956 continue;
3957 for (addr = 0;
3958 addr <
3959 sizeof(useful_addresses) /
3960 sizeof(useful_addresses[0]); addr++)
3961 gav(info.
3962 spd[channel][0][useful_addresses
3963 [addr]] =
3964 smbus_read_byte(spd_addrmap[2 * channel + slot],
3965 useful_addresses
3966 [addr]));
3967 if (info.spd[channel][0][DEVICE_TYPE] != 11)
3968 die("Only DDR3 is supported");
3969
3970 v = info.spd[channel][0][RANKS_AND_DQ];
3971 info.populated_ranks[channel][0][0] = 1;
3972 info.populated_ranks[channel][0][1] =
3973 ((v >> 3) & 7);
3974 if (((v >> 3) & 7) > 1)
3975 die("At most 2 ranks are supported");
3976 if ((v & 7) == 0 || (v & 7) > 2)
3977 die("Only x8 and x16 modules are supported");
3978 if ((info.
3979 spd[channel][slot][MODULE_TYPE] & 0xF) != 2
3980 && (info.
3981 spd[channel][slot][MODULE_TYPE] & 0xF)
3982 != 3)
3983 die("Registered memory is not supported");
3984 info.is_x16_module[channel][0] = (v & 7) - 1;
3985 info.density[channel][slot] =
3986 info.spd[channel][slot][DENSITY] & 0xF;
3987 if (!
3988 (info.
3989 spd[channel][slot][MEMORY_BUS_WIDTH] &
3990 0x18))
3991 info.use_ecc = 0;
3992 }
3993
3994 gav(0x55);
3995
3996 for (channel = 0; channel < NUM_CHANNELS; channel++) {
3997 int v = 0;
3998 for (slot = 0; slot < NUM_SLOTS; slot++)
3999 for (rank = 0; rank < NUM_RANKS; rank++)
4000 v |= info.
4001 populated_ranks[channel][slot][rank]
4002 << (2 * slot + rank);
4003 info.populated_ranks_mask[channel] = v;
4004 }
4005
4006 gav(0x55);
4007
4008 gav(pcie_read_config32(NORTHBRIDGE, D0F0_CAPID0 + 4));
4009 }
4010
4011 /* after SPD */
4012 timestamp_add_now(102);
4013
4014 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) & 0xfc);
4015 #if !REAL
4016 rdmsr (MTRR_PHYS_MASK (3));
4017 #endif
4018
4019 collect_system_info(&info);
4020 calculate_timings(&info);
4021
4022 #if !REAL
4023 pcie_write_config8(NORTHBRIDGE, 0xdf, 0x82);
4024 #endif
4025
4026 if (!s3resume) {
4027 u8 reg8 = pcie_read_config8(SOUTHBRIDGE, GEN_PMCON_2);
4028 if (x2ca8 == 0 && (reg8 & 0x80)) {
4029 /* Don't enable S4-assertion stretch. Makes trouble on roda/rk9.
4030 reg8 = pci_read_config8(PCI_DEV(0, 0x1f, 0), 0xa4);
4031 pci_write_config8(PCI_DEV(0, 0x1f, 0), 0xa4, reg8 | 0x08);
4032 */
4033
4034 /* Clear bit7. */
4035
4036 pci_write_config8(SOUTHBRIDGE, GEN_PMCON_2,
4037 (reg8 & ~(1 << 7)));
4038
4039 printk(BIOS_INFO,
4040 "Interrupted RAM init, reset required.\n");
4041 outb(0x6, 0xcf9);
4042 #if REAL
4043 halt();
4044 #endif
4045 }
4046 }
4047 #if !REAL
4048 gav(read_mchbar8(0x2ca8)); ///!!!!
4049 #endif
4050
4051 if (!s3resume && x2ca8 == 0)
4052 pcie_write_config8(SOUTHBRIDGE, GEN_PMCON_2,
4053 pcie_read_config8(SOUTHBRIDGE, GEN_PMCON_2) | 0x80);
4054
4055 compute_derived_timings(&info);
4056
4057 if (x2ca8 == 0) {
4058 gav(read_mchbar8(0x164));
4059 write_mchbar8(0x164, 0x26);
4060 write_mchbar16(0x2c20, 0x10);
4061 }
4062
4063 write_mchbar32(0x18b4, read_mchbar32(0x18b4) | 0x210000); /* OK */
4064 write_mchbar32(0x1890, read_mchbar32(0x1890) | 0x2000000); /* OK */
4065 write_mchbar32(0x18b4, read_mchbar32(0x18b4) | 0x8000);
4066
4067 gav(pcie_read_config32(PCI_DEV(0xff, 2, 1), 0x50)); // !!!!
4068 pcie_write_config8(PCI_DEV(0xff, 2, 1), 0x54, 0x12);
4069
4070 gav(read_mchbar16(0x2c10)); // !!!!
4071 write_mchbar16(0x2c10, 0x412);
4072 gav(read_mchbar16(0x2c10)); // !!!!
4073 write_mchbar16(0x2c12, read_mchbar16(0x2c12) | 0x100); /* OK */
4074
4075 gav(read_mchbar8(0x2ca8)); // !!!!
4076 write_mchbar32(0x1804,
4077 (read_mchbar32(0x1804) & 0xfffffffc) | 0x8400080);
4078
4079 pcie_read_config32(PCI_DEV(0xff, 2, 1), 0x6c); // !!!!
4080 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0x6c, 0x40a0a0);
4081 gav(read_mchbar32(0x1c04)); // !!!!
4082 gav(read_mchbar32(0x1804)); // !!!!
4083
4084 if (x2ca8 == 0) {
4085 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) | 1);
4086 }
4087
4088 write_mchbar32(0x18d8, 0x120000);
4089 write_mchbar32(0x18dc, 0x30a484a);
4090 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xe0, 0x0);
4091 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xf4, 0x9444a);
4092 write_mchbar32(0x18d8, 0x40000);
4093 write_mchbar32(0x18dc, 0xb000000);
4094 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xe0, 0x60000);
4095 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xf4, 0x0);
4096 write_mchbar32(0x18d8, 0x180000);
4097 write_mchbar32(0x18dc, 0xc0000142);
4098 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xe0, 0x20000);
4099 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xf4, 0x142);
4100 write_mchbar32(0x18d8, 0x1e0000);
4101
4102 gav(read_mchbar32(0x18dc)); // !!!!
4103 write_mchbar32(0x18dc, 0x3);
4104 gav(read_mchbar32(0x18dc)); // !!!!
4105
4106 if (x2ca8 == 0) {
4107 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) | 1); // guess
4108 }
4109
4110 write_mchbar32(0x188c, 0x20bc09);
4111 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xd0, 0x40b0c09);
4112 write_mchbar32(0x1a10, 0x4200010e);
4113 write_mchbar32(0x18b8, read_mchbar32(0x18b8) | 0x200);
4114 gav(read_mchbar32(0x1918)); // !!!!
4115 write_mchbar32(0x1918, 0x332);
4116
4117 gav(read_mchbar32(0x18b8)); // !!!!
4118 write_mchbar32(0x18b8, 0xe00);
4119 gav(read_mchbar32(0x182c)); // !!!!
4120 write_mchbar32(0x182c, 0x10202);
4121 gav(pcie_read_config32(PCI_DEV(0xff, 2, 1), 0x94)); // !!!!
4122 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0x94, 0x10202);
4123 write_mchbar32(0x1a1c, read_mchbar32(0x1a1c) & 0x8fffffff);
4124 write_mchbar32(0x1a70, read_mchbar32(0x1a70) | 0x100000);
4125
4126 write_mchbar32(0x18b4, read_mchbar32(0x18b4) & 0xffff7fff);
4127 gav(read_mchbar32(0x1a68)); // !!!!
4128 write_mchbar32(0x1a68, 0x343800);
4129 gav(read_mchbar32(0x1e68)); // !!!!
4130 gav(read_mchbar32(0x1a68)); // !!!!
4131
4132 if (x2ca8 == 0) {
4133 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) | 1); // guess
4134 }
4135
4136 pcie_read_config32(PCI_DEV(0xff, 2, 0), 0x048); // !!!!
4137 pcie_write_config32(PCI_DEV(0xff, 2, 0), 0x048, 0x140000);
4138 pcie_read_config32(PCI_DEV(0xff, 2, 0), 0x058); // !!!!
4139 pcie_write_config32(PCI_DEV(0xff, 2, 0), 0x058, 0x64555);
4140 pcie_read_config32(PCI_DEV(0xff, 2, 0), 0x058); // !!!!
4141 pcie_read_config32(PCI_DEV (0xff, 0, 0), 0xd0); // !!!!
4142 pcie_write_config32(PCI_DEV (0xff, 0, 0), 0xd0, 0x180);
4143 gav(read_mchbar32(0x1af0)); // !!!!
4144 gav(read_mchbar32(0x1af0)); // !!!!
4145 write_mchbar32(0x1af0, 0x1f020003);
4146 gav(read_mchbar32(0x1af0)); // !!!!
4147
4148 if (x2ca8 == 0) {
4149 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) | 1); // guess
4150 }
4151
4152 gav(read_mchbar32(0x1890)); // !!!!
4153 write_mchbar32(0x1890, 0x80102);
4154 gav(read_mchbar32(0x18b4)); // !!!!
4155 write_mchbar32(0x18b4, 0x216000);
4156 write_mchbar32(0x18a4, 0x22222222);
4157 write_mchbar32(0x18a8, 0x22222222);
4158 write_mchbar32(0x18ac, 0x22222);
4159
4160 udelay(1000);
4161
4162 info.cached_training = get_cached_training();
4163
4164 if (x2ca8 == 0) {
4165 int j;
4166 if (s3resume && info.cached_training) {
4167 restore_274265(&info);
4168 printk(BIOS_DEBUG, "reg2ca9_bit0 = %x\n",
4169 info.cached_training->reg2ca9_bit0);
4170 for (i = 0; i < 2; i++)
4171 for (j = 0; j < 3; j++)
4172 printk(BIOS_DEBUG, "reg274265[%d][%d] = %x\n",
4173 i, j, info.cached_training->reg274265[i][j]);
4174 } else {
4175 set_274265(&info);
4176 printk(BIOS_DEBUG, "reg2ca9_bit0 = %x\n",
4177 info.training.reg2ca9_bit0);
4178 for (i = 0; i < 2; i++)
4179 for (j = 0; j < 3; j++)
4180 printk(BIOS_DEBUG, "reg274265[%d][%d] = %x\n",
4181 i, j, info.training.reg274265[i][j]);
4182 }
4183
4184 set_2dxx_series(&info, s3resume);
4185
4186 if (!(deven & 8)) {
4187 read_mchbar32(0x2cb0);
4188 write_mchbar32(0x2cb0, 0x40);
4189 }
4190
4191 udelay(1000);
4192
4193 if (deven & 8) {
4194 write_mchbar32(0xff8, 0x1800 | read_mchbar32(0xff8));
4195 read_mchbar32(0x2cb0);
4196 write_mchbar32(0x2cb0, 0x00);
4197 pcie_read_config8(PCI_DEV (0, 0x2, 0x0), 0x4c);
4198 pcie_read_config8(PCI_DEV (0, 0x2, 0x0), 0x4c);
4199 pcie_read_config8(PCI_DEV (0, 0x2, 0x0), 0x4e);
4200
4201 read_mchbar8(0x1150);
4202 read_mchbar8(0x1151);
4203 read_mchbar8(0x1022);
4204 read_mchbar8(0x16d0);
4205 write_mchbar32(0x1300, 0x60606060);
4206 write_mchbar32(0x1304, 0x60606060);
4207 write_mchbar32(0x1308, 0x78797a7b);
4208 write_mchbar32(0x130c, 0x7c7d7e7f);
4209 write_mchbar32(0x1310, 0x60606060);
4210 write_mchbar32(0x1314, 0x60606060);
4211 write_mchbar32(0x1318, 0x60606060);
4212 write_mchbar32(0x131c, 0x60606060);
4213 write_mchbar32(0x1320, 0x50515253);
4214 write_mchbar32(0x1324, 0x54555657);
4215 write_mchbar32(0x1328, 0x58595a5b);
4216 write_mchbar32(0x132c, 0x5c5d5e5f);
4217 write_mchbar32(0x1330, 0x40414243);
4218 write_mchbar32(0x1334, 0x44454647);
4219 write_mchbar32(0x1338, 0x48494a4b);
4220 write_mchbar32(0x133c, 0x4c4d4e4f);
4221 write_mchbar32(0x1340, 0x30313233);
4222 write_mchbar32(0x1344, 0x34353637);
4223 write_mchbar32(0x1348, 0x38393a3b);
4224 write_mchbar32(0x134c, 0x3c3d3e3f);
4225 write_mchbar32(0x1350, 0x20212223);
4226 write_mchbar32(0x1354, 0x24252627);
4227 write_mchbar32(0x1358, 0x28292a2b);
4228 write_mchbar32(0x135c, 0x2c2d2e2f);
4229 write_mchbar32(0x1360, 0x10111213);
4230 write_mchbar32(0x1364, 0x14151617);
4231 write_mchbar32(0x1368, 0x18191a1b);
4232 write_mchbar32(0x136c, 0x1c1d1e1f);
4233 write_mchbar32(0x1370, 0x10203);
4234 write_mchbar32(0x1374, 0x4050607);
4235 write_mchbar32(0x1378, 0x8090a0b);
4236 write_mchbar32(0x137c, 0xc0d0e0f);
4237 write_mchbar8(0x11cc, 0x4e);
4238 write_mchbar32(0x1110, 0x73970404);
4239 write_mchbar32(0x1114, 0x72960404);
4240 write_mchbar32(0x1118, 0x6f950404);
4241 write_mchbar32(0x111c, 0x6d940404);
4242 write_mchbar32(0x1120, 0x6a930404);
4243 write_mchbar32(0x1124, 0x68a41404);
4244 write_mchbar32(0x1128, 0x66a21404);
4245 write_mchbar32(0x112c, 0x63a01404);
4246 write_mchbar32(0x1130, 0x609e1404);
4247 write_mchbar32(0x1134, 0x5f9c1404);
4248 write_mchbar32(0x1138, 0x5c961404);
4249 write_mchbar32(0x113c, 0x58a02404);
4250 write_mchbar32(0x1140, 0x54942404);
4251 write_mchbar32(0x1190, 0x900080a);
4252 write_mchbar16(0x11c0, 0xc40b);
4253 write_mchbar16(0x11c2, 0x303);
4254 write_mchbar16(0x11c4, 0x301);
4255 read_mchbar32(0x1190);
4256 write_mchbar32(0x1190, 0x8900080a);
4257 write_mchbar32(0x11b8, 0x70c3000);
4258 write_mchbar8(0x11ec, 0xa);
4259 write_mchbar16(0x1100, 0x800);
4260 read_mchbar32(0x11bc);
4261 write_mchbar32(0x11bc, 0x1e84800);
4262 write_mchbar16(0x11ca, 0xfa);
4263 write_mchbar32(0x11e4, 0x4e20);
4264 write_mchbar8(0x11bc, 0xf);
4265 write_mchbar16(0x11da, 0x19);
4266 write_mchbar16(0x11ba, 0x470c);
4267 write_mchbar32(0x1680, 0xe6ffe4ff);
4268 write_mchbar32(0x1684, 0xdeffdaff);
4269 write_mchbar32(0x1688, 0xd4ffd0ff);
4270 write_mchbar32(0x168c, 0xccffc6ff);
4271 write_mchbar32(0x1690, 0xc0ffbeff);
4272 write_mchbar32(0x1694, 0xb8ffb0ff);
4273 write_mchbar32(0x1698, 0xa8ff0000);
4274 write_mchbar32(0x169c, 0xc00);
4275 write_mchbar32(0x1290, 0x5000000);
4276 }
4277
4278 write_mchbar32(0x124c, 0x15040d00);
4279 write_mchbar32(0x1250, 0x7f0000);
4280 write_mchbar32(0x1254, 0x1e220004);
4281 write_mchbar32(0x1258, 0x4000004);
4282 write_mchbar32(0x1278, 0x0);
4283 write_mchbar32(0x125c, 0x0);
4284 write_mchbar32(0x1260, 0x0);
4285 write_mchbar32(0x1264, 0x0);
4286 write_mchbar32(0x1268, 0x0);
4287 write_mchbar32(0x126c, 0x0);
4288 write_mchbar32(0x1270, 0x0);
4289 write_mchbar32(0x1274, 0x0);
4290 }
4291
4292 if ((deven & 8) && x2ca8 == 0) {
4293 write_mchbar16(0x1214, 0x320);
4294 write_mchbar32(0x1600, 0x40000000);
4295 read_mchbar32(0x11f4);
4296 write_mchbar32(0x11f4, 0x10000000);
4297 read_mchbar16(0x1230);
4298 write_mchbar16(0x1230, 0x8000);
4299 write_mchbar32(0x1400, 0x13040020);
4300 write_mchbar32(0x1404, 0xe090120);
4301 write_mchbar32(0x1408, 0x5120220);
4302 write_mchbar32(0x140c, 0x5120330);
4303 write_mchbar32(0x1410, 0xe090220);
4304 write_mchbar32(0x1414, 0x1010001);
4305 write_mchbar32(0x1418, 0x1110000);
4306 write_mchbar32(0x141c, 0x9020020);
4307 write_mchbar32(0x1420, 0xd090220);
4308 write_mchbar32(0x1424, 0x2090220);
4309 write_mchbar32(0x1428, 0x2090330);
4310 write_mchbar32(0x142c, 0xd090220);
4311 write_mchbar32(0x1430, 0x1010001);
4312 write_mchbar32(0x1434, 0x1110000);
4313 write_mchbar32(0x1438, 0x11040020);
4314 write_mchbar32(0x143c, 0x4030220);
4315 write_mchbar32(0x1440, 0x1060220);
4316 write_mchbar32(0x1444, 0x1060330);
4317 write_mchbar32(0x1448, 0x4030220);
4318 write_mchbar32(0x144c, 0x1010001);
4319 write_mchbar32(0x1450, 0x1110000);
4320 write_mchbar32(0x1454, 0x4010020);
4321 write_mchbar32(0x1458, 0xb090220);
4322 write_mchbar32(0x145c, 0x1090220);
4323 write_mchbar32(0x1460, 0x1090330);
4324 write_mchbar32(0x1464, 0xb090220);
4325 write_mchbar32(0x1468, 0x1010001);
4326 write_mchbar32(0x146c, 0x1110000);
4327 write_mchbar32(0x1470, 0xf040020);
4328 write_mchbar32(0x1474, 0xa090220);
4329 write_mchbar32(0x1478, 0x1120220);
4330 write_mchbar32(0x147c, 0x1120330);
4331 write_mchbar32(0x1480, 0xa090220);
4332 write_mchbar32(0x1484, 0x1010001);
4333 write_mchbar32(0x1488, 0x1110000);
4334 write_mchbar32(0x148c, 0x7020020);
4335 write_mchbar32(0x1490, 0x1010220);
4336 write_mchbar32(0x1494, 0x10210);
4337 write_mchbar32(0x1498, 0x10320);
4338 write_mchbar32(0x149c, 0x1010220);
4339 write_mchbar32(0x14a0, 0x1010001);
4340 write_mchbar32(0x14a4, 0x1110000);
4341 write_mchbar32(0x14a8, 0xd040020);
4342 write_mchbar32(0x14ac, 0x8090220);
4343 write_mchbar32(0x14b0, 0x1111310);
4344 write_mchbar32(0x14b4, 0x1111420);
4345 write_mchbar32(0x14b8, 0x8090220);
4346 write_mchbar32(0x14bc, 0x1010001);
4347 write_mchbar32(0x14c0, 0x1110000);
4348 write_mchbar32(0x14c4, 0x3010020);
4349 write_mchbar32(0x14c8, 0x7090220);
4350 write_mchbar32(0x14cc, 0x1081310);
4351 write_mchbar32(0x14d0, 0x1081420);
4352 write_mchbar32(0x14d4, 0x7090220);
4353 write_mchbar32(0x14d8, 0x1010001);
4354 write_mchbar32(0x14dc, 0x1110000);
4355 write_mchbar32(0x14e0, 0xb040020);
4356 write_mchbar32(0x14e4, 0x2030220);
4357 write_mchbar32(0x14e8, 0x1051310);
4358 write_mchbar32(0x14ec, 0x1051420);
4359 write_mchbar32(0x14f0, 0x2030220);
4360 write_mchbar32(0x14f4, 0x1010001);
4361 write_mchbar32(0x14f8, 0x1110000);
4362 write_mchbar32(0x14fc, 0x5020020);
4363 write_mchbar32(0x1500, 0x5090220);
4364 write_mchbar32(0x1504, 0x2071310);
4365 write_mchbar32(0x1508, 0x2071420);
4366 write_mchbar32(0x150c, 0x5090220);
4367 write_mchbar32(0x1510, 0x1010001);
4368 write_mchbar32(0x1514, 0x1110000);
4369 write_mchbar32(0x1518, 0x7040120);
4370 write_mchbar32(0x151c, 0x2090220);
4371 write_mchbar32(0x1520, 0x70b1210);
4372 write_mchbar32(0x1524, 0x70b1310);
4373 write_mchbar32(0x1528, 0x2090220);
4374 write_mchbar32(0x152c, 0x1010001);
4375 write_mchbar32(0x1530, 0x1110000);
4376 write_mchbar32(0x1534, 0x1010110);
4377 write_mchbar32(0x1538, 0x1081310);
4378 write_mchbar32(0x153c, 0x5041200);
4379 write_mchbar32(0x1540, 0x5041310);
4380 write_mchbar32(0x1544, 0x1081310);
4381 write_mchbar32(0x1548, 0x1010001);
4382 write_mchbar32(0x154c, 0x1110000);
4383 write_mchbar32(0x1550, 0x1040120);
4384 write_mchbar32(0x1554, 0x4051210);
4385 write_mchbar32(0x1558, 0xd051200);
4386 write_mchbar32(0x155c, 0xd051200);
4387 write_mchbar32(0x1560, 0x4051210);
4388 write_mchbar32(0x1564, 0x1010001);
4389 write_mchbar32(0x1568, 0x1110000);
4390 write_mchbar16(0x1222, 0x220a);
4391 write_mchbar16(0x123c, 0x1fc0);
4392 write_mchbar16(0x1220, 0x1388);
4393 }
4394
4395 read_mchbar32(0x2c80); // !!!!
4396 write_mchbar32(0x2c80, 0x1053688);
4397 read_mchbar32(0x1c04); // !!!!
4398 write_mchbar32(0x1804, 0x406080);
4399
4400 read_mchbar8(0x2ca8);
4401
4402 if (x2ca8 == 0) {
4403 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) & ~3);
4404 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) + 4);
4405 write_mchbar32(0x1af0, read_mchbar32(0x1af0) | 0x10);
4406 #if REAL
4407 halt();
4408 #else
4409 printf("CP5\n");
4410 exit(0);
4411 #endif
4412 }
4413
4414 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8));
4415 read_mchbar32(0x2c80); // !!!!
4416 write_mchbar32(0x2c80, 0x53688);
4417 pcie_write_config32(PCI_DEV (0xff, 0, 0), 0x60, 0x20220);
4418 read_mchbar16(0x2c20); // !!!!
4419 read_mchbar16(0x2c10); // !!!!
4420 read_mchbar16(0x2c00); // !!!!
4421 write_mchbar16(0x2c00, 0x8c0);
4422 udelay(1000);
4423 write_1d0(0, 0x33d, 0, 0);
4424 write_500(&info, 0, 0, 0xb61, 0, 0);
4425 write_500(&info, 1, 0, 0xb61, 0, 0);
4426 write_mchbar32(0x1a30, 0x0);
4427 write_mchbar32(0x1a34, 0x0);
4428 write_mchbar16(0x614,
4429 0xb5b | (info.populated_ranks[1][0][0] *
4430 0x404) | (info.populated_ranks[0][0][0] *
4431 0xa0));
4432 write_mchbar16(0x616, 0x26a);
4433 write_mchbar32(0x134, 0x856000);
4434 write_mchbar32(0x160, 0x5ffffff);
4435 read_mchbar32(0x114); // !!!!
4436 write_mchbar32(0x114, 0xc2024440);
4437 read_mchbar32(0x118); // !!!!
4438 write_mchbar32(0x118, 0x4);
4439 for (channel = 0; channel < NUM_CHANNELS; channel++)
4440 write_mchbar32(0x260 + (channel << 10),
4441 0x30809ff |
4442 ((info.
4443 populated_ranks_mask[channel] & 3) << 20));
4444 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4445 write_mchbar16(0x31c + (channel << 10), 0x101);
4446 write_mchbar16(0x360 + (channel << 10), 0x909);
4447 write_mchbar16(0x3a4 + (channel << 10), 0x101);
4448 write_mchbar16(0x3e8 + (channel << 10), 0x101);
4449 write_mchbar32(0x320 + (channel << 10), 0x29002900);
4450 write_mchbar32(0x324 + (channel << 10), 0x0);
4451 write_mchbar32(0x368 + (channel << 10), 0x32003200);
4452 write_mchbar16(0x352 + (channel << 10), 0x505);
4453 write_mchbar16(0x354 + (channel << 10), 0x3c3c);
4454 write_mchbar16(0x356 + (channel << 10), 0x1040);
4455 write_mchbar16(0x39a + (channel << 10), 0x73e4);
4456 write_mchbar16(0x3de + (channel << 10), 0x77ed);
4457 write_mchbar16(0x422 + (channel << 10), 0x1040);
4458 }
4459
4460 write_1d0(0x4, 0x151, 4, 1);
4461 write_1d0(0, 0x142, 3, 1);
4462 rdmsr(0x1ac); // !!!!
4463 write_500(&info, 1, 1, 0x6b3, 4, 1);
4464 write_500(&info, 1, 1, 0x6cf, 4, 1);
4465
4466 rmw_1d0(0x21c, 0x38, 0, 6, 1);
4467
4468 write_1d0(((!info.populated_ranks[1][0][0]) << 1) | ((!info.
4469 populated_ranks[0]
4470 [0][0]) << 0),
4471 0x1d1, 3, 1);
4472 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4473 write_mchbar16(0x38e + (channel << 10), 0x5f5f);
4474 write_mchbar16(0x3d2 + (channel << 10), 0x5f5f);
4475 }
4476
4477 set_334(0);
4478
4479 program_base_timings(&info);
4480
4481 write_mchbar8(0x5ff, read_mchbar8(0x5ff) | 0x80); /* OK */
4482
4483 write_1d0(0x2, 0x1d5, 2, 1);
4484 write_1d0(0x20, 0x166, 7, 1);
4485 write_1d0(0x0, 0xeb, 3, 1);
4486 write_1d0(0x0, 0xf3, 6, 1);
4487
4488 for (channel = 0; channel < NUM_CHANNELS; channel++)
4489 for (lane = 0; lane < 9; lane++) {
4490 u16 addr = 0x125 + get_lane_offset(0, 0, lane);
4491 u8 a;
4492 a = read_500(&info, channel, addr, 6); // = 0x20040080 //!!!!
4493 write_500(&info, channel, a, addr, 6, 1);
4494 }
4495
4496 udelay(1000);
4497
4498 if (s3resume) {
4499 if (info.cached_training == NULL) {
4500 u32 reg32;
4501 printk(BIOS_ERR,
4502 "Couldn't find training data. Rebooting\n");
4503 reg32 = inl(DEFAULT_PMBASE + 0x04);
4504 outl(reg32 & ~(7 << 10), DEFAULT_PMBASE + 0x04);
4505 outb(0xe, 0xcf9);
4506
4507 #if REAL
4508 halt();
4509 #else
4510 printf("CP5\n");
4511 exit(0);
4512 #endif
4513 }
4514 int tm;
4515 info.training = *info.cached_training;
4516 for (tm = 0; tm < 4; tm++)
4517 for (channel = 0; channel < NUM_CHANNELS; channel++)
4518 for (slot = 0; slot < NUM_SLOTS; slot++)
4519 for (rank = 0; rank < NUM_RANKS; rank++)
4520 for (lane = 0; lane < 9; lane++)
4521 write_500(&info,
4522 channel,
4523 info.training.
4524 lane_timings
4525 [tm][channel]
4526 [slot][rank]
4527 [lane],
4528 get_timing_register_addr
4529 (lane, tm,
4530 slot, rank),
4531 9, 0);
4532 write_1d0(info.cached_training->reg_178, 0x178, 7, 1);
4533 write_1d0(info.cached_training->reg_10b, 0x10b, 6, 1);
4534 }
4535
4536 read_mchbar32(0x1f4); // !!!!
4537 write_mchbar32(0x1f4, 0x20000);
4538 write_mchbar32(0x1f0, 0x1d000200);
4539 read_mchbar8(0x1f0); // !!!!
4540 write_mchbar8(0x1f0, 0x1);
4541 read_mchbar8(0x1f0); // !!!!
4542
4543 program_board_delay(&info);
4544
4545 write_mchbar8(0x5ff, 0x0); /* OK */
4546 write_mchbar8(0x5ff, 0x80); /* OK */
4547 write_mchbar8(0x5f4, 0x1); /* OK */
4548
4549 write_mchbar32(0x130, read_mchbar32(0x130) & 0xfffffffd); // | 2 when ?
4550 while (read_mchbar32(0x130) & 1) ;
4551 gav(read_1d0(0x14b, 7)); // = 0x81023100
4552 write_1d0(0x30, 0x14b, 7, 1);
4553 read_1d0(0xd6, 6); // = 0xfa008080 // !!!!
4554 write_1d0(7, 0xd6, 6, 1);
4555 read_1d0(0x328, 6); // = 0xfa018080 // !!!!
4556 write_1d0(7, 0x328, 6, 1);
4557
4558 for (channel = 0; channel < NUM_CHANNELS; channel++)
4559 set_4cf(&info, channel,
4560 info.populated_ranks[channel][0][0] ? 8 : 0);
4561
4562 read_1d0(0x116, 4); // = 0x4040432 // !!!!
4563 write_1d0(2, 0x116, 4, 1);
4564 read_1d0(0xae, 6); // = 0xe8088080 // !!!!
4565 write_1d0(0, 0xae, 6, 1);
4566 read_1d0(0x300, 4); // = 0x48088080 // !!!!
4567 write_1d0(0, 0x300, 6, 1);
4568 read_mchbar16(0x356); // !!!!
4569 write_mchbar16(0x356, 0x1040);
4570 read_mchbar16(0x756); // !!!!
4571 write_mchbar16(0x756, 0x1040);
4572 write_mchbar32(0x140, read_mchbar32(0x140) & ~0x07000000);
4573 write_mchbar32(0x138, read_mchbar32(0x138) & ~0x07000000);
4574 write_mchbar32(0x130, 0x31111301);
4575 /* Wait until REG130b0 is 1. */
4576 while (read_mchbar32(0x130) & 1)
4577 ;
4578
4579 {
4580 u32 t;
4581 u8 val_a1;
4582 val_a1 = read_1d0(0xa1, 6); // = 0x1cf4040 // !!!!
4583 t = read_1d0(0x2f3, 6); // = 0x10a4040 // !!!!
4584 rmw_1d0(0x320, 0x07,
4585 (t & 4) | ((t & 8) >> 2) | ((t & 0x10) >> 4), 6, 1);
4586 rmw_1d0(0x14b, 0x78,
4587 ((((val_a1 >> 2) & 4) | (val_a1 & 8)) >> 2) | (val_a1 &
4588 4), 7,
4589 1);
4590 rmw_1d0(0xce, 0x38,
4591 ((((val_a1 >> 2) & 4) | (val_a1 & 8)) >> 2) | (val_a1 &
4592 4), 6,
4593 1);
4594 }
4595
4596 for (channel = 0; channel < NUM_CHANNELS; channel++)
4597 set_4cf(&info, channel,
4598 info.populated_ranks[channel][0][0] ? 9 : 1);
4599
4600 rmw_1d0(0x116, 0xe, 1, 4, 1); // = 0x4040432 // !!!!
4601 read_mchbar32(0x144); // !!!!
4602 write_1d0(2, 0xae, 6, 1);
4603 write_1d0(2, 0x300, 6, 1);
4604 write_1d0(2, 0x121, 3, 1);
4605 read_1d0(0xd6, 6); // = 0xfa00c0c7 // !!!!
4606 write_1d0(4, 0xd6, 6, 1);
4607 read_1d0(0x328, 6); // = 0xfa00c0c7 // !!!!
4608 write_1d0(4, 0x328, 6, 1);
4609
4610 for (channel = 0; channel < NUM_CHANNELS; channel++)
4611 set_4cf(&info, channel,
4612 info.populated_ranks[channel][0][0] ? 9 : 0);
4613
4614 write_mchbar32(0x130,
4615 0x11111301 | (info.
4616 populated_ranks[1][0][0] << 30) | (info.
4617 populated_ranks
4618 [0][0]
4619 [0] <<
4620 29));
4621 while (read_mchbar8(0x130) & 1) ; // !!!!
4622 read_1d0(0xa1, 6); // = 0x1cf4054 // !!!!
4623 read_1d0(0x2f3, 6); // = 0x10a4054 // !!!!
4624 read_1d0(0x21c, 6); // = 0xafa00c0 // !!!!
4625 write_1d0(0, 0x21c, 6, 1);
4626 read_1d0(0x14b, 7); // = 0x810231b0 // !!!!
4627 write_1d0(0x35, 0x14b, 7, 1);
4628
4629 for (channel = 0; channel < NUM_CHANNELS; channel++)
4630 set_4cf(&info, channel,
4631 info.populated_ranks[channel][0][0] ? 0xb : 0x2);
4632
4633 set_334(1);
4634
4635 write_mchbar8(0x1e8, 0x4); /* OK */
4636
4637 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4638 write_500(&info, channel,
4639 0x3 & ~(info.populated_ranks_mask[channel]), 0x6b7, 2,
4640 1);
4641 write_500(&info, channel, 0x3, 0x69b, 2, 1);
4642 }
4643 write_mchbar32(0x2d0, (read_mchbar32(0x2d0) & 0xff2c01ff) | 0x200000); /* OK */
4644 write_mchbar16(0x6c0, 0x14a0); /* OK */
4645 write_mchbar32(0x6d0, (read_mchbar32(0x6d0) & 0xff0080ff) | 0x8000); /* OK */
4646 write_mchbar16(0x232, 0x8);
4647 write_mchbar32(0x234, (read_mchbar32(0x234) & 0xfffbfffb) | 0x40004); /* 0x40004 or 0 depending on ? */
4648 write_mchbar32(0x34, (read_mchbar32(0x34) & 0xfffffffd) | 5); /* OK */
4649 write_mchbar32(0x128, 0x2150d05);
4650 write_mchbar8(0x12c, 0x1f); /* OK */
4651 write_mchbar8(0x12d, 0x56); /* OK */
4652 write_mchbar8(0x12e, 0x31);
4653 write_mchbar8(0x12f, 0x0); /* OK */
4654 write_mchbar8(0x271, 0x2); /* OK */
4655 write_mchbar8(0x671, 0x2); /* OK */
4656 write_mchbar8(0x1e8, 0x4); /* OK */
4657 for (channel = 0; channel < NUM_CHANNELS; channel++)
4658 write_mchbar32(0x294 + (channel << 10),
4659 (info.populated_ranks_mask[channel] & 3) << 16);
4660 write_mchbar32(0x134, (read_mchbar32(0x134) & 0xfc01ffff) | 0x10000); /* OK */
4661 write_mchbar32(0x134, (read_mchbar32(0x134) & 0xfc85ffff) | 0x850000); /* OK */
4662 for (channel = 0; channel < NUM_CHANNELS; channel++)
4663 write_mchbar32(0x260 + (channel << 10),
4664 (read_mchbar32(0x260 + (channel << 10)) &
4665 ~0xf00000) | 0x8000000 | ((info.
4666 populated_ranks_mask
4667 [channel] & 3) <<
4668 20));
4669
4670 if (!s3resume)
4671 jedec_init(&info);
4672
4673 int totalrank = 0;
4674 for (channel = 0; channel < NUM_CHANNELS; channel++)
4675 for (slot = 0; slot < NUM_SLOTS; slot++)
4676 for (rank = 0; rank < NUM_RANKS; rank++)
4677 if (info.populated_ranks[channel][slot][rank]) {
4678 jedec_read(&info, channel, slot, rank,
4679 totalrank, 0xa, 0x400);
4680 totalrank++;
4681 }
4682
4683 write_mchbar8(0x12c, 0x9f);
4684
4685 read_mchbar8(0x271); // 2 // !!!!
4686 write_mchbar8(0x271, 0xe);
4687 read_mchbar8(0x671); // !!!!
4688 write_mchbar8(0x671, 0xe);
4689
4690 if (!s3resume) {
4691 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4692 write_mchbar32(0x294 + (channel << 10),
4693 (info.
4694 populated_ranks_mask[channel] & 3) <<
4695 16);
4696 write_mchbar16(0x298 + (channel << 10),
4697 (info.
4698 populated_ranks[channel][0][0]) | (info.
4699 populated_ranks
4700 [channel]
4701 [0]
4702 [1]
4703 <<
4704 5));
4705 write_mchbar32(0x29c + (channel << 10), 0x77a);
4706 }
4707 read_mchbar32(0x2c0); /// !!!
4708 write_mchbar32(0x2c0, 0x6009cc00);
4709
4710 {
4711 u8 a, b;
4712 a = read_mchbar8(0x243); // !!!!
4713 b = read_mchbar8(0x643); // !!!!
4714 write_mchbar8(0x243, a | 2);
4715 write_mchbar8(0x643, b | 2);
4716 }
4717
4718 write_1d0(7, 0x19b, 3, 1);
4719 write_1d0(7, 0x1c0, 3, 1);
4720 write_1d0(4, 0x1c6, 4, 1);
4721 write_1d0(4, 0x1cc, 4, 1);
4722 read_1d0(0x151, 4); // = 0x408c6d74 // !!!!
4723 write_1d0(4, 0x151, 4, 1);
4724 write_mchbar32(0x584, 0xfffff);
4725 write_mchbar32(0x984, 0xfffff);
4726
4727 for (channel = 0; channel < NUM_CHANNELS; channel++)
4728 for (slot = 0; slot < NUM_SLOTS; slot++)
4729 for (rank = 0; rank < NUM_RANKS; rank++)
4730 if (info.
4731 populated_ranks[channel][slot]
4732 [rank])
4733 config_rank(&info, s3resume,
4734 channel, slot,
4735 rank);
4736
4737 write_mchbar8(0x243, 0x1);
4738 write_mchbar8(0x643, 0x1);
4739 }
4740
4741 /* was == 1 but is common */
4742 pcie_write_config16(NORTHBRIDGE, 0xc8, 3);
4743 write_26c(0, 0x820);
4744 write_26c(1, 0x820);
4745 write_mchbar32(0x130, read_mchbar32(0x130) | 2);
4746 /* end */
4747
4748 if (s3resume) {
4749 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4750 write_mchbar32(0x294 + (channel << 10),
4751 (info.
4752 populated_ranks_mask[channel] & 3) <<
4753 16);
4754 write_mchbar16(0x298 + (channel << 10),
4755 (info.
4756 populated_ranks[channel][0][0]) | (info.
4757 populated_ranks
4758 [channel]
4759 [0]
4760 [1]
4761 <<
4762 5));
4763 write_mchbar32(0x29c + (channel << 10), 0x77a);
4764 }
4765 read_mchbar32(0x2c0); /// !!!
4766 write_mchbar32(0x2c0, 0x6009cc00);
4767 }
4768
4769 write_mchbar32(0xfa4, read_mchbar32(0xfa4) & ~0x01000002);
4770 write_mchbar32(0xfb0, 0x2000e019);
4771
4772 #if !REAL
4773 printf("CP16\n");
4774 #endif
4775
4776 /* Before training. */
4777 timestamp_add_now(103);
4778
4779 if (!s3resume)
4780 ram_training(&info);
4781
4782 /* After training. */
4783 timestamp_add_now(104);
4784
4785 dump_timings(&info);
4786
4787 #if 0
4788 ram_check(0x100000, 0x200000);
4789 #endif
4790 program_modules_memory_map(&info, 0);
4791 program_total_memory_map(&info);
4792
4793 if (info.non_interleaved_part_mb != 0 && info.interleaved_part_mb != 0)
4794 write_mchbar8(0x111, 0x20 | (0 << 2) | (1 << 6) | (0 << 7));
4795 else if (have_match_ranks(&info, 0, 4) && have_match_ranks(&info, 1, 4))
4796 write_mchbar8(0x111, 0x20 | (3 << 2) | (0 << 6) | (1 << 7));
4797 else if (have_match_ranks(&info, 0, 2) && have_match_ranks(&info, 1, 2))
4798 write_mchbar8(0x111, 0x20 | (3 << 2) | (0 << 6) | (0 << 7));
4799 else
4800 write_mchbar8(0x111, 0x20 | (3 << 2) | (1 << 6) | (0 << 7));
4801
4802 write_mchbar32(0xfac, read_mchbar32(0xfac) & ~0x80000000); // OK
4803 write_mchbar32(0xfb4, 0x4800); // OK
4804 write_mchbar32(0xfb8, (info.revision < 8) ? 0x20 : 0x0); // OK
4805 write_mchbar32(0xe94, 0x7ffff); // OK
4806 write_mchbar32(0xfc0, 0x80002040); // OK
4807 write_mchbar32(0xfc4, 0x701246); // OK
4808 write_mchbar8(0xfc8, read_mchbar8(0xfc8) & ~0x70); // OK
4809 write_mchbar32(0xe5c, 0x1000000 | read_mchbar32(0xe5c)); // OK
4810 write_mchbar32(0x1a70, (read_mchbar32(0x1a70) | 0x00200000) & ~0x00100000); // OK
4811 write_mchbar32(0x50, 0x700b0); // OK
4812 write_mchbar32(0x3c, 0x10); // OK
4813 write_mchbar8(0x1aa8, (read_mchbar8(0x1aa8) & ~0x35) | 0xa); // OK
4814 write_mchbar8(0xff4, read_mchbar8(0xff4) | 0x2); // OK
4815 write_mchbar32(0xff8, (read_mchbar32(0xff8) & ~0xe008) | 0x1020); // OK
4816
4817 #if REAL
4818 write_mchbar32(0xd00, IOMMU_BASE2 | 1);
4819 write_mchbar32(0xd40, IOMMU_BASE1 | 1);
4820 write_mchbar32(0xdc0, IOMMU_BASE4 | 1);
4821
4822 write32p(IOMMU_BASE1 | 0xffc, 0x80000000);
4823 write32p(IOMMU_BASE2 | 0xffc, 0xc0000000);
4824 write32p(IOMMU_BASE4 | 0xffc, 0x80000000);
4825
4826 #else
4827 {
4828 u32 eax;
4829 eax = read32p(0xffc + (read_mchbar32(0xd00) & ~1)) | 0x08000000; // = 0xe911714b// OK
4830 write32p(0xffc + (read_mchbar32(0xd00) & ~1), eax); // OK
4831 eax = read32p(0xffc + (read_mchbar32(0xdc0) & ~1)) | 0x40000000; // = 0xe911714b// OK
4832 write32p(0xffc + (read_mchbar32(0xdc0) & ~1), eax); // OK
4833 }
4834 #endif
4835
4836 {
4837 u32 eax;
4838
4839 eax = info.fsb_frequency / 9;
4840 write_mchbar32(0xfcc, (read_mchbar32(0xfcc) & 0xfffc0000) | (eax * 0x280) | (eax * 0x5000) | eax | 0x40000); // OK
4841 write_mchbar32(0x20, 0x33001); //OK
4842 }
4843
4844 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4845 write_mchbar32(0x220 + (channel << 10), read_mchbar32(0x220 + (channel << 10)) & ~0x7770); //OK
4846 if (info.max_slots_used_in_channel == 1)
4847 write_mchbar16(0x237 + (channel << 10), (read_mchbar16(0x237 + (channel << 10)) | 0x0201)); //OK
4848 else
4849 write_mchbar16(0x237 + (channel << 10), (read_mchbar16(0x237 + (channel << 10)) & ~0x0201)); //OK
4850
4851 write_mchbar8(0x241 + (channel << 10), read_mchbar8(0x241 + (channel << 10)) | 1); // OK
4852
4853 if (info.clock_speed_index <= 1
4854 && (info.silicon_revision == 2
4855 || info.silicon_revision == 3))
4856 write_mchbar32(0x248 + (channel << 10), (read_mchbar32(0x248 + (channel << 10)) | 0x00102000)); // OK
4857 else
4858 write_mchbar32(0x248 + (channel << 10), (read_mchbar32(0x248 + (channel << 10)) & ~0x00102000)); // OK
4859 }
4860
4861 write_mchbar32(0x115, read_mchbar32(0x115) | 0x1000000); // OK
4862
4863 {
4864 u8 al;
4865 al = 0xd;
4866 if (!(info.silicon_revision == 0 || info.silicon_revision == 1))
4867 al += 2;
4868 al |= ((1 << (info.max_slots_used_in_channel - 1)) - 1) << 4;
4869 write_mchbar32(0x210, (al << 16) | 0x20); // OK
4870 }
4871
4872 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4873 write_mchbar32(0x288 + (channel << 10), 0x70605040); // OK
4874 write_mchbar32(0x28c + (channel << 10), 0xfffec080); // OK
4875 write_mchbar32(0x290 + (channel << 10), 0x282091c | ((info.max_slots_used_in_channel - 1) << 0x16)); // OK
4876 }
4877 u32 reg1c;
4878 pcie_read_config32(NORTHBRIDGE, 0x40); // = DEFAULT_EPBAR | 0x001 // OK
4879 reg1c = read32p(DEFAULT_EPBAR | 0x01c); // = 0x8001 // OK
4880 pcie_read_config32(NORTHBRIDGE, 0x40); // = DEFAULT_EPBAR | 0x001 // OK
4881 write32p(DEFAULT_EPBAR | 0x01c, reg1c); // OK
4882 read_mchbar8(0xe08); // = 0x0
4883 pcie_read_config32(NORTHBRIDGE, 0xe4); // = 0x316126
4884 write_mchbar8(0x1210, read_mchbar8(0x1210) | 2); // OK
4885 write_mchbar32(0x1200, 0x8800440); // OK
4886 write_mchbar32(0x1204, 0x53ff0453); // OK
4887 write_mchbar32(0x1208, 0x19002043); // OK
4888 write_mchbar16(0x1214, 0x320); // OK
4889
4890 if (info.revision == 0x10 || info.revision == 0x11) {
4891 write_mchbar16(0x1214, 0x220); // OK
4892 write_mchbar8(0x1210, read_mchbar8(0x1210) | 0x40); // OK
4893 }
4894
4895 write_mchbar8(0x1214, read_mchbar8(0x1214) | 0x4); // OK
4896 write_mchbar8(0x120c, 0x1); // OK
4897 write_mchbar8(0x1218, 0x3); // OK
4898 write_mchbar8(0x121a, 0x3); // OK
4899 write_mchbar8(0x121c, 0x3); // OK
4900 write_mchbar16(0xc14, 0x0); // OK
4901 write_mchbar16(0xc20, 0x0); // OK
4902 write_mchbar32(0x1c, 0x0); // OK
4903
4904 /* revision dependent here. */
4905
4906 write_mchbar16(0x1230, read_mchbar16(0x1230) | 0x1f07); // OK
4907
4908 if (info.uma_enabled)
4909 write_mchbar32(0x11f4, read_mchbar32(0x11f4) | 0x10000000); // OK
4910
4911 write_mchbar16(0x1230, read_mchbar16(0x1230) | 0x8000); // OK
4912 write_mchbar8(0x1214, read_mchbar8(0x1214) | 1); // OK
4913
4914 u8 bl, ebpb;
4915 u16 reg_1020;
4916
4917 reg_1020 = read_mchbar32(0x1020); // = 0x6c733c // OK
4918 write_mchbar8(0x1070, 0x1); // OK
4919
4920 write_mchbar32(0x1000, 0x100); // OK
4921 write_mchbar8(0x1007, 0x0); // OK
4922
4923 if (reg_1020 != 0) {
4924 write_mchbar16(0x1018, 0x0); // OK
4925 bl = reg_1020 >> 8;
4926 ebpb = reg_1020 & 0xff;
4927 } else {
4928 ebpb = 0;
4929 bl = 8;
4930 }
4931
4932 rdmsr(0x1a2);
4933
4934 write_mchbar32(0x1014, 0xffffffff); // OK
4935
4936 write_mchbar32(0x1010, ((((ebpb + 0x7d) << 7) / bl) & 0xff) * (! !reg_1020)); // OK
4937
4938 write_mchbar8(0x101c, 0xb8); // OK
4939
4940 write_mchbar8(0x123e, (read_mchbar8(0x123e) & 0xf) | 0x60); // OK
4941 if (reg_1020 != 0) {
4942 write_mchbar32(0x123c, (read_mchbar32(0x123c) & ~0x00900000) | 0x600000); // OK
4943 write_mchbar8(0x101c, 0xb8); // OK
4944 }
4945
4946 setup_heci_uma(&info);
4947
4948 if (info.uma_enabled) {
4949 u16 ax;
4950 write_mchbar32(0x11b0, read_mchbar32(0x11b0) | 0x4000); // OK
4951 write_mchbar32(0x11b4, read_mchbar32(0x11b4) | 0x4000); // OK
4952 write_mchbar16(0x1190, read_mchbar16(0x1190) | 0x4000); // OK
4953
4954 ax = read_mchbar16(0x1190) & 0xf00; // = 0x480a // OK
4955 write_mchbar16(0x1170, ax | (read_mchbar16(0x1170) & 0x107f) | 0x4080); // OK
4956 write_mchbar16(0x1170, read_mchbar16(0x1170) | 0x1000); // OK
4957 #if REAL
4958 udelay(1000);
4959 #endif
4960 u16 ecx;
4961 for (ecx = 0xffff; ecx && (read_mchbar16(0x1170) & 0x1000); ecx--) ; // OK
4962 write_mchbar16(0x1190, read_mchbar16(0x1190) & ~0x4000); // OK
4963 }
4964
4965 pcie_write_config8(SOUTHBRIDGE, GEN_PMCON_2,
4966 pcie_read_config8(SOUTHBRIDGE, GEN_PMCON_2) & ~0x80);
4967 udelay(10000);
4968 write_mchbar16(0x2ca8, 0x8);
4969
4970 #if REAL
4971 udelay(1000);
4972 dump_timings(&info);
4973 cbmem_wasnot_inited = cbmem_recovery(s3resume);
4974
4975 if (!s3resume)
4976 save_timings(&info);
4977 if (s3resume && cbmem_wasnot_inited) {
4978 u32 reg32;
4979 printk(BIOS_ERR, "Failed S3 resume.\n");
4980 ram_check(0x100000, 0x200000);
4981
4982 /* Clear SLP_TYPE. */
4983 reg32 = inl(DEFAULT_PMBASE + 0x04);
4984 outl(reg32 & ~(7 << 10), DEFAULT_PMBASE + 0x04);
4985
4986 /* Failed S3 resume, reset to come up cleanly */
4987 outb(0xe, 0xcf9);
4988 halt();
4989 }
4990 #endif
4991 }
ast, secure and flexible OpenSource firmware