?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
northbridge/intel: Remove unneeded includes
[coreboot.git] / src / northbridge / intel / sandybridge / raminit_common.c
blobb82075e58bd4542a50404c55a1036ef231a82ac2
1 /*
2 * This file is part of the coreboot project.
3 *
4 * Copyright (C) 2014 Damien Zammit <damien@zamaudio.com>
5 * Copyright (C) 2014 Vladimir Serbinenko <phcoder@gmail.com>
6 * Copyright (C) 2016 Patrick Rudolph <siro@das-labor.org>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; version 2 of the License.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 */
17
18 #include <console/console.h>
19 #include <console/usb.h>
20 #include <string.h>
21 #include <arch/io.h>
22 #include <cbmem.h>
23 #include <arch/cbfs.h>
24 #include <cbfs.h>
25 #include <northbridge/intel/sandybridge/chip.h>
26 #include <device/pci_def.h>
27 #include <delay.h>
28 #include <arch/cpu.h>
29 #include "raminit_native.h"
30 #include "raminit_common.h"
31 #include "sandybridge.h"
32
33 /* FIXME: no ECC support. */
34 /* FIXME: no support for 3-channel chipsets. */
35
36 /*
37 * Register description:
38 * Intel provides a command queue of depth four.
39 * Every command is configured by using multiple registers.
40 * On executing the command queue you have to provide the depth used.
41 *
42 * Known registers:
43 * Channel X = [0, 1]
44 * Command queue index Y = [0, 1, 2, 3]
45 *
46 * DEFAULT_MCHBAR + 0x4220 + 0x400 * X + 4 * Y: command io register
47 * Controls the DRAM command signals
48 * Bit 0: !RAS
49 * Bit 1: !CAS
50 * Bit 2: !WE
51 *
52 * DEFAULT_MCHBAR + 0x4200 + 0x400 * X + 4 * Y: addr bankslot io register
53 * Controls the address, bank address and slotrank signals
54 * Bit 0-15 : Address
55 * Bit 20-22: Bank Address
56 * Bit 24-25: slotrank
57 *
58 * DEFAULT_MCHBAR + 0x4230 + 0x400 * X + 4 * Y: idle register
59 * Controls the idle time after issuing this DRAM command
60 * Bit 16-32: number of clock-cylces to idle
61 *
62 * DEFAULT_MCHBAR + 0x4284 + 0x400 * channel: execute command queue
63 * Starts to execute all queued commands
64 * Bit 0 : start DRAM command execution
65 * Bit 16-20: (number of queued commands - 1) * 4
66 */
67
68 static void sfence(void)
69 {
70 asm volatile ("sfence");
71 }
72
73 static void toggle_io_reset(void) {
74 /* toggle IO reset bit */
75 u32 r32 = read32(DEFAULT_MCHBAR + 0x5030);
76 write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
77 udelay(1);
78 write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
79 udelay(1);
80 }
81
82 static u32 get_XOVER_CLK(u8 rankmap)
83 {
84 return rankmap << 24;
85 }
86
87 static u32 get_XOVER_CMD(u8 rankmap)
88 {
89 u32 reg;
90
91 // enable xover cmd
92 reg = 0x4000;
93
94 // enable xover ctl
95 if (rankmap & 0x3)
96 reg |= 0x20000;
97
98 if (rankmap & 0xc)
99 reg |= 0x4000000;
100
101 return reg;
102 }
103
104 /* CAS write latency. To be programmed in MR2.
105 * See DDR3 SPEC for MR2 documentation. */
106 u8 get_CWL(u32 tCK)
107 {
108 /* Get CWL based on tCK using the following rule: */
109 switch (tCK) {
110 case TCK_1333MHZ:
111 return 12;
112 case TCK_1200MHZ:
113 case TCK_1100MHZ:
114 return 11;
115 case TCK_1066MHZ:
116 case TCK_1000MHZ:
117 return 10;
118 case TCK_933MHZ:
119 case TCK_900MHZ:
120 return 9;
121 case TCK_800MHZ:
122 case TCK_700MHZ:
123 return 8;
124 case TCK_666MHZ:
125 return 7;
126 case TCK_533MHZ:
127 return 6;
128 default:
129 return 5;
130 }
131 }
132
133 void dram_find_common_params(ramctr_timing *ctrl)
134 {
135 size_t valid_dimms;
136 int channel, slot;
137 dimm_info *dimms = &ctrl->info;
138
139 ctrl->cas_supported = (1 << (MAX_CAS - MIN_CAS + 1)) - 1;
140 valid_dimms = 0;
141 FOR_ALL_CHANNELS for (slot = 0; slot < 2; slot++) {
142 const dimm_attr *dimm = &dimms->dimm[channel][slot];
143 if (dimm->dram_type != SPD_MEMORY_TYPE_SDRAM_DDR3)
144 continue;
145 valid_dimms++;
146
147 /* Find all possible CAS combinations */
148 ctrl->cas_supported &= dimm->cas_supported;
149
150 /* Find the smallest common latencies supported by all DIMMs */
151 ctrl->tCK = MAX(ctrl->tCK, dimm->tCK);
152 ctrl->tAA = MAX(ctrl->tAA, dimm->tAA);
153 ctrl->tWR = MAX(ctrl->tWR, dimm->tWR);
154 ctrl->tRCD = MAX(ctrl->tRCD, dimm->tRCD);
155 ctrl->tRRD = MAX(ctrl->tRRD, dimm->tRRD);
156 ctrl->tRP = MAX(ctrl->tRP, dimm->tRP);
157 ctrl->tRAS = MAX(ctrl->tRAS, dimm->tRAS);
158 ctrl->tRFC = MAX(ctrl->tRFC, dimm->tRFC);
159 ctrl->tWTR = MAX(ctrl->tWTR, dimm->tWTR);
160 ctrl->tRTP = MAX(ctrl->tRTP, dimm->tRTP);
161 ctrl->tFAW = MAX(ctrl->tFAW, dimm->tFAW);
162 ctrl->tCWL = MAX(ctrl->tCWL, dimm->tCWL);
163 ctrl->tCMD = MAX(ctrl->tCMD, dimm->tCMD);
164 }
165
166 if (!ctrl->cas_supported)
167 die("Unsupported DIMM combination. "
168 "DIMMS do not support common CAS latency");
169 if (!valid_dimms)
170 die("No valid DIMMs found");
171 }
172
173 void dram_xover(ramctr_timing * ctrl)
174 {
175 u32 reg;
176 int channel;
177
178 FOR_ALL_CHANNELS {
179 // enable xover clk
180 reg = get_XOVER_CLK(ctrl->rankmap[channel]);
181 printram("XOVER CLK [%x] = %x\n", channel * 0x100 + 0xc14,
182 reg);
183 MCHBAR32(channel * 0x100 + 0xc14) = reg;
184
185 // enable xover ctl & xover cmd
186 reg = get_XOVER_CMD(ctrl->rankmap[channel]);
187 printram("XOVER CMD [%x] = %x\n", 0x100 * channel + 0x320c,
188 reg);
189 MCHBAR32(0x100 * channel + 0x320c) = reg;
190 }
191 }
192
193 static void dram_odt_stretch(ramctr_timing *ctrl, int channel)
194 {
195 struct cpuid_result cpures;
196 u32 reg, addr, cpu, stretch;
197
198 stretch = ctrl->ref_card_offset[channel];
199 /* ODT stretch: Delay ODT signal by stretch value.
200 * Useful for multi DIMM setups on the same channel. */
201 cpures = cpuid(1);
202 cpu = cpures.eax;
203 if (IS_SANDY_CPU(cpu) && IS_SANDY_CPU_C(cpu)) {
204 if (stretch == 2)
205 stretch = 3;
206 addr = 0x400 * channel + 0x401c;
207 reg = MCHBAR32(addr) & 0xffffc3ff;
208 reg |= (stretch << 12);
209 reg |= (stretch << 10);
210 MCHBAR32(addr) = reg;
211 printram("OTHP Workaround [%x] = %x\n", addr, reg);
212 } else {
213 // OTHP
214 addr = 0x400 * channel + 0x400c;
215 reg = MCHBAR32(addr) & 0xfff0ffff;
216 reg |= (stretch << 16);
217 reg |= (stretch << 18);
218 MCHBAR32(addr) = reg;
219 printram("OTHP [%x] = %x\n", addr, reg);
220 }
221 }
222
223 void dram_timing_regs(ramctr_timing *ctrl)
224 {
225 u32 reg, addr, val32;
226 int channel;
227
228 FOR_ALL_CHANNELS {
229 // DBP
230 reg = 0;
231 reg |= ctrl->tRCD;
232 reg |= (ctrl->tRP << 4);
233 reg |= (ctrl->CAS << 8);
234 reg |= (ctrl->CWL << 12);
235 reg |= (ctrl->tRAS << 16);
236 printram("DBP [%x] = %x\n", 0x400 * channel + 0x4000, reg);
237 MCHBAR32(0x400 * channel + 0x4000) = reg;
238
239 // RAP
240 reg = 0;
241 reg |= ctrl->tRRD;
242 reg |= (ctrl->tRTP << 4);
243 reg |= (ctrl->tCKE << 8);
244 reg |= (ctrl->tWTR << 12);
245 reg |= (ctrl->tFAW << 16);
246 reg |= (ctrl->tWR << 24);
247 reg |= (3 << 30);
248 printram("RAP [%x] = %x\n", 0x400 * channel + 0x4004, reg);
249 MCHBAR32(0x400 * channel + 0x4004) = reg;
250
251 // OTHP
252 addr = 0x400 * channel + 0x400c;
253 reg = 0;
254 reg |= ctrl->tXPDLL;
255 reg |= (ctrl->tXP << 5);
256 reg |= (ctrl->tAONPD << 8);
257 reg |= 0xa0000;
258 printram("OTHP [%x] = %x\n", addr, reg);
259 MCHBAR32(addr) = reg;
260
261 MCHBAR32(0x400 * channel + 0x4014) = 0;
262
263 MCHBAR32(addr) |= 0x00020000;
264
265 dram_odt_stretch(ctrl, channel);
266
267 // REFI
268 reg = 0;
269 val32 = ctrl->tREFI;
270 reg = (reg & ~0xffff) | val32;
271 val32 = ctrl->tRFC;
272 reg = (reg & ~0x1ff0000) | (val32 << 16);
273 val32 = (u32) (ctrl->tREFI * 9) / 1024;
274 reg = (reg & ~0xfe000000) | (val32 << 25);
275 printram("REFI [%x] = %x\n", 0x400 * channel + 0x4298,
276 reg);
277 MCHBAR32(0x400 * channel + 0x4298) = reg;
278
279 MCHBAR32(0x400 * channel + 0x4294) |= 0xff;
280
281 // SRFTP
282 reg = 0;
283 val32 = tDLLK;
284 reg = (reg & ~0xfff) | val32;
285 val32 = ctrl->tXSOffset;
286 reg = (reg & ~0xf000) | (val32 << 12);
287 val32 = tDLLK - ctrl->tXSOffset;
288 reg = (reg & ~0x3ff0000) | (val32 << 16);
289 val32 = ctrl->tMOD - 8;
290 reg = (reg & ~0xf0000000) | (val32 << 28);
291 printram("SRFTP [%x] = %x\n", 0x400 * channel + 0x42a4,
292 reg);
293 MCHBAR32(0x400 * channel + 0x42a4) = reg;
294 }
295 }
296
297 void dram_dimm_mapping(ramctr_timing *ctrl)
298 {
299 int channel;
300 dimm_info *info = &ctrl->info;
301
302 FOR_ALL_CHANNELS {
303 dimm_attr *dimmA, *dimmB;
304 u32 reg = 0;
305
306 if (info->dimm[channel][0].size_mb >=
307 info->dimm[channel][1].size_mb) {
308 dimmA = &info->dimm[channel][0];
309 dimmB = &info->dimm[channel][1];
310 reg |= 0 << 16;
311 } else {
312 dimmA = &info->dimm[channel][1];
313 dimmB = &info->dimm[channel][0];
314 reg |= 1 << 16;
315 }
316
317 if (dimmA && (dimmA->ranks > 0)) {
318 reg |= dimmA->size_mb / 256;
319 reg |= (dimmA->ranks - 1) << 17;
320 reg |= (dimmA->width / 8 - 1) << 19;
321 }
322
323 if (dimmB && (dimmB->ranks > 0)) {
324 reg |= (dimmB->size_mb / 256) << 8;
325 reg |= (dimmB->ranks - 1) << 18;
326 reg |= (dimmB->width / 8 - 1) << 20;
327 }
328
329 reg |= 1 << 21; /* rank interleave */
330 reg |= 1 << 22; /* enhanced interleave */
331
332 if ((dimmA && (dimmA->ranks > 0))
333 || (dimmB && (dimmB->ranks > 0))) {
334 ctrl->mad_dimm[channel] = reg;
335 } else {
336 ctrl->mad_dimm[channel] = 0;
337 }
338 }
339 }
340
341 void dram_dimm_set_mapping(ramctr_timing * ctrl)
342 {
343 int channel;
344 FOR_ALL_CHANNELS {
345 MCHBAR32(0x5004 + channel * 4) = ctrl->mad_dimm[channel];
346 }
347 }
348
349 void dram_zones(ramctr_timing * ctrl, int training)
350 {
351 u32 reg, ch0size, ch1size;
352 u8 val;
353 reg = 0;
354 val = 0;
355 if (training) {
356 ch0size = ctrl->channel_size_mb[0] ? 256 : 0;
357 ch1size = ctrl->channel_size_mb[1] ? 256 : 0;
358 } else {
359 ch0size = ctrl->channel_size_mb[0];
360 ch1size = ctrl->channel_size_mb[1];
361 }
362
363 if (ch0size >= ch1size) {
364 reg = MCHBAR32(0x5014);
365 val = ch1size / 256;
366 reg = (reg & ~0xff000000) | val << 24;
367 reg = (reg & ~0xff0000) | (2 * val) << 16;
368 MCHBAR32(0x5014) = reg;
369 MCHBAR32(0x5000) = 0x24;
370 } else {
371 reg = MCHBAR32(0x5014);
372 val = ch0size / 256;
373 reg = (reg & ~0xff000000) | val << 24;
374 reg = (reg & ~0xff0000) | (2 * val) << 16;
375 MCHBAR32(0x5014) = reg;
376 MCHBAR32(0x5000) = 0x21;
377 }
378 }
379
380 #define HOST_BRIDGE PCI_DEVFN(0, 0)
381 #define DEFAULT_TCK TCK_800MHZ
382
383 unsigned int get_mem_min_tck(void)
384 {
385 u32 reg32;
386 u8 rev;
387 const struct device *dev;
388 const struct northbridge_intel_sandybridge_config *cfg = NULL;
389
390 dev = dev_find_slot(0, HOST_BRIDGE);
391 if (dev)
392 cfg = dev->chip_info;
393
394 /* If this is zero, it just means devicetree.cb didn't set it */
395 if (!cfg || cfg->max_mem_clock_mhz == 0) {
396 if (IS_ENABLED(CONFIG_NATIVE_RAMINIT_IGNORE_MAX_MEM_FUSES))
397 return TCK_1333MHZ;
398
399 rev = pci_read_config8(PCI_DEV(0, 0, 0), PCI_DEVICE_ID);
400
401 if ((rev & BASE_REV_MASK) == BASE_REV_SNB) {
402 /* read Capabilities A Register DMFC bits */
403 reg32 = pci_read_config32(PCI_DEV(0, 0, 0), CAPID0_A);
404 reg32 &= 0x7;
405
406 switch (reg32) {
407 case 7: return TCK_533MHZ;
408 case 6: return TCK_666MHZ;
409 case 5: return TCK_800MHZ;
410 /* reserved: */
411 default:
412 break;
413 }
414 } else {
415 /* read Capabilities B Register DMFC bits */
416 reg32 = pci_read_config32(PCI_DEV(0, 0, 0), CAPID0_B);
417 reg32 = (reg32 >> 4) & 0x7;
418
419 switch (reg32) {
420 case 7: return TCK_533MHZ;
421 case 6: return TCK_666MHZ;
422 case 5: return TCK_800MHZ;
423 case 4: return TCK_933MHZ;
424 case 3: return TCK_1066MHZ;
425 case 2: return TCK_1200MHZ;
426 case 1: return TCK_1333MHZ;
427 /* reserved: */
428 default:
429 break;
430 }
431 }
432 return DEFAULT_TCK;
433 } else {
434 if (cfg->max_mem_clock_mhz >= 1066)
435 return TCK_1066MHZ;
436 else if (cfg->max_mem_clock_mhz >= 933)
437 return TCK_933MHZ;
438 else if (cfg->max_mem_clock_mhz >= 800)
439 return TCK_800MHZ;
440 else if (cfg->max_mem_clock_mhz >= 666)
441 return TCK_666MHZ;
442 else if (cfg->max_mem_clock_mhz >= 533)
443 return TCK_533MHZ;
444 else
445 return TCK_400MHZ;
446 }
447 }
448
449 #define DEFAULT_PCI_MMIO_SIZE 2048
450
451 static unsigned int get_mmio_size(void)
452 {
453 const struct device *dev;
454 const struct northbridge_intel_sandybridge_config *cfg = NULL;
455
456 dev = dev_find_slot(0, HOST_BRIDGE);
457 if (dev)
458 cfg = dev->chip_info;
459
460 /* If this is zero, it just means devicetree.cb didn't set it */
461 if (!cfg || cfg->pci_mmio_size == 0)
462 return DEFAULT_PCI_MMIO_SIZE;
463 else
464 return cfg->pci_mmio_size;
465 }
466
467 void dram_memorymap(ramctr_timing * ctrl, int me_uma_size)
468 {
469 u32 reg, val, reclaim;
470 u32 tom, gfxstolen, gttsize;
471 size_t tsegsize, mmiosize, toludbase, touudbase, gfxstolenbase, gttbase,
472 tsegbase, mestolenbase;
473 size_t tsegbasedelta, remapbase, remaplimit;
474 uint16_t ggc;
475
476 mmiosize = get_mmio_size();
477
478 ggc = pci_read_config16(NORTHBRIDGE, GGC);
479 if (!(ggc & 2)) {
480 gfxstolen = ((ggc >> 3) & 0x1f) * 32;
481 gttsize = ((ggc >> 8) & 0x3);
482 } else {
483 gfxstolen = 0;
484 gttsize = 0;
485 }
486
487 tsegsize = CONFIG_SMM_TSEG_SIZE >> 20;
488
489 tom = ctrl->channel_size_mb[0] + ctrl->channel_size_mb[1];
490
491 mestolenbase = tom - me_uma_size;
492
493 toludbase = MIN(4096 - mmiosize + gfxstolen + gttsize + tsegsize,
494 tom - me_uma_size);
495 gfxstolenbase = toludbase - gfxstolen;
496 gttbase = gfxstolenbase - gttsize;
497
498 tsegbase = gttbase - tsegsize;
499
500 // Round tsegbase down to nearest address aligned to tsegsize
501 tsegbasedelta = tsegbase & (tsegsize - 1);
502 tsegbase &= ~(tsegsize - 1);
503
504 gttbase -= tsegbasedelta;
505 gfxstolenbase -= tsegbasedelta;
506 toludbase -= tsegbasedelta;
507
508 // Test if it is possible to reclaim a hole in the RAM addressing
509 if (tom - me_uma_size > toludbase) {
510 // Reclaim is possible
511 reclaim = 1;
512 remapbase = MAX(4096, tom - me_uma_size);
513 remaplimit =
514 remapbase + MIN(4096, tom - me_uma_size) - toludbase - 1;
515 touudbase = remaplimit + 1;
516 } else {
517 // Reclaim not possible
518 reclaim = 0;
519 touudbase = tom - me_uma_size;
520 }
521
522 // Update memory map in pci-e configuration space
523 printk(BIOS_DEBUG, "Update PCI-E configuration space:\n");
524
525 // TOM (top of memory)
526 reg = pci_read_config32(PCI_DEV(0, 0, 0), 0xa0);
527 val = tom & 0xfff;
528 reg = (reg & ~0xfff00000) | (val << 20);
529 printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", 0xa0, reg);
530 pci_write_config32(PCI_DEV(0, 0, 0), 0xa0, reg);
531
532 reg = pci_read_config32(PCI_DEV(0, 0, 0), 0xa4);
533 val = tom & 0xfffff000;
534 reg = (reg & ~0x000fffff) | (val >> 12);
535 printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", 0xa4, reg);
536 pci_write_config32(PCI_DEV(0, 0, 0), 0xa4, reg);
537
538 // TOLUD (top of low used dram)
539 reg = pci_read_config32(PCI_DEV(0, 0, 0), 0xbc);
540 val = toludbase & 0xfff;
541 reg = (reg & ~0xfff00000) | (val << 20);
542 printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", 0xbc, reg);
543 pci_write_config32(PCI_DEV(0, 0, 0), 0xbc, reg);
544
545 // TOUUD LSB (top of upper usable dram)
546 reg = pci_read_config32(PCI_DEV(0, 0, 0), 0xa8);
547 val = touudbase & 0xfff;
548 reg = (reg & ~0xfff00000) | (val << 20);
549 printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", 0xa8, reg);
550 pci_write_config32(PCI_DEV(0, 0, 0), 0xa8, reg);
551
552 // TOUUD MSB
553 reg = pci_read_config32(PCI_DEV(0, 0, 0), 0xac);
554 val = touudbase & 0xfffff000;
555 reg = (reg & ~0x000fffff) | (val >> 12);
556 printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", 0xac, reg);
557 pci_write_config32(PCI_DEV(0, 0, 0), 0xac, reg);
558
559 if (reclaim) {
560 // REMAP BASE
561 pci_write_config32(PCI_DEV(0, 0, 0), 0x90, remapbase << 20);
562 pci_write_config32(PCI_DEV(0, 0, 0), 0x94, remapbase >> 12);
563
564 // REMAP LIMIT
565 pci_write_config32(PCI_DEV(0, 0, 0), 0x98, remaplimit << 20);
566 pci_write_config32(PCI_DEV(0, 0, 0), 0x9c, remaplimit >> 12);
567 }
568 // TSEG
569 reg = pci_read_config32(PCI_DEV(0, 0, 0), 0xb8);
570 val = tsegbase & 0xfff;
571 reg = (reg & ~0xfff00000) | (val << 20);
572 printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", 0xb8, reg);
573 pci_write_config32(PCI_DEV(0, 0, 0), 0xb8, reg);
574
575 // GFX stolen memory
576 reg = pci_read_config32(PCI_DEV(0, 0, 0), 0xb0);
577 val = gfxstolenbase & 0xfff;
578 reg = (reg & ~0xfff00000) | (val << 20);
579 printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", 0xb0, reg);
580 pci_write_config32(PCI_DEV(0, 0, 0), 0xb0, reg);
581
582 // GTT stolen memory
583 reg = pci_read_config32(PCI_DEV(0, 0, 0), 0xb4);
584 val = gttbase & 0xfff;
585 reg = (reg & ~0xfff00000) | (val << 20);
586 printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", 0xb4, reg);
587 pci_write_config32(PCI_DEV(0, 0, 0), 0xb4, reg);
588
589 if (me_uma_size) {
590 reg = pci_read_config32(PCI_DEV(0, 0, 0), 0x7c);
591 val = (0x80000 - me_uma_size) & 0xfffff000;
592 reg = (reg & ~0x000fffff) | (val >> 12);
593 printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", 0x7c, reg);
594 pci_write_config32(PCI_DEV(0, 0, 0), 0x7c, reg);
595
596 // ME base
597 reg = pci_read_config32(PCI_DEV(0, 0, 0), 0x70);
598 val = mestolenbase & 0xfff;
599 reg = (reg & ~0xfff00000) | (val << 20);
600 printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", 0x70, reg);
601 pci_write_config32(PCI_DEV(0, 0, 0), 0x70, reg);
602
603 reg = pci_read_config32(PCI_DEV(0, 0, 0), 0x74);
604 val = mestolenbase & 0xfffff000;
605 reg = (reg & ~0x000fffff) | (val >> 12);
606 printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", 0x74, reg);
607 pci_write_config32(PCI_DEV(0, 0, 0), 0x74, reg);
608
609 // ME mask
610 reg = pci_read_config32(PCI_DEV(0, 0, 0), 0x78);
611 val = (0x80000 - me_uma_size) & 0xfff;
612 reg = (reg & ~0xfff00000) | (val << 20);
613 reg = (reg & ~0x400) | (1 << 10); // set lockbit on ME mem
614
615 reg = (reg & ~0x800) | (1 << 11); // set ME memory enable
616 printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", 0x78, reg);
617 pci_write_config32(PCI_DEV(0, 0, 0), 0x78, reg);
618 }
619 }
620
621 static void wait_428c(int channel)
622 {
623 while (1) {
624 if (read32(DEFAULT_MCHBAR + 0x428c + (channel << 10)) & 0x50)
625 return;
626 }
627 }
628
629 static void write_reset(ramctr_timing * ctrl)
630 {
631 int channel, slotrank;
632
633 /* choose a populated channel. */
634 channel = (ctrl->rankmap[0]) ? 0 : 1;
635
636 wait_428c(channel);
637
638 /* choose a populated rank. */
639 slotrank = (ctrl->rankmap[channel] & 1) ? 0 : 2;
640
641 /* DRAM command ZQCS */
642 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
643 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x80c01);
644
645 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
646 (slotrank << 24) | 0x60000);
647
648 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
649
650 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x400001);
651 wait_428c(channel);
652 }
653
654 void dram_jedecreset(ramctr_timing * ctrl)
655 {
656 u32 reg, addr;
657 int channel;
658
659 while (!(MCHBAR32(0x5084) & 0x10000));
660 do {
661 reg = MCHBAR32(0x428c);
662 } while ((reg & 0x14) == 0);
663
664 // Set state of memory controller
665 reg = 0x112;
666 MCHBAR32(0x5030) = reg;
667 MCHBAR32(0x4ea0) = 0;
668 reg |= 2; //ddr reset
669 MCHBAR32(0x5030) = reg;
670
671 // Assert dimm reset signal
672 reg = MCHBAR32(0x5030);
673 reg &= ~0x2;
674 MCHBAR32(0x5030) = reg;
675
676 // Wait 200us
677 udelay(200);
678
679 // Deassert dimm reset signal
680 MCHBAR32(0x5030) |= 2;
681
682 // Wait 500us
683 udelay(500);
684
685 // Enable DCLK
686 MCHBAR32(0x5030) |= 4;
687
688 // XXX Wait 20ns
689 udelay(1);
690
691 FOR_ALL_CHANNELS {
692 // Set valid rank CKE
693 reg = 0;
694 reg = (reg & ~0xf) | ctrl->rankmap[channel];
695 addr = 0x400 * channel + 0x42a0;
696 MCHBAR32(addr) = reg;
697
698 // Wait 10ns for ranks to settle
699 //udelay(0.01);
700
701 reg = (reg & ~0xf0) | (ctrl->rankmap[channel] << 4);
702 MCHBAR32(addr) = reg;
703
704 // Write reset using a NOP
705 write_reset(ctrl);
706 }
707 }
708
709 static odtmap get_ODT(ramctr_timing *ctrl, u8 rank, int channel)
710 {
711 /* Get ODT based on rankmap: */
712 int dimms_per_ch = (ctrl->rankmap[channel] & 1)
713 + ((ctrl->rankmap[channel] >> 2) & 1);
714
715 if (dimms_per_ch == 1) {
716 return (const odtmap){60, 60};
717 } else {
718 return (const odtmap){120, 30};
719 }
720 }
721
722 static void write_mrreg(ramctr_timing *ctrl, int channel, int slotrank,
723 int reg, u32 val)
724 {
725 wait_428c(channel);
726
727 if (ctrl->rank_mirror[channel][slotrank]) {
728 /* DDR3 Rank1 Address mirror
729 * swap the following pins:
730 * A3<->A4, A5<->A6, A7<->A8, BA0<->BA1 */
731 reg = ((reg >> 1) & 1) | ((reg << 1) & 2);
732 val = (val & ~0x1f8) | ((val >> 1) & 0xa8)
733 | ((val & 0xa8) << 1);
734 }
735
736 /* DRAM command MRS */
737 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f000);
738 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
739 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
740 (slotrank << 24) | (reg << 20) | val | 0x60000);
741 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
742
743 /* DRAM command MRS */
744 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f000);
745 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x41001);
746 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
747 (slotrank << 24) | (reg << 20) | val | 0x60000);
748 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
749
750 /* DRAM command MRS */
751 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x0f000);
752 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
753 0x1001 | (ctrl->tMOD << 16));
754 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
755 (slotrank << 24) | (reg << 20) | val | 0x60000);
756 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
757 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x80001);
758 }
759
760 static u32 make_mr0(ramctr_timing * ctrl, u8 rank)
761 {
762 u16 mr0reg, mch_cas, mch_wr;
763 static const u8 mch_wr_t[12] = { 1, 2, 3, 4, 0, 5, 0, 6, 0, 7, 0, 0 };
764
765 /* DLL Reset - self clearing - set after CLK frequency has been changed */
766 mr0reg = 0x100;
767
768 // Convert CAS to MCH register friendly
769 if (ctrl->CAS < 12) {
770 mch_cas = (u16) ((ctrl->CAS - 4) << 1);
771 } else {
772 mch_cas = (u16) (ctrl->CAS - 12);
773 mch_cas = ((mch_cas << 1) | 0x1);
774 }
775
776 // Convert tWR to MCH register friendly
777 mch_wr = mch_wr_t[ctrl->tWR - 5];
778
779 mr0reg = (mr0reg & ~0x4) | ((mch_cas & 0x1) << 2);
780 mr0reg = (mr0reg & ~0x70) | ((mch_cas & 0xe) << 3);
781 mr0reg = (mr0reg & ~0xe00) | (mch_wr << 9);
782
783 // Precharge PD - Fast (desktop) 0x1 or slow (mobile) 0x0 - mostly power-saving feature
784 mr0reg = (mr0reg & ~0x1000) | (!ctrl->mobile << 12);
785 return mr0reg;
786 }
787
788 static void dram_mr0(ramctr_timing *ctrl, u8 rank, int channel)
789 {
790 write_mrreg(ctrl, channel, rank, 0,
791 make_mr0(ctrl, rank));
792 }
793
794 static u32 encode_odt(u32 odt)
795 {
796 switch (odt) {
797 case 30:
798 return (1 << 9) | (1 << 2); // RZQ/8, RZQ/4
799 case 60:
800 return (1 << 2); // RZQ/4
801 case 120:
802 return (1 << 6); // RZQ/2
803 default:
804 case 0:
805 return 0;
806 }
807 }
808
809 static u32 make_mr1(ramctr_timing *ctrl, u8 rank, int channel)
810 {
811 odtmap odt;
812 u32 mr1reg;
813
814 odt = get_ODT(ctrl, rank, channel);
815 mr1reg = 0x2;
816
817 mr1reg |= encode_odt(odt.rttnom);
818
819 return mr1reg;
820 }
821
822 static void dram_mr1(ramctr_timing *ctrl, u8 rank, int channel)
823 {
824 u16 mr1reg;
825
826 mr1reg = make_mr1(ctrl, rank, channel);
827
828 write_mrreg(ctrl, channel, rank, 1, mr1reg);
829 }
830
831 static void dram_mr2(ramctr_timing *ctrl, u8 rank, int channel)
832 {
833 u16 pasr, cwl, mr2reg;
834 odtmap odt;
835 int srt;
836
837 pasr = 0;
838 cwl = ctrl->CWL - 5;
839 odt = get_ODT(ctrl, rank, channel);
840
841 srt = ctrl->extended_temperature_range && !ctrl->auto_self_refresh;
842
843 mr2reg = 0;
844 mr2reg = (mr2reg & ~0x7) | pasr;
845 mr2reg = (mr2reg & ~0x38) | (cwl << 3);
846 mr2reg = (mr2reg & ~0x40) | (ctrl->auto_self_refresh << 6);
847 mr2reg = (mr2reg & ~0x80) | (srt << 7);
848 mr2reg |= (odt.rttwr / 60) << 9;
849
850 write_mrreg(ctrl, channel, rank, 2, mr2reg);
851 }
852
853 static void dram_mr3(ramctr_timing *ctrl, u8 rank, int channel)
854 {
855 write_mrreg(ctrl, channel, rank, 3, 0);
856 }
857
858 void dram_mrscommands(ramctr_timing * ctrl)
859 {
860 u8 slotrank;
861 u32 reg, addr;
862 int channel;
863
864 FOR_ALL_POPULATED_CHANNELS {
865 FOR_ALL_POPULATED_RANKS {
866 // MR2
867 dram_mr2(ctrl, slotrank, channel);
868
869 // MR3
870 dram_mr3(ctrl, slotrank, channel);
871
872 // MR1
873 dram_mr1(ctrl, slotrank, channel);
874
875 // MR0
876 dram_mr0(ctrl, slotrank, channel);
877 }
878 }
879
880 /* DRAM command NOP */
881 write32(DEFAULT_MCHBAR + 0x4e20, 0x7);
882 write32(DEFAULT_MCHBAR + 0x4e30, 0xf1001);
883 write32(DEFAULT_MCHBAR + 0x4e00, 0x60002);
884 write32(DEFAULT_MCHBAR + 0x4e10, 0);
885
886 /* DRAM command ZQCL */
887 write32(DEFAULT_MCHBAR + 0x4e24, 0x1f003);
888 write32(DEFAULT_MCHBAR + 0x4e34, 0x1901001);
889 write32(DEFAULT_MCHBAR + 0x4e04, 0x60400);
890 write32(DEFAULT_MCHBAR + 0x4e14, 0x288);
891
892 /* execute command queue on all channels ? */
893 write32(DEFAULT_MCHBAR + 0x4e84, 0x40004);
894
895 // Drain
896 FOR_ALL_CHANNELS {
897 // Wait for ref drained
898 wait_428c(channel);
899 }
900
901 // Refresh enable
902 MCHBAR32(0x5030) |= 8;
903
904 FOR_ALL_POPULATED_CHANNELS {
905 addr = 0x400 * channel + 0x4020;
906 reg = MCHBAR32(addr);
907 reg &= ~0x200000;
908 MCHBAR32(addr) = reg;
909
910 wait_428c(channel);
911
912 slotrank = (ctrl->rankmap[channel] & 1) ? 0 : 2;
913
914 // Drain
915 wait_428c(channel);
916
917 /* DRAM command ZQCS */
918 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
919 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x659001);
920 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
921 (slotrank << 24) | 0x60000);
922 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
923 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x1);
924
925 // Drain
926 wait_428c(channel);
927 }
928 }
929
930 static const u32 lane_registers[] = {
931 0x0000, 0x0200, 0x0400, 0x0600,
932 0x1000, 0x1200, 0x1400, 0x1600,
933 0x0800
934 };
935
936 void program_timings(ramctr_timing * ctrl, int channel)
937 {
938 u32 reg32, reg_4024, reg_c14, reg_c18, reg_4028;
939 int lane;
940 int slotrank, slot;
941 int full_shift = 0;
942 u16 slot320c[NUM_SLOTS];
943
944 FOR_ALL_POPULATED_RANKS {
945 if (full_shift < -ctrl->timings[channel][slotrank].val_320c)
946 full_shift = -ctrl->timings[channel][slotrank].val_320c;
947 }
948
949 for (slot = 0; slot < NUM_SLOTS; slot++)
950 switch ((ctrl->rankmap[channel] >> (2 * slot)) & 3) {
951 case 0:
952 default:
953 slot320c[slot] = 0x7f;
954 break;
955 case 1:
956 slot320c[slot] =
957 ctrl->timings[channel][2 * slot + 0].val_320c +
958 full_shift;
959 break;
960 case 2:
961 slot320c[slot] =
962 ctrl->timings[channel][2 * slot + 1].val_320c +
963 full_shift;
964 break;
965 case 3:
966 slot320c[slot] =
967 (ctrl->timings[channel][2 * slot].val_320c +
968 ctrl->timings[channel][2 * slot +
969 1].val_320c) / 2 +
970 full_shift;
971 break;
972 }
973
974 /* enable CMD XOVER */
975 reg32 = get_XOVER_CMD(ctrl->rankmap[channel]);
976 reg32 |= ((slot320c[0] & 0x3f) << 6) | ((slot320c[0] & 0x40) << 9);
977 reg32 |= (slot320c[1] & 0x7f) << 18;
978 reg32 |= (full_shift & 0x3f) | ((full_shift & 0x40) << 6);
979
980 MCHBAR32(0x320c + 0x100 * channel) = reg32;
981
982 /* enable CLK XOVER */
983 reg_c14 = get_XOVER_CLK(ctrl->rankmap[channel]);
984 reg_c18 = 0;
985
986 FOR_ALL_POPULATED_RANKS {
987 int shift =
988 ctrl->timings[channel][slotrank].val_320c + full_shift;
989 int offset_val_c14;
990 if (shift < 0)
991 shift = 0;
992 offset_val_c14 = ctrl->reg_c14_offset + shift;
993 /* set CLK phase shift */
994 reg_c14 |= (offset_val_c14 & 0x3f) << (6 * slotrank);
995 reg_c18 |= ((offset_val_c14 >> 6) & 1) << slotrank;
996 }
997
998 MCHBAR32(0xc14 + channel * 0x100) = reg_c14;
999 MCHBAR32(0xc18 + channel * 0x100) = reg_c18;
1000
1001 reg_4028 = MCHBAR32(0x4028 + 0x400 * channel);
1002 reg_4028 &= 0xffff0000;
1003
1004 reg_4024 = 0;
1005
1006 FOR_ALL_POPULATED_RANKS {
1007 int post_timA_min_high = 7, post_timA_max_high = 0;
1008 int pre_timA_min_high = 7, pre_timA_max_high = 0;
1009 int shift_402x = 0;
1010 int shift =
1011 ctrl->timings[channel][slotrank].val_320c + full_shift;
1012
1013 if (shift < 0)
1014 shift = 0;
1015
1016 FOR_ALL_LANES {
1017 post_timA_min_high = MIN(post_timA_min_high,
1018 (ctrl->timings[channel][slotrank].lanes[lane].
1019 timA + shift) >> 6);
1020 pre_timA_min_high = MIN(pre_timA_min_high,
1021 ctrl->timings[channel][slotrank].lanes[lane].
1022 timA >> 6);
1023 post_timA_max_high = MAX(post_timA_max_high,
1024 (ctrl->timings[channel][slotrank].lanes[lane].
1025 timA + shift) >> 6);
1026 pre_timA_max_high = MAX(pre_timA_max_high,
1027 ctrl->timings[channel][slotrank].lanes[lane].
1028 timA >> 6);
1029 }
1030
1031 if (pre_timA_max_high - pre_timA_min_high <
1032 post_timA_max_high - post_timA_min_high)
1033 shift_402x = +1;
1034 else if (pre_timA_max_high - pre_timA_min_high >
1035 post_timA_max_high - post_timA_min_high)
1036 shift_402x = -1;
1037
1038 reg_4028 |=
1039 (ctrl->timings[channel][slotrank].val_4028 + shift_402x -
1040 post_timA_min_high) << (4 * slotrank);
1041 reg_4024 |=
1042 (ctrl->timings[channel][slotrank].val_4024 +
1043 shift_402x) << (8 * slotrank);
1044
1045 FOR_ALL_LANES {
1046 MCHBAR32(lane_registers[lane] + 0x10 + 0x100 * channel +
1047 4 * slotrank)
1048 =
1049 (((ctrl->timings[channel][slotrank].lanes[lane].
1050 timA + shift) & 0x3f)
1051 |
1052 ((ctrl->timings[channel][slotrank].lanes[lane].
1053 rising + shift) << 8)
1054 |
1055 (((ctrl->timings[channel][slotrank].lanes[lane].
1056 timA + shift -
1057 (post_timA_min_high << 6)) & 0x1c0) << 10)
1058 | ((ctrl->timings[channel][slotrank].lanes[lane].
1059 falling + shift) << 20));
1060
1061 MCHBAR32(lane_registers[lane] + 0x20 + 0x100 * channel +
1062 4 * slotrank)
1063 =
1064 (((ctrl->timings[channel][slotrank].lanes[lane].
1065 timC + shift) & 0x3f)
1066 |
1067 (((ctrl->timings[channel][slotrank].lanes[lane].
1068 timB + shift) & 0x3f) << 8)
1069 |
1070 (((ctrl->timings[channel][slotrank].lanes[lane].
1071 timB + shift) & 0x1c0) << 9)
1072 |
1073 (((ctrl->timings[channel][slotrank].lanes[lane].
1074 timC + shift) & 0x40) << 13));
1075 }
1076 }
1077 MCHBAR32(0x4024 + 0x400 * channel) = reg_4024;
1078 MCHBAR32(0x4028 + 0x400 * channel) = reg_4028;
1079 }
1080
1081 static void test_timA(ramctr_timing * ctrl, int channel, int slotrank)
1082 {
1083 wait_428c(channel);
1084
1085 /* DRAM command MRS
1086 * write MR3 MPR enable
1087 * in this mode only RD and RDA are allowed
1088 * all reads return a predefined pattern */
1089 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f000);
1090 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1091 (0xc01 | (ctrl->tMOD << 16)));
1092 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1093 (slotrank << 24) | 0x360004);
1094 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
1095
1096 /* DRAM command RD */
1097 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f105);
1098 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x4040c01);
1099 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel, (slotrank << 24));
1100 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
1101
1102 /* DRAM command RD */
1103 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
1104 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
1105 0x100f | ((ctrl->CAS + 36) << 16));
1106 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
1107 (slotrank << 24) | 0x60000);
1108 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
1109
1110 /* DRAM command MRS
1111 * write MR3 MPR disable */
1112 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f000);
1113 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
1114 (0xc01 | (ctrl->tMOD << 16)));
1115 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
1116 (slotrank << 24) | 0x360000);
1117 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
1118
1119 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
1120
1121 wait_428c(channel);
1122 }
1123
1124 static int does_lane_work(ramctr_timing * ctrl, int channel, int slotrank,
1125 int lane)
1126 {
1127 u32 timA = ctrl->timings[channel][slotrank].lanes[lane].timA;
1128 return ((read32
1129 (DEFAULT_MCHBAR + lane_registers[lane] + channel * 0x100 + 4 +
1130 ((timA / 32) & 1) * 4)
1131 >> (timA % 32)) & 1);
1132 }
1133
1134 struct run {
1135 int middle;
1136 int end;
1137 int start;
1138 int all;
1139 int length;
1140 };
1141
1142 static struct run get_longest_zero_run(int *seq, int sz)
1143 {
1144 int i, ls;
1145 int bl = 0, bs = 0;
1146 struct run ret;
1147
1148 ls = 0;
1149 for (i = 0; i < 2 * sz; i++)
1150 if (seq[i % sz]) {
1151 if (i - ls > bl) {
1152 bl = i - ls;
1153 bs = ls;
1154 }
1155 ls = i + 1;
1156 }
1157 if (bl == 0) {
1158 ret.middle = sz / 2;
1159 ret.start = 0;
1160 ret.end = sz;
1161 ret.all = 1;
1162 return ret;
1163 }
1164
1165 ret.start = bs % sz;
1166 ret.end = (bs + bl - 1) % sz;
1167 ret.middle = (bs + (bl - 1) / 2) % sz;
1168 ret.length = bl;
1169 ret.all = 0;
1170
1171 return ret;
1172 }
1173
1174 static void discover_timA_coarse(ramctr_timing * ctrl, int channel,
1175 int slotrank, int *upperA)
1176 {
1177 int timA;
1178 int statistics[NUM_LANES][128];
1179 int lane;
1180
1181 for (timA = 0; timA < 128; timA++) {
1182 FOR_ALL_LANES {
1183 ctrl->timings[channel][slotrank].lanes[lane].timA = timA;
1184 }
1185 program_timings(ctrl, channel);
1186
1187 test_timA(ctrl, channel, slotrank);
1188
1189 FOR_ALL_LANES {
1190 statistics[lane][timA] =
1191 !does_lane_work(ctrl, channel, slotrank, lane);
1192 }
1193 }
1194 FOR_ALL_LANES {
1195 struct run rn = get_longest_zero_run(statistics[lane], 128);
1196 ctrl->timings[channel][slotrank].lanes[lane].timA = rn.middle;
1197 upperA[lane] = rn.end;
1198 if (upperA[lane] < rn.middle)
1199 upperA[lane] += 128;
1200 printram("timA: %d, %d, %d: 0x%02x-0x%02x-0x%02x\n",
1201 channel, slotrank, lane, rn.start, rn.middle, rn.end);
1202 }
1203 }
1204
1205 static void discover_timA_fine(ramctr_timing * ctrl, int channel, int slotrank,
1206 int *upperA)
1207 {
1208 int timA_delta;
1209 int statistics[NUM_LANES][51];
1210 int lane, i;
1211
1212 memset(statistics, 0, sizeof(statistics));
1213
1214 for (timA_delta = -25; timA_delta <= 25; timA_delta++) {
1215 FOR_ALL_LANES ctrl->timings[channel][slotrank].lanes[lane].
1216 timA = upperA[lane] + timA_delta + 0x40;
1217 program_timings(ctrl, channel);
1218
1219 for (i = 0; i < 100; i++) {
1220 test_timA(ctrl, channel, slotrank);
1221 FOR_ALL_LANES {
1222 statistics[lane][timA_delta + 25] +=
1223 does_lane_work(ctrl, channel, slotrank,
1224 lane);
1225 }
1226 }
1227 }
1228 FOR_ALL_LANES {
1229 int last_zero, first_all;
1230
1231 for (last_zero = -25; last_zero <= 25; last_zero++)
1232 if (statistics[lane][last_zero + 25])
1233 break;
1234 last_zero--;
1235 for (first_all = -25; first_all <= 25; first_all++)
1236 if (statistics[lane][first_all + 25] == 100)
1237 break;
1238
1239 printram("lane %d: %d, %d\n", lane, last_zero,
1240 first_all);
1241
1242 ctrl->timings[channel][slotrank].lanes[lane].timA =
1243 (last_zero + first_all) / 2 + upperA[lane];
1244 printram("Aval: %d, %d, %d: %x\n", channel, slotrank,
1245 lane, ctrl->timings[channel][slotrank].lanes[lane].timA);
1246 }
1247 }
1248
1249 static int discover_402x(ramctr_timing *ctrl, int channel, int slotrank,
1250 int *upperA)
1251 {
1252 int works[NUM_LANES];
1253 int lane;
1254 while (1) {
1255 int all_works = 1, some_works = 0;
1256 program_timings(ctrl, channel);
1257 test_timA(ctrl, channel, slotrank);
1258 FOR_ALL_LANES {
1259 works[lane] =
1260 !does_lane_work(ctrl, channel, slotrank, lane);
1261 if (works[lane])
1262 some_works = 1;
1263 else
1264 all_works = 0;
1265 }
1266 if (all_works)
1267 return 0;
1268 if (!some_works) {
1269 if (ctrl->timings[channel][slotrank].val_4024 < 2) {
1270 printk(BIOS_EMERG, "402x discovery failed (1): %d, %d\n",
1271 channel, slotrank);
1272 return MAKE_ERR;
1273 }
1274 ctrl->timings[channel][slotrank].val_4024 -= 2;
1275 printram("4024 -= 2;\n");
1276 continue;
1277 }
1278 ctrl->timings[channel][slotrank].val_4028 += 2;
1279 printram("4028 += 2;\n");
1280 if (ctrl->timings[channel][slotrank].val_4028 >= 0x10) {
1281 printk(BIOS_EMERG, "402x discovery failed (2): %d, %d\n",
1282 channel, slotrank);
1283 return MAKE_ERR;
1284 }
1285 FOR_ALL_LANES if (works[lane]) {
1286 ctrl->timings[channel][slotrank].lanes[lane].timA +=
1287 128;
1288 upperA[lane] += 128;
1289 printram("increment %d, %d, %d\n", channel,
1290 slotrank, lane);
1291 }
1292 }
1293 return 0;
1294 }
1295
1296 struct timA_minmax {
1297 int timA_min_high, timA_max_high;
1298 };
1299
1300 static void pre_timA_change(ramctr_timing * ctrl, int channel, int slotrank,
1301 struct timA_minmax *mnmx)
1302 {
1303 int lane;
1304 mnmx->timA_min_high = 7;
1305 mnmx->timA_max_high = 0;
1306
1307 FOR_ALL_LANES {
1308 if (mnmx->timA_min_high >
1309 (ctrl->timings[channel][slotrank].lanes[lane].timA >> 6))
1310 mnmx->timA_min_high =
1311 (ctrl->timings[channel][slotrank].lanes[lane].
1312 timA >> 6);
1313 if (mnmx->timA_max_high <
1314 (ctrl->timings[channel][slotrank].lanes[lane].timA >> 6))
1315 mnmx->timA_max_high =
1316 (ctrl->timings[channel][slotrank].lanes[lane].
1317 timA >> 6);
1318 }
1319 }
1320
1321 static void post_timA_change(ramctr_timing * ctrl, int channel, int slotrank,
1322 struct timA_minmax *mnmx)
1323 {
1324 struct timA_minmax post;
1325 int shift_402x = 0;
1326
1327 /* Get changed maxima. */
1328 pre_timA_change(ctrl, channel, slotrank, &post);
1329
1330 if (mnmx->timA_max_high - mnmx->timA_min_high <
1331 post.timA_max_high - post.timA_min_high)
1332 shift_402x = +1;
1333 else if (mnmx->timA_max_high - mnmx->timA_min_high >
1334 post.timA_max_high - post.timA_min_high)
1335 shift_402x = -1;
1336 else
1337 shift_402x = 0;
1338
1339 ctrl->timings[channel][slotrank].val_4028 += shift_402x;
1340 ctrl->timings[channel][slotrank].val_4024 += shift_402x;
1341 printram("4024 += %d;\n", shift_402x);
1342 printram("4028 += %d;\n", shift_402x);
1343 }
1344
1345 /* Compensate the skew between DQS and DQs.
1346 * To ease PCB design a small skew between Data Strobe signals and
1347 * Data Signals is allowed.
1348 * The controller has to measure and compensate this skew for every byte-lane.
1349 * By delaying either all DQs signals or DQS signal, a full phase
1350 * shift can be introduced.
1351 * It is assumed that one byte-lane's DQs signals have the same routing delay.
1352 *
1353 * To measure the actual skew, the DRAM is placed in "read leveling" mode.
1354 * In read leveling mode the DRAM-chip outputs an alternating periodic pattern.
1355 * The memory controller iterates over all possible values to do a full phase shift
1356 * and issues read commands.
1357 * With DQS and DQs in phase the data read is expected to alternate on every byte:
1358 * 0xFF 0x00 0xFF ...
1359 * Once the controller has detected this pattern a bit in the result register is
1360 * set for the current phase shift.
1361 */
1362 int read_training(ramctr_timing * ctrl)
1363 {
1364 int channel, slotrank, lane;
1365 int err;
1366
1367 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
1368 int all_high, some_high;
1369 int upperA[NUM_LANES];
1370 struct timA_minmax mnmx;
1371
1372 wait_428c(channel);
1373
1374 /* DRAM command PREA */
1375 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
1376 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1377 0xc01 | (ctrl->tRP << 16));
1378 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1379 (slotrank << 24) | 0x60400);
1380 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
1381 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
1382
1383 write32(DEFAULT_MCHBAR + 0x3400, (slotrank << 2) | 0x8001);
1384
1385 ctrl->timings[channel][slotrank].val_4028 = 4;
1386 ctrl->timings[channel][slotrank].val_4024 = 55;
1387 program_timings(ctrl, channel);
1388
1389 discover_timA_coarse(ctrl, channel, slotrank, upperA);
1390
1391 all_high = 1;
1392 some_high = 0;
1393 FOR_ALL_LANES {
1394 if (ctrl->timings[channel][slotrank].lanes[lane].
1395 timA >= 0x40)
1396 some_high = 1;
1397 else
1398 all_high = 0;
1399 }
1400
1401 if (all_high) {
1402 ctrl->timings[channel][slotrank].val_4028--;
1403 printram("4028--;\n");
1404 FOR_ALL_LANES {
1405 ctrl->timings[channel][slotrank].lanes[lane].
1406 timA -= 0x40;
1407 upperA[lane] -= 0x40;
1408
1409 }
1410 } else if (some_high) {
1411 ctrl->timings[channel][slotrank].val_4024++;
1412 ctrl->timings[channel][slotrank].val_4028++;
1413 printram("4024++;\n");
1414 printram("4028++;\n");
1415 }
1416
1417 program_timings(ctrl, channel);
1418
1419 pre_timA_change(ctrl, channel, slotrank, &mnmx);
1420
1421 err = discover_402x(ctrl, channel, slotrank, upperA);
1422 if (err)
1423 return err;
1424
1425 post_timA_change(ctrl, channel, slotrank, &mnmx);
1426 pre_timA_change(ctrl, channel, slotrank, &mnmx);
1427
1428 discover_timA_fine(ctrl, channel, slotrank, upperA);
1429
1430 post_timA_change(ctrl, channel, slotrank, &mnmx);
1431 pre_timA_change(ctrl, channel, slotrank, &mnmx);
1432
1433 FOR_ALL_LANES {
1434 ctrl->timings[channel][slotrank].lanes[lane].timA -= mnmx.timA_min_high * 0x40;
1435 }
1436 ctrl->timings[channel][slotrank].val_4028 -= mnmx.timA_min_high;
1437 printram("4028 -= %d;\n", mnmx.timA_min_high);
1438
1439 post_timA_change(ctrl, channel, slotrank, &mnmx);
1440
1441 printram("4/8: %d, %d, %x, %x\n", channel, slotrank,
1442 ctrl->timings[channel][slotrank].val_4024,
1443 ctrl->timings[channel][slotrank].val_4028);
1444
1445 printram("final results:\n");
1446 FOR_ALL_LANES
1447 printram("Aval: %d, %d, %d: %x\n", channel, slotrank,
1448 lane,
1449 ctrl->timings[channel][slotrank].lanes[lane].timA);
1450
1451 write32(DEFAULT_MCHBAR + 0x3400, 0);
1452
1453 toggle_io_reset();
1454 }
1455
1456 FOR_ALL_POPULATED_CHANNELS {
1457 program_timings(ctrl, channel);
1458 }
1459 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
1460 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel
1461 + 4 * lane, 0);
1462 }
1463 return 0;
1464 }
1465
1466 static void test_timC(ramctr_timing * ctrl, int channel, int slotrank)
1467 {
1468 int lane;
1469
1470 FOR_ALL_LANES {
1471 write32(DEFAULT_MCHBAR + 0x4340 + 0x400 * channel + 4 * lane, 0);
1472 read32(DEFAULT_MCHBAR + 0x4140 + 0x400 * channel + 4 * lane);
1473 }
1474
1475 wait_428c(channel);
1476
1477 /* DRAM command ACT */
1478 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
1479 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1480 (max((ctrl->tFAW >> 2) + 1, ctrl->tRRD) << 10)
1481 | 4 | (ctrl->tRCD << 16));
1482
1483 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1484 (slotrank << 24) | (6 << 16));
1485
1486 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x244);
1487
1488 /* DRAM command NOP */
1489 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f207);
1490 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x8041001);
1491 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
1492 (slotrank << 24) | 8);
1493 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x3e0);
1494
1495 /* DRAM command WR */
1496 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f201);
1497 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel, 0x80411f4);
1498 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel, (slotrank << 24));
1499 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x242);
1500
1501 /* DRAM command NOP */
1502 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f207);
1503 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
1504 0x8000c01 | ((ctrl->CWL + ctrl->tWTR + 5) << 16));
1505 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
1506 (slotrank << 24) | 8);
1507 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x3e0);
1508
1509 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
1510
1511 wait_428c(channel);
1512
1513 /* DRAM command PREA */
1514 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
1515 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1516 0xc01 | (ctrl->tRP << 16));
1517 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1518 (slotrank << 24) | 0x60400);
1519 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x240);
1520
1521 /* DRAM command ACT */
1522 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f006);
1523 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
1524 (max(ctrl->tRRD, (ctrl->tFAW >> 2) + 1) << 10)
1525 | 8 | (ctrl->CAS << 16));
1526
1527 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
1528 (slotrank << 24) | 0x60000);
1529
1530 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x244);
1531
1532 /* DRAM command RD */
1533 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
1534 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
1535 0x40011f4 | (max(ctrl->tRTP, 8) << 16));
1536 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel, (slotrank << 24));
1537 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x242);
1538
1539 /* DRAM command PREA */
1540 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f002);
1541 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
1542 0xc01 | (ctrl->tRP << 16));
1543 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
1544 (slotrank << 24) | 0x60400);
1545 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x240);
1546 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
1547 wait_428c(channel);
1548 }
1549
1550 static int discover_timC(ramctr_timing *ctrl, int channel, int slotrank)
1551 {
1552 int timC;
1553 int statistics[NUM_LANES][MAX_TIMC + 1];
1554 int lane;
1555
1556 wait_428c(channel);
1557
1558 /* DRAM command PREA */
1559 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
1560 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1561 0xc01 | (ctrl->tRP << 16));
1562 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1563 (slotrank << 24) | 0x60400);
1564 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x240);
1565 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
1566
1567 for (timC = 0; timC <= MAX_TIMC; timC++) {
1568 FOR_ALL_LANES ctrl->timings[channel][slotrank].lanes[lane].
1569 timC = timC;
1570 program_timings(ctrl, channel);
1571
1572 test_timC(ctrl, channel, slotrank);
1573
1574 FOR_ALL_LANES {
1575 statistics[lane][timC] =
1576 read32(DEFAULT_MCHBAR + 0x4340 + 4 * lane +
1577 0x400 * channel);
1578 }
1579 }
1580 FOR_ALL_LANES {
1581 struct run rn =
1582 get_longest_zero_run(statistics[lane], MAX_TIMC + 1);
1583 ctrl->timings[channel][slotrank].lanes[lane].timC = rn.middle;
1584 if (rn.all) {
1585 printk(BIOS_EMERG, "timC discovery failed: %d, %d, %d\n",
1586 channel, slotrank, lane);
1587 return MAKE_ERR;
1588 }
1589 printram("timC: %d, %d, %d: 0x%02x-0x%02x-0x%02x\n",
1590 channel, slotrank, lane, rn.start, rn.middle, rn.end);
1591 }
1592 return 0;
1593 }
1594
1595 static int get_precedening_channels(ramctr_timing * ctrl, int target_channel)
1596 {
1597 int channel, ret = 0;
1598 FOR_ALL_POPULATED_CHANNELS if (channel < target_channel)
1599 ret++;
1600 return ret;
1601 }
1602
1603 static void fill_pattern0(ramctr_timing * ctrl, int channel, u32 a, u32 b)
1604 {
1605 unsigned j;
1606 unsigned channel_offset =
1607 get_precedening_channels(ctrl, channel) * 0x40;
1608 for (j = 0; j < 16; j++)
1609 write32((void *)(0x04000000 + channel_offset + 4 * j), j & 2 ? b : a);
1610 sfence();
1611 }
1612
1613 static int num_of_channels(const ramctr_timing * ctrl)
1614 {
1615 int ret = 0;
1616 int channel;
1617 FOR_ALL_POPULATED_CHANNELS ret++;
1618 return ret;
1619 }
1620
1621 static void fill_pattern1(ramctr_timing * ctrl, int channel)
1622 {
1623 unsigned j;
1624 unsigned channel_offset =
1625 get_precedening_channels(ctrl, channel) * 0x40;
1626 unsigned channel_step = 0x40 * num_of_channels(ctrl);
1627 for (j = 0; j < 16; j++)
1628 write32((void *)(0x04000000 + channel_offset + j * 4), 0xffffffff);
1629 for (j = 0; j < 16; j++)
1630 write32((void *)(0x04000000 + channel_offset + channel_step + j * 4), 0);
1631 sfence();
1632 }
1633
1634 static void precharge(ramctr_timing * ctrl)
1635 {
1636 int channel, slotrank, lane;
1637
1638 FOR_ALL_POPULATED_CHANNELS {
1639 FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
1640 ctrl->timings[channel][slotrank].lanes[lane].falling =
1641 16;
1642 ctrl->timings[channel][slotrank].lanes[lane].rising =
1643 16;
1644 }
1645
1646 program_timings(ctrl, channel);
1647
1648 FOR_ALL_POPULATED_RANKS {
1649 wait_428c(channel);
1650
1651 /* DRAM command MRS
1652 * write MR3 MPR enable
1653 * in this mode only RD and RDA are allowed
1654 * all reads return a predefined pattern */
1655 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
1656 0x1f000);
1657 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1658 0xc01 | (ctrl->tMOD << 16));
1659 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1660 (slotrank << 24) | 0x360004);
1661 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
1662
1663 /* DRAM command RD */
1664 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
1665 0x1f105);
1666 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
1667 0x4041003);
1668 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
1669 (slotrank << 24) | 0);
1670 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
1671
1672 /* DRAM command RD */
1673 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
1674 0x1f105);
1675 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
1676 0x1001 | ((ctrl->CAS + 8) << 16));
1677 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
1678 (slotrank << 24) | 0x60000);
1679 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
1680
1681 /* DRAM command MRS
1682 * write MR3 MPR disable */
1683 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
1684 0x1f000);
1685 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
1686 0xc01 | (ctrl->tMOD << 16));
1687 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
1688 (slotrank << 24) | 0x360000);
1689 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
1690 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
1691 0xc0001);
1692
1693 wait_428c(channel);
1694 }
1695
1696 FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
1697 ctrl->timings[channel][slotrank].lanes[lane].falling =
1698 48;
1699 ctrl->timings[channel][slotrank].lanes[lane].rising =
1700 48;
1701 }
1702
1703 program_timings(ctrl, channel);
1704
1705 FOR_ALL_POPULATED_RANKS {
1706 wait_428c(channel);
1707 /* DRAM command MRS
1708 * write MR3 MPR enable
1709 * in this mode only RD and RDA are allowed
1710 * all reads return a predefined pattern */
1711 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
1712 0x1f000);
1713 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1714 0xc01 | (ctrl->tMOD << 16));
1715 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1716 (slotrank << 24) | 0x360004);
1717 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
1718
1719 /* DRAM command RD */
1720 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
1721 0x1f105);
1722 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
1723 0x4041003);
1724 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
1725 (slotrank << 24) | 0);
1726 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
1727
1728 /* DRAM command RD */
1729 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
1730 0x1f105);
1731 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
1732 0x1001 | ((ctrl->CAS + 8) << 16));
1733 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
1734 (slotrank << 24) | 0x60000);
1735 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
1736
1737 /* DRAM command MRS
1738 * write MR3 MPR disable */
1739 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
1740 0x1f000);
1741 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
1742 0xc01 | (ctrl->tMOD << 16));
1743
1744 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
1745 (slotrank << 24) | 0x360000);
1746 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
1747
1748 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
1749 0xc0001);
1750 wait_428c(channel);
1751 }
1752 }
1753 }
1754
1755 static void test_timB(ramctr_timing * ctrl, int channel, int slotrank)
1756 {
1757 /* enable DQs on this slotrank */
1758 write_mrreg(ctrl, channel, slotrank, 1,
1759 0x80 | make_mr1(ctrl, slotrank, channel));
1760
1761 wait_428c(channel);
1762 /* DRAM command NOP */
1763 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f207);
1764 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1765 0x8000c01 | ((ctrl->CWL + ctrl->tWLO) << 16));
1766 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1767 8 | (slotrank << 24));
1768 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
1769
1770 /* DRAM command NOP */
1771 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f107);
1772 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
1773 0x4000c01 | ((ctrl->CAS + 38) << 16));
1774 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
1775 (slotrank << 24) | 4);
1776 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
1777
1778 write32(DEFAULT_MCHBAR + 0x400 * channel + 0x4284, 0x40001);
1779 wait_428c(channel);
1780
1781 /* disable DQs on this slotrank */
1782 write_mrreg(ctrl, channel, slotrank, 1,
1783 0x1080 | make_mr1(ctrl, slotrank, channel));
1784 }
1785
1786 static int discover_timB(ramctr_timing *ctrl, int channel, int slotrank)
1787 {
1788 int timB;
1789 int statistics[NUM_LANES][128];
1790 int lane;
1791
1792 write32(DEFAULT_MCHBAR + 0x3400, 0x108052 | (slotrank << 2));
1793
1794 for (timB = 0; timB < 128; timB++) {
1795 FOR_ALL_LANES {
1796 ctrl->timings[channel][slotrank].lanes[lane].timB = timB;
1797 }
1798 program_timings(ctrl, channel);
1799
1800 test_timB(ctrl, channel, slotrank);
1801
1802 FOR_ALL_LANES {
1803 statistics[lane][timB] =
1804 !((read32
1805 (DEFAULT_MCHBAR + lane_registers[lane] +
1806 channel * 0x100 + 4 + ((timB / 32) & 1) * 4)
1807 >> (timB % 32)) & 1);
1808 }
1809 }
1810 FOR_ALL_LANES {
1811 struct run rn = get_longest_zero_run(statistics[lane], 128);
1812 /* timC is a direct function of timB's 6 LSBs.
1813 * Some tests increments the value of timB by a small value,
1814 * which might cause the 6bit value to overflow, if it's close
1815 * to 0x3F. Increment the value by a small offset if it's likely
1816 * to overflow, to make sure it won't overflow while running
1817 * tests and bricks the system due to a non matching timC.
1818 *
1819 * TODO: find out why some tests (edge write discovery)
1820 * increment timB. */
1821 if ((rn.start & 0x3F) == 0x3E)
1822 rn.start += 2;
1823 else if ((rn.start & 0x3F) == 0x3F)
1824 rn.start += 1;
1825 ctrl->timings[channel][slotrank].lanes[lane].timB = rn.start;
1826 if (rn.all) {
1827 printk(BIOS_EMERG, "timB discovery failed: %d, %d, %d\n",
1828 channel, slotrank, lane);
1829 return MAKE_ERR;
1830 }
1831 printram("timB: %d, %d, %d: 0x%02x-0x%02x-0x%02x\n",
1832 channel, slotrank, lane, rn.start, rn.middle, rn.end);
1833 }
1834 return 0;
1835 }
1836
1837 static int get_timB_high_adjust(u64 val)
1838 {
1839 int i;
1840
1841 /* good */
1842 if (val == 0xffffffffffffffffLL)
1843 return 0;
1844
1845 if (val >= 0xf000000000000000LL) {
1846 /* needs negative adjustment */
1847 for (i = 0; i < 8; i++)
1848 if (val << (8 * (7 - i) + 4))
1849 return -i;
1850 } else {
1851 /* needs positive adjustment */
1852 for (i = 0; i < 8; i++)
1853 if (val >> (8 * (7 - i) + 4))
1854 return i;
1855 }
1856 return 8;
1857 }
1858
1859 static void adjust_high_timB(ramctr_timing * ctrl)
1860 {
1861 int channel, slotrank, lane, old;
1862 write32(DEFAULT_MCHBAR + 0x3400, 0x200);
1863 FOR_ALL_POPULATED_CHANNELS {
1864 fill_pattern1(ctrl, channel);
1865 write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 1);
1866 }
1867 FOR_ALL_POPULATED_CHANNELS FOR_ALL_POPULATED_RANKS {
1868
1869 write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x10001);
1870
1871 wait_428c(channel);
1872
1873 /* DRAM command ACT */
1874 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
1875 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1876 0xc01 | (ctrl->tRCD << 16));
1877 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1878 (slotrank << 24) | 0x60000);
1879 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
1880
1881 /* DRAM command NOP */
1882 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f207);
1883 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x8040c01);
1884 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
1885 (slotrank << 24) | 0x8);
1886 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x3e0);
1887
1888 /* DRAM command WR */
1889 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f201);
1890 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel, 0x8041003);
1891 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
1892 (slotrank << 24));
1893 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x3e2);
1894
1895 /* DRAM command NOP */
1896 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f207);
1897 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
1898 0x8000c01 | ((ctrl->CWL + ctrl->tWTR + 5) << 16));
1899 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
1900 (slotrank << 24) | 0x8);
1901 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x3e0);
1902
1903 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
1904
1905 wait_428c(channel);
1906
1907 /* DRAM command PREA */
1908 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
1909 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1910 0xc01 | ((ctrl->tRP) << 16));
1911 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1912 (slotrank << 24) | 0x60400);
1913 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x240);
1914
1915 /* DRAM command ACT */
1916 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f006);
1917 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
1918 0xc01 | ((ctrl->tRCD) << 16));
1919 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
1920 (slotrank << 24) | 0x60000);
1921 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
1922
1923 /* DRAM command RD */
1924 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x3f105);
1925 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
1926 0x4000c01 |
1927 ((ctrl->tRP +
1928 ctrl->timings[channel][slotrank].val_4024 +
1929 ctrl->timings[channel][slotrank].val_4028) << 16));
1930 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
1931 (slotrank << 24) | 0x60008);
1932 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
1933
1934 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x80001);
1935 wait_428c(channel);
1936 FOR_ALL_LANES {
1937 u64 res =
1938 read32(DEFAULT_MCHBAR + lane_registers[lane] +
1939 0x100 * channel + 4);
1940 res |=
1941 ((u64) read32(DEFAULT_MCHBAR + lane_registers[lane] +
1942 0x100 * channel + 8)) << 32;
1943 old = ctrl->timings[channel][slotrank].lanes[lane].timB;
1944 ctrl->timings[channel][slotrank].lanes[lane].timB +=
1945 get_timB_high_adjust(res) * 64;
1946
1947 printram("High adjust %d:%016llx\n", lane, res);
1948 printram("Bval+: %d, %d, %d, %x -> %x\n", channel,
1949 slotrank, lane, old,
1950 ctrl->timings[channel][slotrank].lanes[lane].
1951 timB);
1952 }
1953 }
1954 write32(DEFAULT_MCHBAR + 0x3400, 0);
1955 }
1956
1957 static void write_op(ramctr_timing * ctrl, int channel)
1958 {
1959 int slotrank;
1960
1961 wait_428c(channel);
1962
1963 /* choose an existing rank. */
1964 slotrank = !(ctrl->rankmap[channel] & 1) ? 2 : 0;
1965
1966 /* DRAM command ACT */
1967 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
1968 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
1969
1970 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1971 (slotrank << 24) | 0x60000);
1972
1973 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
1974
1975 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
1976 wait_428c(channel);
1977 }
1978
1979 /* Compensate the skew between CMD/ADDR/CLK and DQ/DQS lanes.
1980 * DDR3 adopted the fly-by topology. The data and strobes signals reach
1981 * the chips at different times with respect to command, address and
1982 * clock signals.
1983 * By delaying either all DQ/DQs or all CMD/ADDR/CLK signals, a full phase
1984 * shift can be introduced.
1985 * It is assumed that the CLK/ADDR/CMD signals have the same routing delay.
1986 *
1987 * To find the required phase shift the DRAM is placed in "write leveling" mode.
1988 * In this mode the DRAM-chip samples the CLK on every DQS edge and feeds back the
1989 * sampled value on the data lanes (DQs).
1990 */
1991 int write_training(ramctr_timing * ctrl)
1992 {
1993 int channel, slotrank, lane;
1994 int err;
1995
1996 FOR_ALL_POPULATED_CHANNELS
1997 write32(DEFAULT_MCHBAR + 0x4008 + 0x400 * channel,
1998 read32(DEFAULT_MCHBAR + 0x4008 +
1999 0x400 * channel) | 0x8000000);
2000
2001 FOR_ALL_POPULATED_CHANNELS {
2002 write_op(ctrl, channel);
2003 write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
2004 read32(DEFAULT_MCHBAR + 0x4020 +
2005 0x400 * channel) | 0x200000);
2006 }
2007
2008 /* refresh disable */
2009 write32(DEFAULT_MCHBAR + 0x5030, read32(DEFAULT_MCHBAR + 0x5030) & ~8);
2010 FOR_ALL_POPULATED_CHANNELS {
2011 write_op(ctrl, channel);
2012 }
2013
2014 /* enable write leveling on all ranks
2015 * disable all DQ outputs
2016 * only NOP is allowed in this mode */
2017 FOR_ALL_CHANNELS
2018 FOR_ALL_POPULATED_RANKS
2019 write_mrreg(ctrl, channel, slotrank, 1,
2020 make_mr1(ctrl, slotrank, channel) | 0x1080);
2021
2022 write32(DEFAULT_MCHBAR + 0x3400, 0x108052);
2023
2024 toggle_io_reset();
2025
2026 /* set any valid value for timB, it gets corrected later */
2027 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
2028 err = discover_timB(ctrl, channel, slotrank);
2029 if (err)
2030 return err;
2031 }
2032
2033 /* disable write leveling on all ranks */
2034 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS
2035 write_mrreg(ctrl, channel,
2036 slotrank, 1, make_mr1(ctrl, slotrank, channel));
2037
2038 write32(DEFAULT_MCHBAR + 0x3400, 0);
2039
2040 FOR_ALL_POPULATED_CHANNELS
2041 wait_428c(channel);
2042
2043 /* refresh enable */
2044 write32(DEFAULT_MCHBAR + 0x5030, read32(DEFAULT_MCHBAR + 0x5030) | 8);
2045
2046 FOR_ALL_POPULATED_CHANNELS {
2047 write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
2048 ~0x00200000 & read32(DEFAULT_MCHBAR + 0x4020 +
2049 0x400 * channel));
2050 read32(DEFAULT_MCHBAR + 0x428c + 0x400 * channel);
2051 wait_428c(channel);
2052
2053 /* DRAM command ZQCS */
2054 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
2055 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x659001);
2056 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel, 0x60000);
2057 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
2058
2059 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
2060 wait_428c(channel);
2061 }
2062
2063 toggle_io_reset();
2064
2065 printram("CPE\n");
2066 precharge(ctrl);
2067 printram("CPF\n");
2068
2069 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2070 read32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane);
2071 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
2072 0);
2073 }
2074
2075 FOR_ALL_POPULATED_CHANNELS {
2076 fill_pattern0(ctrl, channel, 0xaaaaaaaa, 0x55555555);
2077 write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
2078 }
2079
2080 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
2081 err = discover_timC(ctrl, channel, slotrank);
2082 if (err)
2083 return err;
2084 }
2085
2086 FOR_ALL_POPULATED_CHANNELS
2087 program_timings(ctrl, channel);
2088
2089 /* measure and adjust timB timings */
2090 adjust_high_timB(ctrl);
2091
2092 FOR_ALL_POPULATED_CHANNELS
2093 program_timings(ctrl, channel);
2094
2095 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2096 read32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane);
2097 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
2098 0);
2099 }
2100 return 0;
2101 }
2102
2103 static int test_320c(ramctr_timing * ctrl, int channel, int slotrank)
2104 {
2105 struct ram_rank_timings saved_rt = ctrl->timings[channel][slotrank];
2106 int timC_delta;
2107 int lanes_ok = 0;
2108 int ctr = 0;
2109 int lane;
2110
2111 for (timC_delta = -5; timC_delta <= 5; timC_delta++) {
2112 FOR_ALL_LANES {
2113 ctrl->timings[channel][slotrank].lanes[lane].timC =
2114 saved_rt.lanes[lane].timC + timC_delta;
2115 }
2116 program_timings(ctrl, channel);
2117 FOR_ALL_LANES {
2118 write32(DEFAULT_MCHBAR + 4 * lane + 0x4f40, 0);
2119 }
2120
2121 write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
2122
2123 wait_428c(channel);
2124 /* DRAM command ACT */
2125 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
2126 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2127 ((max(ctrl->tRRD, (ctrl->tFAW >> 2) + 1)) << 10)
2128 | 8 | (ctrl->tRCD << 16));
2129
2130 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2131 (slotrank << 24) | ctr | 0x60000);
2132
2133 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x244);
2134 /* DRAM command WR */
2135 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f201);
2136 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
2137 0x8001020 | ((ctrl->CWL + ctrl->tWTR + 8) << 16));
2138 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2139 (slotrank << 24));
2140 write32(DEFAULT_MCHBAR + 0x4244 + 0x400 * channel, 0x389abcd);
2141 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x20e42);
2142
2143 /* DRAM command RD */
2144 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
2145 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
2146 0x4001020 | (max(ctrl->tRTP, 8) << 16));
2147 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
2148 (slotrank << 24));
2149 write32(DEFAULT_MCHBAR + 0x4248 + 0x400 * channel, 0x389abcd);
2150 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x20e42);
2151
2152 /* DRAM command PRE */
2153 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f002);
2154 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel, 0xf1001);
2155 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
2156 (slotrank << 24) | 0x60400);
2157 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x240);
2158
2159 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
2160 wait_428c(channel);
2161 FOR_ALL_LANES {
2162 u32 r32 =
2163 read32(DEFAULT_MCHBAR + 0x4340 + 4 * lane +
2164 0x400 * channel);
2165
2166 if (r32 == 0)
2167 lanes_ok |= 1 << lane;
2168 }
2169 ctr++;
2170 if (lanes_ok == ((1 << NUM_LANES) - 1))
2171 break;
2172 }
2173
2174 ctrl->timings[channel][slotrank] = saved_rt;
2175
2176 return lanes_ok != ((1 << NUM_LANES) - 1);
2177 }
2178
2179 #include "raminit_patterns.h"
2180
2181 static void fill_pattern5(ramctr_timing * ctrl, int channel, int patno)
2182 {
2183 unsigned i, j;
2184 unsigned channel_offset =
2185 get_precedening_channels(ctrl, channel) * 0x40;
2186 unsigned channel_step = 0x40 * num_of_channels(ctrl);
2187
2188 if (patno) {
2189 u8 base8 = 0x80 >> ((patno - 1) % 8);
2190 u32 base = base8 | (base8 << 8) | (base8 << 16) | (base8 << 24);
2191 for (i = 0; i < 32; i++) {
2192 for (j = 0; j < 16; j++) {
2193 u32 val = use_base[patno - 1][i] & (1 << (j / 2)) ? base : 0;
2194 if (invert[patno - 1][i] & (1 << (j / 2)))
2195 val = ~val;
2196 write32((void *)(0x04000000 + channel_offset + i * channel_step +
2197 j * 4), val);
2198 }
2199 }
2200
2201 } else {
2202 for (i = 0; i < sizeof(pattern) / sizeof(pattern[0]); i++) {
2203 for (j = 0; j < 16; j++)
2204 write32((void *)(0x04000000 + channel_offset + i * channel_step +
2205 j * 4), pattern[i][j]);
2206 }
2207 sfence();
2208 }
2209 }
2210
2211 static void reprogram_320c(ramctr_timing * ctrl)
2212 {
2213 int channel, slotrank;
2214
2215 FOR_ALL_POPULATED_CHANNELS {
2216 wait_428c(channel);
2217
2218 /* choose an existing rank. */
2219 slotrank = !(ctrl->rankmap[channel] & 1) ? 2 : 0;
2220
2221 /* DRAM command ZQCS */
2222 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
2223 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
2224
2225 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2226 (slotrank << 24) | 0x60000);
2227
2228 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
2229
2230 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
2231 wait_428c(channel);
2232 write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
2233 read32(DEFAULT_MCHBAR + 0x4020 +
2234 0x400 * channel) | 0x200000);
2235 }
2236
2237 /* refresh disable */
2238 write32(DEFAULT_MCHBAR + 0x5030, read32(DEFAULT_MCHBAR + 0x5030) & ~8);
2239 FOR_ALL_POPULATED_CHANNELS {
2240 wait_428c(channel);
2241
2242 /* choose an existing rank. */
2243 slotrank = !(ctrl->rankmap[channel] & 1) ? 2 : 0;
2244
2245 /* DRAM command ZQCS */
2246 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
2247 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
2248
2249 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2250 (slotrank << 24) | 0x60000);
2251
2252 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
2253
2254 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
2255 wait_428c(channel);
2256 }
2257
2258 /* jedec reset */
2259 dram_jedecreset(ctrl);
2260 /* mrs commands. */
2261 dram_mrscommands(ctrl);
2262
2263 toggle_io_reset();
2264 }
2265
2266 #define MIN_C320C_LEN 13
2267
2268 static int try_cmd_stretch(ramctr_timing *ctrl, int channel, int cmd_stretch)
2269 {
2270 struct ram_rank_timings saved_timings[NUM_CHANNELS][NUM_SLOTRANKS];
2271 int slotrank;
2272 int c320c;
2273 int stat[NUM_SLOTRANKS][256];
2274 int delta = 0;
2275
2276 printram("Trying cmd_stretch %d on channel %d\n", cmd_stretch, channel);
2277
2278 FOR_ALL_POPULATED_RANKS {
2279 saved_timings[channel][slotrank] =
2280 ctrl->timings[channel][slotrank];
2281 }
2282
2283 ctrl->cmd_stretch[channel] = cmd_stretch;
2284
2285 MCHBAR32(0x4004 + 0x400 * channel) =
2286 ctrl->tRRD
2287 | (ctrl->tRTP << 4)
2288 | (ctrl->tCKE << 8)
2289 | (ctrl->tWTR << 12)
2290 | (ctrl->tFAW << 16)
2291 | (ctrl->tWR << 24)
2292 | (ctrl->cmd_stretch[channel] << 30);
2293
2294 if (ctrl->cmd_stretch[channel] == 2)
2295 delta = 2;
2296 else if (ctrl->cmd_stretch[channel] == 0)
2297 delta = 4;
2298
2299 FOR_ALL_POPULATED_RANKS {
2300 ctrl->timings[channel][slotrank].val_4024 -= delta;
2301 }
2302
2303 for (c320c = -127; c320c <= 127; c320c++) {
2304 FOR_ALL_POPULATED_RANKS {
2305 ctrl->timings[channel][slotrank].val_320c = c320c;
2306 }
2307 program_timings(ctrl, channel);
2308 reprogram_320c(ctrl);
2309 FOR_ALL_POPULATED_RANKS {
2310 stat[slotrank][c320c + 127] =
2311 test_320c(ctrl, channel, slotrank);
2312 }
2313 }
2314 FOR_ALL_POPULATED_RANKS {
2315 struct run rn =
2316 get_longest_zero_run(stat[slotrank], 255);
2317 ctrl->timings[channel][slotrank].val_320c =
2318 rn.middle - 127;
2319 printram("cmd_stretch: %d, %d: 0x%02x-0x%02x-0x%02x\n",
2320 channel, slotrank, rn.start, rn.middle, rn.end);
2321 if (rn.all || rn.length < MIN_C320C_LEN) {
2322 FOR_ALL_POPULATED_RANKS {
2323 ctrl->timings[channel][slotrank] =
2324 saved_timings[channel][slotrank];
2325 }
2326 return MAKE_ERR;
2327 }
2328 }
2329
2330 return 0;
2331 }
2332
2333 /* Adjust CMD phase shift and try multiple command rates.
2334 * A command rate of 2T doubles the time needed for address and
2335 * command decode. */
2336 int command_training(ramctr_timing *ctrl)
2337 {
2338 int channel;
2339
2340 FOR_ALL_POPULATED_CHANNELS {
2341 fill_pattern5(ctrl, channel, 0);
2342 write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
2343 }
2344
2345 FOR_ALL_POPULATED_CHANNELS {
2346 int cmdrate, err;
2347
2348 /*
2349 * Dual DIMM per channel:
2350 * Issue: While c320c discovery seems to succeed raminit
2351 * will fail in write training.
2352 * Workaround: Skip 1T in dual DIMM mode, that's only
2353 * supported by a few DIMMs.
2354 * Only try 1T mode for XMP DIMMs that request it in dual DIMM
2355 * mode.
2356 *
2357 * Single DIMM per channel:
2358 * Try command rate 1T and 2T
2359 */
2360 cmdrate = ((ctrl->rankmap[channel] & 0x5) == 0x5);
2361 if (ctrl->tCMD)
2362 /* XMP gives the CMD rate in clock ticks, not ns */
2363 cmdrate = MIN(DIV_ROUND_UP(ctrl->tCMD, 256) - 1, 1);
2364
2365 for (; cmdrate < 2; cmdrate++) {
2366 err = try_cmd_stretch(ctrl, channel, cmdrate << 1);
2367
2368 if (!err)
2369 break;
2370 }
2371
2372 if (err) {
2373 printk(BIOS_EMERG, "c320c discovery failed\n");
2374 return err;
2375 }
2376
2377 printram("Using CMD rate %uT on channel %u\n",
2378 cmdrate + 1, channel);
2379 }
2380
2381 FOR_ALL_POPULATED_CHANNELS
2382 program_timings(ctrl, channel);
2383
2384 reprogram_320c(ctrl);
2385 return 0;
2386 }
2387
2388
2389 static int discover_edges_real(ramctr_timing *ctrl, int channel, int slotrank,
2390 int *edges)
2391 {
2392 int edge;
2393 int statistics[NUM_LANES][MAX_EDGE_TIMING + 1];
2394 int lane;
2395
2396 for (edge = 0; edge <= MAX_EDGE_TIMING; edge++) {
2397 FOR_ALL_LANES {
2398 ctrl->timings[channel][slotrank].lanes[lane].rising =
2399 edge;
2400 ctrl->timings[channel][slotrank].lanes[lane].falling =
2401 edge;
2402 }
2403 program_timings(ctrl, channel);
2404
2405 FOR_ALL_LANES {
2406 write32(DEFAULT_MCHBAR + 0x4340 + 0x400 * channel +
2407 4 * lane, 0);
2408 read32(DEFAULT_MCHBAR + 0x400 * channel + 4 * lane +
2409 0x4140);
2410 }
2411
2412 wait_428c(channel);
2413 /* DRAM command MRS
2414 * write MR3 MPR enable
2415 * in this mode only RD and RDA are allowed
2416 * all reads return a predefined pattern */
2417 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f000);
2418 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2419 (0xc01 | (ctrl->tMOD << 16)));
2420 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2421 (slotrank << 24) | 0x360004);
2422 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
2423
2424 /* DRAM command RD */
2425 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f105);
2426 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x40411f4);
2427 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2428 (slotrank << 24));
2429 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
2430
2431 /* DRAM command RD */
2432 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
2433 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
2434 0x1001 | ((ctrl->CAS + 8) << 16));
2435 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
2436 (slotrank << 24) | 0x60000);
2437 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
2438
2439 /* DRAM command MRS
2440 * MR3 disable MPR */
2441 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f000);
2442 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
2443 (0xc01 | (ctrl->tMOD << 16)));
2444 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
2445 (slotrank << 24) | 0x360000);
2446 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
2447
2448 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
2449
2450 wait_428c(channel);
2451
2452 FOR_ALL_LANES {
2453 statistics[lane][edge] =
2454 read32(DEFAULT_MCHBAR + 0x4340 + 0x400 * channel +
2455 lane * 4);
2456 }
2457 }
2458 FOR_ALL_LANES {
2459 struct run rn =
2460 get_longest_zero_run(statistics[lane], MAX_EDGE_TIMING + 1);
2461 edges[lane] = rn.middle;
2462 if (rn.all) {
2463 printk(BIOS_EMERG, "edge discovery failed: %d, %d, %d\n",
2464 channel, slotrank, lane);
2465 return MAKE_ERR;
2466 }
2467 printram("eval %d, %d, %d: %02x\n", channel, slotrank,
2468 lane, edges[lane]);
2469 }
2470 return 0;
2471 }
2472
2473 int discover_edges(ramctr_timing *ctrl)
2474 {
2475 int falling_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2476 int rising_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2477 int channel, slotrank, lane;
2478 int err;
2479
2480 write32(DEFAULT_MCHBAR + 0x3400, 0);
2481
2482 toggle_io_reset();
2483
2484 FOR_ALL_POPULATED_CHANNELS FOR_ALL_LANES {
2485 write32(DEFAULT_MCHBAR + 4 * lane +
2486 0x400 * channel + 0x4080, 0);
2487 }
2488
2489 FOR_ALL_POPULATED_CHANNELS {
2490 fill_pattern0(ctrl, channel, 0, 0);
2491 write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
2492 FOR_ALL_LANES {
2493 read32(DEFAULT_MCHBAR + 0x400 * channel +
2494 lane * 4 + 0x4140);
2495 }
2496
2497 FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2498 ctrl->timings[channel][slotrank].lanes[lane].falling =
2499 16;
2500 ctrl->timings[channel][slotrank].lanes[lane].rising =
2501 16;
2502 }
2503
2504 program_timings(ctrl, channel);
2505
2506 FOR_ALL_POPULATED_RANKS {
2507 wait_428c(channel);
2508
2509 /* DRAM command MRS
2510 * MR3 enable MPR
2511 * write MR3 MPR enable
2512 * in this mode only RD and RDA are allowed
2513 * all reads return a predefined pattern */
2514 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
2515 0x1f000);
2516 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2517 0xc01 | (ctrl->tMOD << 16));
2518 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2519 (slotrank << 24) | 0x360004);
2520 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
2521
2522 /* DRAM command RD */
2523 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
2524 0x1f105);
2525 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
2526 0x4041003);
2527 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2528 (slotrank << 24) | 0);
2529 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
2530
2531 /* DRAM command RD */
2532 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
2533 0x1f105);
2534 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
2535 0x1001 | ((ctrl->CAS + 8) << 16));
2536 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
2537 (slotrank << 24) | 0x60000);
2538 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
2539
2540 /* DRAM command MRS
2541 * MR3 disable MPR */
2542 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
2543 0x1f000);
2544 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
2545 0xc01 | (ctrl->tMOD << 16));
2546 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
2547 (slotrank << 24) | 0x360000);
2548 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
2549 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
2550 0xc0001);
2551
2552 wait_428c(channel);
2553 }
2554
2555 /* XXX: check any measured value ? */
2556
2557 FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2558 ctrl->timings[channel][slotrank].lanes[lane].falling =
2559 48;
2560 ctrl->timings[channel][slotrank].lanes[lane].rising =
2561 48;
2562 }
2563
2564 program_timings(ctrl, channel);
2565
2566 FOR_ALL_POPULATED_RANKS {
2567 wait_428c(channel);
2568
2569 /* DRAM command MRS
2570 * MR3 enable MPR
2571 * write MR3 MPR enable
2572 * in this mode only RD and RDA are allowed
2573 * all reads return a predefined pattern */
2574 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
2575 0x1f000);
2576 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2577 0xc01 | (ctrl->tMOD << 16));
2578 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2579 (slotrank << 24) | 0x360004);
2580 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
2581
2582 /* DRAM command RD */
2583 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
2584 0x1f105);
2585 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
2586 0x4041003);
2587 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2588 (slotrank << 24) | 0);
2589 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
2590
2591 /* DRAM command RD */
2592 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
2593 0x1f105);
2594 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
2595 0x1001 | ((ctrl->CAS + 8) << 16));
2596 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
2597 (slotrank << 24) | 0x60000);
2598 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
2599
2600 /* DRAM command MRS
2601 * MR3 disable MPR */
2602 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
2603 0x1f000);
2604 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
2605 0xc01 | (ctrl->tMOD << 16));
2606 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
2607 (slotrank << 24) | 0x360000);
2608 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
2609
2610 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
2611 0xc0001);
2612 wait_428c(channel);
2613 }
2614
2615 /* XXX: check any measured value ? */
2616
2617 FOR_ALL_LANES {
2618 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel +
2619 lane * 4,
2620 ~read32(DEFAULT_MCHBAR + 0x4040 +
2621 0x400 * channel + lane * 4) & 0xff);
2622 }
2623
2624 fill_pattern0(ctrl, channel, 0, 0xffffffff);
2625 write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
2626 }
2627
2628 /* FIXME: under some conditions (older chipsets?) vendor BIOS sets both edges to the same value. */
2629 write32(DEFAULT_MCHBAR + 0x4eb0, 0x300);
2630 printram("discover falling edges:\n[%x] = %x\n", 0x4eb0, 0x300);
2631
2632 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
2633 err = discover_edges_real(ctrl, channel, slotrank,
2634 falling_edges[channel][slotrank]);
2635 if (err)
2636 return err;
2637 }
2638
2639 write32(DEFAULT_MCHBAR + 0x4eb0, 0x200);
2640 printram("discover rising edges:\n[%x] = %x\n", 0x4eb0, 0x200);
2641
2642 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
2643 err = discover_edges_real(ctrl, channel, slotrank,
2644 rising_edges[channel][slotrank]);
2645 if (err)
2646 return err;
2647 }
2648
2649 write32(DEFAULT_MCHBAR + 0x4eb0, 0);
2650
2651 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2652 ctrl->timings[channel][slotrank].lanes[lane].falling =
2653 falling_edges[channel][slotrank][lane];
2654 ctrl->timings[channel][slotrank].lanes[lane].rising =
2655 rising_edges[channel][slotrank][lane];
2656 }
2657
2658 FOR_ALL_POPULATED_CHANNELS {
2659 program_timings(ctrl, channel);
2660 }
2661
2662 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2663 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
2664 0);
2665 }
2666 return 0;
2667 }
2668
2669 static int discover_edges_write_real(ramctr_timing *ctrl, int channel,
2670 int slotrank, int *edges)
2671 {
2672 int edge;
2673 u32 raw_statistics[MAX_EDGE_TIMING + 1];
2674 int statistics[MAX_EDGE_TIMING + 1];
2675 const int reg3000b24[] = { 0, 0xc, 0x2c };
2676 int lane, i;
2677 int lower[NUM_LANES];
2678 int upper[NUM_LANES];
2679 int pat;
2680
2681 FOR_ALL_LANES {
2682 lower[lane] = 0;
2683 upper[lane] = MAX_EDGE_TIMING;
2684 }
2685
2686 for (i = 0; i < 3; i++) {
2687 write32(DEFAULT_MCHBAR + 0x3000 + 0x100 * channel,
2688 reg3000b24[i] << 24);
2689 printram("[%x] = 0x%08x\n",
2690 0x3000 + 0x100 * channel, reg3000b24[i] << 24);
2691 for (pat = 0; pat < NUM_PATTERNS; pat++) {
2692 fill_pattern5(ctrl, channel, pat);
2693 write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
2694 printram("using pattern %d\n", pat);
2695 for (edge = 0; edge <= MAX_EDGE_TIMING; edge++) {
2696 FOR_ALL_LANES {
2697 ctrl->timings[channel][slotrank].lanes[lane].
2698 rising = edge;
2699 ctrl->timings[channel][slotrank].lanes[lane].
2700 falling = edge;
2701 }
2702 program_timings(ctrl, channel);
2703
2704 FOR_ALL_LANES {
2705 write32(DEFAULT_MCHBAR + 0x4340 +
2706 0x400 * channel + 4 * lane, 0);
2707 read32(DEFAULT_MCHBAR + 0x400 * channel +
2708 4 * lane + 0x4140);
2709 }
2710 wait_428c(channel);
2711
2712 /* DRAM command ACT */
2713 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
2714 0x1f006);
2715 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2716 0x4 | (ctrl->tRCD << 16)
2717 | (max(ctrl->tRRD, (ctrl->tFAW >> 2) + 1) <<
2718 10));
2719 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2720 (slotrank << 24) | 0x60000);
2721 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel,
2722 0x240);
2723
2724 /* DRAM command WR */
2725 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
2726 0x1f201);
2727 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
2728 0x8005020 | ((ctrl->tWTR + ctrl->CWL + 8) <<
2729 16));
2730 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2731 (slotrank << 24));
2732 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel,
2733 0x242);
2734
2735 /* DRAM command RD */
2736 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
2737 0x1f105);
2738 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
2739 0x4005020 | (max(ctrl->tRTP, 8) << 16));
2740 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
2741 (slotrank << 24));
2742 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel,
2743 0x242);
2744
2745 /* DRAM command PRE */
2746 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
2747 0x1f002);
2748 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
2749 0xc01 | (ctrl->tRP << 16));
2750 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
2751 (slotrank << 24) | 0x60400);
2752 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
2753
2754 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
2755 0xc0001);
2756 wait_428c(channel);
2757 FOR_ALL_LANES {
2758 read32(DEFAULT_MCHBAR + 0x4340 +
2759 0x400 * channel + lane * 4);
2760 }
2761
2762 raw_statistics[edge] =
2763 MCHBAR32(0x436c + 0x400 * channel);
2764 }
2765 FOR_ALL_LANES {
2766 struct run rn;
2767 for (edge = 0; edge <= MAX_EDGE_TIMING; edge++)
2768 statistics[edge] =
2769 ! !(raw_statistics[edge] & (1 << lane));
2770 rn = get_longest_zero_run(statistics,
2771 MAX_EDGE_TIMING + 1);
2772 printram("edges: %d, %d, %d: 0x%02x-0x%02x-0x%02x, 0x%02x-0x%02x\n",
2773 channel, slotrank, i, rn.start, rn.middle,
2774 rn.end, rn.start + ctrl->edge_offset[i],
2775 rn.end - ctrl->edge_offset[i]);
2776 lower[lane] =
2777 max(rn.start + ctrl->edge_offset[i], lower[lane]);
2778 upper[lane] =
2779 min(rn.end - ctrl->edge_offset[i], upper[lane]);
2780 edges[lane] = (lower[lane] + upper[lane]) / 2;
2781 if (rn.all || (lower[lane] > upper[lane])) {
2782 printk(BIOS_EMERG, "edge write discovery failed: %d, %d, %d\n",
2783 channel, slotrank, lane);
2784 return MAKE_ERR;
2785 }
2786 }
2787 }
2788 }
2789
2790 write32(DEFAULT_MCHBAR + 0x3000, 0);
2791 printram("CPA\n");
2792 return 0;
2793 }
2794
2795 int discover_edges_write(ramctr_timing *ctrl)
2796 {
2797 int falling_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2798 int rising_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2799 int channel, slotrank, lane;
2800 int err;
2801
2802 /* FIXME: under some conditions (older chipsets?) vendor BIOS sets both edges to the same value. */
2803 write32(DEFAULT_MCHBAR + 0x4eb0, 0x300);
2804 printram("discover falling edges write:\n[%x] = %x\n", 0x4eb0, 0x300);
2805
2806 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
2807 err = discover_edges_write_real(ctrl, channel, slotrank,
2808 falling_edges[channel][slotrank]);
2809 if (err)
2810 return err;
2811 }
2812
2813 write32(DEFAULT_MCHBAR + 0x4eb0, 0x200);
2814 printram("discover rising edges write:\n[%x] = %x\n", 0x4eb0, 0x200);
2815
2816 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
2817 err = discover_edges_write_real(ctrl, channel, slotrank,
2818 rising_edges[channel][slotrank]);
2819 if (err)
2820 return err;
2821 }
2822
2823 write32(DEFAULT_MCHBAR + 0x4eb0, 0);
2824
2825 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2826 ctrl->timings[channel][slotrank].lanes[lane].falling =
2827 falling_edges[channel][slotrank][lane];
2828 ctrl->timings[channel][slotrank].lanes[lane].rising =
2829 rising_edges[channel][slotrank][lane];
2830 }
2831
2832 FOR_ALL_POPULATED_CHANNELS
2833 program_timings(ctrl, channel);
2834
2835 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2836 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
2837 0);
2838 }
2839 return 0;
2840 }
2841
2842 static void test_timC_write(ramctr_timing *ctrl, int channel, int slotrank)
2843 {
2844 wait_428c(channel);
2845 /* DRAM command ACT */
2846 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
2847 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2848 (max((ctrl->tFAW >> 2) + 1, ctrl->tRRD)
2849 << 10) | (ctrl->tRCD << 16) | 4);
2850 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2851 (slotrank << 24) | 0x60000);
2852 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x244);
2853
2854 /* DRAM command WR */
2855 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f201);
2856 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
2857 0x80011e0 |
2858 ((ctrl->tWTR + ctrl->CWL + 8) << 16));
2859 write32(DEFAULT_MCHBAR + 0x4204 +
2860 0x400 * channel, (slotrank << 24));
2861 write32(DEFAULT_MCHBAR + 0x4214 +
2862 0x400 * channel, 0x242);
2863
2864 /* DRAM command RD */
2865 write32(DEFAULT_MCHBAR + 0x4228 +
2866 0x400 * channel, 0x1f105);
2867 write32(DEFAULT_MCHBAR + 0x4238 +
2868 0x400 * channel,
2869 0x40011e0 | (max(ctrl->tRTP, 8) << 16));
2870 write32(DEFAULT_MCHBAR + 0x4208 +
2871 0x400 * channel, (slotrank << 24));
2872 write32(DEFAULT_MCHBAR + 0x4218 +
2873 0x400 * channel, 0x242);
2874
2875 /* DRAM command PRE */
2876 write32(DEFAULT_MCHBAR + 0x422c +
2877 0x400 * channel, 0x1f002);
2878 write32(DEFAULT_MCHBAR + 0x423c +
2879 0x400 * channel,
2880 0x1001 | (ctrl->tRP << 16));
2881 write32(DEFAULT_MCHBAR + 0x420c +
2882 0x400 * channel,
2883 (slotrank << 24) | 0x60400);
2884 write32(DEFAULT_MCHBAR + 0x421c +
2885 0x400 * channel, 0);
2886
2887 write32(DEFAULT_MCHBAR + 0x4284 +
2888 0x400 * channel, 0xc0001);
2889 wait_428c(channel);
2890 }
2891
2892 int discover_timC_write(ramctr_timing *ctrl)
2893 {
2894 const u8 rege3c_b24[3] = { 0, 0xf, 0x2f };
2895 int i, pat;
2896
2897 int lower[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2898 int upper[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2899 int channel, slotrank, lane;
2900
2901 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2902 lower[channel][slotrank][lane] = 0;
2903 upper[channel][slotrank][lane] = MAX_TIMC;
2904 }
2905
2906 write32(DEFAULT_MCHBAR + 0x4ea8, 1);
2907 printram("discover timC write:\n");
2908
2909 for (i = 0; i < 3; i++)
2910 FOR_ALL_POPULATED_CHANNELS {
2911 write32(DEFAULT_MCHBAR + 0xe3c + (channel * 0x100),
2912 (rege3c_b24[i] << 24)
2913 | (read32(DEFAULT_MCHBAR + 0xe3c + (channel * 0x100))
2914 & ~0x3f000000));
2915 udelay(2);
2916 for (pat = 0; pat < NUM_PATTERNS; pat++) {
2917 FOR_ALL_POPULATED_RANKS {
2918 int timC;
2919 u32 raw_statistics[MAX_TIMC + 1];
2920 int statistics[MAX_TIMC + 1];
2921
2922 /* Make sure rn.start < rn.end */
2923 statistics[MAX_TIMC] = 1;
2924
2925 fill_pattern5(ctrl, channel, pat);
2926 write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
2927 for (timC = 0; timC < MAX_TIMC; timC++) {
2928 FOR_ALL_LANES
2929 ctrl->timings[channel][slotrank].lanes[lane].timC = timC;
2930 program_timings(ctrl, channel);
2931
2932 test_timC_write (ctrl, channel, slotrank);
2933
2934 raw_statistics[timC] =
2935 MCHBAR32(0x436c + 0x400 * channel);
2936 }
2937 FOR_ALL_LANES {
2938 struct run rn;
2939 for (timC = 0; timC < MAX_TIMC; timC++)
2940 statistics[timC] =
2941 !!(raw_statistics[timC] &
2942 (1 << lane));
2943
2944 rn = get_longest_zero_run(statistics,
2945 MAX_TIMC + 1);
2946 if (rn.all) {
2947 printk(BIOS_EMERG, "timC write discovery failed: %d, %d, %d\n",
2948 channel, slotrank, lane);
2949 return MAKE_ERR;
2950 }
2951 printram("timC: %d, %d, %d: 0x%02x-0x%02x-0x%02x, 0x%02x-0x%02x\n",
2952 channel, slotrank, i, rn.start,
2953 rn.middle, rn.end,
2954 rn.start + ctrl->timC_offset[i],
2955 rn.end - ctrl->timC_offset[i]);
2956 lower[channel][slotrank][lane] =
2957 max(rn.start + ctrl->timC_offset[i],
2958 lower[channel][slotrank][lane]);
2959 upper[channel][slotrank][lane] =
2960 min(rn.end - ctrl->timC_offset[i],
2961 upper[channel][slotrank][lane]);
2962
2963 }
2964 }
2965 }
2966 }
2967
2968 FOR_ALL_CHANNELS {
2969 write32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c,
2970 0 | (read32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c) &
2971 ~0x3f000000));
2972 udelay(2);
2973 }
2974
2975 write32(DEFAULT_MCHBAR + 0x4ea8, 0);
2976
2977 printram("CPB\n");
2978
2979 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2980 printram("timC %d, %d, %d: %x\n", channel,
2981 slotrank, lane,
2982 (lower[channel][slotrank][lane] +
2983 upper[channel][slotrank][lane]) / 2);
2984 ctrl->timings[channel][slotrank].lanes[lane].timC =
2985 (lower[channel][slotrank][lane] +
2986 upper[channel][slotrank][lane]) / 2;
2987 }
2988 FOR_ALL_POPULATED_CHANNELS {
2989 program_timings(ctrl, channel);
2990 }
2991 return 0;
2992 }
2993
2994 void normalize_training(ramctr_timing * ctrl)
2995 {
2996 int channel, slotrank, lane;
2997 int mat;
2998
2999 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
3000 int delta;
3001 mat = 0;
3002 FOR_ALL_LANES mat =
3003 max(ctrl->timings[channel][slotrank].lanes[lane].timA, mat);
3004 printram("normalize %d, %d, %d: mat %d\n",
3005 channel, slotrank, lane, mat);
3006
3007 delta = (mat >> 6) - ctrl->timings[channel][slotrank].val_4028;
3008 printram("normalize %d, %d, %d: delta %d\n",
3009 channel, slotrank, lane, delta);
3010
3011 ctrl->timings[channel][slotrank].val_4024 += delta;
3012 ctrl->timings[channel][slotrank].val_4028 += delta;
3013 }
3014
3015 FOR_ALL_POPULATED_CHANNELS {
3016 program_timings(ctrl, channel);
3017 }
3018 }
3019
3020 void write_controller_mr(ramctr_timing * ctrl)
3021 {
3022 int channel, slotrank;
3023
3024 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
3025 write32(DEFAULT_MCHBAR + 0x0004 + (channel << 8) +
3026 lane_registers[slotrank], make_mr0(ctrl, slotrank));
3027 write32(DEFAULT_MCHBAR + 0x0008 + (channel << 8) +
3028 lane_registers[slotrank],
3029 make_mr1(ctrl, slotrank, channel));
3030 }
3031 }
3032
3033 int channel_test(ramctr_timing *ctrl)
3034 {
3035 int channel, slotrank, lane;
3036
3037 slotrank = 0;
3038 FOR_ALL_POPULATED_CHANNELS
3039 if (read32(DEFAULT_MCHBAR + 0x42a0 + (channel << 10)) & 0xa000) {
3040 printk(BIOS_EMERG, "Mini channel test failed (1): %d\n",
3041 channel);
3042 return MAKE_ERR;
3043 }
3044 FOR_ALL_POPULATED_CHANNELS {
3045 fill_pattern0(ctrl, channel, 0x12345678, 0x98765432);
3046
3047 write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
3048 }
3049
3050 for (slotrank = 0; slotrank < 4; slotrank++)
3051 FOR_ALL_CHANNELS
3052 if (ctrl->rankmap[channel] & (1 << slotrank)) {
3053 FOR_ALL_LANES {
3054 write32(DEFAULT_MCHBAR + (0x4f40 + 4 * lane), 0);
3055 write32(DEFAULT_MCHBAR + (0x4d40 + 4 * lane), 0);
3056 }
3057 wait_428c(channel);
3058 /* DRAM command ACT */
3059 write32(DEFAULT_MCHBAR + 0x4220 + (channel << 10), 0x0001f006);
3060 write32(DEFAULT_MCHBAR + 0x4230 + (channel << 10), 0x0028a004);
3061 write32(DEFAULT_MCHBAR + 0x4200 + (channel << 10),
3062 0x00060000 | (slotrank << 24));
3063 write32(DEFAULT_MCHBAR + 0x4210 + (channel << 10), 0x00000244);
3064 /* DRAM command WR */
3065 write32(DEFAULT_MCHBAR + 0x4224 + (channel << 10), 0x0001f201);
3066 write32(DEFAULT_MCHBAR + 0x4234 + (channel << 10), 0x08281064);
3067 write32(DEFAULT_MCHBAR + 0x4204 + (channel << 10),
3068 0x00000000 | (slotrank << 24));
3069 write32(DEFAULT_MCHBAR + 0x4214 + (channel << 10), 0x00000242);
3070 /* DRAM command RD */
3071 write32(DEFAULT_MCHBAR + 0x4228 + (channel << 10), 0x0001f105);
3072 write32(DEFAULT_MCHBAR + 0x4238 + (channel << 10), 0x04281064);
3073 write32(DEFAULT_MCHBAR + 0x4208 + (channel << 10),
3074 0x00000000 | (slotrank << 24));
3075 write32(DEFAULT_MCHBAR + 0x4218 + (channel << 10), 0x00000242);
3076 /* DRAM command PRE */
3077 write32(DEFAULT_MCHBAR + 0x422c + (channel << 10), 0x0001f002);
3078 write32(DEFAULT_MCHBAR + 0x423c + (channel << 10), 0x00280c01);
3079 write32(DEFAULT_MCHBAR + 0x420c + (channel << 10),
3080 0x00060400 | (slotrank << 24));
3081 write32(DEFAULT_MCHBAR + 0x421c + (channel << 10), 0x00000240);
3082 write32(DEFAULT_MCHBAR + 0x4284 + (channel << 10), 0x000c0001);
3083 wait_428c(channel);
3084 FOR_ALL_LANES
3085 if (read32(DEFAULT_MCHBAR + 0x4340 + (channel << 10) + 4 * lane)) {
3086 printk(BIOS_EMERG, "Mini channel test failed (2): %d, %d, %d\n",
3087 channel, slotrank, lane);
3088 return MAKE_ERR;
3089 }
3090 }
3091 return 0;
3092 }
3093
3094 void set_scrambling_seed(ramctr_timing * ctrl)
3095 {
3096 int channel;
3097
3098 /* FIXME: we hardcode seeds. Do we need to use some PRNG for them?
3099 I don't think so. */
3100 static u32 seeds[NUM_CHANNELS][3] = {
3101 {0x00009a36, 0xbafcfdcf, 0x46d1ab68},
3102 {0x00028bfa, 0x53fe4b49, 0x19ed5483}
3103 };
3104 FOR_ALL_POPULATED_CHANNELS {
3105 MCHBAR32(0x4020 + 0x400 * channel) &= ~0x10000000;
3106 MCHBAR32(0x4034 + 0x400 * channel) = seeds[channel][0];
3107 MCHBAR32(0x403c + 0x400 * channel) = seeds[channel][1];
3108 MCHBAR32(0x4038 + 0x400 * channel) = seeds[channel][2];
3109 }
3110 }
3111
3112 void set_4f8c(void)
3113 {
3114 struct cpuid_result cpures;
3115 u32 cpu;
3116
3117 cpures = cpuid(1);
3118 cpu = (cpures.eax);
3119 if (IS_SANDY_CPU(cpu) && (IS_SANDY_CPU_D0(cpu) || IS_SANDY_CPU_D1(cpu))) {
3120 MCHBAR32(0x4f8c) = 0x141D1519;
3121 } else {
3122 MCHBAR32(0x4f8c) = 0x551D1519;
3123 }
3124 }
3125
3126 void prepare_training(ramctr_timing * ctrl)
3127 {
3128 int channel;
3129
3130 FOR_ALL_POPULATED_CHANNELS {
3131 // Always drive command bus
3132 MCHBAR32(0x4004 + 0x400 * channel) |= 0x20000000;
3133 }
3134
3135 udelay(1);
3136
3137 FOR_ALL_POPULATED_CHANNELS {
3138 wait_428c(channel);
3139 }
3140 }
3141
3142 void set_4008c(ramctr_timing * ctrl)
3143 {
3144 int channel, slotrank;
3145
3146 FOR_ALL_POPULATED_CHANNELS {
3147 u32 b20, b4_8_12;
3148 int min_320c = 10000;
3149 int max_320c = -10000;
3150
3151 FOR_ALL_POPULATED_RANKS {
3152 max_320c = max(ctrl->timings[channel][slotrank].val_320c, max_320c);
3153 min_320c = min(ctrl->timings[channel][slotrank].val_320c, min_320c);
3154 }
3155
3156 if (max_320c - min_320c > 51)
3157 b20 = 0;
3158 else
3159 b20 = ctrl->ref_card_offset[channel];
3160
3161 if (ctrl->reg_320c_range_threshold < max_320c - min_320c)
3162 b4_8_12 = 0x3330;
3163 else
3164 b4_8_12 = 0x2220;
3165
3166 dram_odt_stretch(ctrl, channel);
3167
3168 write32(DEFAULT_MCHBAR + 0x4008 + (channel << 10),
3169 0x0a000000
3170 | (b20 << 20)
3171 | ((ctrl->ref_card_offset[channel] + 2) << 16)
3172 | b4_8_12);
3173 }
3174 }
3175
3176 void set_42a0(ramctr_timing * ctrl)
3177 {
3178 int channel;
3179 FOR_ALL_POPULATED_CHANNELS {
3180 write32(DEFAULT_MCHBAR + (0x42a0 + 0x400 * channel),
3181 0x00001000 | ctrl->rankmap[channel]);
3182 MCHBAR32(0x4004 + 0x400 * channel) &= ~0x20000000; // OK
3183 }
3184 }
3185
3186 static int encode_5d10(int ns)
3187 {
3188 return (ns + 499) / 500;
3189 }
3190
3191 /* FIXME: values in this function should be hardware revision-dependent. */
3192 void final_registers(ramctr_timing * ctrl)
3193 {
3194 int channel;
3195 int t1_cycles = 0, t1_ns = 0, t2_ns;
3196 int t3_ns;
3197 u32 r32;
3198
3199 write32(DEFAULT_MCHBAR + 0x4cd4, 0x00000046);
3200
3201 write32(DEFAULT_MCHBAR + 0x400c, (read32(DEFAULT_MCHBAR + 0x400c) & 0xFFFFCFFF) | 0x1000); // OK
3202 write32(DEFAULT_MCHBAR + 0x440c, (read32(DEFAULT_MCHBAR + 0x440c) & 0xFFFFCFFF) | 0x1000); // OK
3203 write32(DEFAULT_MCHBAR + 0x4cb0, 0x00000740);
3204 write32(DEFAULT_MCHBAR + 0x4380, 0x00000aaa); // OK
3205 write32(DEFAULT_MCHBAR + 0x4780, 0x00000aaa); // OK
3206 write32(DEFAULT_MCHBAR + 0x4f88, 0x5f7003ff); // OK
3207 write32(DEFAULT_MCHBAR + 0x5064, 0x00073000 | ctrl->reg_5064b0); // OK
3208
3209 FOR_ALL_CHANNELS {
3210 switch (ctrl->rankmap[channel]) {
3211 /* Unpopulated channel. */
3212 case 0:
3213 write32(DEFAULT_MCHBAR + 0x4384 + channel * 0x400, 0);
3214 break;
3215 /* Only single-ranked dimms. */
3216 case 1:
3217 case 4:
3218 case 5:
3219 write32(DEFAULT_MCHBAR + 0x4384 + channel * 0x400, 0x373131);
3220 break;
3221 /* Dual-ranked dimms present. */
3222 default:
3223 write32(DEFAULT_MCHBAR + 0x4384 + channel * 0x400, 0x9b6ea1);
3224 break;
3225 }
3226 }
3227
3228 write32 (DEFAULT_MCHBAR + 0x5880, 0xca9171e5);
3229 write32 (DEFAULT_MCHBAR + 0x5888,
3230 (read32 (DEFAULT_MCHBAR + 0x5888) & ~0xffffff) | 0xe4d5d0);
3231 write32 (DEFAULT_MCHBAR + 0x58a8, read32 (DEFAULT_MCHBAR + 0x58a8) & ~0x1f);
3232 write32 (DEFAULT_MCHBAR + 0x4294,
3233 (read32 (DEFAULT_MCHBAR + 0x4294) & ~0x30000)
3234 | (1 << 16));
3235 write32 (DEFAULT_MCHBAR + 0x4694,
3236 (read32 (DEFAULT_MCHBAR + 0x4694) & ~0x30000)
3237 | (1 << 16));
3238
3239 MCHBAR32(0x5030) |= 1; // OK
3240 MCHBAR32(0x5030) |= 0x80; // OK
3241 MCHBAR32(0x5f18) = 0xfa; // OK
3242
3243 /* Find a populated channel. */
3244 FOR_ALL_POPULATED_CHANNELS
3245 break;
3246
3247 t1_cycles = ((read32(DEFAULT_MCHBAR + 0x4290 + channel * 0x400) >> 8) & 0xff);
3248 r32 = read32(DEFAULT_MCHBAR + 0x5064);
3249 if (r32 & 0x20000)
3250 t1_cycles += (r32 & 0xfff);
3251 t1_cycles += (read32(DEFAULT_MCHBAR + channel * 0x400 + 0x42a4) & 0xfff);
3252 t1_ns = t1_cycles * ctrl->tCK / 256 + 544;
3253 if (!(r32 & 0x20000))
3254 t1_ns += 500;
3255
3256 t2_ns = 10 * ((read32(DEFAULT_MCHBAR + 0x5f10) >> 8) & 0xfff);
3257 if ( read32(DEFAULT_MCHBAR + 0x5f00) & 8 )
3258 {
3259 t3_ns = 10 * ((read32(DEFAULT_MCHBAR + 0x5f20) >> 8) & 0xfff);
3260 t3_ns += 10 * (read32(DEFAULT_MCHBAR + 0x5f18) & 0xff);
3261 }
3262 else
3263 {
3264 t3_ns = 500;
3265 }
3266 printk(BIOS_DEBUG, "t123: %d, %d, %d\n",
3267 t1_ns, t2_ns, t3_ns);
3268 write32 (DEFAULT_MCHBAR + 0x5d10,
3269 ((encode_5d10(t1_ns) + encode_5d10(t2_ns)) << 16)
3270 | (encode_5d10(t1_ns) << 8)
3271 | ((encode_5d10(t3_ns) + encode_5d10(t2_ns) + encode_5d10(t1_ns)) << 24)
3272 | (read32(DEFAULT_MCHBAR + 0x5d10) & 0xC0C0C0C0)
3273 | 0xc);
3274 }
3275
3276 void restore_timings(ramctr_timing * ctrl)
3277 {
3278 int channel, slotrank, lane;
3279
3280 FOR_ALL_POPULATED_CHANNELS
3281 MCHBAR32(0x4004 + 0x400 * channel) =
3282 ctrl->tRRD
3283 | (ctrl->tRTP << 4)
3284 | (ctrl->tCKE << 8)
3285 | (ctrl->tWTR << 12)
3286 | (ctrl->tFAW << 16)
3287 | (ctrl->tWR << 24)
3288 | (ctrl->cmd_stretch[channel] << 30);
3289
3290 udelay(1);
3291
3292 FOR_ALL_POPULATED_CHANNELS {
3293 wait_428c(channel);
3294 }
3295
3296 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
3297 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel
3298 + 4 * lane, 0);
3299 }
3300
3301 FOR_ALL_POPULATED_CHANNELS
3302 write32(DEFAULT_MCHBAR + 0x4008 + 0x400 * channel,
3303 read32(DEFAULT_MCHBAR + 0x4008 +
3304 0x400 * channel) | 0x8000000);
3305
3306 FOR_ALL_POPULATED_CHANNELS {
3307 udelay (1);
3308 write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
3309 read32(DEFAULT_MCHBAR + 0x4020 +
3310 0x400 * channel) | 0x200000);
3311 }
3312
3313 printram("CPE\n");
3314
3315 write32(DEFAULT_MCHBAR + 0x3400, 0);
3316 write32(DEFAULT_MCHBAR + 0x4eb0, 0);
3317
3318 printram("CP5b\n");
3319
3320 FOR_ALL_POPULATED_CHANNELS {
3321 program_timings(ctrl, channel);
3322 }
3323
3324 u32 reg, addr;
3325
3326 while (!(MCHBAR32(0x5084) & 0x10000));
3327 do {
3328 reg = MCHBAR32(0x428c);
3329 } while ((reg & 0x14) == 0);
3330
3331 // Set state of memory controller
3332 MCHBAR32(0x5030) = 0x116;
3333 MCHBAR32(0x4ea0) = 0;
3334
3335 // Wait 500us
3336 udelay(500);
3337
3338 FOR_ALL_CHANNELS {
3339 // Set valid rank CKE
3340 reg = 0;
3341 reg = (reg & ~0xf) | ctrl->rankmap[channel];
3342 addr = 0x400 * channel + 0x42a0;
3343 MCHBAR32(addr) = reg;
3344
3345 // Wait 10ns for ranks to settle
3346 //udelay(0.01);
3347
3348 reg = (reg & ~0xf0) | (ctrl->rankmap[channel] << 4);
3349 MCHBAR32(addr) = reg;
3350
3351 // Write reset using a NOP
3352 write_reset(ctrl);
3353 }
3354
3355 /* mrs commands. */
3356 dram_mrscommands(ctrl);
3357
3358 printram("CP5c\n");
3359
3360 write32(DEFAULT_MCHBAR + 0x3000, 0);
3361
3362 FOR_ALL_CHANNELS {
3363 write32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c,
3364 0 | (read32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c) &
3365 ~0x3f000000));
3366 udelay(2);
3367 }
3368
3369 write32(DEFAULT_MCHBAR + 0x4ea8, 0);
3370 }
ast, secure and flexible OpenSource firmware