firmware/target/arm/imx31/ata-imx31.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2007 by Will Robertson
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21 #include "config.h"
  22 #include "cpu.h"
  23 #include "kernel.h"
  24 #include "thread.h"
  25 #include "system.h"
  26 #include "power.h"
  27 #include "panic.h"
  28 #include "ata.h"
  29 #include "ata-target.h"
  30 #include "ccm-imx31.h"
  31 #ifdef HAVE_ATA_DMA
  32 #include "sdma-imx31.h"
  33 #include "mmu-imx31.h"
  34 #endif
  35
  36 /* PIO modes timing info */
  37 static const struct ata_pio_timings
  38 {
  39     uint16_t time_2w;       /* t2 during write */
  40     uint16_t time_2r;       /* t2 during read */
  41     uint8_t  time_ax;       /* tA */
  42     uint8_t  time_1;        /* t1 */
  43     uint8_t  time_4;        /* t4 */
  44     uint8_t  time_9;        /* t9 */
  45 } pio_timings[5] =
  46 {
  47     [0] =  /* PIO mode 0 */
  48     {
  49         .time_1 = 70,
  50         .time_2w = 290,
  51         .time_2r = 290,
  52         .time_ax = 35,
  53         .time_4 = 30,
  54         .time_9 = 20
  55     },
  56     [1] = /* PIO mode 1 */
  57     {
  58         .time_1 = 50,
  59         .time_2w = 290,
  60         .time_2r = 290,
  61         .time_ax = 35,
  62         .time_4 = 20,
  63         .time_9 = 15
  64     },
  65     [2] = /* PIO mode 2 */
  66     {
  67         .time_1 = 30,
  68         .time_2w = 290,
  69         .time_2r = 290,
  70         .time_ax = 35,
  71         .time_4 = 15,
  72         .time_9 = 10
  73     },
  74     [3] = /* PIO mode 3 */
  75     {
  76         .time_1 = 30,
  77         .time_2w = 80,
  78         .time_2r = 80,
  79         .time_ax = 35,
  80         .time_4 = 10,
  81         .time_9 = 10
  82     },
  83     [4] = /* PIO mode 4 */
  84     {
  85         .time_1 = 25,
  86         .time_2w = 70,
  87         .time_2r = 70,
  88         .time_ax = 35,
  89         .time_4 = 10,
  90         .time_9 = 10
  91     }
  92 };
  93
  94 /* Track first init */
  95 static bool initialized = false;
  96
  97 #ifdef HAVE_ATA_DMA
  98 /* One DMA channel for reads, the other for writes othewise one channel would
  99  * have to be reinitialized every time the direction changed. (Different
 100  * SDMA scripts are used for reading or writing) */
 101 #define ATA_DMA_CH_NUM_RD   3
 102 #define ATA_DMA_CH_NUM_WR   4
 103 /* Use default priority for these channels (1) - ATA isn't realtime urgent. */
 104 /* Maximum DMA size per buffer descriptor (32-byte aligned) */
 105 #define ATA_MAX_BD_SIZE     (65534 & ~31) /* 65504 */
 106
 107 /* Number of buffer descriptors required for a maximum sector count trasfer.
 108  * NOTE: Assumes LBA28 and 512-byte sectors! */
 109 #define ATA_BASE_BD_COUNT   ((256*512 + (ATA_MAX_BD_SIZE-1)) / ATA_MAX_BD_SIZE)
 110 #define ATA_BD_COUNT        (ATA_BASE_BD_COUNT + 2)
 111
 112 static const struct ata_mdma_timings
 113 {
 114     uint8_t time_m;         /* tM */
 115     uint8_t time_jn;        /* tH */
 116     uint8_t time_d;         /* tD */
 117     uint8_t time_k;         /* tKW */
 118 } mdma_timings[] =
 119 {
 120     [0] = /* MDMA mode 0 */
 121     {
 122         .time_m = 50,
 123         .time_jn = 20,
 124         .time_d = 215,
 125         .time_k = 215
 126     },
 127     [1] = /* MDMA mode 1 */
 128     {
 129         .time_m = 30,
 130         .time_jn = 15,
 131         .time_d = 80,
 132         .time_k = 50
 133     },
 134     [2] = /* MDMA mode 2 */
 135     {
 136         .time_m = 25,
 137         .time_jn = 10,
 138         .time_d = 70,
 139         .time_k = 25
 140     }
 141 };
 142
 143 static const struct ata_udma_timings
 144 {
 145     uint8_t time_ack;       /* tACK */
 146     uint8_t time_env;       /* tENV */
 147     uint8_t time_rpx;       /* tRP */
 148     uint8_t time_zah;       /* tZAH */
 149     uint8_t time_mlix;      /* tMLI */
 150     uint8_t time_dvh;       /* tDVH */
 151     uint8_t time_dzfs;      /* tDVS+tDVH? */
 152     uint8_t time_dvs;       /* tDVS */
 153     uint8_t time_cvh;       /* ?? */
 154     uint8_t time_ss;        /* tSS */
 155     uint8_t time_cyc;       /* tCYC */
 156 } udma_timings[] =
 157 {
 158     [0] = /* UDMA mode 0 */
 159     {
 160         .time_ack = 20,
 161         .time_env = 20,
 162         .time_rpx = 160,
 163         .time_zah = 20,
 164         .time_mlix = 20,
 165         .time_dvh = 6,
 166         .time_dzfs = 80,
 167         .time_dvs = 70,
 168         .time_cvh = 6,
 169         .time_ss = 50,
 170         .time_cyc = 114
 171     },
 172     [1] = /* UDMA mode 1 */
 173     {
 174         .time_ack = 20,
 175         .time_env = 20,
 176         .time_rpx = 125,
 177         .time_zah = 20,
 178         .time_mlix = 20,
 179         .time_dvh = 6,
 180         .time_dzfs = 63,
 181         .time_dvs = 48,
 182         .time_cvh = 6,
 183         .time_ss = 50,
 184         .time_cyc = 75
 185     },
 186     [2] = /* UDMA mode 2 */
 187     {
 188         .time_ack = 20,
 189         .time_env = 20,
 190         .time_rpx = 100,
 191         .time_zah = 20,
 192         .time_mlix = 20,
 193         .time_dvh = 6,
 194         .time_dzfs = 47,
 195         .time_dvs = 34,
 196         .time_cvh = 6,
 197         .time_ss = 50,
 198         .time_cyc = 55
 199     },
 200     [3] = /* UDMA mode 3 */
 201     {
 202         .time_ack = 20,
 203         .time_env = 20,
 204         .time_rpx = 100,
 205         .time_zah = 20,
 206         .time_mlix = 20,
 207         .time_dvh = 6,
 208         .time_dzfs = 35,
 209         .time_dvs = 20,
 210         .time_cvh = 6,
 211         .time_ss = 50,
 212         .time_cyc = 39
 213     },
 214     [4] = /* UDMA mode 4 */
 215     {
 216         .time_ack = 20,
 217         .time_env = 20,
 218         .time_rpx = 100,
 219         .time_zah = 20,
 220         .time_mlix = 20,
 221         .time_dvh = 6,
 222         .time_dzfs = 25,
 223         .time_dvs = 7,
 224         .time_cvh = 6,
 225         .time_ss = 50,
 226         .time_cyc = 25
 227     },
 228 #if 0
 229     [5] = /* UDMA mode 5 (bus clock 80MHz or higher only) */
 230     {
 231         .time_ack = 20,
 232         .time_env = 20,
 233         .time_rpx = 85,
 234         .time_zah = 20,
 235         .time_mlix = 20,
 236         .time_dvh = 6,
 237         .time_dzfs = 40,
 238         .time_dvs = 5,
 239         .time_cvh = 10,
 240         .time_ss = 50,
 241         .time_cyc = 17
 242     }
 243 #endif
 244 };
 245
 246 /** Threading **/
 247 /* Signal to tell thread when DMA is done */
 248 static struct wakeup ata_dma_wakeup;
 249
 250 /** SDMA **/
 251 /* Array of buffer descriptors for large transfers and alignnment */
 252 static struct buffer_descriptor ata_bda[ATA_BD_COUNT] NOCACHEBSS_ATTR;
 253 /* ATA channel descriptors */
 254 /* Read/write channels share buffer descriptors and callbacks */
 255 static void ata_dma_callback(void);
 256
 257 static struct channel_descriptor ata_cd_rd = /* read channel */
 258 {
 259     .bd_count = ATA_BD_COUNT,
 260     .callback = ata_dma_callback,
 261     .shp_addr = SDMA_PER_ADDR_ATA_RX,
 262     .wml      = SDMA_ATA_WML,
 263     .per_type = SDMA_PER_ATA,
 264     .tran_type = SDMA_TRAN_PER_2_EMI,
 265     .event_id1 = SDMA_REQ_ATA_TXFER_END,
 266     .event_id2 = SDMA_REQ_ATA_RX,
 267 };
 268
 269 static struct channel_descriptor ata_cd_wr = /* write channel */
 270 {
 271     .bd_count = ATA_BD_COUNT,
 272     .callback = ata_dma_callback,
 273     .shp_addr = SDMA_PER_ADDR_ATA_TX,
 274     .wml      = SDMA_ATA_WML,
 275     .per_type = SDMA_PER_ATA,
 276     .tran_type = SDMA_TRAN_EMI_2_PER,
 277     .event_id1 = SDMA_REQ_ATA_TXFER_END,
 278     .event_id2 = SDMA_REQ_ATA_TX,
 279 };
 280
 281 /* DMA channel to be started for transfer */
 282 static unsigned int current_channel = 0;
 283
 284 /** Buffers **/
 285 /* Scatter buffer for first and last 32 bytes of a non cache-aligned transfer
 286  * to cached RAM. */
 287 static uint32_t scatter_buffer[32/4*2] NOCACHEBSS_ATTR;
 288 /* Address of ends in destination buffer for unaligned reads - copied after
 289  * DMA completes. */
 290 static void *sb_dst[2] = { NULL, NULL };
 291
 292 /** Modes **/
 293 #define ATA_DMA_MWDMA    0x00000000             /* Using multiword DMA */
 294 #define ATA_DMA_UDMA     ATA_DMA_ULTRA_SELECTED /* Using Ultra DMA */
 295 #define ATA_DMA_PIO      0x80000000             /* Using PIO */
 296 #define ATA_DMA_DISABLED 0x80000001             /* DMA init error - use PIO */
 297 static unsigned long ata_dma_selected = ATA_DMA_PIO;
 298 #endif /* HAVE_ATA_DMA */
 299
 300 static unsigned int get_T(void)
 301 {
 302     /* T = ATA clock period in nanoseconds */
 303     return 1000 * 1000 * 1000 / ccm_get_ata_clk();
 304 }
 305
 306 static void ata_wait_for_idle(void)
 307 {
 308     while (!(ATA_INTERRUPT_PENDING & ATA_CONTROLLER_IDLE));
 309 }
 310
 311 /* Route the INTRQ to either the MCU or SDMA depending upon whether there is
 312  * a DMA transfer in progress. */
 313 static inline void ata_set_intrq(bool to_dma)
 314 {
 315      ATA_INTERRUPT_ENABLE =
 316         (ATA_INTERRUPT_ENABLE & ~(ATA_INTRQ1 | ATA_INTRQ2)) |
 317             (to_dma ? ATA_INTRQ1 : ATA_INTRQ2);
 318 }
 319
 320 /* Setup the timing for PIO mode */
 321 void ata_set_pio_timings(int mode)
 322 {
 323     const struct ata_pio_timings * const timings = &pio_timings[mode];
 324     unsigned int T = get_T();
 325
 326     ata_wait_for_idle();
 327
 328     ATA_TIME_1 = (timings->time_1 + T) / T;
 329     ATA_TIME_2W = (timings->time_2w + T) / T;
 330     ATA_TIME_2R = (timings->time_2r + T) / T;
 331     ATA_TIME_AX = (timings->time_ax + T) / T + 2; /* 1.5 + tAX */
 332     ATA_TIME_PIO_RDX = 1;
 333     ATA_TIME_4 = (timings->time_4 + T) / T;
 334     ATA_TIME_9 = (timings->time_9 + T) / T;
 335 }
 336
 337 void ata_reset(void)
 338 {
 339     /* Be sure we're not busy */
 340     ata_wait_for_idle();
 341
 342     ATA_INTF_CONTROL &= ~(ATA_ATA_RST | ATA_FIFO_RST);
 343     sleep(HZ/100);
 344     ATA_INTF_CONTROL = ATA_ATA_RST | ATA_FIFO_RST;
 345     sleep(HZ/100);
 346
 347     ata_wait_for_idle();
 348 }
 349
 350 void ata_enable(bool on)
 351 {
 352     /* Unconditionally clock module before writing regs */
 353     ccm_module_clock_gating(CG_ATA, CGM_ON_RUN_WAIT);
 354     ata_wait_for_idle();
 355
 356     if (on)
 357     {
 358         ATA_INTF_CONTROL = ATA_ATA_RST | ATA_FIFO_RST;
 359         sleep(HZ/100);
 360     }
 361     else
 362     {
 363         ATA_INTF_CONTROL &= ~(ATA_ATA_RST | ATA_FIFO_RST);
 364         sleep(HZ/100);
 365
 366         /* Disable off - unclock ATA module */
 367         ccm_module_clock_gating(CG_ATA, CGM_OFF);
 368     }
 369 }
 370
 371 bool ata_is_coldstart(void)
 372 {
 373     return true;
 374 }
 375
 376 #ifdef HAVE_ATA_DMA
 377 static void ata_set_mdma_timings(unsigned int mode)
 378 {
 379     const struct ata_mdma_timings * const timings = &mdma_timings[mode];
 380     unsigned int T = get_T();
 381
 382     ATA_TIME_M = (timings->time_m + T) / T;
 383     ATA_TIME_JN = (timings->time_jn + T) / T;
 384     ATA_TIME_D = (timings->time_d + T) / T;
 385     ATA_TIME_K = (timings->time_k + T) / T;
 386 }
 387
 388 static void ata_set_udma_timings(unsigned int mode)
 389 {
 390     const struct ata_udma_timings * const timings = &udma_timings[mode];
 391     unsigned int T = get_T();
 392
 393     ATA_TIME_ACK = (timings->time_ack + T) / T;
 394     ATA_TIME_ENV = (timings->time_env + T) / T;
 395     ATA_TIME_RPX = (timings->time_rpx + T) / T;
 396     ATA_TIME_ZAH = (timings->time_zah + T) / T;
 397     ATA_TIME_MLIX = (timings->time_mlix + T) / T;
 398     ATA_TIME_DVH = (timings->time_dvh + T) / T + 1;
 399     ATA_TIME_DZFS = (timings->time_dzfs + T) / T;
 400     ATA_TIME_DVS = (timings->time_dvs + T) / T;
 401     ATA_TIME_CVH = (timings->time_cvh + T) / T;
 402     ATA_TIME_SS = (timings->time_ss + T) / T;
 403     ATA_TIME_CYC = (timings->time_cyc + T) / T;
 404 }
 405
 406 void ata_dma_set_mode(unsigned char mode)
 407 {
 408     unsigned int modeidx = mode & 0x07;
 409     unsigned int dmamode = mode & 0xf8;
 410
 411     ata_wait_for_idle();
 412
 413     if (ata_dma_selected == ATA_DMA_DISABLED)
 414     {
 415         /* Configuration error - no DMA */
 416     }
 417     else if (dmamode == 0x40 && modeidx <= ATA_MAX_UDMA)
 418     {
 419         /* Using Ultra DMA */
 420         ata_set_udma_timings(dmamode);
 421         ata_dma_selected = ATA_DMA_UDMA;
 422     }
 423     else if (dmamode == 0x20 && modeidx <= ATA_MAX_MWDMA)
 424     {
 425         /* Using Multiword DMA */
 426         ata_set_mdma_timings(dmamode);
 427         ata_dma_selected = ATA_DMA_MWDMA;
 428     }
 429     else
 430     {
 431         /* Don't understand this - force PIO. */
 432         ata_dma_selected = ATA_DMA_PIO;
 433     }
 434 }
 435
 436 /* Called by SDMA when transfer is complete */
 437 static void ata_dma_callback(void)
 438 {
 439     /* Clear FIFO if not empty - shouldn't happen */
 440     while (ATA_FIFO_FILL != 0)
 441         ATA_FIFO_DATA_32;
 442
 443     /* Clear FIFO interrupts (the only ones that can be) */
 444     ATA_INTERRUPT_CLEAR = ATA_INTERRUPT_PENDING;
 445
 446     ata_set_intrq(false);           /* Return INTRQ to MCU */
 447     wakeup_signal(&ata_dma_wakeup); /* Signal waiting thread */
 448 }
 449
 450 bool ata_dma_setup(void *addr, unsigned long bytes, bool write)
 451 {
 452     struct buffer_descriptor *bd_p;
 453     unsigned char *buf;
 454
 455     if (UNLIKELY(bytes > ATA_BASE_BD_COUNT*ATA_MAX_BD_SIZE ||
 456                  (ata_dma_selected & ATA_DMA_PIO)))
 457     {
 458         /* Too much? Implies BD count should be reevaluated since this
 459          * shouldn't be reached based upon size. Otherwise we simply didn't
 460          * understand the DMA mode setup. Force PIO in both cases. */
 461         ATA_INTF_CONTROL = ATA_FIFO_RST | ATA_ATA_RST;
 462         return false;
 463     }
 464
 465     bd_p = &ata_bda[0];
 466     buf = (unsigned char *)addr_virt_to_phys((unsigned long)addr);
 467     sb_dst[0] = NULL; /* Assume not needed */
 468
 469     if (write)
 470     {
 471         /* No cache alignment concerns */
 472         current_channel = ATA_DMA_CH_NUM_WR;
 473
 474         if (LIKELY(buf != addr))
 475         {
 476             /* addr is virtual */
 477             clean_dcache_range(addr, bytes);
 478         }
 479
 480         /* Setup ATA controller for DMA transmit */
 481         ATA_INTF_CONTROL = ATA_FIFO_RST | ATA_ATA_RST | ATA_FIFO_TX_EN |
 482             ATA_DMA_PENDING | ata_dma_selected | ATA_DMA_WRITE;
 483         ATA_FIFO_ALARM = SDMA_ATA_WML / 2;
 484     }
 485     else
 486     {
 487         current_channel = ATA_DMA_CH_NUM_RD;
 488
 489         /* Setup ATA controller for DMA receive */
 490         ATA_INTF_CONTROL = ATA_FIFO_RST | ATA_ATA_RST | ATA_FIFO_RCV_EN |
 491             ATA_DMA_PENDING | ata_dma_selected;
 492         ATA_FIFO_ALARM = SDMA_ATA_WML / 2;
 493
 494         if (LIKELY(buf != addr))
 495         {
 496             /* addr is virtual */
 497             dump_dcache_range(addr, bytes);
 498
 499             if ((unsigned long)addr & 31)
 500             {
 501                 /* Not cache aligned, must use scatter buffers for first and
 502                  * last 32 bytes. */
 503                 unsigned char *bufstart = buf;
 504
 505                 sb_dst[0] = addr;
 506                 bd_p->buf_addr = scatter_buffer;
 507                 bd_p->mode.count = 32;
 508                 bd_p->mode.status = BD_DONE | BD_CONT;
 509
 510                 buf += 32;
 511                 bytes -= 32;
 512                 bd_p++;
 513
 514                 while (bytes > ATA_MAX_BD_SIZE)
 515                 {
 516                     bd_p->buf_addr = buf;
 517                     bd_p->mode.count = ATA_MAX_BD_SIZE;
 518                     bd_p->mode.status = BD_DONE | BD_CONT;
 519                     buf += ATA_MAX_BD_SIZE;
 520                     bytes -= ATA_MAX_BD_SIZE;
 521                     bd_p++;
 522                 }
 523
 524                 if (bytes > 32)
 525                 {
 526                     unsigned long size = bytes - 32;
 527                     bd_p->buf_addr = buf;
 528                     bd_p->mode.count = size;
 529                     bd_p->mode.status = BD_DONE | BD_CONT;
 530                     buf += size;
 531                     bd_p++;
 532                 }
 533
 534                 /* There will be exactly 32 bytes left */
 535
 536                 /* Final buffer - wrap to base bd, interrupt */
 537                 sb_dst[1] = addr + (buf - bufstart);
 538                 bd_p->buf_addr = &scatter_buffer[32/4];
 539                 bd_p->mode.count = 32;
 540                 bd_p->mode.status = BD_DONE | BD_WRAP | BD_INTR;
 541
 542                 return true;
 543             }
 544         }
 545     }
 546
 547     /* Setup buffer descriptors for both cache-aligned reads and all write
 548      * operations. */
 549     while (bytes > ATA_MAX_BD_SIZE)
 550     {
 551         bd_p->buf_addr = buf;
 552         bd_p->mode.count = ATA_MAX_BD_SIZE;
 553         bd_p->mode.status = BD_DONE | BD_CONT;
 554         buf += ATA_MAX_BD_SIZE;
 555         bytes -= ATA_MAX_BD_SIZE;
 556         bd_p++;
 557     }
 558
 559     /* Final buffer - wrap to base bd, interrupt */
 560     bd_p->buf_addr = buf;
 561     bd_p->mode.count = bytes;
 562     bd_p->mode.status = BD_DONE | BD_WRAP | BD_INTR;
 563
 564     return true;
 565 }
 566
 567 bool ata_dma_finish(void)
 568 {
 569     unsigned int channel = current_channel;
 570     long timeout = current_tick + HZ*10;
 571
 572     current_channel = 0;
 573
 574     ata_set_intrq(true);       /* Give INTRQ to DMA */
 575     sdma_channel_run(channel); /* Kick the channel to wait for events */
 576
 577     while (1)
 578     {
 579         int oldirq;
 580
 581         if (LIKELY(wakeup_wait(&ata_dma_wakeup, HZ/2) == OBJ_WAIT_SUCCEEDED))
 582             break;
 583
 584         ata_keep_active();
 585
 586         if (TIME_BEFORE(current_tick, timeout))
 587             continue;
 588
 589         /* Epic fail - timed out - maybe. */
 590         oldirq = disable_irq_save();
 591         ata_set_intrq(false);       /* Strip INTRQ from DMA */
 592         sdma_channel_stop(channel); /* Stop DMA */
 593         restore_irq(oldirq);
 594
 595         if (wakeup_wait(&ata_dma_wakeup, TIMEOUT_NOBLOCK) == OBJ_WAIT_SUCCEEDED)
 596             break; /* DMA really did finish after timeout */
 597
 598         sdma_channel_reset(channel); /* Reset everything + clear error */
 599         return false;
 600     }
 601
 602     if (sdma_channel_is_error(channel))
 603     {
 604         /* Channel error in one or more descriptors */
 605         sdma_channel_reset(channel); /* Reset everything + clear error */
 606         return false;
 607     }
 608
 609     if (sb_dst[0] != NULL)
 610     {
 611         /* NOTE: This requires that unaligned access support be enabled! */
 612         register void *sbs = scatter_buffer;
 613         register void *sbd0 = sb_dst[0];
 614         register void *sbd1 = sb_dst[1];
 615         asm volatile(
 616             "add    r0, %1, #32         \n" /* Prefetch at DMA-direct boundaries */
 617             "mcrr   p15, 2, r0, r0, c12 \n"
 618             "mcrr   p15, 2, %2, %2, c12 \n"
 619             "ldmia  %0!, { r0-r3 }      \n" /* Copy the 32-bytes to destination */
 620             "str    r0, [%1], #4        \n" /* stmia doesn't work unaligned */
 621             "str    r1, [%1], #4        \n"
 622             "str    r2, [%1], #4        \n"
 623             "str    r3, [%1], #4        \n"
 624             "ldmia  %0!, { r0-r3 }      \n"
 625             "str    r0, [%1], #4        \n"
 626             "str    r1, [%1], #4        \n"
 627             "str    r2, [%1], #4        \n"
 628             "str    r3, [%1]            \n"
 629             "ldmia  %0!, { r0-r3 }      \n" /* Copy the 32-bytes to destination */
 630             "str    r0, [%2], #4        \n" /* stmia doesn't work unaligned */
 631             "str    r1, [%2], #4        \n"
 632             "str    r2, [%2], #4        \n"
 633             "str    r3, [%2], #4        \n"
 634             "ldmia  %0!, { r0-r3 }      \n"
 635             "str    r0, [%2], #4        \n"
 636             "str    r1, [%2], #4        \n"
 637             "str    r2, [%2], #4        \n"
 638             "str    r3, [%2]            \n"
 639             : "+r"(sbs), "+r"(sbd0), "+r"(sbd1)
 640             :
 641             : "r0", "r1", "r2", "r3");
 642     }
 643
 644     return true;
 645 }
 646 #endif /* HAVE_ATA_DMA */
 647
 648 void ata_device_init(void)
 649 {
 650     /* Make sure we're not in reset mode */
 651     ata_enable(true);
 652
 653     if (!initialized)
 654     {
 655         ATA_INTERRUPT_ENABLE = 0;
 656         ATA_INTERRUPT_CLEAR = ATA_INTERRUPT_PENDING;
 657     }
 658
 659     ata_set_intrq(false);
 660
 661     if (initialized)
 662         return;
 663
 664     /* All modes use same tOFF/tON */
 665     ATA_TIME_OFF = 3;
 666     ATA_TIME_ON = 3;
 667
 668     /* Setup mode 0 for all by default
 669      * Mode may be switched later once identify info is ready in which
 670      * case the main driver calls back */
 671     ata_set_pio_timings(0);
 672
 673 #ifdef HAVE_ATA_DMA
 674     ata_set_mdma_timings(0);
 675     ata_set_udma_timings(0);
 676
 677     ata_dma_selected = ATA_DMA_PIO;
 678
 679     /* Called for first time at startup */
 680     wakeup_init(&ata_dma_wakeup);
 681
 682     if (!sdma_channel_init(ATA_DMA_CH_NUM_RD, &ata_cd_rd, ata_bda) ||
 683         !sdma_channel_init(ATA_DMA_CH_NUM_WR, &ata_cd_wr, ata_bda))
 684     {
 685         /* Channel init error - disable DMA forever */
 686         ata_dma_selected = ATA_DMA_DISABLED;
 687     }
 688 #endif /* HAVE_ATA_DMA */
 689
 690     initialized = true;
 691 }