uspace/drv/block/ahci/ahci.c

   1 /*
   2  * Copyright (c) 2012 Petr Jerman
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  *
   9  * - Redistributions of source code must retain the above copyright
  10  *   notice, this list of conditions and the following disclaimer.
  11  * - Redistributions in binary form must reproduce the above copyright
  12  *   notice, this list of conditions and the following disclaimer in the
  13  *   documentation and/or other materials provided with the distribution.
  14  * - The name of the author may not be used to endorse or promote products
  15  *   derived from this software without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 /** @file
  30  * AHCI SATA driver implementation.
  31  */
  32
  33 #include <as.h>
  34 #include <errno.h>
  35 #include <stdio.h>
  36 #include <devman.h>
  37 #include <ddf/interrupt.h>
  38 #include <ddf/log.h>
  39 #include <device/hw_res_parsed.h>
  40 #include <device/pci.h>
  41 #include <sysinfo.h>
  42 #include <ipc/irc.h>
  43 #include <ns.h>
  44 #include <ahci_iface.h>
  45 #include "ahci.h"
  46 #include "ahci_hw.h"
  47 #include "ahci_sata.h"
  48
  49 #define NAME  "ahci"
  50
  51 #define AHCI_TIMER_TICKS  1000000000
  52
  53 #define LO(ptr) \
  54         ((uint32_t) (((uint64_t) ((uintptr_t) (ptr))) & 0xffffffff))
  55
  56 #define HI(ptr) \
  57         ((uint32_t) (((uint64_t) ((uintptr_t) (ptr))) >> 32))
  58
  59 static int ahci_get_sata_device_name(ddf_fun_t *, size_t, char *);
  60 static int ahci_get_num_blocks(ddf_fun_t *, uint64_t *);
  61 static int ahci_get_block_size(ddf_fun_t *, size_t *);
  62 static int ahci_read_blocks(ddf_fun_t *, uint64_t, size_t, void *);
  63 static int ahci_write_blocks(ddf_fun_t *, uint64_t, size_t, void *);
  64
  65 static int ahci_identify_device(sata_dev_t *);
  66 static int ahci_set_highest_ultra_dma_mode(sata_dev_t *);
  67 static int ahci_rb_fpdma(sata_dev_t *, void *, uint64_t);
  68 static int ahci_wb_fpdma(sata_dev_t *, void *, uint64_t);
  69
  70 static void ahci_sata_devices_create(ahci_dev_t *, ddf_dev_t *);
  71 static ahci_dev_t *ahci_ahci_create(ddf_dev_t *);
  72 static void ahci_ahci_init(ahci_dev_t *);
  73
  74 static int ahci_dev_add(ddf_dev_t *);
  75
  76 static void ahci_get_model_name(uint16_t *, char *);
  77 static int ahci_pciintel_enable_interrupt(int);
  78
  79 static fibril_mutex_t sata_devices_count_lock;
  80 static int sata_devices_count = 0;
  81
  82 /*----------------------------------------------------------------------------*/
  83 /*-- AHCI Interface ----------------------------------------------------------*/
  84 /*----------------------------------------------------------------------------*/
  85
  86 static ahci_iface_t ahci_interface = {
  87         .get_sata_device_name = &ahci_get_sata_device_name,
  88         .get_num_blocks = &ahci_get_num_blocks,
  89         .get_block_size = &ahci_get_block_size,
  90         .read_blocks = &ahci_read_blocks,
  91         .write_blocks = &ahci_write_blocks
  92 };
  93
  94 static ddf_dev_ops_t ahci_ops = {
  95         .interfaces[AHCI_DEV_IFACE] = &ahci_interface
  96 };
  97
  98 static driver_ops_t driver_ops = {
  99         .dev_add = &ahci_dev_add
 100 };
 101
 102 static driver_t ahci_driver = {
 103         .name = NAME,
 104         .driver_ops = &driver_ops
 105 };
 106
 107 static int ahci_get_sata_device_name(ddf_fun_t *fun,
 108     size_t sata_dev_name_length, char *sata_dev_name)
 109 {
 110         sata_dev_t *sata = (sata_dev_t *) fun->driver_data;
 111         str_cpy(sata_dev_name, sata_dev_name_length, sata->model);
 112         return EOK;
 113 }
 114
 115 static int ahci_get_num_blocks(ddf_fun_t *fun, uint64_t *num_blocks)
 116 {
 117         sata_dev_t *sata = (sata_dev_t *) fun->driver_data;
 118         *num_blocks = sata->blocks;
 119         return EOK;
 120 }
 121
 122 static int ahci_get_block_size(ddf_fun_t *fun, size_t *block_size)
 123 {
 124         sata_dev_t *sata = (sata_dev_t *) fun->driver_data;
 125         *block_size = sata->block_size;
 126         return EOK;
 127 }
 128
 129 static int ahci_read_blocks(ddf_fun_t *fun, uint64_t blocknum,
 130     size_t count, void *buf)
 131 {
 132         int rc = EOK;
 133         sata_dev_t *sata = (sata_dev_t *) fun->driver_data;
 134
 135         void *phys;
 136         void *ibuf;
 137
 138         dmamem_map_anonymous(sata->block_size, AS_AREA_READ | AS_AREA_WRITE,
 139             0, &phys, (void **) &ibuf);
 140         bzero(buf, sata->block_size);
 141
 142         fibril_mutex_lock(&sata->lock);
 143
 144         for (size_t cur = 0; cur < count; cur++) {
 145                 rc = ahci_rb_fpdma(sata, phys, blocknum + cur);
 146                 if (rc != EOK)
 147                         break;
 148
 149                 memcpy((void *) (((uint8_t *) buf) + (sata->block_size * cur)),
 150                     ibuf, sata->block_size);
 151         }
 152
 153         fibril_mutex_unlock(&sata->lock);
 154         dmamem_unmap_anonymous(ibuf);
 155
 156         return rc;
 157 }
 158
 159 static int ahci_write_blocks(ddf_fun_t *fun, uint64_t blocknum,
 160     size_t count, void *buf)
 161 {
 162         int rc = EOK;
 163         sata_dev_t *sata = (sata_dev_t *) fun->driver_data;
 164
 165         void *phys;
 166         void *ibuf;
 167
 168         dmamem_map_anonymous(sata->block_size, AS_AREA_READ | AS_AREA_WRITE,
 169             0, &phys, (void **) &ibuf);
 170
 171         fibril_mutex_lock(&sata->lock);
 172
 173         for (size_t cur = 0; cur < count; cur++) {
 174                 memcpy(ibuf, (void *) (((uint8_t *) buf) + (sata->block_size * cur)),
 175                     sata->block_size);
 176                 rc = ahci_wb_fpdma(sata, phys, blocknum + cur);
 177                 if (rc != EOK)
 178                         break;
 179         }
 180
 181         fibril_mutex_unlock(&sata->lock);
 182         dmamem_unmap_anonymous(ibuf);
 183         return rc;
 184 }
 185
 186 /*----------------------------------------------------------------------------*/
 187 /*-- AHCI Commands -----------------------------------------------------------*/
 188 /*----------------------------------------------------------------------------*/
 189
 190 static void ahci_identify_device_cmd(sata_dev_t *sata, void *phys)
 191 {
 192         volatile std_command_frame_t *cmd =
 193             (std_command_frame_t *) sata->cmd_table;
 194
 195         cmd->fis_type = 0x27;
 196         cmd->c = 0x80;
 197         cmd->command = 0xec;
 198         cmd->features = 0;
 199         cmd->lba_lower = 0;
 200         cmd->device = 0;
 201         cmd->lba_upper = 0;
 202         cmd->features_upper = 0;
 203         cmd->count = 0;
 204         cmd->reserved1 = 0;
 205         cmd->control = 0;
 206         cmd->reserved2 = 0;
 207
 208         volatile ahci_cmd_prdt_t *prdt =
 209             (ahci_cmd_prdt_t *) (&sata->cmd_table[0x20]);
 210
 211         prdt->data_address_low = LO(phys);
 212         prdt->data_address_upper = HI(phys);
 213         prdt->reserved1 = 0;
 214         prdt->dbc = 511;
 215         prdt->reserved2 = 0;
 216         prdt->ioc = 0;
 217
 218         sata->cmd_header->prdtl = 1;
 219         sata->cmd_header->flags = 0x402;
 220         sata->cmd_header->bytesprocessed = 0;
 221
 222         sata->port->pxsact |= 1;
 223         sata->port->pxci |= 1;
 224 }
 225
 226 static void ahci_identify_packet_device_cmd(sata_dev_t *sata, void *phys)
 227 {
 228         volatile std_command_frame_t *cmd =
 229             (std_command_frame_t *) sata->cmd_table;
 230
 231         cmd->fis_type = 0x27;
 232         cmd->c = 0x80;
 233         cmd->command = 0xa1;
 234         cmd->features = 0;
 235         cmd->lba_lower = 0;
 236         cmd->device = 0;
 237         cmd->lba_upper = 0;
 238         cmd->features_upper = 0;
 239         cmd->count = 0;
 240         cmd->reserved1 = 0;
 241         cmd->control = 0;
 242         cmd->reserved2 = 0;
 243
 244         volatile ahci_cmd_prdt_t *prdt =
 245             (ahci_cmd_prdt_t *) (&sata->cmd_table[0x20]);
 246
 247         prdt->data_address_low = LO(phys);
 248         prdt->data_address_upper = HI(phys);
 249         prdt->reserved1 = 0;
 250         prdt->dbc = 511;
 251         prdt->reserved2 = 0;
 252         prdt->ioc = 0;
 253
 254         sata->cmd_header->prdtl = 1;
 255         sata->cmd_header->flags = 0x402;
 256         sata->cmd_header->bytesprocessed = 0;
 257
 258         sata->port->pxsact |= 1;
 259         sata->port->pxci |= 1;
 260 }
 261
 262 static int ahci_identify_device(sata_dev_t *sata)
 263 {
 264         if (sata->invalid_device) {
 265                 ddf_msg(LVL_ERROR,
 266                     "Identify command device on invalid device");
 267                 return EINTR;
 268         }
 269
 270         void *phys;
 271         identify_data_t *idata;
 272
 273         dmamem_map_anonymous(512, AS_AREA_READ | AS_AREA_WRITE, 0, &phys,
 274             (void **) &idata);
 275         bzero(idata, 512);
 276
 277         fibril_mutex_lock(&sata->lock);
 278
 279         ahci_identify_device_cmd(sata, phys);
 280
 281         fibril_mutex_lock(&sata->event_lock);
 282         fibril_condvar_wait(&sata->event_condvar, &sata->event_lock);
 283         fibril_mutex_unlock(&sata->event_lock);
 284
 285         ahci_port_is_t pxis = sata->shadow_pxis;
 286         sata->shadow_pxis.u32 &= ~pxis.u32;
 287
 288         if (sata->invalid_device) {
 289                 ddf_msg(LVL_ERROR,
 290                     "Unrecoverable error during ata identify device");
 291                 goto error;
 292         }
 293
 294         if (ahci_port_is_tfes(pxis)) {
 295                 ahci_identify_packet_device_cmd(sata, phys);
 296
 297                 fibril_mutex_lock(&sata->event_lock);
 298                 fibril_condvar_wait(&sata->event_condvar, &sata->event_lock);
 299                 fibril_mutex_unlock(&sata->event_lock);
 300
 301                 pxis = sata->shadow_pxis;
 302                 sata->shadow_pxis.u32 &= ~pxis.u32;
 303
 304                 if ((sata->invalid_device) || (ahci_port_is_error(pxis))) {
 305                         ddf_msg(LVL_ERROR,
 306                             "Unrecoverable error during ata identify packet device");
 307                         goto error;
 308                 }
 309
 310                 sata->packet_device = true;
 311         } else
 312                 sata->packet_device = false;
 313
 314         ahci_get_model_name(idata->model_name, sata->model);
 315
 316         /*
 317          * Due to QEMU limitation (as of 2012-06-22),
 318          * only NCQ FPDMA mode is supported.
 319          */
 320         if ((idata->sata_cap & np_cap_ncq) == 0) {
 321                 ddf_msg(LVL_ERROR, "%s: NCQ must be supported", sata->model);
 322                 goto error;
 323         }
 324
 325         if (sata->packet_device) {
 326                 /*
 327                  * Due to QEMU limitation (as of 2012-06-22),
 328                  * only NCQ FPDMA mode supported - block size is 512 B,
 329                  * not 2048 B!
 330                  */
 331                 sata->block_size = 512;
 332                 sata->blocks = 0;
 333         } else {
 334                 sata->block_size = 512;
 335
 336                 if ((idata->caps & rd_cap_lba) == 0) {
 337                         ddf_msg(LVL_ERROR, "%s: LBA for NCQ must be supported",
 338                             sata->model);
 339                         goto error;
 340                 } else if ((idata->cmd_set1 & cs1_addr48) == 0) {
 341                         sata->blocks = (uint32_t) idata->total_lba28_0 |
 342                             ((uint32_t) idata->total_lba28_1 << 16);
 343                 } else {
 344                         /* Device supports LBA-48 addressing. */
 345                         sata->blocks = (uint64_t) idata->total_lba48_0 |
 346                             ((uint64_t) idata->total_lba48_1 << 16) |
 347                             ((uint64_t) idata->total_lba48_2 << 32) |
 348                             ((uint64_t) idata->total_lba48_3 << 48);
 349                 }
 350         }
 351
 352         uint8_t udma_mask = idata->udma & 0x007f;
 353         if (udma_mask == 0) {
 354                 ddf_msg(LVL_ERROR,
 355                     "%s: UDMA mode for NCQ FPDMA mode must be supported",
 356                     sata->model);
 357                 goto error;
 358         } else {
 359                 for (unsigned int i = 0; i < 7; i++) {
 360                         if (udma_mask & (1 << i))
 361                                 sata->highest_udma_mode = i;
 362                 }
 363         }
 364
 365         fibril_mutex_unlock(&sata->lock);
 366         dmamem_unmap_anonymous(idata);
 367
 368         return EOK;
 369
 370 error:
 371         fibril_mutex_unlock(&sata->lock);
 372         dmamem_unmap_anonymous(idata);
 373
 374         return EINTR;
 375 }
 376
 377 static void ahci_set_mode_cmd(sata_dev_t *sata, void* phys, uint8_t mode)
 378 {
 379         volatile std_command_frame_t *cmd =
 380             (std_command_frame_t *) sata->cmd_table;
 381
 382         cmd->fis_type = 0x27;
 383         cmd->c = 0x80;
 384         cmd->command = 0xef;
 385         cmd->features = 0x03;
 386         cmd->lba_lower = 0;
 387         cmd->device = 0;
 388         cmd->lba_upper = 0;
 389         cmd->features_upper = 0;
 390         cmd->count = mode;
 391         cmd->reserved1 = 0;
 392         cmd->control = 0;
 393         cmd->reserved2 = 0;
 394
 395         volatile ahci_cmd_prdt_t* prdt =
 396             (ahci_cmd_prdt_t *) (&sata->cmd_table[0x20]);
 397
 398         prdt->data_address_low = LO(phys);
 399         prdt->data_address_upper = HI(phys);
 400         prdt->reserved1 = 0;
 401         prdt->dbc = 511;
 402         prdt->reserved2 = 0;
 403         prdt->ioc = 0;
 404
 405         sata->cmd_header->prdtl = 1;
 406         sata->cmd_header->flags = 0x402;
 407         sata->cmd_header->bytesprocessed = 0;
 408
 409         sata->port->pxsact |= 1;
 410         sata->port->pxci |= 1;
 411 }
 412
 413 static int ahci_set_highest_ultra_dma_mode(sata_dev_t *sata)
 414 {
 415         if (sata->invalid_device) {
 416                 ddf_msg(LVL_ERROR,
 417                     "%s: Setting highest UDMA mode on invalid device",
 418                     sata->model);
 419                 return EINTR;
 420         }
 421
 422         void *phys;
 423         identify_data_t *idata;
 424
 425         dmamem_map_anonymous(512, AS_AREA_READ | AS_AREA_WRITE, 0, &phys,
 426             (void **) &idata);
 427         bzero(idata, 512);
 428
 429         fibril_mutex_lock(&sata->lock);
 430
 431         uint8_t mode = 0x40 | (sata->highest_udma_mode & 0x07);
 432         ahci_set_mode_cmd(sata, phys, mode);
 433
 434         fibril_mutex_lock(&sata->event_lock);
 435         fibril_condvar_wait(&sata->event_condvar, &sata->event_lock);
 436         fibril_mutex_unlock(&sata->event_lock);
 437
 438         ahci_port_is_t pxis = sata->shadow_pxis;
 439         sata->shadow_pxis.u32 &= ~pxis.u32;
 440
 441         if (sata->invalid_device) {
 442                 ddf_msg(LVL_ERROR,
 443                     "%s: Unrecoverable error during set highest UDMA mode",
 444                     sata->model);
 445                 goto error;
 446         }
 447
 448         if (ahci_port_is_error(pxis)) {
 449                 ddf_msg(LVL_ERROR,
 450                     "%s: Error during set highest UDMA mode", sata->model);
 451                 goto error;
 452         }
 453
 454         fibril_mutex_unlock(&sata->lock);
 455         dmamem_unmap_anonymous(idata);
 456
 457         return EOK;
 458
 459 error:
 460         fibril_mutex_unlock(&sata->lock);
 461         dmamem_unmap_anonymous(idata);
 462
 463         return EINTR;
 464 }
 465
 466 static void ahci_rb_fpdma_cmd(sata_dev_t *sata, void *phys, uint64_t blocknum)
 467 {
 468         volatile ncq_command_frame_t *cmd =
 469             (ncq_command_frame_t *) sata->cmd_table;
 470
 471         cmd->fis_type = 0x27;
 472         cmd->c = 0x80;
 473         cmd->command = 0x60;
 474         cmd->tag = 0;
 475         cmd->control = 0;
 476
 477         cmd->reserved1 = 0;
 478         cmd->reserved2 = 0;
 479         cmd->reserved3 = 0;
 480         cmd->reserved4 = 0;
 481         cmd->reserved5 = 0;
 482         cmd->reserved6 = 0;
 483
 484         cmd->sector_count_low = 1;
 485         cmd->sector_count_high = 0;
 486
 487         cmd->lba0 = blocknum & 0xff;
 488         cmd->lba1 = (blocknum >> 8) & 0xff;
 489         cmd->lba2 = (blocknum >> 16) & 0xff;
 490         cmd->lba3 = (blocknum >> 24) & 0xff;
 491         cmd->lba4 = (blocknum >> 32) & 0xff;
 492         cmd->lba5 = (blocknum >> 40) & 0xff;
 493
 494         volatile ahci_cmd_prdt_t *prdt =
 495             (ahci_cmd_prdt_t *) (&sata->cmd_table[0x20]);
 496
 497         prdt->data_address_low = LO(phys);
 498         prdt->data_address_upper = HI(phys);
 499         prdt->reserved1 = 0;
 500         prdt->dbc = sata->block_size - 1;
 501         prdt->reserved2 = 0;
 502         prdt->ioc = 0;
 503
 504         sata->cmd_header->prdtl = 1;
 505         sata->cmd_header->flags = 0x405;
 506         sata->cmd_header->bytesprocessed = 0;
 507
 508         sata->port->pxsact |= 1;
 509         sata->port->pxci |= 1;
 510 }
 511
 512 static int ahci_rb_fpdma(sata_dev_t *sata, void *phys, uint64_t blocknum)
 513 {
 514         if (sata->invalid_device) {
 515                 ddf_msg(LVL_ERROR,
 516                     "%s: FPDMA read from invalid device", sata->model);
 517                 return EINTR;
 518         }
 519
 520         ahci_rb_fpdma_cmd(sata, phys, blocknum);
 521
 522         fibril_mutex_lock(&sata->event_lock);
 523         fibril_condvar_wait(&sata->event_condvar, &sata->event_lock);
 524         fibril_mutex_unlock(&sata->event_lock);
 525
 526         ahci_port_is_t pxis = sata->shadow_pxis;
 527         sata->shadow_pxis.u32 &= ~pxis.u32;
 528
 529         if ((sata->invalid_device) || (ahci_port_is_error(pxis))_ {
 530                 ddf_msg(LVL_ERROR,
 531                     "%s: Unrecoverable error during FPDMA read", sata->model);
 532                 return EINTR;
 533         }
 534
 535         return EOK;
 536 }
 537
 538 static void ahci_wb_fpdma_cmd(sata_dev_t *sata, void *phys, uint64_t blocknum)
 539 {
 540         volatile ncq_command_frame_t *cmd =
 541             (ncq_command_frame_t *) sata->cmd_table;
 542
 543         cmd->fis_type = 0x27;
 544         cmd->c = 0x80;
 545         cmd->command = 0x61;
 546         cmd->tag = 0;
 547         cmd->control = 0;
 548
 549         cmd->reserved1 = 0;
 550         cmd->reserved2 = 0;
 551         cmd->reserved3 = 0;
 552         cmd->reserved4 = 0;
 553         cmd->reserved5 = 0;
 554         cmd->reserved6 = 0;
 555
 556         cmd->sector_count_low = 1;
 557         cmd->sector_count_high = 0;
 558
 559         cmd->lba0 = blocknum & 0xff;
 560         cmd->lba1 = (blocknum >> 8) & 0xff;
 561         cmd->lba2 = (blocknum >> 16) & 0xff;
 562         cmd->lba3 = (blocknum >> 24) & 0xff;
 563         cmd->lba4 = (blocknum >> 32) & 0xff;
 564         cmd->lba5 = (blocknum >> 40) & 0xff;
 565
 566         volatile ahci_cmd_prdt_t * prdt =
 567             (ahci_cmd_prdt_t *) (&sata->cmd_table[0x20]);
 568
 569         prdt->data_address_low = LO(phys);
 570         prdt->data_address_upper = HI(phys);
 571         prdt->reserved1 = 0;
 572         prdt->dbc = sata->block_size - 1;
 573         prdt->reserved2 = 0;
 574         prdt->ioc = 0;
 575
 576         sata->cmd_header->prdtl = 1;
 577         sata->cmd_header->flags = 0x445;
 578         sata->cmd_header->bytesprocessed = 0;
 579
 580         sata->port->pxsact |= 1;
 581         sata->port->pxci |= 1;
 582 }
 583
 584 static int ahci_wb_fpdma(sata_dev_t *sata, void *phys, uint64_t blocknum)
 585 {
 586         if (sata->invalid_device) {
 587                 ddf_msg(LVL_ERROR,
 588                     "%s: FPDMA write to invalid device", sata->model);
 589                 return EINTR;
 590         }
 591
 592         ahci_wb_fpdma_cmd(sata, phys, blocknum);
 593
 594         fibril_mutex_lock(&sata->event_lock);
 595         fibril_condvar_wait(&sata->event_condvar, &sata->event_lock);
 596         fibril_mutex_unlock(&sata->event_lock);
 597
 598         ahci_port_is_t pxis = sata->shadow_pxis;
 599         sata->shadow_pxis.u32 &= ~pxis.u32;
 600
 601         if ((sata->invalid_device) || (ahci_port_is_error(pxis))) {
 602                 ddf_msg(LVL_ERROR,
 603                     "%s: Unrecoverable error during fpdma write", sata->model);
 604                 return EINTR;
 605         }
 606
 607         return EOK;
 608 }
 609
 610 /*----------------------------------------------------------------------------*/
 611 /*-- Interrupts and timer unified handling -----------------------------------*/
 612 /*----------------------------------------------------------------------------*/
 613
 614 static irq_pio_range_t ahci_ranges[] = {
 615         {
 616                 .base = 0,
 617                 .size = 32,
 618         }
 619 };
 620
 621 static irq_cmd_t ahci_cmds[] = {
 622         {
 623                 /* Disable interrupt - interrupt is deasserted in qemu 1.0.1 */
 624                 .cmd = CMD_MEM_WRITE_32,
 625                 .addr = NULL,
 626                 .value = AHCI_GHC_GHC_AE
 627         },
 628         {
 629                 /* Clear interrupt status register - for vbox and real hw */
 630                 .cmd = CMD_MEM_REWRITE_32,
 631                 .addr = NULL
 632         },
 633         {
 634                 .cmd = CMD_ACCEPT
 635         }
 636 };
 637
 638 /** Unified AHCI interrupt and timer interrupt handler.
 639  *
 640  * @param ahci     AHCI device.
 641  * @param is_timer Indicate timer interrupt.
 642  *
 643  */
 644 static void ahci_interrupt_or_timer(ahci_dev_t *ahci, bool is_timer)
 645 {
 646         /*
 647          * Get current value of hardware interrupt state register,
 648          * clear hardware register (write to clear behavior).
 649          */
 650         ahci_ghc_is_t is;
 651
 652         is.u32 = ahci->memregs->ghc.is;
 653         ahci->memregs->ghc.is = is.u32;
 654         is.u32 = ahci->memregs->ghc.is;
 655
 656         uint32_t port_event_flags = 0;
 657         uint32_t port_mask = 1;
 658         for (unsigned int i = 0; i < 32; i++) {
 659                 /*
 660                  * Get current value of hardware port interrupt state register,
 661                  * clear hardware register (write to clear behavior).
 662                  */
 663                 ahci_port_is_t pxis;
 664
 665                 pxis.u32 = ahci->memregs->ports[i].pxis;
 666                 ahci->memregs->ports[i].pxis = pxis.u32;
 667
 668                 sata_dev_t *sata = (sata_dev_t *) ahci->sata_devs[i];
 669                 if (sata != NULL) {
 670                         /* add value to shadow copy of port interrupt state register. */
 671                         sata->shadow_pxis.u32 |= pxis.u32;
 672
 673                         /* Evaluate port event. */
 674                         if ((ahci_port_is_end_of_operation(pxis)) ||
 675                             (ahci_port_is_error(pxis)))
 676                                 port_event_flags |= port_mask;
 677
 678                         if (ahci_port_is_permanent_error(pxis))
 679                                 sata->invalid_device = true;
 680                 }
 681
 682                 port_mask <<= 1;
 683         }
 684
 685         port_mask = 1;
 686         for (unsigned int i = 0; i < 32; i++) {
 687                 sata_dev_t *sata = (sata_dev_t *) ahci->sata_devs[i];
 688                 if ((port_event_flags & port_mask) && (sata != NULL)) {
 689                         fibril_mutex_lock(&sata->event_lock);
 690                         fibril_condvar_signal(&sata->event_condvar);
 691                         fibril_mutex_unlock(&sata->event_lock);
 692                 }
 693
 694                 port_mask <<= 1;
 695         }
 696 }
 697
 698 /** AHCI timer interrupt handler.
 699  *
 700  * @param arg Pointer to AHCI device.
 701  *
 702  */
 703 static void ahci_timer(void *arg)
 704 {
 705         ahci_dev_t *ahci = (ahci_dev_t *) arg;
 706
 707         ahci_interrupt_or_timer(ahci, 1);
 708         fibril_timer_set(ahci->timer, AHCI_TIMER_TICKS, ahci_timer, ahci);
 709 }
 710
 711 /** AHCI interrupt handler.
 712  *
 713  * @param dev Pointer to device driver handler.
 714  *
 715  */
 716 static void ahci_interrupt(ddf_dev_t *dev, ipc_callid_t iid, ipc_call_t *icall)
 717 {
 718         ahci_dev_t *ahci = (ahci_dev_t *) dev->driver_data;
 719
 720         ahci_interrupt_or_timer(ahci, 0);
 721
 722         /* Enable interrupt. */
 723         ahci->memregs->ghc.ghc |= 2;
 724 }
 725
 726 /*----------------------------------------------------------------------------*/
 727 /*-- AHCI and SATA device creating and initializing routines -----------------*/
 728 /*----------------------------------------------------------------------------*/
 729
 730 static sata_dev_t *ahci_sata_device_allocate(volatile ahci_port_t *port)
 731 {
 732         size_t size = 4096;
 733         void* phys = NULL;
 734         void* virt_fb = NULL;
 735         void* virt_cmd = NULL;
 736         void* virt_table = NULL;
 737
 738         sata_dev_t *sata = malloc(sizeof(sata_dev_t));
 739         if (sata == NULL)
 740                 return NULL;
 741
 742         bzero(sata, sizeof(sata_dev_t));
 743
 744         sata->port = port;
 745
 746         /* Allocate and init retfis structure. */
 747         int rc = dmamem_map_anonymous(size, AS_AREA_READ | AS_AREA_WRITE, 0,
 748             &phys, &virt_fb);
 749         if (rc != EOK)
 750                 goto error_retfis;
 751
 752         bzero(virt_fb, size);
 753         sata->port->pxfbu = HI(phys);
 754         sata->port->pxfb = LO(phys);
 755
 756         /* Allocate and init command header structure. */
 757         rc = dmamem_map_anonymous(size, AS_AREA_READ | AS_AREA_WRITE, 0,
 758             &phys, &virt_cmd);
 759
 760         if (rc != EOK)
 761                 goto error_cmd;
 762
 763         bzero(virt_cmd, size);
 764         sata->port->pxclbu = HI(phys);
 765         sata->port->pxclb = LO(phys);
 766         sata->cmd_header = (ahci_cmdhdr_t *) virt_cmd;
 767
 768         /* Allocate and init command table structure. */
 769         rc = dmamem_map_anonymous(size, AS_AREA_READ | AS_AREA_WRITE, 0,
 770             &phys, &virt_table);
 771
 772         if (rc != EOK)
 773                 goto error_table;
 774
 775         bzero(virt_table, size);
 776         sata->cmd_header->cmdtableu = HI(phys);
 777         sata->cmd_header->cmdtable = LO(phys);
 778         sata->cmd_table = (uint32_t*) virt_table;
 779
 780         return sata;
 781
 782 error_table:
 783         dmamem_unmap(virt_cmd, size);
 784 error_cmd:
 785         dmamem_unmap(virt_fb, size);
 786 error_retfis:
 787         free(sata);
 788         return NULL;
 789
 790         /*
 791          * Deleting of pointers in memory hardware mapped register
 792          * unneccessary, hardware port is not in operational state.
 793          */
 794 }
 795
 796 static int ahci_sata_device_create(ahci_dev_t *ahci, ddf_dev_t *dev,
 797     volatile ahci_port_t *port, unsigned int port_num)
 798 {
 799         ddf_fun_t *fun = NULL;
 800         sata_dev_t *sata = ahci_sata_device_allocate(port);
 801
 802         if (sata == NULL)
 803                 return EINTR;
 804
 805         /* Set pointers between SATA and AHCI structures. */
 806         sata->ahci = ahci;
 807         sata->port_num = port_num;
 808         ahci->sata_devs[port_num] = sata;
 809
 810         fibril_mutex_initialize(&sata->lock);
 811         fibril_mutex_initialize(&sata->event_lock);
 812         fibril_condvar_initialize(&sata->event_condvar);
 813
 814         /* Initialize SATA port operational registers. */
 815         sata->port->pxis = 0;
 816         sata->port->pxie = 0xffffffff;
 817         sata->port->pxserr = 0;
 818         sata->port->pxcmd |= 0x10;
 819         sata->port->pxcmd |= 0x01;
 820
 821         if (ahci_identify_device(sata) != EOK)
 822                 goto error;
 823
 824         if (ahci_set_highest_ultra_dma_mode(sata) != EOK)
 825                 goto error;
 826
 827         /* Add sata device to system. */
 828         char sata_dev_name[1024];
 829         snprintf(sata_dev_name, 1024, "ahci_%u", sata_devices_count);
 830
 831         fibril_mutex_lock(&sata_devices_count_lock);
 832         sata_devices_count++;
 833         fibril_mutex_unlock(&sata_devices_count_lock);
 834
 835         fun = ddf_fun_create(dev, fun_exposed, sata_dev_name);
 836         if (fun == NULL) {
 837                 ddf_msg(LVL_ERROR, "Failed creating function.");
 838                 goto error;
 839         }
 840
 841         fun->ops = &ahci_ops;
 842         fun->driver_data = sata;
 843         int rc = ddf_fun_bind(fun);
 844         if (rc != EOK) {
 845                 ddf_msg(LVL_ERROR, "Failed binding function.");
 846                 goto error;
 847         }
 848
 849         return EOK;
 850
 851 error:
 852         sata->invalid_device = true;
 853         if (fun != NULL)
 854                 ddf_fun_destroy(fun);
 855
 856         return EINTR;
 857 }
 858
 859 static void ahci_sata_devices_create(ahci_dev_t *ahci, ddf_dev_t *dev)
 860 {
 861         for (unsigned int port_num = 0; port_num < 32; port_num++) {
 862                 /* Active ports only */
 863                 if (!(ahci->memregs->ghc.pi & (1 << port_num)))
 864                         continue;
 865
 866                 volatile ahci_port_t *port = ahci->memregs->ports + port_num;
 867
 868                 /* Active devices only */
 869                 if ((port->pxssts & 0x0f) != 3)
 870                         continue;
 871
 872                 ahci_sata_device_create(ahci, dev, port, port_num);
 873         }
 874 }
 875
 876 static ahci_dev_t *ahci_ahci_create(ddf_dev_t *dev)
 877 {
 878         ahci_dev_t *ahci = malloc(sizeof(ahci_dev_t));
 879         if (!ahci)
 880                 return NULL;
 881
 882         bzero(ahci, sizeof(ahci_dev_t));
 883
 884         ahci->dev = dev;
 885
 886         hw_res_list_parsed_t hw_res_parsed;
 887         hw_res_list_parsed_init(&hw_res_parsed);
 888         if (hw_res_get_list_parsed(dev->parent_sess, &hw_res_parsed, 0) != EOK)
 889                 goto error_get_res_parsed;
 890
 891         /* Register interrupt handler */
 892         ahci_ranges[0].base = (size_t) hw_res_parsed.mem_ranges.ranges[0].address;
 893         ahci_ranges[0].size = sizeof(ahci_dev_t);
 894         ahci_cmds[0].addr =
 895             ((uint32_t *) (size_t) hw_res_parsed.mem_ranges.ranges[0].address) + 1;
 896         ahci_cmds[1].addr =
 897             ((uint32_t *) (size_t) hw_res_parsed.mem_ranges.ranges[0].address) + 2;
 898
 899         irq_code_t ct;
 900         ct.cmdcount = 3;
 901         ct.cmds = ahci_cmds;
 902         ct.rangecount = 1;
 903         ct.ranges = ahci_ranges;
 904
 905         int rc = register_interrupt_handler(dev, hw_res_parsed.irqs.irqs[0],
 906             ahci_interrupt, &ct);
 907
 908         if (rc != EOK) {
 909                 ddf_msg(LVL_ERROR, "Failed register_interrupt_handler function.");
 910                 goto error_register_interrupt_handler;
 911         }
 912
 913         if (ahci_pciintel_enable_interrupt(hw_res_parsed.irqs.irqs[0]) != EOK) {
 914                 ddf_msg(LVL_ERROR, "Failed enable interupt.");
 915                 goto error_enable_interrupt;
 916         }
 917
 918         /* Map AHCI register. */
 919         physmem_map((void *) (size_t) (hw_res_parsed.mem_ranges.ranges[0].address),
 920             8, AS_AREA_READ | AS_AREA_WRITE, (void **) &ahci->memregs);
 921         hw_res_list_parsed_clean(&hw_res_parsed);
 922
 923         if (ahci->memregs == NULL)
 924                 goto error_map_registers;
 925
 926         ahci->timer = fibril_timer_create();
 927
 928         return ahci;
 929
 930 error_map_registers:
 931 error_enable_interrupt:
 932 error_register_interrupt_handler:
 933         hw_res_list_parsed_clean(&hw_res_parsed);
 934 error_get_res_parsed:
 935         return NULL;
 936 }
 937
 938 static void ahci_ahci_init(ahci_dev_t *ahci)
 939 {
 940         /* Enable interrupt and bus mastering */
 941         ahci_pcireg_cmd_t cmd;
 942
 943         pci_config_space_read_16(ahci->dev->parent_sess, AHCI_PCI_CMD, &cmd.u16);
 944         cmd.id = 0;
 945         cmd.bme = 1;
 946         pci_config_space_write_16(ahci->dev->parent_sess, AHCI_PCI_CMD, cmd.u16);
 947
 948         /* Set master latency timer */
 949         pci_config_space_write_8(ahci->dev->parent_sess, AHCI_PCI_MLT, 32);
 950
 951         /* Disable command completion coalescing feature. */
 952         ahci_ghc_ccc_ctl_t ccc;
 953         ccc.u32 = ahci->memregs->ghc.ccc_ctl;
 954         ccc.en = 0;
 955         ahci->memregs->ghc.ccc_ctl = ccc.u32;
 956
 957         /* Enable AHCI and interrupt. */
 958         ahci->memregs->ghc.ghc = AHCI_GHC_GHC_AE | AHCI_GHC_GHC_IE;
 959
 960         /* Enable timer. */
 961         fibril_timer_set(ahci->timer, AHCI_TIMER_TICKS, ahci_timer, ahci);
 962 }
 963
 964 static int ahci_dev_add(ddf_dev_t *dev)
 965 {
 966         dev->parent_sess = devman_parent_device_connect(EXCHANGE_SERIALIZE,
 967             dev->handle, IPC_FLAG_BLOCKING);
 968         if (dev->parent_sess == NULL)
 969                 return EINTR;
 970
 971         ahci_dev_t *ahci = ahci_ahci_create(dev);
 972         if (ahci == NULL)
 973                 return EINTR;
 974
 975         dev->driver_data = ahci;
 976         ahci_ahci_init(ahci);
 977         ahci_sata_devices_create(ahci, dev);
 978
 979         return EOK;
 980 }
 981
 982 /*----------------------------------------------------------------------------*/
 983 /*-- Helpers and utilities ---------------------------------------------------*/
 984 /*----------------------------------------------------------------------------*/
 985
 986 static void ahci_get_model_name(uint16_t *src, char *dst)
 987 {
 988         uint8_t model[40];
 989         bzero(model, 40);
 990
 991         for (unsigned int i = 0; i < 20; i++) {
 992                 uint16_t w = src[i];
 993                 model[2 * i] = w >> 8;
 994                 model[2 * i + 1] = w & 0x00ff;
 995         }
 996
 997         uint32_t len = 40;
 998         while ((len > 0) && (model[len - 1] == 0x20))
 999                 len--;
1000
1001         size_t pos = 0;
1002         for (unsigned int i = 0; i < len; i++) {
1003                 uint8_t c = model[i];
1004                 if (c >= 0x80)
1005                         c = '?';
1006
1007                 chr_encode(c, dst, &pos, 40);
1008         }
1009
1010         dst[pos] = '\0';
1011 }
1012
1013 static int ahci_pciintel_enable_interrupt(int irq)
1014 {
1015         async_sess_t *irc_sess = NULL;
1016         irc_sess = service_connect_blocking(EXCHANGE_SERIALIZE, SERVICE_IRC, 0, 0);
1017
1018         if (!irc_sess)
1019                 return EINTR;
1020
1021         async_exch_t *exch = async_exchange_begin(irc_sess);
1022         const int rc = async_req_1_0(exch, IRC_ENABLE_INTERRUPT, irq);
1023         async_exchange_end(exch);
1024
1025         async_hangup(irc_sess);
1026         return rc;
1027 }
1028
1029 /*----------------------------------------------------------------------------*/
1030 /*-- AHCI Main routine -------------------------------------------------------*/
1031 /*----------------------------------------------------------------------------*/
1032
1033 int main(int argc, char *argv[])
1034 {
1035         printf("%s: HelenOS AHCI device driver\n", NAME);
1036         ddf_log_init(NAME, LVL_ERROR);
1037         fibril_mutex_initialize(&sata_devices_count_lock);
1038         return ddf_driver_main(&ahci_driver);
1039 }