4 * Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19 * Copyright (c) 2009 The DragonFly Project. All rights reserved.
21 * This code is derived from software contributed to The DragonFly Project
22 * by Matthew Dillon <dillon@backplane.com>
24 * Redistribution and use in source and binary forms, with or without
25 * modification, are permitted provided that the following conditions
28 * 1. Redistributions of source code must retain the above copyright
29 * notice, this list of conditions and the following disclaimer.
30 * 2. Redistributions in binary form must reproduce the above copyright
31 * notice, this list of conditions and the following disclaimer in
32 * the documentation and/or other materials provided with the
34 * 3. Neither the name of The DragonFly Project nor the names of its
35 * contributors may be used to endorse or promote products derived
36 * from this software without specific, prior written permission.
38 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
39 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
40 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
41 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
42 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
43 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
44 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
45 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
46 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
47 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
48 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51 * $OpenBSD: ahci.c,v 1.147 2009/02/16 21:19:07 miod Exp $
56 void ahci_port_interrupt_enable(struct ahci_port
*ap
);
58 int ahci_load_prdt(struct ahci_ccb
*);
59 void ahci_unload_prdt(struct ahci_ccb
*);
60 static void ahci_load_prdt_callback(void *info
, bus_dma_segment_t
*segs
,
61 int nsegs
, int error
);
62 void ahci_start(struct ahci_ccb
*);
63 int ahci_port_softreset(struct ahci_port
*ap
);
64 int ahci_port_hardreset(struct ahci_port
*ap
, int hard
);
65 void ahci_port_hardstop(struct ahci_port
*ap
);
67 static void ahci_ata_cmd_timeout_unserialized(void *);
68 void ahci_check_active_timeouts(struct ahci_port
*ap
);
70 void ahci_beg_exclusive_access(struct ahci_port
*ap
, struct ata_port
*at
);
71 void ahci_end_exclusive_access(struct ahci_port
*ap
, struct ata_port
*at
);
72 void ahci_issue_pending_commands(struct ahci_port
*ap
, struct ahci_ccb
*ccb
);
73 void ahci_issue_saved_commands(struct ahci_port
*ap
, u_int32_t mask
);
75 int ahci_port_read_ncq_error(struct ahci_port
*, int);
77 struct ahci_dmamem
*ahci_dmamem_alloc(struct ahci_softc
*, bus_dma_tag_t tag
);
78 void ahci_dmamem_free(struct ahci_softc
*, struct ahci_dmamem
*);
79 static void ahci_dmamem_saveseg(void *info
, bus_dma_segment_t
*segs
, int nsegs
, int error
);
81 static void ahci_dummy_done(struct ata_xfer
*xa
);
82 static void ahci_empty_done(struct ahci_ccb
*ccb
);
83 static void ahci_ata_cmd_done(struct ahci_ccb
*ccb
);
84 static u_int32_t
ahci_pactive(struct ahci_port
*ap
);
87 * Initialize the global AHCI hardware. This code does not set up any of
91 ahci_init(struct ahci_softc
*sc
)
94 u_int32_t bios_cap
, vers
;
98 DPRINTF(AHCI_D_VERBOSE
, " GHC 0x%b",
99 ahci_read(sc
, AHCI_REG_GHC
), AHCI_FMT_GHC
);
104 vers
= ahci_read(sc
, AHCI_REG_VS
);
107 * save BIOS initialised parameters, enable staggered spin up
109 bios_cap
= ahci_read(sc
, AHCI_REG_CAP
);
110 bios_cap
&= AHCI_REG_CAP_SMPS
| AHCI_REG_CAP_SSS
;
112 pi
= ahci_read(sc
, AHCI_REG_PI
);
115 * Unconditionally reset the controller, do not conditionalize on
116 * trying to figure it if it was previously active or not.
118 * NOTE: On AE before HR. The AHCI-1.1 spec has a note in section
119 * 5.2.2.1 regarding this. HR should be set to 1 only after
120 * AE is set to 1. The reset sequence will clear HR when
121 * it completes, and will also clear AE if SAM is 0. AE must
122 * then be set again. When SAM is 1 the AE bit typically reads
123 * as 1 (and is read-only).
125 * NOTE: Avoid PCI[e] transaction burst by issuing dummy reads,
126 * otherwise the writes will only be separated by a few
131 * If you have a port multiplier and it does not have a device
132 * in target 0, and it probes normally, but a later operation
133 * mis-probes a target behind that PM, it is possible for the
134 * port to brick such that only (a) a power cycle of the host
135 * or (b) placing a device in target 0 will fix the problem.
136 * Power cycling the PM has no effect (it works fine on another
137 * host port). This issue is unrelated to CLO.
140 * Wait for any prior reset sequence to complete
142 if (ahci_wait_ne(sc
, AHCI_REG_GHC
,
143 AHCI_REG_GHC_HR
, AHCI_REG_GHC_HR
) != 0) {
144 device_printf(sc
->sc_dev
, "Controller is stuck in reset\n");
147 ahci_write(sc
, AHCI_REG_GHC
, AHCI_REG_GHC_AE
);
149 ahci_read(sc
, AHCI_REG_GHC
); /* flush */
150 ahci_write(sc
, AHCI_REG_GHC
, AHCI_REG_GHC_AE
| AHCI_REG_GHC_HR
);
152 ahci_read(sc
, AHCI_REG_GHC
); /* flush */
153 if (ahci_wait_ne(sc
, AHCI_REG_GHC
,
154 AHCI_REG_GHC_HR
, AHCI_REG_GHC_HR
) != 0) {
155 device_printf(sc
->sc_dev
, "unable to reset controller\n");
158 if (ahci_read(sc
, AHCI_REG_GHC
) & AHCI_REG_GHC_AE
) {
159 device_printf(sc
->sc_dev
, "AE did not auto-clear!\n");
160 ahci_write(sc
, AHCI_REG_GHC
, 0);
165 * Enable ahci (global interrupts disabled)
167 * Restore saved parameters. Avoid pci transaction burst write
168 * by issuing dummy reads.
171 ahci_write(sc
, AHCI_REG_GHC
, AHCI_REG_GHC_AE
);
174 ahci_read(sc
, AHCI_REG_GHC
); /* flush */
176 bios_cap
|= AHCI_REG_CAP_SSS
;
177 ahci_write(sc
, AHCI_REG_CAP
, ahci_read(sc
, AHCI_REG_CAP
) | bios_cap
);
178 ahci_write(sc
, AHCI_REG_PI
, pi
);
179 ahci_read(sc
, AHCI_REG_GHC
); /* flush */
182 * Intel hocus pocus in case the BIOS has not set the chip up
183 * properly for AHCI operation.
185 if (pci_get_vendor(sc
->sc_dev
) == PCI_VENDOR_INTEL
) {
186 if ((pci_read_config(sc
->sc_dev
, 0x92, 2) & 0x0F) != 0x0F)
187 device_printf(sc
->sc_dev
, "Intel hocus pocus\n");
188 pci_write_config(sc
->sc_dev
, 0x92,
189 pci_read_config(sc
->sc_dev
, 0x92, 2) | 0x0F, 2);
193 * This is a hack that currently does not appear to have
194 * a significant effect, but I noticed the port registers
195 * do not appear to be completely cleared after the host
196 * controller is reset.
198 * Use a temporary ap structure so we can call ahci_pwrite().
200 * We must be sure to stop the port
202 ap
= kmalloc(sizeof(*ap
), M_DEVBUF
, M_WAITOK
| M_ZERO
);
205 for (i
= 0; i
< AHCI_MAX_PORTS
; ++i
) {
208 if ((pi
& (1 << i
)) == 0)
210 if (bus_space_subregion(sc
->sc_iot
, sc
->sc_ioh
,
211 AHCI_PORT_REGION(i
), AHCI_PORT_SIZE
, &ap
->ap_ioh
) != 0) {
212 device_printf(sc
->sc_dev
, "can't map port\n");
216 * NOTE! Setting AHCI_PREG_SCTL_DET_DISABLE on AHCI1.0 or
217 * AHCI1.1 can brick the chipset. Not only brick it,
218 * but also crash the PC. The bit seems unreliable
219 * on AHCI1.2 as well.
221 ahci_port_stop(ap
, 1);
222 ahci_pwrite(ap
, AHCI_PREG_SCTL
, ap
->ap_sc
->sc_ipm_disable
);
223 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
224 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
225 ahci_write(ap
->ap_sc
, AHCI_REG_IS
, 1 << i
);
226 ahci_pwrite(ap
, AHCI_PREG_CMD
, 0);
227 ahci_pwrite(ap
, AHCI_PREG_IS
, -1);
228 sc
->sc_portmask
|= (1 << i
);
238 * Allocate and initialize an AHCI port.
241 ahci_port_alloc(struct ahci_softc
*sc
, u_int port
)
243 struct ahci_port
*ap
;
245 struct ahci_ccb
*ccb
;
249 struct ahci_cmd_hdr
*hdr
;
250 struct ahci_cmd_table
*table
;
255 ap
= kmalloc(sizeof(*ap
), M_DEVBUF
, M_WAITOK
| M_ZERO
);
256 ap
->ap_err_scratch
= kmalloc(512, M_DEVBUF
, M_WAITOK
| M_ZERO
);
258 ksnprintf(ap
->ap_name
, sizeof(ap
->ap_name
), "%s%d.%d",
259 device_get_name(sc
->sc_dev
),
260 device_get_unit(sc
->sc_dev
),
262 sc
->sc_ports
[port
] = ap
;
265 * Allocate enough so we never have to reallocate, it makes
268 * ap_pmcount will be reduced by the scan if we encounter the
269 * port multiplier port prior to target 15.
271 * kmalloc power-of-2 allocations are guaranteed not to cross
272 * a page boundary. Make sure the identify sub-structure in the
273 * at structure does not cross a page boundary, just in case the
274 * part is AHCI-1.1 and can't handle multiple DRQ blocks.
276 if (ap
->ap_ata
[0] == NULL
) {
279 for (pw2
= 1; pw2
< sizeof(*at
); pw2
<<= 1)
281 for (i
= 0; i
< AHCI_MAX_PMPORTS
; ++i
) {
282 at
= kmalloc(pw2
, M_DEVBUF
, M_INTWAIT
| M_ZERO
);
284 at
->at_ahci_port
= ap
;
286 at
->at_probe
= ATA_PROBE_NEED_INIT
;
287 at
->at_features
|= ATA_PORT_F_RESCAN
;
288 ksnprintf(at
->at_name
, sizeof(at
->at_name
),
289 "%s.%d", ap
->ap_name
, i
);
292 if (bus_space_subregion(sc
->sc_iot
, sc
->sc_ioh
,
293 AHCI_PORT_REGION(port
), AHCI_PORT_SIZE
, &ap
->ap_ioh
) != 0) {
294 device_printf(sc
->sc_dev
,
295 "unable to create register window for port %d\n",
302 ap
->ap_probe
= ATA_PROBE_NEED_INIT
;
303 ap
->link_pwr_mgmt
= AHCI_LINK_PWR_MGMT_NONE
;
304 ap
->sysctl_tree
= NULL
;
305 TAILQ_INIT(&ap
->ap_ccb_free
);
306 TAILQ_INIT(&ap
->ap_ccb_pending
);
307 lockinit(&ap
->ap_ccb_lock
, "ahcipo", 0, 0);
309 /* Disable port interrupts */
310 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
311 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
314 * Sec 10.1.2 - deinitialise port if it is already running
316 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
318 if ((cmd
& (AHCI_PREG_CMD_ST
| AHCI_PREG_CMD_CR
|
319 AHCI_PREG_CMD_FRE
| AHCI_PREG_CMD_FR
)) ||
320 (ahci_pread(ap
, AHCI_PREG_SCTL
) & AHCI_PREG_SCTL_DET
)) {
323 r
= ahci_port_stop(ap
, 1);
325 device_printf(sc
->sc_dev
,
326 "unable to disable %s, ignoring port %d\n",
327 ((r
== 2) ? "CR" : "FR"), port
);
332 /* Write DET to zero */
333 ahci_pwrite(ap
, AHCI_PREG_SCTL
, ap
->ap_sc
->sc_ipm_disable
);
337 ap
->ap_dmamem_rfis
= ahci_dmamem_alloc(sc
, sc
->sc_tag_rfis
);
338 if (ap
->ap_dmamem_rfis
== NULL
) {
339 kprintf("%s: NORFIS\n", PORTNAME(ap
));
343 /* Setup RFIS base address */
344 ap
->ap_rfis
= (struct ahci_rfis
*) AHCI_DMA_KVA(ap
->ap_dmamem_rfis
);
345 bzero(ap
->ap_rfis
, sc
->sc_rfis_size
);
347 dva
= AHCI_DMA_DVA(ap
->ap_dmamem_rfis
);
348 ahci_pwrite(ap
, AHCI_PREG_FB
, (u_int32_t
)dva
);
349 ahci_pwrite(ap
, AHCI_PREG_FBU
, (u_int32_t
)(dva
>> 32));
351 /* Clear SERR before starting FIS reception or ST or anything */
353 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
356 * Power up any device sitting on the port.
358 * Don't turn on FIS reception here, it will be handled in the first
361 * Don't make the ICC ACTIVE here, it will be handled in port_init.
363 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
) & ~AHCI_PREG_CMD_ICC
;
364 cmd
&= ~(AHCI_PREG_CMD_CLO
| AHCI_PREG_CMD_PMA
);
365 cmd
|= AHCI_PREG_CMD_POD
| AHCI_PREG_CMD_SUD
;
367 /* this will be done in ahci_pm_port_probe() */
368 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SPM
)
369 cmd
|= AHCI_PREG_CMD_PMA
;
372 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
374 /* Allocate a CCB for each command slot */
375 ap
->ap_ccbs
= kmalloc(sizeof(struct ahci_ccb
) * sc
->sc_ncmds
, M_DEVBUF
,
377 if (ap
->ap_ccbs
== NULL
) {
378 device_printf(sc
->sc_dev
,
379 "unable to allocate command list for port %d\n",
384 /* Command List Structures and Command Tables */
385 ap
->ap_dmamem_cmd_list
= ahci_dmamem_alloc(sc
, sc
->sc_tag_cmdh
);
386 ap
->ap_dmamem_cmd_table
= ahci_dmamem_alloc(sc
, sc
->sc_tag_cmdt
);
387 if (ap
->ap_dmamem_cmd_table
== NULL
||
388 ap
->ap_dmamem_cmd_list
== NULL
) {
390 device_printf(sc
->sc_dev
,
391 "unable to allocate DMA memory for port %d\n",
396 /* Setup command list base address */
397 dva
= AHCI_DMA_DVA(ap
->ap_dmamem_cmd_list
);
398 ahci_pwrite(ap
, AHCI_PREG_CLB
, (u_int32_t
)dva
);
399 ahci_pwrite(ap
, AHCI_PREG_CLBU
, (u_int32_t
)(dva
>> 32));
401 /* Split CCB allocation into CCBs and assign to command header/table */
402 hdr
= AHCI_DMA_KVA(ap
->ap_dmamem_cmd_list
);
403 table
= AHCI_DMA_KVA(ap
->ap_dmamem_cmd_table
);
404 bzero(hdr
, sc
->sc_cmdlist_size
);
406 for (i
= 0; i
< sc
->sc_ncmds
; i
++) {
407 ccb
= &ap
->ap_ccbs
[i
];
409 error
= bus_dmamap_create(sc
->sc_tag_data
, BUS_DMA_ALLOCNOW
,
412 device_printf(sc
->sc_dev
,
413 "unable to create dmamap for port %d "
414 "ccb %d\n", port
, i
);
418 callout_init_mp(&ccb
->ccb_timeout
);
421 ccb
->ccb_cmd_hdr
= &hdr
[i
];
422 ccb
->ccb_cmd_table
= &table
[i
];
423 dva
= AHCI_DMA_DVA(ap
->ap_dmamem_cmd_table
) +
424 ccb
->ccb_slot
* sizeof(struct ahci_cmd_table
);
425 ccb
->ccb_cmd_hdr
->ctba_hi
= htole32((u_int32_t
)(dva
>> 32));
426 ccb
->ccb_cmd_hdr
->ctba_lo
= htole32((u_int32_t
)dva
);
429 (struct ata_fis_h2d
*)ccb
->ccb_cmd_table
->cfis
;
430 ccb
->ccb_xa
.packetcmd
= ccb
->ccb_cmd_table
->acmd
;
433 ccb
->ccb_xa
.state
= ATA_S_COMPLETE
;
436 * CCB[1] is the error CCB and is not get or put. It is
437 * also used for probing. Numerous HBAs only load the
438 * signature from CCB[1] so it MUST be used for the second
442 ap
->ap_err_ccb
= ccb
;
448 * Wait for ICC change to complete
450 ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_ICC
);
453 * Calculate the interrupt mask
455 data
= AHCI_PREG_IE_TFEE
| AHCI_PREG_IE_HBFE
|
456 AHCI_PREG_IE_IFE
| AHCI_PREG_IE_OFE
|
457 AHCI_PREG_IE_DPE
| AHCI_PREG_IE_UFE
|
458 AHCI_PREG_IE_PCE
| AHCI_PREG_IE_PRCE
|
459 AHCI_PREG_IE_DHRE
| AHCI_PREG_IE_SDBE
;
460 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SSNTF
)
461 data
|= AHCI_PREG_IE_IPME
;
463 if (sc
->sc_ccc_ports
& (1 << port
)
464 data
&= ~(AHCI_PREG_IE_SDBE
| AHCI_PREG_IE_DHRE
);
466 ap
->ap_intmask
= data
;
469 * Start the port helper thread. The helper thread will call
470 * ahci_port_init() so the ports can all be started in parallel.
471 * A failure by ahci_port_init() does not deallocate the port
472 * since we still want hot-plug events.
474 ahci_os_start_port(ap
);
477 ahci_port_free(sc
, port
);
482 * [re]initialize an idle port. No CCBs should be active. (from port thread)
484 * This function is called during the initial port allocation sequence
485 * and is also called on hot-plug insertion. We take no chances and
486 * use a portreset instead of a softreset.
488 * This function is the only way to move a failed port back to active
491 * Returns 0 if a device is successfully detected.
494 ahci_port_init(struct ahci_port
*ap
)
499 * Register [re]initialization
501 * Flush the TFD and SERR and make sure the port is stopped before
502 * enabling its interrupt. We no longer cycle the port start as
503 * the port should not be started unless a device is present.
505 * XXX should we enable FIS reception? (FRE)?
507 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
508 ahci_port_stop(ap
, 0);
509 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SSNTF
)
510 ahci_pwrite(ap
, AHCI_PREG_SNTF
, -1);
512 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
515 * If we are being harsh try to kill the port completely. Normally
516 * we would want to hold on to some of the state the BIOS may have
517 * set, such as SUD (spin up device).
519 * AP_F_HARSH_REINIT is cleared in the hard reset state
521 if (ap
->ap_flags
& AP_F_HARSH_REINIT
) {
522 ahci_pwrite(ap
, AHCI_PREG_SCTL
, ap
->ap_sc
->sc_ipm_disable
);
523 ahci_pwrite(ap
, AHCI_PREG_CMD
, 0);
527 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
) & ~AHCI_PREG_CMD_ICC
;
528 cmd
&= ~(AHCI_PREG_CMD_CLO
| AHCI_PREG_CMD_PMA
);
529 cmd
|= AHCI_PREG_CMD_POD
| AHCI_PREG_CMD_SUD
;
530 cmd
|= AHCI_PREG_CMD_ICC_ACTIVE
;
531 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
532 ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_ICC
);
537 * Clear any pending garbage and re-enable the interrupt before
538 * going to the next stage.
540 ap
->ap_probe
= ATA_PROBE_NEED_HARD_RESET
;
543 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SSNTF
)
544 ahci_pwrite(ap
, AHCI_PREG_SNTF
, -1);
546 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
547 ahci_pwrite(ap
, AHCI_PREG_IS
, -1);
549 ahci_port_interrupt_enable(ap
);
555 * Enable or re-enable interrupts on a port.
557 * This routine is called from the port initialization code or from the
558 * helper thread as the real interrupt may be forced to turn off certain
562 ahci_port_interrupt_enable(struct ahci_port
*ap
)
564 ahci_pwrite(ap
, AHCI_PREG_IE
, ap
->ap_intmask
);
568 * Manage the agressive link power management capability.
571 ahci_port_link_pwr_mgmt(struct ahci_port
*ap
, int link_pwr_mgmt
)
575 if (link_pwr_mgmt
== ap
->link_pwr_mgmt
)
578 if ((ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SALP
) == 0) {
579 kprintf("%s: link power management not supported.\n",
584 ahci_os_lock_port(ap
);
586 if (link_pwr_mgmt
== AHCI_LINK_PWR_MGMT_AGGR
&&
587 (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SSC
)) {
588 kprintf("%s: enabling aggressive link power management.\n",
591 ap
->link_pwr_mgmt
= link_pwr_mgmt
;
593 ap
->ap_intmask
&= ~AHCI_PREG_IE_PRCE
;
594 ahci_port_interrupt_enable(ap
);
596 sctl
= ahci_pread(ap
, AHCI_PREG_SCTL
);
597 sctl
&= ~(AHCI_PREG_SCTL_IPM
);
598 if (ap
->ap_sc
->sc_cap2
& AHCI_REG_CAP2_SDS
)
599 sctl
|= AHCI_PREG_SCTL_IPM_NODEVSLP
;
600 ahci_pwrite(ap
, AHCI_PREG_SCTL
, sctl
);
603 * Enable device initiated link power management for
604 * directly attached devices that support it.
606 if (ap
->ap_type
!= ATA_PORT_T_PM
&&
607 (ap
->ap_ata
[0]->at_identify
.satafsup
&
608 SATA_FEATURE_SUP_DEVIPS
)) {
609 if (ahci_set_feature(ap
, NULL
, ATA_SATAFT_DEVIPS
, 1))
610 kprintf("%s: Could not enable device initiated "
611 "link power management.\n",
615 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
616 cmd
|= AHCI_PREG_CMD_ASP
;
617 cmd
|= AHCI_PREG_CMD_ALPE
;
618 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
619 } else if (link_pwr_mgmt
== AHCI_LINK_PWR_MGMT_MEDIUM
&&
620 (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_PSC
)) {
621 kprintf("%s: enabling medium link power management.\n",
624 ap
->link_pwr_mgmt
= link_pwr_mgmt
;
626 ap
->ap_intmask
&= ~AHCI_PREG_IE_PRCE
;
627 ahci_port_interrupt_enable(ap
);
629 sctl
= ahci_pread(ap
, AHCI_PREG_SCTL
);
630 sctl
&= ~(AHCI_PREG_SCTL_IPM
);
631 sctl
|= AHCI_PREG_SCTL_IPM_NOSLUMBER
;
632 if (ap
->ap_sc
->sc_cap2
& AHCI_REG_CAP2_SDS
)
633 sctl
|= AHCI_PREG_SCTL_IPM_NODEVSLP
;
634 ahci_pwrite(ap
, AHCI_PREG_SCTL
, sctl
);
636 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
637 cmd
&= ~AHCI_PREG_CMD_ASP
;
638 cmd
|= AHCI_PREG_CMD_ALPE
;
639 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
641 } else if (link_pwr_mgmt
== AHCI_LINK_PWR_MGMT_NONE
) {
642 kprintf("%s: disabling link power management.\n",
645 /* Disable device initiated link power management */
646 if (ap
->ap_type
!= ATA_PORT_T_PM
&&
647 (ap
->ap_ata
[0]->at_identify
.satafsup
&
648 SATA_FEATURE_SUP_DEVIPS
)) {
649 ahci_set_feature(ap
, NULL
, ATA_SATAFT_DEVIPS
, 0);
652 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
653 cmd
&= ~(AHCI_PREG_CMD_ALPE
| AHCI_PREG_CMD_ASP
);
654 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
656 sctl
= ahci_pread(ap
, AHCI_PREG_SCTL
);
657 sctl
&= ~(AHCI_PREG_SCTL_IPM
);
658 sctl
|= ap
->ap_sc
->sc_ipm_disable
;
659 ahci_pwrite(ap
, AHCI_PREG_SCTL
, sctl
);
661 /* let the drive come back to avoid PRCS interrupts later */
662 ahci_os_unlock_port(ap
);
664 ahci_os_lock_port(ap
);
666 ahci_pwrite(ap
, AHCI_PREG_SERR
,
667 AHCI_PREG_SERR_DIAG_N
| AHCI_PREG_SERR_DIAG_W
);
668 ahci_pwrite(ap
, AHCI_PREG_IS
, AHCI_PREG_IS_PRCS
);
670 ap
->ap_intmask
|= AHCI_PREG_IE_PRCE
;
671 ahci_port_interrupt_enable(ap
);
673 ap
->link_pwr_mgmt
= link_pwr_mgmt
;
675 kprintf("%s: unsupported link power management state %d.\n",
676 PORTNAME(ap
), link_pwr_mgmt
);
679 ahci_os_unlock_port(ap
);
683 * Return current link power state.
686 ahci_port_link_pwr_state(struct ahci_port
*ap
)
690 r
= ahci_pread(ap
, AHCI_PREG_SSTS
);
691 switch (r
& AHCI_PREG_SSTS_IPM
) {
692 case AHCI_PREG_SSTS_IPM_ACTIVE
:
694 case AHCI_PREG_SSTS_IPM_PARTIAL
:
696 case AHCI_PREG_SSTS_IPM_SLUMBER
:
698 case AHCI_PREG_SSTS_IPM_DEVSLEEP
:
706 * Run the port / target state machine from a main context.
708 * The state machine for the port is always run.
710 * If atx is non-NULL run the state machine for a particular target.
711 * If atx is NULL run the state machine for all targets.
714 ahci_port_state_machine(struct ahci_port
*ap
, int initial
)
723 * State machine for port. Note that CAM is not yet associated
724 * during the initial parallel probe and the port's probe state
725 * will not get past ATA_PROBE_NEED_IDENT.
728 if (initial
== 0 && ap
->ap_probe
<= ATA_PROBE_NEED_HARD_RESET
) {
729 kprintf("%s: Waiting 10 seconds on insertion\n",
731 ahci_os_sleep(10000);
734 if (ap
->ap_probe
== ATA_PROBE_NEED_INIT
)
736 if (ap
->ap_probe
== ATA_PROBE_NEED_HARD_RESET
)
737 ahci_port_reset(ap
, NULL
, 1);
738 if (ap
->ap_probe
== ATA_PROBE_NEED_SOFT_RESET
)
739 ahci_port_reset(ap
, NULL
, 0);
740 if (ap
->ap_probe
== ATA_PROBE_NEED_IDENT
)
741 ahci_cam_probe(ap
, NULL
);
743 if (ap
->ap_type
!= ATA_PORT_T_PM
) {
744 if (ap
->ap_probe
== ATA_PROBE_FAILED
) {
745 ahci_cam_changed(ap
, NULL
, 0);
746 } else if (ap
->ap_probe
>= ATA_PROBE_NEED_IDENT
) {
747 ahci_cam_changed(ap
, NULL
, 1);
753 * Port Multiplier state machine.
755 * Get a mask of changed targets and combine with any runnable
756 * states already present.
758 for (loop
= 0; ;++loop
) {
759 if (ahci_pm_read(ap
, 15, SATA_PMREG_EINFO
, &data
)) {
760 kprintf("%s: PM unable to read hot-plug bitmap\n",
766 * Do at least one loop, then stop if no more state changes
767 * have occured. The PM might not generate a new
768 * notification until we clear the entire bitmap.
770 if (loop
&& data
== 0)
774 * New devices showing up in the bitmap require some spin-up
775 * time before we start probing them. Reset didsleep. The
776 * first new device we detect will sleep before probing.
778 * This only applies to devices whos change bit is set in
779 * the data, and does not apply to the initial boot-time
784 for (target
= 0; target
< ap
->ap_pmcount
; ++target
) {
785 at
= ap
->ap_ata
[target
];
788 * Check the target state for targets behind the PM
789 * which have changed state. This will adjust
790 * at_probe and set ATA_PORT_F_RESCAN
792 * We want to wait at least 10 seconds before probing
793 * a newly inserted device. If the check status
794 * indicates a device is present and in need of a
795 * hard reset, we make sure we have slept before
798 * We also need to wait at least 1 second for the
799 * PHY state to change after insertion, if we
800 * haven't already waited the 10 seconds.
802 * NOTE: When pm_check_good finds a good port it
803 * typically starts us in probe state
804 * NEED_HARD_RESET rather than INIT.
806 if (data
& (1 << target
)) {
807 if (initial
== 0 && didsleep
== 0)
809 ahci_pm_check_good(ap
, target
);
810 if (initial
== 0 && didsleep
== 0 &&
811 at
->at_probe
<= ATA_PROBE_NEED_HARD_RESET
814 kprintf("%s: Waiting 10 seconds on insertion\n", PORTNAME(ap
));
815 ahci_os_sleep(10000);
820 * Report hot-plug events before the probe state
821 * really gets hot. Only actual events are reported
822 * here to reduce spew.
824 if (data
& (1 << target
)) {
825 kprintf("%s: HOTPLUG (PM) - ", ATANAME(ap
, at
));
826 switch(at
->at_probe
) {
827 case ATA_PROBE_NEED_INIT
:
828 case ATA_PROBE_NEED_HARD_RESET
:
829 kprintf("Device inserted\n");
831 case ATA_PROBE_FAILED
:
832 kprintf("Device removed\n");
835 kprintf("Device probe in progress\n");
841 * Run through the state machine as necessary if
842 * the port is not marked failed.
844 * The state machine may stop at NEED_IDENT if
845 * CAM is not yet attached.
847 * Acquire exclusive access to the port while we
848 * are doing this. This prevents command-completion
849 * from queueing commands for non-polled targets
850 * inbetween our probe steps. We need to do this
851 * because the reset probes can generate severe PHY
852 * and protocol errors and soft-brick the port.
854 if (at
->at_probe
!= ATA_PROBE_FAILED
&&
855 at
->at_probe
!= ATA_PROBE_GOOD
) {
856 ahci_beg_exclusive_access(ap
, at
);
857 if (at
->at_probe
== ATA_PROBE_NEED_INIT
)
858 ahci_pm_port_init(ap
, at
);
859 if (at
->at_probe
== ATA_PROBE_NEED_HARD_RESET
)
860 ahci_port_reset(ap
, at
, 1);
861 if (at
->at_probe
== ATA_PROBE_NEED_SOFT_RESET
)
862 ahci_port_reset(ap
, at
, 0);
863 if (at
->at_probe
== ATA_PROBE_NEED_IDENT
)
864 ahci_cam_probe(ap
, at
);
865 ahci_end_exclusive_access(ap
, at
);
869 * Add or remove from CAM
871 if (at
->at_features
& ATA_PORT_F_RESCAN
) {
872 at
->at_features
&= ~ATA_PORT_F_RESCAN
;
873 if (at
->at_probe
== ATA_PROBE_FAILED
) {
874 ahci_cam_changed(ap
, at
, 0);
875 } else if (at
->at_probe
>= ATA_PROBE_NEED_IDENT
) {
876 ahci_cam_changed(ap
, at
, 1);
879 data
&= ~(1 << target
);
882 kprintf("%s: WARNING (PM): extra bits set in "
883 "EINFO: %08x\n", PORTNAME(ap
), data
);
884 while (target
< AHCI_MAX_PMPORTS
) {
885 ahci_pm_check_good(ap
, target
);
894 * De-initialize and detach a port.
897 ahci_port_free(struct ahci_softc
*sc
, u_int port
)
899 struct ahci_port
*ap
= sc
->sc_ports
[port
];
900 struct ahci_ccb
*ccb
;
904 * Ensure port is disabled and its interrupts are all flushed.
907 ahci_port_stop(ap
, 1);
908 ahci_os_stop_port(ap
);
909 ahci_pwrite(ap
, AHCI_PREG_CMD
, 0);
910 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
911 ahci_pwrite(ap
, AHCI_PREG_IS
, ahci_pread(ap
, AHCI_PREG_IS
));
912 ahci_write(sc
, AHCI_REG_IS
, 1 << port
);
916 while ((ccb
= ahci_get_ccb(ap
)) != NULL
) {
917 if (ccb
->ccb_dmamap
) {
918 bus_dmamap_destroy(sc
->sc_tag_data
,
920 ccb
->ccb_dmamap
= NULL
;
923 if ((ccb
= ap
->ap_err_ccb
) != NULL
) {
924 if (ccb
->ccb_dmamap
) {
925 bus_dmamap_destroy(sc
->sc_tag_data
,
927 ccb
->ccb_dmamap
= NULL
;
929 ap
->ap_err_ccb
= NULL
;
931 kfree(ap
->ap_ccbs
, M_DEVBUF
);
935 if (ap
->ap_dmamem_cmd_list
) {
936 ahci_dmamem_free(sc
, ap
->ap_dmamem_cmd_list
);
937 ap
->ap_dmamem_cmd_list
= NULL
;
939 if (ap
->ap_dmamem_rfis
) {
940 ahci_dmamem_free(sc
, ap
->ap_dmamem_rfis
);
941 ap
->ap_dmamem_rfis
= NULL
;
943 if (ap
->ap_dmamem_cmd_table
) {
944 ahci_dmamem_free(sc
, ap
->ap_dmamem_cmd_table
);
945 ap
->ap_dmamem_cmd_table
= NULL
;
948 for (i
= 0; i
< AHCI_MAX_PMPORTS
; ++i
) {
950 kfree(ap
->ap_ata
[i
], M_DEVBUF
);
951 ap
->ap_ata
[i
] = NULL
;
955 if (ap
->ap_err_scratch
) {
956 kfree(ap
->ap_err_scratch
, M_DEVBUF
);
957 ap
->ap_err_scratch
= NULL
;
960 /* bus_space(9) says we dont free the subregions handle */
963 sc
->sc_ports
[port
] = NULL
;
968 ahci_pactive(struct ahci_port
*ap
)
972 mask
= ahci_pread(ap
, AHCI_PREG_CI
);
973 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SNCQ
)
974 mask
|= ahci_pread(ap
, AHCI_PREG_SACT
);
979 * Start high-level command processing on the port
982 ahci_port_start(struct ahci_port
*ap
)
984 u_int32_t r
, s
, is
, tfd
;
987 * FRE must be turned on before ST. Wait for FR to go active
988 * before turning on ST. The spec doesn't seem to think this
989 * is necessary but waiting here avoids an on-off race in the
990 * ahci_port_stop() code.
992 r
= ahci_pread(ap
, AHCI_PREG_CMD
);
993 if ((r
& AHCI_PREG_CMD_FRE
) == 0) {
994 r
|= AHCI_PREG_CMD_FRE
;
995 ahci_pwrite(ap
, AHCI_PREG_CMD
, r
);
997 if ((ap
->ap_sc
->sc_flags
& AHCI_F_IGN_FR
) == 0) {
998 if (ahci_pwait_set(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_FR
)) {
999 kprintf("%s: Cannot start FIS reception\n",
1008 * Turn on ST, wait for CR to come up.
1010 r
|= AHCI_PREG_CMD_ST
;
1011 ahci_pwrite(ap
, AHCI_PREG_CMD
, r
);
1013 if ((ap
->ap_sc
->sc_flags
& AHCI_F_IGN_CR
) == 0 &&
1014 ahci_pwait_set_to(ap
, 2000, AHCI_PREG_CMD
, AHCI_PREG_CMD_CR
)) {
1015 s
= ahci_pread(ap
, AHCI_PREG_SERR
);
1016 is
= ahci_pread(ap
, AHCI_PREG_IS
);
1017 tfd
= ahci_pread(ap
, AHCI_PREG_TFD
);
1018 kprintf("%s: Cannot start command DMA\n"
1019 "NCMP=%b NSERR=%b\n"
1023 r
, AHCI_PFMT_CMD
, s
, AHCI_PFMT_SERR
,
1025 tfd
, AHCI_PFMT_TFD_STS
);
1029 #ifdef AHCI_COALESCE
1031 * (Re-)enable coalescing on the port.
1033 if (ap
->ap_sc
->sc_ccc_ports
& (1 << ap
->ap_num
)) {
1034 ap
->ap_sc
->sc_ccc_ports_cur
|= (1 << ap
->ap_num
);
1035 ahci_write(ap
->ap_sc
, AHCI_REG_CCC_PORTS
,
1036 ap
->ap_sc
->sc_ccc_ports_cur
);
1044 * Stop high-level command processing on a port
1046 * WARNING! If the port is stopped while CR is still active our saved
1047 * CI/SACT will race any commands completed by the command
1048 * processor prior to being able to stop. Thus we never call
1049 * this function unless we intend to dispose of any remaining
1050 * active commands. In particular, this complicates the timeout
1054 ahci_port_stop(struct ahci_port
*ap
, int stop_fis_rx
)
1058 #ifdef AHCI_COALESCE
1060 * Disable coalescing on the port while it is stopped.
1062 if (ap
->ap_sc
->sc_ccc_ports
& (1 << ap
->ap_num
)) {
1063 ap
->ap_sc
->sc_ccc_ports_cur
&= ~(1 << ap
->ap_num
);
1064 ahci_write(ap
->ap_sc
, AHCI_REG_CCC_PORTS
,
1065 ap
->ap_sc
->sc_ccc_ports_cur
);
1070 * Turn off ST, then wait for CR to go off.
1072 r
= ahci_pread(ap
, AHCI_PREG_CMD
) & ~AHCI_PREG_CMD_ICC
;
1073 if (r
& AHCI_PREG_CMD_ST
) {
1074 r
&= ~AHCI_PREG_CMD_ST
;
1075 ahci_pwrite(ap
, AHCI_PREG_CMD
, r
);
1078 if (ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_CR
)) {
1079 kprintf("%s: Port bricked, unable to stop (ST)\n",
1086 * Turn off FRE, then wait for FR to go off. FRE cannot
1087 * be turned off until CR transitions to 0.
1089 if ((r
& AHCI_PREG_CMD_FR
) == 0) {
1090 kprintf("%s: FR stopped, clear FRE for next start\n",
1096 r
&= ~AHCI_PREG_CMD_FRE
;
1097 ahci_pwrite(ap
, AHCI_PREG_CMD
, r
);
1098 if (ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_FR
)) {
1099 kprintf("%s: Port bricked, unable to stop (FRE)\n",
1108 * AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ}
1111 ahci_port_clo(struct ahci_port
*ap
)
1113 struct ahci_softc
*sc
= ap
->ap_sc
;
1116 /* Only attempt CLO if supported by controller */
1117 if ((sc
->sc_cap
& AHCI_REG_CAP_SCLO
) == 0)
1121 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
) & ~AHCI_PREG_CMD_ICC
;
1122 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
| AHCI_PREG_CMD_CLO
);
1124 /* Wait for completion */
1125 if (ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_CLO
)) {
1126 kprintf("%s: CLO did not complete\n", PORTNAME(ap
));
1136 * If hard is 0 perform a softreset of the port.
1137 * If hard is 1 perform a hard reset of the port.
1139 * If at is non-NULL an indirect port via a port-multiplier is being
1140 * reset, otherwise a direct port is being reset.
1142 * NOTE: Indirect ports can only be soft-reset.
1145 ahci_port_reset(struct ahci_port
*ap
, struct ata_port
*at
, int hard
)
1151 rc
= ahci_pm_hardreset(ap
, at
->at_target
, hard
);
1153 rc
= ahci_port_hardreset(ap
, hard
);
1156 rc
= ahci_pm_softreset(ap
, at
->at_target
);
1158 rc
= ahci_port_softreset(ap
);
1164 * AHCI soft reset, Section 10.4.1
1166 * (at) will be NULL when soft-resetting a directly-attached device, and
1167 * non-NULL when soft-resetting a device through a port multiplier.
1169 * This function keeps port communications intact and attempts to generate
1170 * a reset to the connected device using device commands.
1173 ahci_port_softreset(struct ahci_port
*ap
)
1175 struct ahci_ccb
*ccb
= NULL
;
1176 struct ahci_cmd_hdr
*cmd_slot
;
1183 kprintf("%s: START SOFTRESET %b\n", PORTNAME(ap
),
1184 ahci_pread(ap
, AHCI_PREG_CMD
), AHCI_PFMT_CMD
);
1187 DPRINTF(AHCI_D_VERBOSE
, "%s: soft reset\n", PORTNAME(ap
));
1190 ap
->ap_flags
|= AP_F_IN_RESET
;
1191 ap
->ap_state
= AP_S_NORMAL
;
1194 * Remember port state in cmd (main to restore start/stop)
1198 if (ahci_port_stop(ap
, 0)) {
1199 kprintf("%s: failed to stop port, cannot softreset\n",
1205 * Request CLO if device appears hung.
1207 if (ahci_pread(ap
, AHCI_PREG_TFD
) &
1208 (AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
1213 * This is an attempt to clear errors so a new signature will
1214 * be latched. It isn't working properly. XXX
1217 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
1220 if (ahci_port_start(ap
)) {
1221 kprintf("%s: failed to start port, cannot softreset\n",
1226 /* Check whether CLO worked */
1227 if (ahci_pwait_clr(ap
, AHCI_PREG_TFD
,
1228 AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
1229 kprintf("%s: CLO %s, need port reset\n",
1231 (ahci_read(ap
->ap_sc
, AHCI_REG_CAP
) & AHCI_REG_CAP_SCLO
)
1232 ? "failed" : "unsupported");
1238 * Prep first D2H command with SRST feature & clear busy/reset flags
1240 * It is unclear which other fields in the FIS are used. Just zero
1243 * NOTE! This CCB is used for both the first and second commands.
1244 * The second command must use CCB slot 1 to properly load
1247 ccb
= ahci_get_err_ccb(ap
);
1248 ccb
->ccb_xa
.complete
= ahci_dummy_done
;
1249 ccb
->ccb_xa
.flags
= ATA_F_POLL
| ATA_F_EXCLUSIVE
;
1250 KKASSERT(ccb
->ccb_slot
== 1);
1251 ccb
->ccb_xa
.at
= NULL
;
1252 cmd_slot
= ccb
->ccb_cmd_hdr
;
1254 fis
= ccb
->ccb_cmd_table
->cfis
;
1255 bzero(fis
, sizeof(ccb
->ccb_cmd_table
->cfis
));
1256 fis
[0] = ATA_FIS_TYPE_H2D
;
1257 fis
[15] = ATA_FIS_CONTROL_SRST
|ATA_FIS_CONTROL_4BIT
;
1259 cmd_slot
->prdtl
= 0;
1260 cmd_slot
->flags
= htole16(5); /* FIS length: 5 DWORDS */
1261 cmd_slot
->flags
|= htole16(AHCI_CMD_LIST_FLAG_C
); /* Clear busy on OK */
1262 cmd_slot
->flags
|= htole16(AHCI_CMD_LIST_FLAG_R
); /* Reset */
1264 ccb
->ccb_xa
.state
= ATA_S_PENDING
;
1266 if (ahci_poll(ccb
, 1000, ahci_quick_timeout
) != ATA_S_COMPLETE
) {
1267 kprintf("%s: First FIS failed\n", PORTNAME(ap
));
1272 * WARNING! TIME SENSITIVE SPACE! WARNING!
1274 * The two FISes are supposed to be back to back. Don't issue other
1275 * commands or even delay if we can help it.
1279 * Prep second D2H command to read status and complete reset sequence
1280 * AHCI 10.4.1 and "Serial ATA Revision 2.6". I can't find the ATA
1281 * Rev 2.6 and it is unclear how the second FIS should be set up
1282 * from the AHCI document.
1284 * It is unclear which other fields in the FIS are used. Just zero
1287 ccb
->ccb_xa
.flags
= ATA_F_POLL
| ATA_F_AUTOSENSE
| ATA_F_EXCLUSIVE
;
1289 bzero(fis
, sizeof(ccb
->ccb_cmd_table
->cfis
));
1290 fis
[0] = ATA_FIS_TYPE_H2D
;
1291 fis
[15] = ATA_FIS_CONTROL_4BIT
;
1293 cmd_slot
->prdtl
= 0;
1294 cmd_slot
->flags
= htole16(5); /* FIS length: 5 DWORDS */
1296 ccb
->ccb_xa
.state
= ATA_S_PENDING
;
1297 if (ahci_poll(ccb
, 1000, ahci_quick_timeout
) != ATA_S_COMPLETE
) {
1298 kprintf("%s: Second FIS failed\n", PORTNAME(ap
));
1302 if (ahci_pwait_clr(ap
, AHCI_PREG_TFD
,
1303 AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
1304 kprintf("%s: device didn't come ready after reset, TFD: 0x%b\n",
1306 ahci_pread(ap
, AHCI_PREG_TFD
), AHCI_PFMT_TFD_STS
);
1312 * If the softreset is trying to clear a BSY condition after a
1313 * normal portreset we assign the port type.
1315 * If the softreset is being run first as part of the ccb error
1316 * processing code then report if the device signature changed
1320 if (ap
->ap_type
== ATA_PORT_T_NONE
) {
1321 ap
->ap_type
= ahci_port_signature_detect(ap
, NULL
);
1323 if (ahci_port_signature_detect(ap
, NULL
) != ap
->ap_type
) {
1324 kprintf("%s: device signature unexpectedly "
1325 "changed\n", PORTNAME(ap
));
1326 error
= EBUSY
; /* XXX */
1334 ahci_put_err_ccb(ccb
);
1337 * If the target is busy use CLO to clear the busy
1338 * condition. The BSY should be cleared on the next
1341 if (ahci_pread(ap
, AHCI_PREG_TFD
) &
1342 (AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
1348 * If we failed to softreset make the port quiescent, otherwise
1349 * make sure the port's start/stop state matches what it was on
1352 * Don't kill the port if the softreset is on a port multiplier
1353 * target, that would kill all the targets!
1356 ahci_port_hardstop(ap
);
1357 /* ap_probe set to failed */
1359 ap
->ap_probe
= ATA_PROBE_NEED_IDENT
;
1361 ahci_port_start(ap
);
1363 ap
->ap_flags
&= ~AP_F_IN_RESET
;
1367 kprintf("%s: END SOFTRESET\n", PORTNAME(ap
));
1373 * Issue just do the core COMRESET and basic device detection on a port.
1375 * NOTE: Only called by ahci_port_hardreset().
1378 ahci_comreset(struct ahci_port
*ap
, int *pmdetectp
)
1387 * Idle the port. We must cycle FRE for certain chips that silently
1388 * clear FR on disconnect. Normally we do not want to cycle FRE
1389 * because other chipsets might react badly to that.
1392 if (ap
->ap_sc
->sc_flags
& AHCI_F_CYCLE_FR
)
1393 ahci_port_stop(ap
, 1);
1395 ahci_port_stop(ap
, 0);
1396 ap
->ap_state
= AP_S_NORMAL
;
1400 * FIS-based switching must be turned off when doing a hardware
1401 * reset, and will be turned on again during the PM probe.
1403 if (ap
->ap_flags
& AP_F_FBSS_ENABLED
) {
1404 ap
->ap_flags
&= ~AP_F_FBSS_ENABLED
;
1405 cmd
= ahci_pread(ap
, AHCI_PREG_FBS
);
1406 cmd
&= ~AHCI_PREG_FBS_EN
;
1407 cmd
|= AHCI_PREG_FBS_DEC
;
1408 ahci_pwrite(ap
, AHCI_PREG_FBS
, cmd
);
1412 * The port may have been quiescent with its SUD bit cleared, so
1413 * set the SUD (spin up device). Also POD (Power up device),
1414 * and issue an ICC_ACTIVE request to bring up communications.
1416 * NOTE: I do not know if SUD is a hardware pin/low-level signal
1417 * or if it is messaged.
1419 r
= ap
->ap_sc
->sc_ipm_disable
;
1420 ahci_pwrite(ap
, AHCI_PREG_SCTL
, r
);
1422 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
) & ~AHCI_PREG_CMD_ICC
;
1423 cmd
|= AHCI_PREG_CMD_SUD
| AHCI_PREG_CMD_POD
;
1424 cmd
|= AHCI_PREG_CMD_ICC_ACTIVE
;
1425 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
1426 ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_ICC
);
1429 * Some parts need FIS reception enabled to be able to COMINIT at
1430 * all, so we can't delay FRE until port-start. Even though that
1431 * isn't what the spec says.
1433 * This is typically the first enablement of FRE, but in most cases
1434 * we never turn it off making this a NOP for later calls.
1436 cmd
|= AHCI_PREG_CMD_FRE
;
1437 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
1438 if ((ap
->ap_sc
->sc_flags
& AHCI_F_IGN_FR
) == 0)
1439 ahci_pwait_set(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_FR
);
1442 * Make sure that all power management is disabled.
1444 * NOTE! AHCI_PREG_SCTL_DET_DISABLE seems to be highly unreliable
1445 * on multiple chipsets and can brick the chipset or even
1446 * the whole PC. Never use it.
1448 ap
->ap_type
= ATA_PORT_T_NONE
;
1452 * Give the new power management state time to settle, then clear
1455 ahci_os_sleep(1000);
1457 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
1460 * Start transmitting COMRESET. The spec says that COMRESET must
1461 * be sent for at least 1ms but in actual fact numerous devices
1462 * appear to take much longer. Delay a whole second here.
1464 * In addition, SATA-3 ports can take longer to train, so even
1465 * SATA-2 devices which would normally detect very quickly may
1466 * take longer when plugged into a SATA-3 port.
1468 r
|= AHCI_PREG_SCTL_DET_INIT
;
1470 switch(AhciForceGen
) {
1472 r
|= AHCI_PREG_SCTL_SPD_ANY
;
1475 r
|= AHCI_PREG_SCTL_SPD_GEN1
;
1478 r
|= AHCI_PREG_SCTL_SPD_GEN2
;
1481 r
|= AHCI_PREG_SCTL_SPD_GEN3
;
1484 r
|= AHCI_PREG_SCTL_SPD_GEN3
;
1487 ahci_pwrite(ap
, AHCI_PREG_SCTL
, r
);
1488 ahci_os_sleep(1000);
1490 ap
->ap_flags
&= ~AP_F_HARSH_REINIT
;
1493 * Only SERR_DIAG_X needs to be cleared for TFD updates, but
1494 * since we are hard-resetting the port we might as well clear
1495 * the whole enchillada. Also be sure to clear any spurious BSY
1496 * prior to clearing INIT.
1498 * Wait 1 whole second after clearing INIT before checking
1499 * the device detection bits in an attempt to work around chipsets
1500 * which do not properly mask PCS/PRCS during low level init.
1503 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
1504 /* ahci_port_clo(ap);*/
1507 r
&= ~AHCI_PREG_SCTL_SPD
;
1508 r
&= ~AHCI_PREG_SCTL_DET_INIT
;
1509 r
|= AHCI_PREG_SCTL_DET_NONE
;
1510 ahci_pwrite(ap
, AHCI_PREG_SCTL
, r
);
1511 ahci_os_sleep(1000);
1514 * Try to determine if there is a device on the port. This operation
1515 * typically runs in parallel on all ports belonging to an AHCI
1518 * 3/10 of a second (loop = 300) is plenty for directly attached
1519 * devices, but not enough for some port multipliers, particularly
1520 * if powered-on cold. Since this operation runs in parallel,
1521 * give us 2 seconds to detect.
1523 * NOTE: The 10-second hot-swap delay prior to the COMRESET is not
1524 * sufficient, since the first COMRESET after a cold power-on
1525 * of a port-multiplier can take extra time.
1527 * If we fail clear PRCS (phy detect) since we may cycled
1528 * the phy and probably caused another PRCS interrupt.
1532 r
= ahci_pread(ap
, AHCI_PREG_SSTS
);
1533 if (r
& AHCI_PREG_SSTS_DET
)
1535 loop
-= ahci_os_softsleep();
1538 ahci_pwrite(ap
, AHCI_PREG_IS
, AHCI_PREG_IS_PRCS
);
1540 kprintf("%s: Port appears to be unplugged\n",
1548 * There is something on the port. Regardless of what happens
1549 * after this tell the caller to try to detect a port multiplier.
1551 * Give the device 3 seconds to fully negotiate.
1555 if (ahci_pwait_eq(ap
, 3000, AHCI_PREG_SSTS
,
1556 AHCI_PREG_SSTS_DET
, AHCI_PREG_SSTS_DET_DEV
)) {
1558 kprintf("%s: Device may be powered down\n",
1566 * We got something that definitely looks like a device. Give
1567 * the device time to send us its first D2H FIS. Waiting for
1568 * BSY to clear accomplishes this.
1570 * The target device might be hung in a BSY state depending on
1571 * the order things are power cycled. We want to retry the COMRESET
1572 * at least once if we find the device BSY for reliable operation.
1574 * NOTE: A port multiplier may or may not clear BSY here,
1575 * particularly if it was previously configured and now
1576 * its cable has been unplugged and plugged back in,
1577 * and also depending on what is sitting in target 0 behind it.
1579 * NOTE: Intel SSDs seem to have compatibility problems with Intel
1580 * mobo's on cold boots and may leave BSY set. A single
1581 * retry works around the problem. This is definitely a bug
1582 * with the mobo and/or the SSD and does not appear to occur
1583 * with other devices connected to the same port.
1586 if (ahci_pwait_clr_to(ap
, 8000, AHCI_PREG_TFD
,
1587 AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
1588 kprintf("%s: Device BUSY: %b\n",
1590 ahci_pread(ap
, AHCI_PREG_TFD
),
1593 kprintf("%s: Retrying\n", PORTNAME(ap
));
1600 kprintf("%s: Device Unbusied after retry\n",
1612 * AHCI port reset, Section 10.4.2
1614 * This function does a hard reset of the port. Note that the device
1615 * connected to the port could still end-up hung.
1618 ahci_port_hardreset(struct ahci_port
*ap
, int hard
)
1625 kprintf("%s: START HARDRESET\n", PORTNAME(ap
));
1626 ap
->ap_flags
|= AP_F_IN_RESET
;
1628 error
= ahci_comreset(ap
, &pmdetect
);
1631 * We may be asked to perform a port multiplier check even if the
1632 * comreset failed. This typically occurs when the PM has nothing
1633 * in slot 0, which can cause BSY to remain set.
1635 * If the PM detection is successful it will override (error),
1636 * otherwise (error) is retained. If an error does occur it
1637 * is possible that a normal device has blown up on us DUE to
1638 * the PM detection code, so re-run the comreset and assume
1642 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SPM
) {
1643 error
= ahci_pm_port_probe(ap
, error
);
1645 error
= ahci_comreset(ap
, &pmdetect
);
1658 * All good, make sure the port is running and set the
1659 * probe state. Ignore the signature junk (it's unreliable)
1660 * until we get to the softreset code.
1662 if (ahci_port_start(ap
)) {
1663 kprintf("%s: failed to start command DMA on port, "
1664 "disabling\n", PORTNAME(ap
));
1668 if (ap
->ap_type
== ATA_PORT_T_PM
)
1669 ap
->ap_probe
= ATA_PROBE_GOOD
;
1671 ap
->ap_probe
= ATA_PROBE_NEED_SOFT_RESET
;
1675 * Normal device probe failure
1677 data
= ahci_pread(ap
, AHCI_PREG_SSTS
);
1679 switch(data
& AHCI_PREG_SSTS_DET
) {
1680 case AHCI_PREG_SSTS_DET_DEV_NE
:
1681 kprintf("%s: Device not communicating\n",
1684 case AHCI_PREG_SSTS_DET_PHYOFFLINE
:
1685 kprintf("%s: PHY offline\n",
1689 kprintf("%s: No device detected\n",
1693 ahci_port_hardstop(ap
);
1697 * Abnormal probe (EBUSY)
1699 kprintf("%s: Device on port is bricked\n",
1701 ahci_port_hardstop(ap
);
1703 rc
= ahci_port_reset(ap
, atx
, 0);
1705 kprintf("%s: Unable unbrick device\n",
1708 kprintf("%s: Successfully unbricked\n",
1718 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
1719 ahci_pwrite(ap
, AHCI_PREG_IS
, AHCI_PREG_IS_PCS
| AHCI_PREG_IS_PRCS
);
1721 ap
->ap_flags
&= ~AP_F_IN_RESET
;
1724 kprintf("%s: END HARDRESET %d\n", PORTNAME(ap
), error
);
1729 * Hard-stop on hot-swap device removal. See 10.10.1
1731 * Place the port in a mode that will allow it to detect hot-swap insertions.
1732 * This is a bit imprecise because just setting-up SCTL to DET_INIT doesn't
1733 * seem to do the job.
1735 * FIS reception is left enabled but command processing is disabled.
1736 * Cycling FIS reception (FRE) can brick ports.
1739 ahci_port_hardstop(struct ahci_port
*ap
)
1741 struct ahci_ccb
*ccb
;
1742 struct ata_port
*at
;
1750 * Stop the port. We can't modify things like SUD if the port
1753 ap
->ap_state
= AP_S_FATAL_ERROR
;
1754 ap
->ap_probe
= ATA_PROBE_FAILED
;
1755 ap
->ap_type
= ATA_PORT_T_NONE
;
1756 ahci_port_stop(ap
, 0);
1757 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
1758 cmd
&= ~(AHCI_PREG_CMD_CLO
| AHCI_PREG_CMD_PMA
| AHCI_PREG_CMD_ICC
);
1759 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
1762 * Clean up AT sub-ports on SATA port.
1764 for (i
= 0; ap
->ap_ata
&& i
< AHCI_MAX_PMPORTS
; ++i
) {
1766 at
->at_type
= ATA_PORT_T_NONE
;
1767 at
->at_probe
= ATA_PROBE_FAILED
;
1771 * 10.10.1 place us in the Listen state.
1773 * 10.10.3 DET must be set to 0 and found to be 0 before
1776 * Deactivating SUD only applies if the controller supports SUD, it
1777 * is a bit unclear what happens w/regards to detecting hotplug
1780 * NOTE: AHCI_PREG_SCTL_SPM_* bits are not implemented by the spec
1783 r
= ap
->ap_sc
->sc_ipm_disable
;
1784 ahci_pwrite(ap
, AHCI_PREG_SCTL
, r
);
1787 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
1788 cmd
&= ~AHCI_PREG_CMD_SUD
;
1789 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
1795 * Transition su to the spin-up state. HBA shall send COMRESET and
1796 * begin initialization sequence (whatever that means). Presumably
1797 * this is edge-triggered. Following the spin-up state the HBA
1798 * will automatically transition to the Normal state.
1800 * This only applies if the controller supports SUD.
1801 * NEVER use AHCI_PREG_DET_DISABLE.
1803 cmd
|= AHCI_PREG_CMD_POD
|
1805 AHCI_PREG_CMD_ICC_ACTIVE
;
1806 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
1807 ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_ICC
);
1810 * Flush SERR_DIAG_X so the TFD can update.
1815 * Clean out pending ccbs
1818 while (ap
->ap_active
) {
1819 slot
= ffs(ap
->ap_active
) - 1;
1820 ap
->ap_active
&= ~(1 << slot
);
1821 --ap
->ap_active_cnt
;
1822 ccb
= &ap
->ap_ccbs
[slot
];
1823 if (ccb
->ccb_xa
.flags
& ATA_F_TIMEOUT_RUNNING
) {
1824 serial
= ccb
->ccb_xa
.serial
;
1825 callout_stop_sync(&ccb
->ccb_timeout
);
1826 if (serial
!= ccb
->ccb_xa
.serial
) {
1827 kprintf("%s: Warning: timeout race ccb %p\n",
1831 ccb
->ccb_xa
.flags
&= ~ATA_F_TIMEOUT_RUNNING
;
1833 ap
->ap_expired
&= ~(1 << slot
);
1834 ccb
->ccb_xa
.flags
&= ~(ATA_F_TIMEOUT_DESIRED
|
1835 ATA_F_TIMEOUT_EXPIRED
);
1836 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
1838 ccb
->ccb_xa
.complete(&ccb
->ccb_xa
);
1840 while (ap
->ap_sactive
) {
1841 slot
= ffs(ap
->ap_sactive
) - 1;
1842 ap
->ap_sactive
&= ~(1 << slot
);
1843 ccb
= &ap
->ap_ccbs
[slot
];
1844 if (ccb
->ccb_xa
.flags
& ATA_F_TIMEOUT_RUNNING
) {
1845 serial
= ccb
->ccb_xa
.serial
;
1846 callout_stop_sync(&ccb
->ccb_timeout
);
1847 if (serial
!= ccb
->ccb_xa
.serial
) {
1848 kprintf("%s: Warning: timeout race ccb %p\n",
1852 ccb
->ccb_xa
.flags
&= ~ATA_F_TIMEOUT_RUNNING
;
1854 ap
->ap_expired
&= ~(1 << slot
);
1855 ccb
->ccb_xa
.flags
&= ~(ATA_F_TIMEOUT_DESIRED
|
1856 ATA_F_TIMEOUT_EXPIRED
);
1857 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
1859 ccb
->ccb_xa
.complete(&ccb
->ccb_xa
);
1861 KKASSERT(ap
->ap_active_cnt
== 0);
1863 while ((ccb
= TAILQ_FIRST(&ap
->ap_ccb_pending
)) != NULL
) {
1864 TAILQ_REMOVE(&ap
->ap_ccb_pending
, ccb
, ccb_entry
);
1865 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
1866 ccb
->ccb_xa
.flags
&= ~ATA_F_TIMEOUT_DESIRED
;
1868 ccb
->ccb_xa
.complete(&ccb
->ccb_xa
);
1872 * Hot-plug device detection should work at this point. e.g. on
1873 * AMD chipsets Spin-Up/Normal state is sufficient for hot-plug
1874 * detection and entering RESET (continuous COMRESET by setting INIT)
1875 * will actually prevent hot-plug detection from working properly.
1877 * There may be cases where this will fail to work, I have some
1878 * additional code to place the HBA in RESET (send continuous
1879 * COMRESET) and hopefully get DIAG.X or other events when something
1880 * is plugged in. Unfortunately this isn't universal and can
1881 * also prevent events from generating interrupts.
1886 * Transition us to the Reset state. Theoretically we send a
1887 * continuous stream of COMRESETs in this state.
1889 r
|= AHCI_PREG_SCTL_DET_INIT
;
1890 if (AhciForceGen1
& (1 << ap
->ap_num
)) {
1891 kprintf("%s: Force 1.5Gbits\n", PORTNAME(ap
));
1892 r
|= AHCI_PREG_SCTL_SPD_GEN1
;
1894 r
|= AHCI_PREG_SCTL_SPD_ANY
;
1896 ahci_pwrite(ap
, AHCI_PREG_SCTL
, r
);
1900 * Flush SERR_DIAG_X so the TFD can update.
1908 * We can't loop on the X bit, a continuous COMINIT received will make
1909 * it loop forever. Just assume one event has built up and clear X
1910 * so the task file descriptor can update.
1913 ahci_flush_tfd(struct ahci_port
*ap
)
1917 r
= ahci_pread(ap
, AHCI_PREG_SERR
);
1918 if (r
& AHCI_PREG_SERR_DIAG_X
)
1919 ahci_pwrite(ap
, AHCI_PREG_SERR
, AHCI_PREG_SERR_DIAG_X
);
1923 * Figure out what type of device is connected to the port, ATAPI or
1927 ahci_port_signature_detect(struct ahci_port
*ap
, struct ata_port
*at
)
1931 sig
= ahci_pread(ap
, AHCI_PREG_SIG
);
1933 kprintf("%s: SIG %08x\n", ATANAME(ap
, at
), sig
);
1934 if ((sig
& 0xffff0000) == (SATA_SIGNATURE_ATAPI
& 0xffff0000)) {
1935 return(ATA_PORT_T_ATAPI
);
1936 } else if ((sig
& 0xffff0000) ==
1937 (SATA_SIGNATURE_PORT_MULTIPLIER
& 0xffff0000)) {
1938 return(ATA_PORT_T_PM
);
1940 return(ATA_PORT_T_DISK
);
1945 * Load the DMA descriptor table for a CCB's buffer.
1948 ahci_load_prdt(struct ahci_ccb
*ccb
)
1950 struct ahci_port
*ap
= ccb
->ccb_port
;
1951 struct ahci_softc
*sc
= ap
->ap_sc
;
1952 struct ata_xfer
*xa
= &ccb
->ccb_xa
;
1953 struct ahci_prdt
*prdt
= ccb
->ccb_cmd_table
->prdt
;
1954 bus_dmamap_t dmap
= ccb
->ccb_dmamap
;
1955 struct ahci_cmd_hdr
*cmd_slot
= ccb
->ccb_cmd_hdr
;
1958 if (xa
->datalen
== 0) {
1959 ccb
->ccb_cmd_hdr
->prdtl
= 0;
1963 error
= bus_dmamap_load(sc
->sc_tag_data
, dmap
,
1964 xa
->data
, xa
->datalen
,
1965 ahci_load_prdt_callback
,
1967 ((xa
->flags
& ATA_F_NOWAIT
) ?
1968 BUS_DMA_NOWAIT
: BUS_DMA_WAITOK
));
1970 kprintf("%s: error %d loading dmamap\n", PORTNAME(ap
), error
);
1974 if (xa
->flags
& ATA_F_PIO
)
1975 prdt
->flags
|= htole32(AHCI_PRDT_FLAG_INTR
);
1978 cmd_slot
->prdtl
= htole16(prdt
- ccb
->ccb_cmd_table
->prdt
+ 1);
1980 if (xa
->flags
& ATA_F_READ
)
1981 bus_dmamap_sync(sc
->sc_tag_data
, dmap
, BUS_DMASYNC_PREREAD
);
1982 if (xa
->flags
& ATA_F_WRITE
)
1983 bus_dmamap_sync(sc
->sc_tag_data
, dmap
, BUS_DMASYNC_PREWRITE
);
1989 * Callback from BUSDMA system to load the segment list. The passed segment
1990 * list is a temporary structure.
1994 ahci_load_prdt_callback(void *info
, bus_dma_segment_t
*segs
, int nsegs
,
1997 struct ahci_prdt
*prd
= *(void **)info
;
2000 KKASSERT(nsegs
<= AHCI_MAX_PRDT
);
2003 addr
= segs
->ds_addr
;
2004 prd
->dba_hi
= htole32((u_int32_t
)(addr
>> 32));
2005 prd
->dba_lo
= htole32((u_int32_t
)addr
);
2006 prd
->flags
= htole32(segs
->ds_len
- 1);
2012 *(void **)info
= prd
; /* return last valid segment */
2016 ahci_unload_prdt(struct ahci_ccb
*ccb
)
2018 struct ahci_port
*ap
= ccb
->ccb_port
;
2019 struct ahci_softc
*sc
= ap
->ap_sc
;
2020 struct ata_xfer
*xa
= &ccb
->ccb_xa
;
2021 bus_dmamap_t dmap
= ccb
->ccb_dmamap
;
2023 if (xa
->datalen
!= 0) {
2024 if (xa
->flags
& ATA_F_READ
) {
2025 bus_dmamap_sync(sc
->sc_tag_data
, dmap
,
2026 BUS_DMASYNC_POSTREAD
);
2028 if (xa
->flags
& ATA_F_WRITE
) {
2029 bus_dmamap_sync(sc
->sc_tag_data
, dmap
,
2030 BUS_DMASYNC_POSTWRITE
);
2032 bus_dmamap_unload(sc
->sc_tag_data
, dmap
);
2035 * prdbc is only updated by hardware for non-NCQ commands.
2037 if (ccb
->ccb_xa
.flags
& ATA_F_NCQ
) {
2040 if (ccb
->ccb_cmd_hdr
->prdbc
== 0 &&
2041 ccb
->ccb_xa
.state
== ATA_S_COMPLETE
) {
2042 kprintf("%s: WARNING! Unload prdbc resid "
2043 "was zero! tag=%d\n",
2044 ATANAME(ap
, xa
->at
), ccb
->ccb_slot
);
2046 xa
->resid
= xa
->datalen
-
2047 le32toh(ccb
->ccb_cmd_hdr
->prdbc
);
2053 * Start a command and poll for completion.
2055 * timeout is in ms and only counts once the command gets on-chip.
2057 * Returns ATA_S_* state, compare against ATA_S_COMPLETE to determine
2058 * that no error occured.
2060 * NOTE: If the caller specifies a NULL timeout function the caller is
2061 * responsible for clearing hardware state on failure, but we will
2062 * deal with removing the ccb from any pending queue.
2064 * NOTE: NCQ should never be used with this function.
2066 * NOTE: If the port is in a failed state and stopped we do not try
2067 * to activate the ccb.
2070 ahci_poll(struct ahci_ccb
*ccb
, int timeout
,
2071 void (*timeout_fn
)(struct ahci_ccb
*))
2073 struct ahci_port
*ap
= ccb
->ccb_port
;
2075 if (ccb
->ccb_port
->ap_state
== AP_S_FATAL_ERROR
) {
2076 ccb
->ccb_xa
.state
= ATA_S_ERROR
;
2077 return(ccb
->ccb_xa
.state
);
2081 kprintf("%s: Start command %02x tag=%d\n",
2082 ATANAME(ccb
->ccb_port
, ccb
->ccb_xa
.at
),
2083 ccb
->ccb_xa
.fis
->command
, ccb
->ccb_slot
);
2088 ahci_port_intr(ap
, 1);
2089 switch(ccb
->ccb_xa
.state
) {
2091 timeout
-= ahci_os_softsleep();
2094 timeout
-= ahci_os_softsleep();
2095 ahci_check_active_timeouts(ap
);
2099 return (ccb
->ccb_xa
.state
);
2101 } while (timeout
> 0);
2103 if ((ccb
->ccb_xa
.flags
& ATA_F_SILENT
) == 0) {
2104 kprintf("%s: Poll timeout slot %d CMD: %b TFD: 0x%b SERR: %b\n",
2105 ATANAME(ap
, ccb
->ccb_xa
.at
), ccb
->ccb_slot
,
2106 ahci_pread(ap
, AHCI_PREG_CMD
), AHCI_PFMT_CMD
,
2107 ahci_pread(ap
, AHCI_PREG_TFD
), AHCI_PFMT_TFD_STS
,
2108 ahci_pread(ap
, AHCI_PREG_SERR
), AHCI_PFMT_SERR
);
2115 return(ccb
->ccb_xa
.state
);
2119 * When polling we have to check if the currently active CCB(s)
2120 * have timed out as the callout will be deadlocked while we
2121 * hold the port lock.
2124 ahci_check_active_timeouts(struct ahci_port
*ap
)
2126 struct ahci_ccb
*ccb
;
2130 mask
= ap
->ap_active
| ap
->ap_sactive
;
2132 tag
= ffs(mask
) - 1;
2133 mask
&= ~(1 << tag
);
2134 ccb
= &ap
->ap_ccbs
[tag
];
2135 if (ccb
->ccb_xa
.flags
& ATA_F_TIMEOUT_EXPIRED
) {
2136 ahci_ata_cmd_timeout(ccb
);
2144 ahci_start_timeout(struct ahci_ccb
*ccb
)
2146 if (ccb
->ccb_xa
.flags
& ATA_F_TIMEOUT_DESIRED
) {
2147 ccb
->ccb_xa
.flags
|= ATA_F_TIMEOUT_RUNNING
;
2148 callout_reset(&ccb
->ccb_timeout
,
2149 (ccb
->ccb_xa
.timeout
* hz
+ 999) / 1000,
2150 ahci_ata_cmd_timeout_unserialized
, ccb
);
2155 ahci_start(struct ahci_ccb
*ccb
)
2157 struct ahci_port
*ap
= ccb
->ccb_port
;
2158 struct ahci_softc
*sc
= ap
->ap_sc
;
2160 KKASSERT(ccb
->ccb_xa
.state
== ATA_S_PENDING
);
2162 /* Zero transferred byte count before transfer */
2163 ccb
->ccb_cmd_hdr
->prdbc
= 0;
2165 /* Sync command list entry and corresponding command table entry */
2166 bus_dmamap_sync(sc
->sc_tag_cmdh
,
2167 AHCI_DMA_MAP(ap
->ap_dmamem_cmd_list
),
2168 BUS_DMASYNC_PREWRITE
);
2169 bus_dmamap_sync(sc
->sc_tag_cmdt
,
2170 AHCI_DMA_MAP(ap
->ap_dmamem_cmd_table
),
2171 BUS_DMASYNC_PREWRITE
);
2173 /* Prepare RFIS area for write by controller */
2174 bus_dmamap_sync(sc
->sc_tag_rfis
,
2175 AHCI_DMA_MAP(ap
->ap_dmamem_rfis
),
2176 BUS_DMASYNC_PREREAD
);
2179 * There's no point trying to optimize this, it only shaves a few
2180 * nanoseconds so just queue the command and call our generic issue.
2182 ahci_issue_pending_commands(ap
, ccb
);
2186 * While holding the port lock acquire exclusive access to the port.
2188 * This is used when running the state machine to initialize and identify
2189 * targets over a port multiplier. Setting exclusive access prevents
2190 * ahci_port_intr() from activating any requests sitting on the pending
2194 ahci_beg_exclusive_access(struct ahci_port
*ap
, struct ata_port
*at
)
2196 KKASSERT((ap
->ap_flags
& AP_F_EXCLUSIVE_ACCESS
) == 0);
2197 ap
->ap_flags
|= AP_F_EXCLUSIVE_ACCESS
;
2198 while (ap
->ap_active
|| ap
->ap_sactive
) {
2199 ahci_port_intr(ap
, 1);
2200 ahci_os_softsleep();
2205 ahci_end_exclusive_access(struct ahci_port
*ap
, struct ata_port
*at
)
2207 KKASSERT((ap
->ap_flags
& AP_F_EXCLUSIVE_ACCESS
) != 0);
2208 ap
->ap_flags
&= ~AP_F_EXCLUSIVE_ACCESS
;
2209 ahci_issue_pending_commands(ap
, NULL
);
2213 * If ccb is not NULL enqueue and/or issue it.
2215 * If ccb is NULL issue whatever we can from the queue. However, nothing
2216 * new is issued if the exclusive access flag is set or expired ccb's are
2219 * If existing commands are still active (ap_active/ap_sactive) we can only
2220 * issue matching new commands.
2223 ahci_issue_pending_commands(struct ahci_port
*ap
, struct ahci_ccb
*ccb
)
2227 struct ata_port
*ccb_at
;
2232 * If just running the queue and in exclusive access mode we
2233 * just return. Also in this case if there are any expired ccb's
2234 * we want to clear the queue so the port can be safely stopped.
2237 TAILQ_INSERT_TAIL(&ap
->ap_ccb_pending
, ccb
, ccb_entry
);
2238 } else if ((ap
->ap_flags
& AP_F_EXCLUSIVE_ACCESS
) || ap
->ap_expired
) {
2243 * Pull the next ccb off the queue and run it if possible.
2245 * The error CCB supercedes all normal queue operations and
2246 * implies exclusive access while the error CCB is active.
2248 if (ccb
!= ap
->ap_err_ccb
) {
2249 if ((ccb
= TAILQ_FIRST(&ap
->ap_ccb_pending
)) == NULL
)
2251 if (ap
->ap_flags
& AP_F_ERR_CCB_RESERVED
) {
2252 kprintf("DELAY CCB slot %d\n", ccb
->ccb_slot
);
2258 * Handle exclusivity requirements.
2260 * ATA_F_EXCLUSIVE is used when we want to be the only command
2263 * ATA_F_AUTOSENSE is used when we want the D2H rfis loaded
2264 * back into the ccb on a normal (non-errored) command completion.
2265 * For example, for PM requests to target 15. Because the AHCI
2266 * spec does not stop the command processor and has only one rfis
2267 * area (for non-FBSS anyway), AUTOSENSE currently implies EXCLUSIVE.
2268 * Otherwise multiple completions can destroy the rfis data before
2269 * we have a chance to copy it.
2271 if (ap
->ap_active
& ~ap
->ap_expired
) {
2273 * There may be multiple ccb's already running,
2274 * if any are running and ap_run_flags sets
2275 * one of these flags then we know only one is
2278 * XXX Current AUTOSENSE code forces exclusivity
2279 * to simplify the code.
2281 if (ap
->ap_run_flags
&
2282 (ATA_F_EXCLUSIVE
| ATA_F_AUTOSENSE
)) {
2286 if (ccb
->ccb_xa
.flags
&
2287 (ATA_F_EXCLUSIVE
| ATA_F_AUTOSENSE
)) {
2292 if (ccb
->ccb_xa
.flags
& ATA_F_NCQ
) {
2294 * The next command is a NCQ command and can be issued as
2295 * long as currently active commands are not standard.
2297 if (ap
->ap_active
) {
2298 KKASSERT(ap
->ap_active_cnt
> 0);
2301 KKASSERT(ap
->ap_active_cnt
== 0);
2305 TAILQ_REMOVE(&ap
->ap_ccb_pending
, ccb
, ccb_entry
);
2306 KKASSERT((mask
& (1 << ccb
->ccb_slot
)) == 0);
2307 mask
|= 1 << ccb
->ccb_slot
;
2308 KKASSERT(ccb
->ccb_xa
.state
== ATA_S_PENDING
);
2309 KKASSERT(ccb
== &ap
->ap_ccbs
[ccb
->ccb_slot
]);
2310 ccb
->ccb_xa
.state
= ATA_S_ONCHIP
;
2311 ahci_start_timeout(ccb
);
2312 ap
->ap_run_flags
= ccb
->ccb_xa
.flags
;
2314 ccb_at
= ccb
->ccb_xa
.at
;
2315 if (ap
->ap_flags
& AP_F_FBSS_ENABLED
) {
2316 ap
->ap_sactive
|= mask
;
2317 ahci_pwrite(ap
, AHCI_PREG_SACT
, mask
);
2319 ahci_pwrite(ap
, AHCI_PREG_FBS
,
2320 (ccb_at
->at_target
<<
2321 AHCI_PREG_FBS_DEV_SHIFT
) |
2324 ahci_pwrite(ap
, AHCI_PREG_FBS
,
2327 ahci_pwrite(ap
, AHCI_PREG_CI
, mask
);
2330 ccb
= TAILQ_FIRST(&ap
->ap_ccb_pending
);
2331 } while (ccb
&& (ccb
->ccb_xa
.flags
& ATA_F_NCQ
) &&
2333 (ATA_F_EXCLUSIVE
| ATA_F_AUTOSENSE
)) == 0);
2335 KKASSERT(((ap
->ap_active
| ap
->ap_sactive
) & mask
) == 0);
2338 ap
->ap_sactive
|= mask
;
2339 ahci_pwrite(ap
, AHCI_PREG_SACT
, mask
);
2340 ahci_pwrite(ap
, AHCI_PREG_CI
, mask
);
2344 * The next command is a standard command and can be issued
2345 * as long as currently active commands are not NCQ.
2347 * We limit ourself to 1 command if we have a port multiplier,
2348 * (at least without FBSS support), otherwise timeouts on
2349 * one port can race completions on other ports (see
2350 * ahci_ata_cmd_timeout() for more information).
2352 * If not on a port multiplier generally allow up to 4
2353 * standard commands to be enqueued. Remember that the
2354 * command processor will still process them sequentially.
2358 if (ap
->ap_type
== ATA_PORT_T_PM
&&
2359 (ap
->ap_flags
& AP_F_FBSS_ENABLED
) == 0) {
2361 } else if (ap
->ap_sc
->sc_ncmds
> 4) {
2367 while (ap
->ap_active_cnt
< limit
&& ccb
&&
2368 (ccb
->ccb_xa
.flags
& ATA_F_NCQ
) == 0) {
2369 ccb_at
= ccb
->ccb_xa
.at
;
2370 TAILQ_REMOVE(&ap
->ap_ccb_pending
, ccb
, ccb_entry
);
2371 KKASSERT(((ap
->ap_active
| ap
->ap_sactive
) &
2372 (1 << ccb
->ccb_slot
)) == 0);
2373 ap
->ap_active
|= 1 << ccb
->ccb_slot
;
2374 ap
->ap_active_cnt
++;
2375 ap
->ap_run_flags
= ccb
->ccb_xa
.flags
;
2376 ccb
->ccb_xa
.state
= ATA_S_ONCHIP
;
2377 ahci_start_timeout(ccb
);
2378 if (ap
->ap_flags
& AP_F_FBSS_ENABLED
) {
2380 ahci_pwrite(ap
, AHCI_PREG_FBS
,
2381 (ccb_at
->at_target
<<
2382 AHCI_PREG_FBS_DEV_SHIFT
) |
2385 ahci_pwrite(ap
, AHCI_PREG_FBS
,
2389 ahci_pwrite(ap
, AHCI_PREG_CI
, 1 << ccb
->ccb_slot
);
2390 if ((ap
->ap_run_flags
&
2391 (ATA_F_EXCLUSIVE
| ATA_F_AUTOSENSE
)) == 0) {
2394 ccb
= TAILQ_FIRST(&ap
->ap_ccb_pending
);
2395 if (ccb
&& (ccb
->ccb_xa
.flags
&
2396 (ATA_F_EXCLUSIVE
| ATA_F_AUTOSENSE
))) {
2404 ahci_intr(void *arg
)
2406 struct ahci_softc
*sc
= arg
;
2407 struct ahci_port
*ap
;
2413 * Check if the master enable is up, and whether any interrupts are
2416 if ((sc
->sc_flags
& AHCI_F_INT_GOOD
) == 0)
2418 is
= ahci_read(sc
, AHCI_REG_IS
);
2419 if (is
== 0 || is
== 0xffffffff) {
2422 is
&= sc
->sc_portmask
;
2424 #ifdef AHCI_COALESCE
2425 /* Check coalescing interrupt first */
2426 if (is
& sc
->sc_ccc_mask
) {
2427 DPRINTF(AHCI_D_INTR
, "%s: command coalescing interrupt\n",
2429 is
&= ~sc
->sc_ccc_mask
;
2430 is
|= sc
->sc_ccc_ports_cur
;
2435 * Process interrupts for each port in a non-blocking fashion.
2437 * The global IS bit is supposed to be forced on if any unmasked
2438 * port interrupt is pending, even if we clear it.
2440 * However it would appear that it is simply latched on some parts,
2441 * which means we have to clear it BEFORE processing the status bits
2444 ahci_write(sc
, AHCI_REG_IS
, is
);
2445 for (ack
= 0; is
; is
&= ~(1 << port
)) {
2449 ap
= sc
->sc_ports
[port
];
2453 if (ahci_os_lock_port_nb(ap
) == 0) {
2454 ahci_port_intr(ap
, 0);
2455 ahci_os_unlock_port(ap
);
2457 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
2458 ahci_os_signal_port_thread(ap
, AP_SIGF_PORTINT
);
2464 * Core called from helper thread.
2467 ahci_port_thread_core(struct ahci_port
*ap
, int mask
)
2470 * Process any expired timedouts.
2472 ahci_os_lock_port(ap
);
2473 if (mask
& AP_SIGF_TIMEOUT
) {
2474 ahci_check_active_timeouts(ap
);
2478 * Process port interrupts which require a higher level of
2481 if (mask
& AP_SIGF_PORTINT
) {
2482 ahci_port_intr(ap
, 1);
2483 ahci_port_interrupt_enable(ap
);
2484 } else if (ap
->ap_probe
!= ATA_PROBE_FAILED
) {
2485 ahci_port_intr(ap
, 1);
2486 ahci_port_interrupt_enable(ap
);
2488 ahci_os_unlock_port(ap
);
2492 * Core per-port interrupt handler.
2494 * If blockable is 0 we cannot call ahci_os_sleep() at all and we can only
2495 * deal with normal command completions which do not require blocking.
2498 ahci_port_intr(struct ahci_port
*ap
, int blockable
)
2500 struct ahci_softc
*sc
= ap
->ap_sc
;
2501 u_int32_t is
, ci_saved
, ci_masked
;
2504 struct ahci_ccb
*ccb
= NULL
;
2505 struct ata_port
*ccb_at
= NULL
;
2506 volatile u_int32_t
*active
;
2507 const u_int32_t blockable_mask
= AHCI_PREG_IS_TFES
|
2515 enum { NEED_NOTHING
, NEED_REINIT
, NEED_RESTART
,
2516 NEED_HOTPLUG_INSERT
, NEED_HOTPLUG_REMOVE
} need
= NEED_NOTHING
;
2519 * All basic command completions are always processed.
2521 is
= ahci_pread(ap
, AHCI_PREG_IS
);
2522 if (is
& AHCI_PREG_IS_DPS
)
2523 ahci_pwrite(ap
, AHCI_PREG_IS
, is
& AHCI_PREG_IS_DPS
);
2526 * If we can't block then we can't handle these here. Disable
2527 * the interrupts in question so we don't live-lock, the helper
2528 * thread will re-enable them.
2530 * If the port is in a completely failed state we do not want
2531 * to drop through to failed-command-processing if blockable is 0,
2532 * just let the thread deal with it all.
2534 * Otherwise we fall through and still handle DHRS and any commands
2535 * which completed normally. Even if we are errored we haven't
2536 * stopped the port yet so CI/SACT are still good.
2538 if (blockable
== 0) {
2539 if (ap
->ap_state
== AP_S_FATAL_ERROR
) {
2540 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
2541 ahci_os_signal_port_thread(ap
, AP_SIGF_PORTINT
);
2544 if (is
& blockable_mask
) {
2545 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
2546 ahci_os_signal_port_thread(ap
, AP_SIGF_PORTINT
);
2552 * Either NCQ or non-NCQ commands will be active, never both.
2554 if (ap
->ap_sactive
) {
2555 KKASSERT(ap
->ap_active
== 0);
2556 KKASSERT(ap
->ap_active_cnt
== 0);
2557 ci_saved
= ahci_pread(ap
, AHCI_PREG_SACT
);
2558 active
= &ap
->ap_sactive
;
2560 ci_saved
= ahci_pread(ap
, AHCI_PREG_CI
);
2561 active
= &ap
->ap_active
;
2563 KKASSERT(!(ap
->ap_sactive
&& ap
->ap_active
));
2564 KKASSERT((ci_saved
& (ap
->ap_sactive
| ap
->ap_active
)) == ci_saved
);
2566 kprintf("CHECK act=%08x/%08x sact=%08x/%08x\n",
2567 ap
->ap_active
, ahci_pread(ap
, AHCI_PREG_CI
),
2568 ap
->ap_sactive
, ahci_pread(ap
, AHCI_PREG_SACT
));
2572 * Ignore AHCI_PREG_IS_PRCS when link power management is on
2574 if (ap
->link_pwr_mgmt
!= AHCI_LINK_PWR_MGMT_NONE
) {
2575 is
&= ~AHCI_PREG_IS_PRCS
;
2576 ahci_pwrite(ap
, AHCI_PREG_SERR
,
2577 AHCI_PREG_SERR_DIAG_N
| AHCI_PREG_SERR_DIAG_W
);
2581 * Command failed (blockable).
2583 * See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2.
2585 * This stops command processing.
2587 if (is
& AHCI_PREG_IS_TFES
) {
2588 u_int32_t tfd
, serr
;
2592 tfd
= ahci_pread(ap
, AHCI_PREG_TFD
);
2593 serr
= ahci_pread(ap
, AHCI_PREG_SERR
);
2596 * Load the error slot and restart command processing.
2597 * CLO if we need to. The error slot may not be valid.
2598 * MUST BE DONE BEFORE CLEARING ST!
2602 * It is unclear but we may have to clear SERR to reenable
2605 err_slot
= AHCI_PREG_CMD_CCS(ahci_pread(ap
, AHCI_PREG_CMD
));
2606 ahci_pwrite(ap
, AHCI_PREG_IS
, AHCI_PREG_IS_TFES
|
2610 is
&= ~(AHCI_PREG_IS_TFES
| AHCI_PREG_IS_PSS
|
2611 AHCI_PREG_IS_DHRS
| AHCI_PREG_IS_SDBS
);
2612 ahci_pwrite(ap
, AHCI_PREG_SERR
, serr
);
2613 ahci_port_stop(ap
, 0);
2614 ahci_os_hardsleep(10);
2615 if (tfd
& (AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
2616 kprintf("%s: Issuing CLO\n", PORTNAME(ap
));
2621 * We are now stopped and need a restart. If we have to
2622 * process a NCQ error we will temporarily start and then
2623 * stop the port again, so this condition holds.
2626 need
= NEED_RESTART
;
2629 * ATAPI errors are fairly common from probing, just
2630 * report disk errors or if bootverbose is on.
2632 if (bootverbose
|| ap
->ap_type
!= ATA_PORT_T_ATAPI
) {
2633 kprintf("%s: TFES slot %d ci_saved = %08x\n",
2634 PORTNAME(ap
), err_slot
, ci_saved
);
2638 * If we got an error on an error CCB just complete it
2639 * with an error. ci_saved has the mask to restart
2640 * (the err_ccb will be removed from it by finish_error).
2642 if (ap
->ap_flags
& AP_F_ERR_CCB_RESERVED
) {
2643 err_slot
= ap
->ap_err_ccb
->ccb_slot
;
2648 * If NCQ commands were active get the error slot from
2649 * the log page. NCQ is not supported for PM's so this
2650 * is a direct-attached target.
2652 * Otherwise if no commands were active we have a problem.
2654 * Otherwise if the error slot is bad we have a problem.
2656 * Otherwise process the error for the slot.
2658 if (ap
->ap_sactive
) {
2659 ahci_port_start(ap
);
2660 err_slot
= ahci_port_read_ncq_error(ap
, 0);
2661 ahci_port_stop(ap
, 0);
2662 } else if (ap
->ap_active
== 0) {
2663 kprintf("%s: TFES with no commands pending\n",
2666 } else if (err_slot
< 0 || err_slot
>= ap
->ap_sc
->sc_ncmds
) {
2667 kprintf("%s: bad error slot %d\n",
2668 PORTNAME(ap
), err_slot
);
2671 ccb
= &ap
->ap_ccbs
[err_slot
];
2674 * Validate the errored ccb. Note that ccb_at can
2675 * be NULL for direct-attached ccb's.
2677 * Copy received taskfile data from the RFIS.
2679 if (ccb
->ccb_xa
.state
== ATA_S_ONCHIP
) {
2683 struct ata_fis_d2h
*rfis
;
2685 ccb_at
= ccb
->ccb_xa
.at
;
2687 (ap
->ap_flags
& AP_F_FBSS_ENABLED
))
2688 fis_target
= ccb_at
->at_target
;
2692 memcpy(&ccb
->ccb_xa
.rfis
,
2693 ap
->ap_rfis
[fis_target
].rfis
,
2694 sizeof(struct ata_fis_d2h
));
2695 rfis
= &ccb
->ccb_xa
.rfis
;
2697 offset
= (intmax_t)rfis
->lba_low
|
2698 ((intmax_t)rfis
->lba_mid
<< 8) |
2699 ((intmax_t)rfis
->lba_high
<< 16) |
2700 ((intmax_t)rfis
->lba_low_exp
<< 24) |
2701 ((intmax_t)rfis
->lba_mid_exp
<< 32) |
2702 ((intmax_t)rfis
->lba_high_exp
<< 40);
2704 bytes
= rfis
->sector_count
* 512;
2706 /* NOTE: expect type == 0x34 */
2707 kprintf("%s: TFES RFIS-%02x flg=%02x "
2708 "st=%02x err=%02x dev=%02x "
2718 kprintf("%s: Cannot copy rfis, CCB slot "
2719 "%d is not on-chip (state=%d)\n",
2720 ATANAME(ap
, ccb
->ccb_xa
.at
),
2721 err_slot
, ccb
->ccb_xa
.state
);
2727 * If we could not determine the errored slot then
2731 kprintf("%s: TFES: Unable to determine errored slot\n",
2733 if (ap
->ap_flags
& AP_F_IN_RESET
)
2739 * Finish error on slot. We will restart ci_saved
2740 * commands except the errored slot which we generate
2744 ccb
= &ap
->ap_ccbs
[err_slot
];
2745 ci_saved
&= ~(1 << err_slot
);
2746 KKASSERT(ccb
->ccb_xa
.state
== ATA_S_ONCHIP
);
2747 ccb
->ccb_xa
.state
= ATA_S_ERROR
;
2748 } else if (is
& AHCI_PREG_IS_DHRS
) {
2750 * Command posted D2H register FIS to the rfis (non-blocking).
2752 * A normal completion with an error may set DHRS instead
2753 * of TFES. The CCS bits are only valid if ERR was set.
2754 * If ERR is set command processing was probably stopped.
2756 * If ERR was not set we can only copy-back data for
2757 * exclusive-mode commands because otherwise we won't know
2758 * which tag the rfis belonged to.
2760 * err_slot must be read from the CCS before any other port
2761 * action, such as stopping the port.
2763 * WARNING! This is not well documented in the AHCI spec.
2764 * It can be found in the state machine tables
2765 * but not in the explanations.
2771 tfd
= ahci_pread(ap
, AHCI_PREG_TFD
);
2772 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
2774 ahci_pwrite(ap
, AHCI_PREG_IS
, AHCI_PREG_IS_DHRS
);
2777 * If command processing is turned off we can process the
2778 * error immediately. Use the ST bit here instead of the
2779 * CR bit in case the CR bit is not implemented via the
2782 if ((tfd
& AHCI_PREG_TFD_STS_ERR
) &&
2783 (cmd
& AHCI_PREG_CMD_ST
) == 0) {
2784 err_slot
= AHCI_PREG_CMD_CCS(
2785 ahci_pread(ap
, AHCI_PREG_CMD
));
2786 ccb
= &ap
->ap_ccbs
[err_slot
];
2787 kprintf("%s: DHRS tfd=%b err_slot=%d cmd=%02x\n",
2789 tfd
, AHCI_PFMT_TFD_STS
,
2790 err_slot
, ccb
->ccb_xa
.fis
->command
);
2794 * NO ELSE... copy back is in the normal command completion
2795 * code and only if no error occured and ATA_F_AUTOSENSE
2801 * Device notification to us (non-blocking)
2803 * NOTE! On some parts notification bits can cause an IPMS
2804 * interrupt instead of a SDBS interrupt.
2806 * NOTE! On some parts (e.g. VBOX, probably intel ICHx),
2807 * SDBS notifies us of the completion of a NCQ command
2810 if (is
& (AHCI_PREG_IS_SDBS
| AHCI_PREG_IS_IPMS
)) {
2813 ahci_pwrite(ap
, AHCI_PREG_IS
,
2814 AHCI_PREG_IS_SDBS
| AHCI_PREG_IS_IPMS
);
2815 if (sc
->sc_cap
& AHCI_REG_CAP_SSNTF
) {
2816 data
= ahci_pread(ap
, AHCI_PREG_SNTF
);
2818 ahci_pwrite(ap
, AHCI_PREG_IS
,
2820 kprintf("%s: NOTIFY %08x\n",
2821 PORTNAME(ap
), data
);
2822 ahci_pwrite(ap
, AHCI_PREG_SERR
,
2823 AHCI_PREG_SERR_DIAG_N
);
2824 ahci_pwrite(ap
, AHCI_PREG_SNTF
, data
);
2825 ahci_cam_changed(ap
, NULL
, -1);
2828 is
&= ~(AHCI_PREG_IS_SDBS
| AHCI_PREG_IS_IPMS
);
2832 * Spurious IFS errors (blockable) - when AP_F_IGNORE_IFS is set.
2834 * Spurious IFS errors can occur while we are doing a reset
2835 * sequence through a PM, probably due to an unexpected FIS
2836 * being received during the PM target reset sequence. Chipsets
2837 * are supposed to mask these events but some do not.
2839 * Try to recover from the condition.
2841 if ((is
& AHCI_PREG_IS_IFS
) && (ap
->ap_flags
& AP_F_IGNORE_IFS
)) {
2842 u_int32_t serr
= ahci_pread(ap
, AHCI_PREG_SERR
);
2843 if ((ap
->ap_flags
& AP_F_IFS_IGNORED
) == 0) {
2844 kprintf("%s: IFS during PM probe (ignored) "
2848 serr
, AHCI_PFMT_SERR
);
2849 ap
->ap_flags
|= AP_F_IFS_IGNORED
;
2853 * Try to clear the error condition. The IFS error killed
2854 * the port so stop it so we can restart it.
2856 ahci_pwrite(ap
, AHCI_PREG_IS
, AHCI_PREG_IS_IFS
);
2857 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
2858 is
&= ~AHCI_PREG_IS_IFS
;
2859 need
= NEED_RESTART
;
2864 * Port change (hot-plug) (blockable).
2866 * A PRCS interrupt can occur:
2867 * (1) On hot-unplug / normal-unplug (phy lost)
2868 * (2) Sometimes on hot-plug too.
2870 * A PCS interrupt can occur in a number of situations:
2871 * (1) On hot-plug once communication is established
2872 * (2) On hot-unplug sometimes.
2873 * (3) For chipsets with badly written firmware it can occur
2874 * during INIT/RESET sequences due to the device reset.
2875 * (4) For chipsets with badly written firmware it can occur
2876 * when it thinks an unsolicited COMRESET is received
2877 * during a INIT/RESET sequence, even though we actually
2880 * XXX We can then check the CPS (Cold Presence State) bit, if
2881 * supported, to determine if a device is plugged in or not and do
2884 * PCS interrupts are cleared by clearing DIAG_X. If this occurs
2885 * command processing is automatically stopped (CR goes inactive)
2886 * and the port must be stopped and restarted.
2888 * WARNING: AMD parts (e.g. 880G chipset, probably others) can
2889 * generate PCS on initialization even when device is
2890 * already connected up. It is unclear why this happens.
2891 * Depending on the state of the device detect this can
2892 * cause us to go into harsh reinit or hot-plug insertion
2895 * WARNING: PCS errors can be repetitive (e.g. unsolicited COMRESET
2896 * continues to flow in from the device), we must clear the
2897 * interrupt in all cases and enforce a delay to prevent
2898 * a livelock and give the port time to settle down.
2899 * Only print something if we aren't in INIT/HARD-RESET.
2901 if (is
& (AHCI_PREG_IS_PCS
| AHCI_PREG_IS_PRCS
)) {
2902 ahci_pwrite(ap
, AHCI_PREG_IS
,
2903 is
& (AHCI_PREG_IS_PCS
| AHCI_PREG_IS_PRCS
));
2905 * Try to clear the error. Because of the repetitiveness
2906 * of this interrupt avoid any harsh action if the port is
2907 * already in the init or hard-reset probe state.
2909 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
2910 /* (AHCI_PREG_SERR_DIAG_N | AHCI_PREG_SERR_DIAG_X) */
2913 * Ignore PCS/PRCS errors during probes (but still clear the
2914 * interrupt to avoid a livelock). The AMD 880/890/SB850
2915 * chipsets do not mask PCS/PRCS internally during reset
2918 if (ap
->ap_flags
& AP_F_IN_RESET
)
2921 if (ap
->ap_probe
== ATA_PROBE_NEED_INIT
||
2922 ap
->ap_probe
== ATA_PROBE_NEED_HARD_RESET
) {
2923 is
&= ~(AHCI_PREG_IS_PCS
| AHCI_PREG_IS_PRCS
);
2924 need
= NEED_NOTHING
;
2925 ahci_os_sleep(1000);
2928 kprintf("%s: Transient Errors: %b (%d)\n",
2929 PORTNAME(ap
), is
, AHCI_PFMT_IS
, ap
->ap_probe
);
2930 is
&= ~(AHCI_PREG_IS_PCS
| AHCI_PREG_IS_PRCS
);
2934 * Stop the port and figure out what to do next.
2936 ahci_port_stop(ap
, 0);
2939 switch (ahci_pread(ap
, AHCI_PREG_SSTS
) & AHCI_PREG_SSTS_DET
) {
2940 case AHCI_PREG_SSTS_DET_DEV
:
2944 if (ap
->ap_probe
== ATA_PROBE_FAILED
) {
2945 need
= NEED_HOTPLUG_INSERT
;
2948 need
= NEED_RESTART
;
2950 case AHCI_PREG_SSTS_DET_DEV_NE
:
2952 * Device not communicating. AMD parts seem to
2953 * like to throw this error on initialization
2954 * for no reason that I can fathom.
2956 kprintf("%s: Device present but not communicating, "
2957 "attempting port restart\n",
2962 if (ap
->ap_probe
!= ATA_PROBE_FAILED
) {
2963 need
= NEED_HOTPLUG_REMOVE
;
2966 need
= NEED_RESTART
;
2974 * Check for remaining errors - they are fatal. (blockable)
2976 if (is
& (AHCI_PREG_IS_TFES
| AHCI_PREG_IS_HBFS
| AHCI_PREG_IS_IFS
|
2977 AHCI_PREG_IS_OFS
| AHCI_PREG_IS_UFS
)) {
2980 ahci_pwrite(ap
, AHCI_PREG_IS
,
2981 is
& (AHCI_PREG_IS_TFES
| AHCI_PREG_IS_HBFS
|
2982 AHCI_PREG_IS_IFS
| AHCI_PREG_IS_OFS
|
2984 serr
= ahci_pread(ap
, AHCI_PREG_SERR
);
2985 kprintf("%s: Unrecoverable errors (IS: %b, SERR: %b), "
2986 "disabling port.\n",
2989 serr
, AHCI_PFMT_SERR
2991 is
&= ~(AHCI_PREG_IS_TFES
| AHCI_PREG_IS_HBFS
|
2992 AHCI_PREG_IS_IFS
| AHCI_PREG_IS_OFS
|
2996 * Fail all commands but then what? For now try to
2997 * reinitialize the port.
3004 * Fail all outstanding commands if we know the port won't recover.
3006 * We may have a ccb_at if the failed command is known and was
3007 * being sent to a device over a port multiplier (PM). In this
3008 * case if the port itself has not completely failed we fail just
3009 * the commands related to that target.
3011 * ci_saved contains the mask of active commands as of when the
3012 * error occured, prior to any port stops.
3014 if (ap
->ap_state
== AP_S_FATAL_ERROR
) {
3016 ap
->ap_state
= AP_S_FATAL_ERROR
;
3018 ahci_port_stop(ap
, 0);
3022 * Error all the active slots not already errored.
3024 ci_masked
= ci_saved
& *active
& ~ap
->ap_expired
;
3026 kprintf("%s: Failing all commands: %08x\n",
3027 PORTNAME(ap
), ci_masked
);
3031 slot
= ffs(ci_masked
) - 1;
3032 ccb
= &ap
->ap_ccbs
[slot
];
3033 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
3034 ap
->ap_expired
|= 1 << slot
;
3035 ci_saved
&= ~(1 << slot
);
3036 ci_masked
&= ~(1 << slot
);
3040 * Clear bits in ci_saved (cause completions to be run)
3041 * for all slots which are not active.
3043 ci_saved
&= ~*active
;
3046 * Don't restart the port if our problems were deemed fatal.
3048 * Also acknowlege all fatal interrupt sources to prevent
3051 if (ap
->ap_state
== AP_S_FATAL_ERROR
) {
3052 if (need
== NEED_RESTART
)
3053 need
= NEED_NOTHING
;
3054 ahci_pwrite(ap
, AHCI_PREG_IS
,
3055 AHCI_PREG_IS_TFES
| AHCI_PREG_IS_HBFS
|
3056 AHCI_PREG_IS_IFS
| AHCI_PREG_IS_OFS
|
3062 * If we are stopped the AHCI chipset is supposed to have cleared
3063 * CI and SACT. Did it? If it didn't we try very hard to clear
3064 * the fields otherwise we may end up completing CCBs which are
3065 * actually still active.
3067 * IFS errors on (at least) AMD chipsets create this confusion.
3071 if ((mask
= ahci_pactive(ap
)) != 0) {
3072 kprintf("%s: chipset failed to clear "
3073 "active cmds %08x\n",
3074 PORTNAME(ap
), mask
);
3075 ahci_port_start(ap
);
3076 ahci_port_stop(ap
, 0);
3077 if ((mask
= ahci_pactive(ap
)) != 0) {
3078 kprintf("%s: unable to prod the chip into "
3079 "clearing active cmds %08x\n",
3080 PORTNAME(ap
), mask
);
3081 /* what do we do now? */
3087 * CCB completion (non blocking).
3089 * CCB completion is detected by noticing its slot's bit in CI has
3090 * changed to zero some time after we activated it.
3091 * If we are polling, we may only be interested in particular slot(s).
3093 * Any active bits not saved are completed within the restrictions
3094 * imposed by the caller.
3096 ci_masked
= ~ci_saved
& *active
;
3098 slot
= ffs(ci_masked
) - 1;
3099 ccb
= &ap
->ap_ccbs
[slot
];
3100 ci_masked
&= ~(1 << slot
);
3102 DPRINTF(AHCI_D_INTR
, "%s: slot %d is complete%s\n",
3103 PORTNAME(ap
), slot
, ccb
->ccb_xa
.state
== ATA_S_ERROR
?
3106 bus_dmamap_sync(sc
->sc_tag_cmdh
,
3107 AHCI_DMA_MAP(ap
->ap_dmamem_cmd_list
),
3108 BUS_DMASYNC_POSTWRITE
);
3110 bus_dmamap_sync(sc
->sc_tag_cmdt
,
3111 AHCI_DMA_MAP(ap
->ap_dmamem_cmd_table
),
3112 BUS_DMASYNC_POSTWRITE
);
3114 bus_dmamap_sync(sc
->sc_tag_rfis
,
3115 AHCI_DMA_MAP(ap
->ap_dmamem_rfis
),
3116 BUS_DMASYNC_POSTREAD
);
3118 *active
&= ~(1 << ccb
->ccb_slot
);
3119 if (active
== &ap
->ap_active
) {
3120 KKASSERT(ap
->ap_active_cnt
> 0);
3121 --ap
->ap_active_cnt
;
3125 * Complete the ccb. If the ccb was marked expired it
3126 * was probably already removed from the command processor,
3127 * so don't take the clear ci_saved bit as meaning the
3128 * command actually succeeded, it didn't.
3130 if (ap
->ap_expired
& (1 << ccb
->ccb_slot
)) {
3131 ap
->ap_expired
&= ~(1 << ccb
->ccb_slot
);
3132 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
3134 ccb
->ccb_xa
.complete(&ccb
->ccb_xa
);
3136 if (ccb
->ccb_xa
.state
== ATA_S_ONCHIP
) {
3137 ccb
->ccb_xa
.state
= ATA_S_COMPLETE
;
3138 if (ccb
->ccb_xa
.flags
& ATA_F_AUTOSENSE
) {
3141 ccb_at
= ccb
->ccb_xa
.at
;
3143 (ap
->ap_flags
& AP_F_FBSS_ENABLED
))
3144 fis_target
= ccb_at
->at_target
;
3147 memcpy(&ccb
->ccb_xa
.rfis
,
3148 ap
->ap_rfis
[fis_target
].rfis
,
3149 sizeof(struct ata_fis_d2h
));
3150 if (ccb
->ccb_xa
.state
== ATA_S_TIMEOUT
)
3151 ccb
->ccb_xa
.state
= ATA_S_ERROR
;
3159 * Cleanup. Will not be set if non-blocking.
3164 * If operating normally and not stopped the interrupt was
3165 * probably just a normal completion and we may be able to
3166 * issue more commands.
3168 if (stopped
== 0 && ap
->ap_state
!= AP_S_FATAL_ERROR
)
3169 ahci_issue_pending_commands(ap
, NULL
);
3173 * A recoverable error occured and we can restart outstanding
3174 * commands on the port.
3176 ci_saved
&= ~ap
->ap_expired
;
3178 kprintf("%s: Restart %08x\n", PORTNAME(ap
), ci_saved
);
3179 ahci_issue_saved_commands(ap
, ci_saved
);
3183 * Potentially issue new commands if not in a failed
3186 if (ap
->ap_state
!= AP_S_FATAL_ERROR
) {
3187 ahci_port_start(ap
);
3188 ahci_issue_pending_commands(ap
, NULL
);
3193 * Something horrible happened to the port and we
3194 * need to reinitialize it.
3196 kprintf("%s: REINIT - Attempting to reinitialize the port "
3197 "after it had a horrible accident\n",
3199 ap
->ap_flags
|= AP_F_IN_RESET
;
3200 ap
->ap_flags
|= AP_F_HARSH_REINIT
;
3201 ap
->ap_probe
= ATA_PROBE_NEED_INIT
;
3202 ahci_cam_changed(ap
, NULL
, -1);
3204 case NEED_HOTPLUG_INSERT
:
3206 * A hot-plug insertion event has occured and all
3207 * outstanding commands have already been revoked.
3209 * Don't recurse if this occurs while we are
3210 * resetting the port.
3212 if ((ap
->ap_flags
& AP_F_IN_RESET
) == 0) {
3213 kprintf("%s: HOTPLUG - Device inserted\n",
3215 ap
->ap_probe
= ATA_PROBE_NEED_INIT
;
3216 ahci_cam_changed(ap
, NULL
, -1);
3219 case NEED_HOTPLUG_REMOVE
:
3221 * A hot-plug removal event has occured and all
3222 * outstanding commands have already been revoked.
3224 * Don't recurse if this occurs while we are
3225 * resetting the port.
3227 if ((ap
->ap_flags
& AP_F_IN_RESET
) == 0) {
3228 kprintf("%s: HOTPLUG - Device removed\n",
3230 ahci_port_hardstop(ap
);
3231 /* ap_probe set to failed */
3232 ahci_cam_changed(ap
, NULL
, -1);
3241 ahci_get_ccb(struct ahci_port
*ap
)
3243 struct ahci_ccb
*ccb
;
3245 lockmgr(&ap
->ap_ccb_lock
, LK_EXCLUSIVE
);
3246 ccb
= TAILQ_FIRST(&ap
->ap_ccb_free
);
3248 KKASSERT((ap
->ap_sactive
& (1 << ccb
->ccb_slot
)) == 0);
3249 KKASSERT(ccb
->ccb_xa
.state
== ATA_S_PUT
);
3250 TAILQ_REMOVE(&ap
->ap_ccb_free
, ccb
, ccb_entry
);
3251 ccb
->ccb_xa
.state
= ATA_S_SETUP
;
3252 ccb
->ccb_xa
.flags
= 0;
3253 ccb
->ccb_xa
.at
= NULL
;
3255 lockmgr(&ap
->ap_ccb_lock
, LK_RELEASE
);
3261 ahci_put_ccb(struct ahci_ccb
*ccb
)
3263 struct ahci_port
*ap
= ccb
->ccb_port
;
3265 KKASSERT(ccb
->ccb_xa
.state
!= ATA_S_PUT
);
3266 KKASSERT((ap
->ap_sactive
& (1 << ccb
->ccb_slot
)) == 0);
3267 lockmgr(&ap
->ap_ccb_lock
, LK_EXCLUSIVE
);
3268 ccb
->ccb_xa
.state
= ATA_S_PUT
;
3269 ++ccb
->ccb_xa
.serial
;
3270 TAILQ_INSERT_TAIL(&ap
->ap_ccb_free
, ccb
, ccb_entry
);
3271 lockmgr(&ap
->ap_ccb_lock
, LK_RELEASE
);
3275 ahci_get_err_ccb(struct ahci_port
*ap
)
3277 struct ahci_ccb
*err_ccb
;
3281 /* No commands may be active on the chip. */
3283 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SNCQ
) {
3284 sact
= ahci_pread(ap
, AHCI_PREG_SACT
);
3286 kprintf("%s: ahci_get_err_ccb but SACT %08x != 0?\n",
3287 PORTNAME(ap
), sact
);
3290 ci
= ahci_pread(ap
, AHCI_PREG_CI
);
3292 kprintf("%s: ahci_get_err_ccb: ci not 0 (%08x)\n",
3296 KKASSERT((ap
->ap_flags
& AP_F_ERR_CCB_RESERVED
) == 0);
3297 ap
->ap_flags
|= AP_F_ERR_CCB_RESERVED
;
3299 /* Save outstanding command state. */
3300 ap
->ap_err_saved_active
= ap
->ap_active
;
3301 ap
->ap_err_saved_active_cnt
= ap
->ap_active_cnt
;
3302 ap
->ap_err_saved_sactive
= ap
->ap_sactive
;
3305 * Pretend we have no commands outstanding, so that completions won't
3308 ap
->ap_active
= ap
->ap_active_cnt
= ap
->ap_sactive
= 0;
3311 * Grab a CCB to use for error recovery. This should never fail, as
3312 * we ask atascsi to reserve one for us at init time.
3314 err_ccb
= ap
->ap_err_ccb
;
3315 KKASSERT(err_ccb
!= NULL
);
3316 err_ccb
->ccb_xa
.flags
= 0;
3317 err_ccb
->ccb_done
= ahci_empty_done
;
3323 ahci_put_err_ccb(struct ahci_ccb
*ccb
)
3325 struct ahci_port
*ap
= ccb
->ccb_port
;
3329 KKASSERT((ap
->ap_flags
& AP_F_ERR_CCB_RESERVED
) != 0);
3332 * No commands may be active on the chip
3334 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SNCQ
) {
3335 sact
= ahci_pread(ap
, AHCI_PREG_SACT
);
3337 panic("ahci_port_err_ccb(%d) but SACT %08x != 0",
3338 ccb
->ccb_slot
, sact
);
3341 ci
= ahci_pread(ap
, AHCI_PREG_CI
);
3343 panic("ahci_put_err_ccb(%d) but CI %08x != 0 "
3344 "(act=%08x sact=%08x)\n",
3346 ap
->ap_active
, ap
->ap_sactive
);
3349 KKASSERT(ccb
== ap
->ap_err_ccb
);
3351 /* Restore outstanding command state */
3352 ap
->ap_sactive
= ap
->ap_err_saved_sactive
;
3353 ap
->ap_active_cnt
= ap
->ap_err_saved_active_cnt
;
3354 ap
->ap_active
= ap
->ap_err_saved_active
;
3356 ap
->ap_flags
&= ~AP_F_ERR_CCB_RESERVED
;
3360 * Read log page to get NCQ error.
3362 * NOTE: NCQ not currently supported on port multipliers. XXX
3365 ahci_port_read_ncq_error(struct ahci_port
*ap
, int target
)
3367 struct ata_log_page_10h
*log
;
3368 struct ahci_ccb
*ccb
;
3369 struct ahci_ccb
*ccb2
;
3370 struct ahci_cmd_hdr
*cmd_slot
;
3371 struct ata_fis_h2d
*fis
;
3375 kprintf("%s: READ LOG PAGE target %d\n", PORTNAME(ap
),
3380 * Prep error CCB for READ LOG EXT, page 10h, 1 sector.
3382 * Getting err_ccb clears active/sactive/active_cnt, putting
3383 * it back restores the fields.
3385 ccb
= ahci_get_err_ccb(ap
);
3386 ccb
->ccb_xa
.flags
= ATA_F_READ
| ATA_F_POLL
;
3387 ccb
->ccb_xa
.data
= ap
->ap_err_scratch
;
3388 ccb
->ccb_xa
.datalen
= 512;
3389 ccb
->ccb_xa
.complete
= ahci_dummy_done
;
3390 ccb
->ccb_xa
.at
= ap
->ap_ata
[target
];
3392 fis
= (struct ata_fis_h2d
*)ccb
->ccb_cmd_table
->cfis
;
3393 bzero(fis
, sizeof(*fis
));
3394 fis
->type
= ATA_FIS_TYPE_H2D
;
3395 fis
->flags
= ATA_H2D_FLAGS_CMD
| target
;
3396 fis
->command
= ATA_C_READ_LOG_EXT
;
3397 fis
->lba_low
= 0x10; /* queued error log page (10h) */
3398 fis
->sector_count
= 1; /* number of sectors (1) */
3399 fis
->sector_count_exp
= 0;
3400 fis
->lba_mid
= 0; /* starting offset */
3401 fis
->lba_mid_exp
= 0;
3404 cmd_slot
= ccb
->ccb_cmd_hdr
;
3405 cmd_slot
->flags
= htole16(5); /* FIS length: 5 DWORDS */
3407 if (ahci_load_prdt(ccb
) != 0) {
3412 ccb
->ccb_xa
.state
= ATA_S_PENDING
;
3413 if (ahci_poll(ccb
, 1000, ahci_quick_timeout
) != ATA_S_COMPLETE
) {
3415 ahci_unload_prdt(ccb
);
3418 ahci_unload_prdt(ccb
);
3421 * Success, extract failed register set and tags from the scratch
3424 log
= (struct ata_log_page_10h
*)ap
->ap_err_scratch
;
3425 if (log
->err_regs
.type
& ATA_LOG_10H_TYPE_NOTQUEUED
) {
3426 /* Not queued bit was set - wasn't an NCQ error? */
3427 kprintf("%s: read NCQ error page, but not an NCQ error?\n",
3431 /* Copy back the log record as a D2H register FIS. */
3432 err_slot
= log
->err_regs
.type
& ATA_LOG_10H_TYPE_TAG_MASK
;
3434 ccb2
= &ap
->ap_ccbs
[err_slot
];
3435 if (ccb2
->ccb_xa
.state
== ATA_S_ONCHIP
) {
3436 kprintf("%s: read NCQ error page slot=%d\n",
3437 ATANAME(ap
, ccb2
->ccb_xa
.at
),
3439 memcpy(&ccb2
->ccb_xa
.rfis
, &log
->err_regs
,
3440 sizeof(struct ata_fis_d2h
));
3441 ccb2
->ccb_xa
.rfis
.type
= ATA_FIS_TYPE_D2H
;
3442 ccb2
->ccb_xa
.rfis
.flags
= 0;
3444 kprintf("%s: read NCQ error page slot=%d, "
3445 "slot does not match any cmds\n",
3446 ATANAME(ccb2
->ccb_port
, ccb2
->ccb_xa
.at
),
3452 ahci_put_err_ccb(ccb
);
3453 kprintf("%s: DONE log page target %d err_slot=%d\n",
3454 PORTNAME(ap
), target
, err_slot
);
3459 * Allocate memory for various structures DMAd by hardware. The maximum
3460 * number of segments for these tags is 1 so the DMA memory will have a
3461 * single physical base address.
3463 struct ahci_dmamem
*
3464 ahci_dmamem_alloc(struct ahci_softc
*sc
, bus_dma_tag_t tag
)
3466 struct ahci_dmamem
*adm
;
3469 adm
= kmalloc(sizeof(*adm
), M_DEVBUF
, M_INTWAIT
| M_ZERO
);
3471 error
= bus_dmamem_alloc(tag
, (void **)&adm
->adm_kva
,
3472 BUS_DMA_ZERO
, &adm
->adm_map
);
3475 error
= bus_dmamap_load(tag
, adm
->adm_map
,
3477 bus_dma_tag_getmaxsize(tag
),
3478 ahci_dmamem_saveseg
, &adm
->adm_busaddr
,
3483 bus_dmamap_destroy(tag
, adm
->adm_map
);
3484 adm
->adm_map
= NULL
;
3485 adm
->adm_tag
= NULL
;
3486 adm
->adm_kva
= NULL
;
3488 kfree(adm
, M_DEVBUF
);
3496 ahci_dmamem_saveseg(void *info
, bus_dma_segment_t
*segs
, int nsegs
, int error
)
3498 KKASSERT(error
== 0);
3499 KKASSERT(nsegs
== 1);
3500 *(bus_addr_t
*)info
= segs
->ds_addr
;
3505 ahci_dmamem_free(struct ahci_softc
*sc
, struct ahci_dmamem
*adm
)
3508 bus_dmamap_unload(adm
->adm_tag
, adm
->adm_map
);
3509 bus_dmamap_destroy(adm
->adm_tag
, adm
->adm_map
);
3510 adm
->adm_map
= NULL
;
3511 adm
->adm_tag
= NULL
;
3512 adm
->adm_kva
= NULL
;
3514 kfree(adm
, M_DEVBUF
);
3518 ahci_read(struct ahci_softc
*sc
, bus_size_t r
)
3520 bus_space_barrier(sc
->sc_iot
, sc
->sc_ioh
, r
, 4,
3521 BUS_SPACE_BARRIER_READ
);
3522 return (bus_space_read_4(sc
->sc_iot
, sc
->sc_ioh
, r
));
3526 ahci_write(struct ahci_softc
*sc
, bus_size_t r
, u_int32_t v
)
3528 bus_space_write_4(sc
->sc_iot
, sc
->sc_ioh
, r
, v
);
3529 bus_space_barrier(sc
->sc_iot
, sc
->sc_ioh
, r
, 4,
3530 BUS_SPACE_BARRIER_WRITE
);
3534 ahci_pread(struct ahci_port
*ap
, bus_size_t r
)
3536 bus_space_barrier(ap
->ap_sc
->sc_iot
, ap
->ap_ioh
, r
, 4,
3537 BUS_SPACE_BARRIER_READ
);
3538 return (bus_space_read_4(ap
->ap_sc
->sc_iot
, ap
->ap_ioh
, r
));
3542 ahci_pwrite(struct ahci_port
*ap
, bus_size_t r
, u_int32_t v
)
3544 bus_space_write_4(ap
->ap_sc
->sc_iot
, ap
->ap_ioh
, r
, v
);
3545 bus_space_barrier(ap
->ap_sc
->sc_iot
, ap
->ap_ioh
, r
, 4,
3546 BUS_SPACE_BARRIER_WRITE
);
3550 * Wait up to (timeout) milliseconds for the masked port register to
3553 * Timeout is in milliseconds.
3556 ahci_pwait_eq(struct ahci_port
*ap
, int timeout
,
3557 bus_size_t r
, u_int32_t mask
, u_int32_t target
)
3562 * Loop hard up to 100uS
3564 for (t
= 0; t
< 100; ++t
) {
3565 if ((ahci_pread(ap
, r
) & mask
) == target
)
3567 ahci_os_hardsleep(1); /* us */
3571 timeout
-= ahci_os_softsleep();
3572 if ((ahci_pread(ap
, r
) & mask
) == target
)
3574 } while (timeout
> 0);
3579 ahci_wait_ne(struct ahci_softc
*sc
, bus_size_t r
, u_int32_t mask
,
3585 * Loop hard up to 100uS
3587 for (t
= 0; t
< 100; ++t
) {
3588 if ((ahci_read(sc
, r
) & mask
) != target
)
3590 ahci_os_hardsleep(1); /* us */
3594 * And one millisecond the slow way
3598 t
-= ahci_os_softsleep();
3599 if ((ahci_read(sc
, r
) & mask
) != target
)
3608 * Acquire an ata transfer.
3610 * Pass a NULL at for direct-attached transfers, and a non-NULL at for
3611 * targets that go through the port multiplier.
3614 ahci_ata_get_xfer(struct ahci_port
*ap
, struct ata_port
*at
)
3616 struct ahci_ccb
*ccb
;
3618 ccb
= ahci_get_ccb(ap
);
3620 DPRINTF(AHCI_D_XFER
, "%s: ahci_ata_get_xfer: NULL ccb\n",
3625 DPRINTF(AHCI_D_XFER
, "%s: ahci_ata_get_xfer got slot %d\n",
3626 PORTNAME(ap
), ccb
->ccb_slot
);
3628 bzero(ccb
->ccb_xa
.fis
, sizeof(*ccb
->ccb_xa
.fis
));
3629 ccb
->ccb_xa
.at
= at
;
3630 ccb
->ccb_xa
.fis
->type
= ATA_FIS_TYPE_H2D
;
3632 return (&ccb
->ccb_xa
);
3636 ahci_ata_put_xfer(struct ata_xfer
*xa
)
3638 struct ahci_ccb
*ccb
= (struct ahci_ccb
*)xa
;
3640 DPRINTF(AHCI_D_XFER
, "ahci_ata_put_xfer slot %d\n", ccb
->ccb_slot
);
3646 ahci_ata_cmd(struct ata_xfer
*xa
)
3648 struct ahci_ccb
*ccb
= (struct ahci_ccb
*)xa
;
3649 struct ahci_cmd_hdr
*cmd_slot
;
3651 KKASSERT(xa
->state
== ATA_S_SETUP
);
3653 if (ccb
->ccb_port
->ap_state
== AP_S_FATAL_ERROR
)
3655 ccb
->ccb_done
= ahci_ata_cmd_done
;
3657 cmd_slot
= ccb
->ccb_cmd_hdr
;
3658 cmd_slot
->flags
= htole16(5); /* FIS length (in DWORDs) */
3659 if (ccb
->ccb_xa
.at
) {
3660 cmd_slot
->flags
|= htole16(ccb
->ccb_xa
.at
->at_target
<<
3661 AHCI_CMD_LIST_FLAG_PMP_SHIFT
);
3664 if (xa
->flags
& ATA_F_WRITE
)
3665 cmd_slot
->flags
|= htole16(AHCI_CMD_LIST_FLAG_W
);
3667 if (xa
->flags
& ATA_F_PACKET
)
3668 cmd_slot
->flags
|= htole16(AHCI_CMD_LIST_FLAG_A
);
3670 if (ahci_load_prdt(ccb
) != 0)
3673 xa
->state
= ATA_S_PENDING
;
3675 if (xa
->flags
& ATA_F_POLL
)
3676 return (ahci_poll(ccb
, xa
->timeout
, ahci_ata_cmd_timeout
));
3679 KKASSERT((xa
->flags
& ATA_F_TIMEOUT_EXPIRED
) == 0);
3680 xa
->flags
|= ATA_F_TIMEOUT_DESIRED
;
3687 xa
->state
= ATA_S_ERROR
;
3690 return (ATA_S_ERROR
);
3694 ahci_ata_cmd_done(struct ahci_ccb
*ccb
)
3696 struct ata_xfer
*xa
= &ccb
->ccb_xa
;
3700 * NOTE: Callout does not lock port and may race us modifying
3701 * the flags, so make sure its stopped.
3703 * A callout race can clean up the ccb. A change in the
3704 * serial number should catch this condition.
3706 if (xa
->flags
& ATA_F_TIMEOUT_RUNNING
) {
3707 serial
= ccb
->ccb_xa
.serial
;
3708 callout_stop_sync(&ccb
->ccb_timeout
);
3709 if (serial
!= ccb
->ccb_xa
.serial
) {
3710 kprintf("%s: Warning: timeout race ccb %p\n",
3711 PORTNAME(ccb
->ccb_port
), ccb
);
3714 xa
->flags
&= ~ATA_F_TIMEOUT_RUNNING
;
3716 xa
->flags
&= ~(ATA_F_TIMEOUT_DESIRED
| ATA_F_TIMEOUT_EXPIRED
);
3717 ccb
->ccb_port
->ap_expired
&= ~(1 << ccb
->ccb_slot
);
3719 KKASSERT(xa
->state
!= ATA_S_ONCHIP
&& xa
->state
!= ATA_S_PUT
);
3720 ahci_unload_prdt(ccb
);
3722 if (xa
->state
!= ATA_S_TIMEOUT
)
3727 * Timeout from callout, MPSAFE - nothing can mess with the CCB's flags
3728 * while the callout is runing.
3730 * We can't safely get the port lock here or delay, we could block
3731 * the callout thread.
3734 ahci_ata_cmd_timeout_unserialized(void *arg
)
3736 struct ahci_ccb
*ccb
= arg
;
3737 struct ahci_port
*ap
= ccb
->ccb_port
;
3739 KKASSERT(ccb
->ccb_xa
.flags
& ATA_F_TIMEOUT_RUNNING
);
3740 ccb
->ccb_xa
.flags
&= ~ATA_F_TIMEOUT_RUNNING
;
3741 ccb
->ccb_xa
.flags
|= ATA_F_TIMEOUT_EXPIRED
;
3742 ahci_os_signal_port_thread(ap
, AP_SIGF_TIMEOUT
);
3746 * Timeout code, typically called when the port command processor is running.
3748 * We have to be very very careful here. We cannot stop the port unless
3749 * CR is already clear or the only active commands remaining are timed-out
3750 * ones. Otherwise stopping the port will race the command processor and
3751 * we can lose events. While we can theoretically just restart everything
3752 * that could result in a double-issue which will not work for ATAPI commands.
3755 ahci_ata_cmd_timeout(struct ahci_ccb
*ccb
)
3757 struct ata_xfer
*xa
= &ccb
->ccb_xa
;
3758 struct ahci_port
*ap
= ccb
->ccb_port
;
3759 struct ata_port
*at
;
3764 at
= ccb
->ccb_xa
.at
;
3766 kprintf("%s: CMD TIMEOUT state=%d slot=%d\n"
3767 "\tglb-status 0x%08x\n"
3769 "\tport_status 0x%b\n"
3770 "\tsactive=%08x active=%08x expired=%08x\n"
3771 "\t sact=%08x ci=%08x\n"
3774 ccb
->ccb_xa
.state
, ccb
->ccb_slot
,
3775 ahci_read(ap
->ap_sc
, AHCI_REG_IS
),
3776 ahci_pread(ap
, AHCI_PREG_CMD
), AHCI_PFMT_CMD
,
3777 ahci_pread(ap
, AHCI_PREG_IS
), AHCI_PFMT_IS
,
3778 ap
->ap_sactive
, ap
->ap_active
, ap
->ap_expired
,
3779 ahci_pread(ap
, AHCI_PREG_SACT
),
3780 ahci_pread(ap
, AHCI_PREG_CI
),
3781 ahci_pread(ap
, AHCI_PREG_TFD
), AHCI_PFMT_TFD_STS
3786 * NOTE: Timeout will not be running if the command was polled.
3787 * If we got here at least one of these flags should be set.
3789 KKASSERT(xa
->flags
& (ATA_F_POLL
| ATA_F_TIMEOUT_DESIRED
|
3790 ATA_F_TIMEOUT_RUNNING
));
3791 xa
->flags
&= ~(ATA_F_TIMEOUT_RUNNING
| ATA_F_TIMEOUT_EXPIRED
);
3793 if (ccb
->ccb_xa
.state
== ATA_S_PENDING
) {
3794 TAILQ_REMOVE(&ap
->ap_ccb_pending
, ccb
, ccb_entry
);
3795 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
3798 ahci_issue_pending_commands(ap
, NULL
);
3801 if (ccb
->ccb_xa
.state
!= ATA_S_ONCHIP
) {
3802 kprintf("%s: Unexpected state during timeout: %d\n",
3803 ATANAME(ap
, at
), ccb
->ccb_xa
.state
);
3808 * Ok, we can only get this command off the chip if CR is inactive
3809 * or if the only commands running on the chip are all expired.
3810 * Otherwise we have to wait until the port is in a safe state.
3811 * Use the ST bit here instead of the CR bit in case the CR bit is
3812 * not implemented via the F_IGN_CR quirk.
3814 * Do not set state here, it will cause polls to return when the
3815 * ccb is not yet off the chip.
3817 ap
->ap_expired
|= 1 << ccb
->ccb_slot
;
3819 if ((ahci_pread(ap
, AHCI_PREG_CMD
) & AHCI_PREG_CMD_ST
) &&
3820 (ap
->ap_active
| ap
->ap_sactive
) != ap
->ap_expired
) {
3822 * If using FBSS or NCQ we can't safely stop the port
3825 kprintf("%s: Deferred timeout until its safe, slot %d\n",
3826 ATANAME(ap
, at
), ccb
->ccb_slot
);
3831 * We can safely stop the port and process all expired ccb's,
3832 * which will include our current ccb.
3834 ci_saved
= (ap
->ap_sactive
) ? ahci_pread(ap
, AHCI_PREG_SACT
) :
3835 ahci_pread(ap
, AHCI_PREG_CI
);
3836 ahci_port_stop(ap
, 0);
3838 while (ap
->ap_expired
) {
3839 slot
= ffs(ap
->ap_expired
) - 1;
3840 ap
->ap_expired
&= ~(1 << slot
);
3841 ci_saved
&= ~(1 << slot
);
3842 ccb
= &ap
->ap_ccbs
[slot
];
3843 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
3844 if (ccb
->ccb_xa
.flags
& ATA_F_NCQ
) {
3845 KKASSERT(ap
->ap_sactive
& (1 << slot
));
3846 ap
->ap_sactive
&= ~(1 << slot
);
3848 KKASSERT(ap
->ap_active
& (1 << slot
));
3849 ap
->ap_active
&= ~(1 << slot
);
3850 --ap
->ap_active_cnt
;
3853 ccb
->ccb_xa
.complete(&ccb
->ccb_xa
);
3855 /* ccb invalid now */
3858 * We can safely CLO the port to clear any BSY/DRQ, a case which
3859 * can occur with port multipliers. This will unbrick the port
3860 * and allow commands to other targets behind the PM continue.
3863 * Finally, once the port has been restarted we can issue any
3864 * previously saved pending commands, and run the port interrupt
3865 * code to handle any completions which may have occured when
3868 if (ahci_pread(ap
, AHCI_PREG_TFD
) &
3869 (AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
3870 kprintf("%s: Warning, issuing CLO after timeout\n",
3874 ahci_port_start(ap
);
3877 * We absolutely must make sure the chipset cleared activity on
3878 * all slots. This sometimes might not happen due to races with
3879 * a chipset interrupt which stops the port before we can manage
3880 * to. For some reason some chipsets don't clear the active
3881 * commands when we turn off CMD_ST after the chip has stopped
3882 * operations itself.
3884 if (ahci_pactive(ap
) != 0) {
3885 ahci_port_stop(ap
, 0);
3886 ahci_port_start(ap
);
3887 if ((mask
= ahci_pactive(ap
)) != 0) {
3888 kprintf("%s: quick-timeout: chipset failed "
3889 "to clear active cmds %08x\n",
3890 PORTNAME(ap
), mask
);
3893 ahci_issue_saved_commands(ap
, ci_saved
& ~ap
->ap_expired
);
3894 ahci_issue_pending_commands(ap
, NULL
);
3895 ahci_port_intr(ap
, 0);
3899 * Issue a previously saved set of commands
3902 ahci_issue_saved_commands(struct ahci_port
*ap
, u_int32_t ci_saved
)
3904 if (ci_saved
&& (ap
->ap_flags
& AP_F_FBSS_ENABLED
) == 0) {
3905 KKASSERT(!((ap
->ap_active
& ci_saved
) &&
3906 (ap
->ap_sactive
& ci_saved
)));
3907 KKASSERT((ci_saved
& ap
->ap_expired
) == 0);
3908 if (ap
->ap_sactive
& ci_saved
)
3909 ahci_pwrite(ap
, AHCI_PREG_SACT
, ci_saved
);
3910 ahci_pwrite(ap
, AHCI_PREG_CI
, ci_saved
);
3911 } else if (ci_saved
) {
3912 struct ata_port
*ccb_at
;
3916 for (i
= 0; i
< 32; ++i
) {
3917 if ((ci_saved
& (1 << i
)) == 0)
3919 ccb_at
= ap
->ap_ccbs
[i
].ccb_xa
.at
;
3921 fis_target
= ccb_at
->at_target
;
3924 ahci_pwrite(ap
, AHCI_PREG_FBS
,
3926 AHCI_PREG_FBS_DEV_SHIFT
) |
3928 if (ap
->ap_sactive
& (1 << i
))
3929 ahci_pwrite(ap
, AHCI_PREG_SACT
, (1 << i
));
3930 ahci_pwrite(ap
, AHCI_PREG_CI
, 1 << i
);
3936 * Used by the softreset, pmprobe, and read_ncq_error only, in very
3937 * specialized, controlled circumstances.
3939 * Only one command may be pending.
3942 ahci_quick_timeout(struct ahci_ccb
*ccb
)
3944 struct ahci_port
*ap
= ccb
->ccb_port
;
3947 switch (ccb
->ccb_xa
.state
) {
3949 TAILQ_REMOVE(&ap
->ap_ccb_pending
, ccb
, ccb_entry
);
3950 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
3954 * We have to clear the command on-chip.
3956 KKASSERT(ap
->ap_active
== (1 << ccb
->ccb_slot
) &&
3957 ap
->ap_sactive
== 0);
3958 ahci_port_stop(ap
, 0);
3959 ahci_port_start(ap
);
3960 if (ahci_pactive(ap
) != 0) {
3961 ahci_port_stop(ap
, 0);
3962 ahci_port_start(ap
);
3963 if ((mask
= ahci_pactive(ap
)) != 0) {
3964 kprintf("%s: quick-timeout: chipset failed "
3965 "to clear active cmds %08x\n",
3966 PORTNAME(ap
), mask
);
3970 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
3971 ap
->ap_active
&= ~(1 << ccb
->ccb_slot
);
3972 KKASSERT(ap
->ap_active_cnt
> 0);
3973 --ap
->ap_active_cnt
;
3976 panic("%s: ahci_quick_timeout: ccb in bad state %d",
3977 ATANAME(ap
, ccb
->ccb_xa
.at
), ccb
->ccb_xa
.state
);
3982 ahci_dummy_done(struct ata_xfer
*xa
)
3987 ahci_empty_done(struct ahci_ccb
*ccb
)
3992 ahci_set_feature(struct ahci_port
*ap
, struct ata_port
*atx
,
3993 int feature
, int enable
)
3995 struct ata_port
*at
;
3996 struct ata_xfer
*xa
;
3999 at
= atx
? atx
: ap
->ap_ata
[0];
4001 xa
= ahci_ata_get_xfer(ap
, atx
);
4003 xa
->fis
->type
= ATA_FIS_TYPE_H2D
;
4004 xa
->fis
->flags
= ATA_H2D_FLAGS_CMD
| at
->at_target
;
4005 xa
->fis
->command
= ATA_C_SET_FEATURES
;
4006 xa
->fis
->features
= enable
? ATA_SF_SATAFT_ENA
: ATA_SF_SATAFT_DIS
;
4007 xa
->fis
->sector_count
= feature
;
4008 xa
->fis
->control
= ATA_FIS_CONTROL_4BIT
;
4010 xa
->complete
= ahci_dummy_done
;
4012 xa
->flags
= ATA_F_POLL
;
4015 if (ahci_ata_cmd(xa
) == ATA_S_COMPLETE
)
4019 ahci_ata_put_xfer(xa
);