4 * Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19 * Copyright (c) 2009 The DragonFly Project. All rights reserved.
21 * This code is derived from software contributed to The DragonFly Project
22 * by Matthew Dillon <dillon@backplane.com>
24 * Redistribution and use in source and binary forms, with or without
25 * modification, are permitted provided that the following conditions
28 * 1. Redistributions of source code must retain the above copyright
29 * notice, this list of conditions and the following disclaimer.
30 * 2. Redistributions in binary form must reproduce the above copyright
31 * notice, this list of conditions and the following disclaimer in
32 * the documentation and/or other materials provided with the
34 * 3. Neither the name of The DragonFly Project nor the names of its
35 * contributors may be used to endorse or promote products derived
36 * from this software without specific, prior written permission.
38 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
39 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
40 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
41 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
42 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
43 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
44 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
45 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
46 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
47 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
48 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51 * $OpenBSD: ahci.c,v 1.147 2009/02/16 21:19:07 miod Exp $
56 void ahci_port_interrupt_enable(struct ahci_port
*ap
);
58 int ahci_load_prdt(struct ahci_ccb
*);
59 void ahci_unload_prdt(struct ahci_ccb
*);
60 static void ahci_load_prdt_callback(void *info
, bus_dma_segment_t
*segs
,
61 int nsegs
, int error
);
62 void ahci_start(struct ahci_ccb
*);
63 int ahci_port_softreset(struct ahci_port
*ap
);
64 int ahci_port_hardreset(struct ahci_port
*ap
, int hard
);
65 void ahci_port_hardstop(struct ahci_port
*ap
);
67 static void ahci_ata_cmd_timeout_unserialized(void *);
68 void ahci_check_active_timeouts(struct ahci_port
*ap
);
70 void ahci_beg_exclusive_access(struct ahci_port
*ap
, struct ata_port
*at
);
71 void ahci_end_exclusive_access(struct ahci_port
*ap
, struct ata_port
*at
);
72 void ahci_issue_pending_commands(struct ahci_port
*ap
, struct ahci_ccb
*ccb
);
73 void ahci_issue_saved_commands(struct ahci_port
*ap
, u_int32_t mask
);
75 int ahci_port_read_ncq_error(struct ahci_port
*, int);
77 struct ahci_dmamem
*ahci_dmamem_alloc(struct ahci_softc
*, bus_dma_tag_t tag
);
78 void ahci_dmamem_free(struct ahci_softc
*, struct ahci_dmamem
*);
79 static void ahci_dmamem_saveseg(void *info
, bus_dma_segment_t
*segs
, int nsegs
, int error
);
81 static void ahci_dummy_done(struct ata_xfer
*xa
);
82 static void ahci_empty_done(struct ahci_ccb
*ccb
);
83 static void ahci_ata_cmd_done(struct ahci_ccb
*ccb
);
84 static u_int32_t
ahci_pactive(struct ahci_port
*ap
);
87 * Initialize the global AHCI hardware. This code does not set up any of
91 ahci_init(struct ahci_softc
*sc
)
94 u_int32_t bios_cap
, vers
;
98 DPRINTF(AHCI_D_VERBOSE
, " GHC 0x%b",
99 ahci_read(sc
, AHCI_REG_GHC
), AHCI_FMT_GHC
);
104 vers
= ahci_read(sc
, AHCI_REG_VS
);
107 * save BIOS initialised parameters, enable staggered spin up
109 bios_cap
= ahci_read(sc
, AHCI_REG_CAP
);
110 bios_cap
&= AHCI_REG_CAP_SMPS
| AHCI_REG_CAP_SSS
;
112 pi
= ahci_read(sc
, AHCI_REG_PI
);
115 * Unconditionally reset the controller, do not conditionalize on
116 * trying to figure it if it was previously active or not.
118 * NOTE: On AE before HR. The AHCI-1.1 spec has a note in section
119 * 5.2.2.1 regarding this. HR should be set to 1 only after
120 * AE is set to 1. The reset sequence will clear HR when
121 * it completes, and will also clear AE if SAM is 0. AE must
122 * then be set again. When SAM is 1 the AE bit typically reads
123 * as 1 (and is read-only).
125 * NOTE: Avoid PCI[e] transaction burst by issuing dummy reads,
126 * otherwise the writes will only be separated by a few
131 * If you have a port multiplier and it does not have a device
132 * in target 0, and it probes normally, but a later operation
133 * mis-probes a target behind that PM, it is possible for the
134 * port to brick such that only (a) a power cycle of the host
135 * or (b) placing a device in target 0 will fix the problem.
136 * Power cycling the PM has no effect (it works fine on another
137 * host port). This issue is unrelated to CLO.
140 * Wait for any prior reset sequence to complete
142 if (ahci_wait_ne(sc
, AHCI_REG_GHC
,
143 AHCI_REG_GHC_HR
, AHCI_REG_GHC_HR
) != 0) {
144 device_printf(sc
->sc_dev
, "Controller is stuck in reset\n");
147 ahci_write(sc
, AHCI_REG_GHC
, AHCI_REG_GHC_AE
);
149 ahci_read(sc
, AHCI_REG_GHC
); /* flush */
150 ahci_write(sc
, AHCI_REG_GHC
, AHCI_REG_GHC_AE
| AHCI_REG_GHC_HR
);
152 ahci_read(sc
, AHCI_REG_GHC
); /* flush */
153 if (ahci_wait_ne(sc
, AHCI_REG_GHC
,
154 AHCI_REG_GHC_HR
, AHCI_REG_GHC_HR
) != 0) {
155 device_printf(sc
->sc_dev
, "unable to reset controller\n");
158 if (ahci_read(sc
, AHCI_REG_GHC
) & AHCI_REG_GHC_AE
) {
159 device_printf(sc
->sc_dev
, "AE did not auto-clear!\n");
160 ahci_write(sc
, AHCI_REG_GHC
, 0);
165 * Enable ahci (global interrupts disabled)
167 * Restore saved parameters. Avoid pci transaction burst write
168 * by issuing dummy reads.
171 ahci_write(sc
, AHCI_REG_GHC
, AHCI_REG_GHC_AE
);
174 ahci_read(sc
, AHCI_REG_GHC
); /* flush */
176 bios_cap
|= AHCI_REG_CAP_SSS
;
177 ahci_write(sc
, AHCI_REG_CAP
, ahci_read(sc
, AHCI_REG_CAP
) | bios_cap
);
178 ahci_write(sc
, AHCI_REG_PI
, pi
);
179 ahci_read(sc
, AHCI_REG_GHC
); /* flush */
182 * Intel hocus pocus in case the BIOS has not set the chip up
183 * properly for AHCI operation.
185 if (pci_get_vendor(sc
->sc_dev
) == PCI_VENDOR_INTEL
) {
186 if ((pci_read_config(sc
->sc_dev
, 0x92, 2) & 0x0F) != 0x0F)
187 device_printf(sc
->sc_dev
, "Intel hocus pocus\n");
188 pci_write_config(sc
->sc_dev
, 0x92,
189 pci_read_config(sc
->sc_dev
, 0x92, 2) | 0x0F, 2);
193 * This is a hack that currently does not appear to have
194 * a significant effect, but I noticed the port registers
195 * do not appear to be completely cleared after the host
196 * controller is reset.
198 * Use a temporary ap structure so we can call ahci_pwrite().
200 * We must be sure to stop the port
202 ap
= kmalloc(sizeof(*ap
), M_DEVBUF
, M_WAITOK
| M_ZERO
);
205 for (i
= 0; i
< AHCI_MAX_PORTS
; ++i
) {
208 if ((pi
& (1 << i
)) == 0)
210 if (bus_space_subregion(sc
->sc_iot
, sc
->sc_ioh
,
211 AHCI_PORT_REGION(i
), AHCI_PORT_SIZE
, &ap
->ap_ioh
) != 0) {
212 device_printf(sc
->sc_dev
, "can't map port\n");
216 * NOTE! Setting AHCI_PREG_SCTL_DET_DISABLE on AHCI1.0 or
217 * AHCI1.1 can brick the chipset. Not only brick it,
218 * but also crash the PC. The bit seems unreliable
219 * on AHCI1.2 as well.
221 ahci_port_stop(ap
, 1);
222 ahci_pwrite(ap
, AHCI_PREG_SCTL
, ap
->ap_sc
->sc_ipm_disable
);
223 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
224 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
225 ahci_write(ap
->ap_sc
, AHCI_REG_IS
, 1 << i
);
226 ahci_pwrite(ap
, AHCI_PREG_CMD
, 0);
227 ahci_pwrite(ap
, AHCI_PREG_IS
, -1);
228 sc
->sc_portmask
|= (1 << i
);
238 * Allocate and initialize an AHCI port.
241 ahci_port_alloc(struct ahci_softc
*sc
, u_int port
)
243 struct ahci_port
*ap
;
245 struct ahci_ccb
*ccb
;
249 struct ahci_cmd_hdr
*hdr
;
250 struct ahci_cmd_table
*table
;
255 ap
= kmalloc(sizeof(*ap
), M_DEVBUF
, M_WAITOK
| M_ZERO
);
256 ap
->ap_err_scratch
= kmalloc(512, M_DEVBUF
, M_WAITOK
| M_ZERO
);
258 ksnprintf(ap
->ap_name
, sizeof(ap
->ap_name
), "%s%d.%d",
259 device_get_name(sc
->sc_dev
),
260 device_get_unit(sc
->sc_dev
),
262 sc
->sc_ports
[port
] = ap
;
265 * Allocate enough so we never have to reallocate, it makes
268 * ap_pmcount will be reduced by the scan if we encounter the
269 * port multiplier port prior to target 15.
271 * kmalloc power-of-2 allocations are guaranteed not to cross
272 * a page boundary. Make sure the identify sub-structure in the
273 * at structure does not cross a page boundary, just in case the
274 * part is AHCI-1.1 and can't handle multiple DRQ blocks.
276 if (ap
->ap_ata
[0] == NULL
) {
279 for (pw2
= 1; pw2
< sizeof(*at
); pw2
<<= 1)
281 for (i
= 0; i
< AHCI_MAX_PMPORTS
; ++i
) {
282 at
= kmalloc(pw2
, M_DEVBUF
, M_INTWAIT
| M_ZERO
);
284 at
->at_ahci_port
= ap
;
286 at
->at_probe
= ATA_PROBE_NEED_INIT
;
287 at
->at_features
|= ATA_PORT_F_RESCAN
;
288 ksnprintf(at
->at_name
, sizeof(at
->at_name
),
289 "%s.%d", ap
->ap_name
, i
);
292 if (bus_space_subregion(sc
->sc_iot
, sc
->sc_ioh
,
293 AHCI_PORT_REGION(port
), AHCI_PORT_SIZE
, &ap
->ap_ioh
) != 0) {
294 device_printf(sc
->sc_dev
,
295 "unable to create register window for port %d\n",
302 ap
->ap_probe
= ATA_PROBE_NEED_INIT
;
303 ap
->link_pwr_mgmt
= AHCI_LINK_PWR_MGMT_NONE
;
304 ap
->sysctl_tree
= NULL
;
305 TAILQ_INIT(&ap
->ap_ccb_free
);
306 TAILQ_INIT(&ap
->ap_ccb_pending
);
307 lockinit(&ap
->ap_ccb_lock
, "ahcipo", 0, 0);
309 /* Disable port interrupts */
310 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
311 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
314 * Sec 10.1.2 - deinitialise port if it is already running
316 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
317 kprintf("%s: Cmdreg %b\n", PORTNAME(ap
), cmd
, AHCI_PFMT_CMD
);
319 if ((cmd
& (AHCI_PREG_CMD_ST
| AHCI_PREG_CMD_CR
|
320 AHCI_PREG_CMD_FRE
| AHCI_PREG_CMD_FR
)) ||
321 (ahci_pread(ap
, AHCI_PREG_SCTL
) & AHCI_PREG_SCTL_DET
)) {
324 r
= ahci_port_stop(ap
, 1);
326 device_printf(sc
->sc_dev
,
327 "unable to disable %s, ignoring port %d\n",
328 ((r
== 2) ? "CR" : "FR"), port
);
333 /* Write DET to zero */
334 ahci_pwrite(ap
, AHCI_PREG_SCTL
, ap
->ap_sc
->sc_ipm_disable
);
338 ap
->ap_dmamem_rfis
= ahci_dmamem_alloc(sc
, sc
->sc_tag_rfis
);
339 if (ap
->ap_dmamem_rfis
== NULL
) {
340 kprintf("%s: NORFIS\n", PORTNAME(ap
));
344 /* Setup RFIS base address */
345 ap
->ap_rfis
= (struct ahci_rfis
*) AHCI_DMA_KVA(ap
->ap_dmamem_rfis
);
346 bzero(ap
->ap_rfis
, sc
->sc_rfis_size
);
348 dva
= AHCI_DMA_DVA(ap
->ap_dmamem_rfis
);
349 ahci_pwrite(ap
, AHCI_PREG_FB
, (u_int32_t
)dva
);
350 ahci_pwrite(ap
, AHCI_PREG_FBU
, (u_int32_t
)(dva
>> 32));
352 /* Clear SERR before starting FIS reception or ST or anything */
354 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
357 * Power up any device sitting on the port. Leave FIS reception
358 * turned off. Don't make the ICC ACTIVE here, it will be handled
361 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
) & ~AHCI_PREG_CMD_ICC
;
362 cmd
&= ~(AHCI_PREG_CMD_CLO
| AHCI_PREG_CMD_PMA
);
363 cmd
|= AHCI_PREG_CMD_POD
| AHCI_PREG_CMD_SUD
;
364 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
366 /* Allocate a CCB for each command slot */
367 ap
->ap_ccbs
= kmalloc(sizeof(struct ahci_ccb
) * sc
->sc_ncmds
, M_DEVBUF
,
369 if (ap
->ap_ccbs
== NULL
) {
370 device_printf(sc
->sc_dev
,
371 "unable to allocate command list for port %d\n",
376 /* Command List Structures and Command Tables */
377 ap
->ap_dmamem_cmd_list
= ahci_dmamem_alloc(sc
, sc
->sc_tag_cmdh
);
378 ap
->ap_dmamem_cmd_table
= ahci_dmamem_alloc(sc
, sc
->sc_tag_cmdt
);
379 if (ap
->ap_dmamem_cmd_table
== NULL
||
380 ap
->ap_dmamem_cmd_list
== NULL
) {
382 device_printf(sc
->sc_dev
,
383 "unable to allocate DMA memory for port %d\n",
388 /* Setup command list base address */
389 dva
= AHCI_DMA_DVA(ap
->ap_dmamem_cmd_list
);
390 ahci_pwrite(ap
, AHCI_PREG_CLB
, (u_int32_t
)dva
);
391 ahci_pwrite(ap
, AHCI_PREG_CLBU
, (u_int32_t
)(dva
>> 32));
393 /* Split CCB allocation into CCBs and assign to command header/table */
394 hdr
= AHCI_DMA_KVA(ap
->ap_dmamem_cmd_list
);
395 table
= AHCI_DMA_KVA(ap
->ap_dmamem_cmd_table
);
396 bzero(hdr
, sc
->sc_cmdlist_size
);
398 for (i
= 0; i
< sc
->sc_ncmds
; i
++) {
399 ccb
= &ap
->ap_ccbs
[i
];
401 error
= bus_dmamap_create(sc
->sc_tag_data
, BUS_DMA_ALLOCNOW
,
404 device_printf(sc
->sc_dev
,
405 "unable to create dmamap for port %d "
406 "ccb %d\n", port
, i
);
410 callout_init_mp(&ccb
->ccb_timeout
);
413 ccb
->ccb_cmd_hdr
= &hdr
[i
];
414 ccb
->ccb_cmd_table
= &table
[i
];
415 dva
= AHCI_DMA_DVA(ap
->ap_dmamem_cmd_table
) +
416 ccb
->ccb_slot
* sizeof(struct ahci_cmd_table
);
417 ccb
->ccb_cmd_hdr
->ctba_hi
= htole32((u_int32_t
)(dva
>> 32));
418 ccb
->ccb_cmd_hdr
->ctba_lo
= htole32((u_int32_t
)dva
);
421 (struct ata_fis_h2d
*)ccb
->ccb_cmd_table
->cfis
;
422 ccb
->ccb_xa
.packetcmd
= ccb
->ccb_cmd_table
->acmd
;
425 ccb
->ccb_xa
.state
= ATA_S_COMPLETE
;
428 * CCB[1] is the error CCB and is not get or put. It is
429 * also used for probing. Numerous HBAs only load the
430 * signature from CCB[1] so it MUST be used for the second
434 ap
->ap_err_ccb
= ccb
;
440 * Wait for ICC change to complete
442 ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_ICC
);
445 * Calculate the interrupt mask
447 data
= AHCI_PREG_IE_TFEE
| AHCI_PREG_IE_HBFE
|
448 AHCI_PREG_IE_IFE
| AHCI_PREG_IE_OFE
|
449 AHCI_PREG_IE_DPE
| AHCI_PREG_IE_UFE
|
450 AHCI_PREG_IE_PCE
| AHCI_PREG_IE_PRCE
|
451 AHCI_PREG_IE_DHRE
| AHCI_PREG_IE_SDBE
;
452 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SSNTF
)
453 data
|= AHCI_PREG_IE_IPME
;
455 if (sc
->sc_ccc_ports
& (1 << port
)
456 data
&= ~(AHCI_PREG_IE_SDBE
| AHCI_PREG_IE_DHRE
);
458 ap
->ap_intmask
= data
;
461 * Start the port helper thread. The helper thread will call
462 * ahci_port_init() so the ports can all be started in parallel.
463 * A failure by ahci_port_init() does not deallocate the port
464 * since we still want hot-plug events.
466 ahci_os_start_port(ap
);
469 ahci_port_free(sc
, port
);
474 * [re]initialize an idle port. No CCBs should be active. (from port thread)
476 * This function is called during the initial port allocation sequence
477 * and is also called on hot-plug insertion. We take no chances and
478 * use a portreset instead of a softreset.
480 * This function is the only way to move a failed port back to active
483 * Returns 0 if a device is successfully detected.
486 ahci_port_init(struct ahci_port
*ap
)
491 * Register [re]initialization
493 * Flush the TFD and SERR and make sure the port is stopped before
494 * enabling its interrupt. We no longer cycle the port start as
495 * the port should not be started unless a device is present.
497 * XXX should we enable FIS reception? (FRE)?
499 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
500 ahci_port_stop(ap
, 0);
501 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SSNTF
)
502 ahci_pwrite(ap
, AHCI_PREG_SNTF
, -1);
504 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
507 * If we are being harsh try to kill the port completely. Normally
508 * we would want to hold on to some of the state the BIOS may have
509 * set, such as SUD (spin up device).
511 * AP_F_HARSH_REINIT is cleared in the hard reset state
513 if (ap
->ap_flags
& AP_F_HARSH_REINIT
) {
514 ahci_pwrite(ap
, AHCI_PREG_SCTL
, ap
->ap_sc
->sc_ipm_disable
);
515 ahci_pwrite(ap
, AHCI_PREG_CMD
, 0);
519 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
) & ~AHCI_PREG_CMD_ICC
;
520 cmd
&= ~(AHCI_PREG_CMD_CLO
| AHCI_PREG_CMD_PMA
);
521 cmd
|= AHCI_PREG_CMD_POD
| AHCI_PREG_CMD_SUD
;
522 cmd
|= AHCI_PREG_CMD_ICC_ACTIVE
;
523 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
524 ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_ICC
);
529 * Clear any pending garbage and re-enable the interrupt before
530 * going to the next stage.
532 ap
->ap_probe
= ATA_PROBE_NEED_HARD_RESET
;
535 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SSNTF
)
536 ahci_pwrite(ap
, AHCI_PREG_SNTF
, -1);
538 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
539 ahci_pwrite(ap
, AHCI_PREG_IS
, -1);
541 ahci_port_interrupt_enable(ap
);
547 * Enable or re-enable interrupts on a port.
549 * This routine is called from the port initialization code or from the
550 * helper thread as the real interrupt may be forced to turn off certain
554 ahci_port_interrupt_enable(struct ahci_port
*ap
)
556 ahci_pwrite(ap
, AHCI_PREG_IE
, ap
->ap_intmask
);
560 * Manage the agressive link power management capability.
563 ahci_port_link_pwr_mgmt(struct ahci_port
*ap
, int link_pwr_mgmt
)
567 if (link_pwr_mgmt
== ap
->link_pwr_mgmt
)
570 if ((ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SALP
) == 0) {
571 kprintf("%s: link power management not supported.\n",
576 ahci_os_lock_port(ap
);
578 if (link_pwr_mgmt
== AHCI_LINK_PWR_MGMT_AGGR
&&
579 (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SSC
)) {
580 kprintf("%s: enabling aggressive link power management.\n",
583 ap
->link_pwr_mgmt
= link_pwr_mgmt
;
585 ap
->ap_intmask
&= ~AHCI_PREG_IE_PRCE
;
586 ahci_port_interrupt_enable(ap
);
588 sctl
= ahci_pread(ap
, AHCI_PREG_SCTL
);
589 sctl
&= ~(AHCI_PREG_SCTL_IPM
);
590 if (ap
->ap_sc
->sc_cap2
& AHCI_REG_CAP2_SDS
)
591 sctl
|= AHCI_PREG_SCTL_IPM_NODEVSLP
;
592 ahci_pwrite(ap
, AHCI_PREG_SCTL
, sctl
);
595 * Enable device initiated link power management for
596 * directly attached devices that support it.
598 if (ap
->ap_type
!= ATA_PORT_T_PM
&&
599 (ap
->ap_ata
[0]->at_identify
.satafsup
&
600 SATA_FEATURE_SUP_DEVIPS
)) {
601 if (ahci_set_feature(ap
, NULL
, ATA_SATAFT_DEVIPS
, 1))
602 kprintf("%s: Could not enable device initiated "
603 "link power management.\n",
607 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
608 cmd
|= AHCI_PREG_CMD_ASP
;
609 cmd
|= AHCI_PREG_CMD_ALPE
;
610 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
611 } else if (link_pwr_mgmt
== AHCI_LINK_PWR_MGMT_MEDIUM
&&
612 (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_PSC
)) {
613 kprintf("%s: enabling medium link power management.\n",
616 ap
->link_pwr_mgmt
= link_pwr_mgmt
;
618 ap
->ap_intmask
&= ~AHCI_PREG_IE_PRCE
;
619 ahci_port_interrupt_enable(ap
);
621 sctl
= ahci_pread(ap
, AHCI_PREG_SCTL
);
622 sctl
&= ~(AHCI_PREG_SCTL_IPM
);
623 sctl
|= AHCI_PREG_SCTL_IPM_NOSLUMBER
;
624 if (ap
->ap_sc
->sc_cap2
& AHCI_REG_CAP2_SDS
)
625 sctl
|= AHCI_PREG_SCTL_IPM_NODEVSLP
;
626 ahci_pwrite(ap
, AHCI_PREG_SCTL
, sctl
);
628 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
629 cmd
&= ~AHCI_PREG_CMD_ASP
;
630 cmd
|= AHCI_PREG_CMD_ALPE
;
631 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
633 } else if (link_pwr_mgmt
== AHCI_LINK_PWR_MGMT_NONE
) {
634 kprintf("%s: disabling link power management.\n",
637 /* Disable device initiated link power management */
638 if (ap
->ap_type
!= ATA_PORT_T_PM
&&
639 (ap
->ap_ata
[0]->at_identify
.satafsup
&
640 SATA_FEATURE_SUP_DEVIPS
)) {
641 ahci_set_feature(ap
, NULL
, ATA_SATAFT_DEVIPS
, 0);
644 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
645 cmd
&= ~(AHCI_PREG_CMD_ALPE
| AHCI_PREG_CMD_ASP
);
646 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
648 sctl
= ahci_pread(ap
, AHCI_PREG_SCTL
);
649 sctl
&= ~(AHCI_PREG_SCTL_IPM
);
650 sctl
|= ap
->ap_sc
->sc_ipm_disable
;
651 ahci_pwrite(ap
, AHCI_PREG_SCTL
, sctl
);
653 /* let the drive come back to avoid PRCS interrupts later */
654 ahci_os_unlock_port(ap
);
656 ahci_os_lock_port(ap
);
658 ahci_pwrite(ap
, AHCI_PREG_SERR
,
659 AHCI_PREG_SERR_DIAG_N
| AHCI_PREG_SERR_DIAG_W
);
660 ahci_pwrite(ap
, AHCI_PREG_IS
, AHCI_PREG_IS_PRCS
);
662 ap
->ap_intmask
|= AHCI_PREG_IE_PRCE
;
663 ahci_port_interrupt_enable(ap
);
665 ap
->link_pwr_mgmt
= link_pwr_mgmt
;
667 kprintf("%s: unsupported link power management state %d.\n",
668 PORTNAME(ap
), link_pwr_mgmt
);
671 ahci_os_unlock_port(ap
);
675 * Return current link power state.
678 ahci_port_link_pwr_state(struct ahci_port
*ap
)
682 r
= ahci_pread(ap
, AHCI_PREG_SSTS
);
683 switch (r
& AHCI_PREG_SSTS_IPM
) {
684 case AHCI_PREG_SSTS_IPM_ACTIVE
:
686 case AHCI_PREG_SSTS_IPM_PARTIAL
:
688 case AHCI_PREG_SSTS_IPM_SLUMBER
:
690 case AHCI_PREG_SSTS_IPM_DEVSLEEP
:
698 * Run the port / target state machine from a main context.
700 * The state machine for the port is always run.
702 * If atx is non-NULL run the state machine for a particular target.
703 * If atx is NULL run the state machine for all targets.
706 ahci_port_state_machine(struct ahci_port
*ap
, int initial
)
715 * State machine for port. Note that CAM is not yet associated
716 * during the initial parallel probe and the port's probe state
717 * will not get past ATA_PROBE_NEED_IDENT.
720 if (initial
== 0 && ap
->ap_probe
<= ATA_PROBE_NEED_HARD_RESET
) {
721 kprintf("%s: Waiting 10 seconds on insertion\n",
723 ahci_os_sleep(10000);
726 if (ap
->ap_probe
== ATA_PROBE_NEED_INIT
)
728 if (ap
->ap_probe
== ATA_PROBE_NEED_HARD_RESET
)
729 ahci_port_reset(ap
, NULL
, 1);
730 if (ap
->ap_probe
== ATA_PROBE_NEED_SOFT_RESET
)
731 ahci_port_reset(ap
, NULL
, 0);
732 if (ap
->ap_probe
== ATA_PROBE_NEED_IDENT
)
733 ahci_cam_probe(ap
, NULL
);
735 if (ap
->ap_type
!= ATA_PORT_T_PM
) {
736 if (ap
->ap_probe
== ATA_PROBE_FAILED
) {
737 ahci_cam_changed(ap
, NULL
, 0);
738 } else if (ap
->ap_probe
>= ATA_PROBE_NEED_IDENT
) {
739 ahci_cam_changed(ap
, NULL
, 1);
745 * Port Multiplier state machine.
747 * Get a mask of changed targets and combine with any runnable
748 * states already present.
750 for (loop
= 0; ;++loop
) {
751 if (ahci_pm_read(ap
, 15, SATA_PMREG_EINFO
, &data
)) {
752 kprintf("%s: PM unable to read hot-plug bitmap\n",
758 * Do at least one loop, then stop if no more state changes
759 * have occured. The PM might not generate a new
760 * notification until we clear the entire bitmap.
762 if (loop
&& data
== 0)
766 * New devices showing up in the bitmap require some spin-up
767 * time before we start probing them. Reset didsleep. The
768 * first new device we detect will sleep before probing.
770 * This only applies to devices whos change bit is set in
771 * the data, and does not apply to the initial boot-time
776 for (target
= 0; target
< ap
->ap_pmcount
; ++target
) {
777 at
= ap
->ap_ata
[target
];
780 * Check the target state for targets behind the PM
781 * which have changed state. This will adjust
782 * at_probe and set ATA_PORT_F_RESCAN
784 * We want to wait at least 10 seconds before probing
785 * a newly inserted device. If the check status
786 * indicates a device is present and in need of a
787 * hard reset, we make sure we have slept before
790 * We also need to wait at least 1 second for the
791 * PHY state to change after insertion, if we
792 * haven't already waited the 10 seconds.
794 * NOTE: When pm_check_good finds a good port it
795 * typically starts us in probe state
796 * NEED_HARD_RESET rather than INIT.
798 if (data
& (1 << target
)) {
799 if (initial
== 0 && didsleep
== 0)
801 ahci_pm_check_good(ap
, target
);
802 if (initial
== 0 && didsleep
== 0 &&
803 at
->at_probe
<= ATA_PROBE_NEED_HARD_RESET
806 kprintf("%s: Waiting 10 seconds on insertion\n", PORTNAME(ap
));
807 ahci_os_sleep(10000);
812 * Report hot-plug events before the probe state
813 * really gets hot. Only actual events are reported
814 * here to reduce spew.
816 if (data
& (1 << target
)) {
817 kprintf("%s: HOTPLUG (PM) - ", ATANAME(ap
, at
));
818 switch(at
->at_probe
) {
819 case ATA_PROBE_NEED_INIT
:
820 case ATA_PROBE_NEED_HARD_RESET
:
821 kprintf("Device inserted\n");
823 case ATA_PROBE_FAILED
:
824 kprintf("Device removed\n");
827 kprintf("Device probe in progress\n");
833 * Run through the state machine as necessary if
834 * the port is not marked failed.
836 * The state machine may stop at NEED_IDENT if
837 * CAM is not yet attached.
839 * Acquire exclusive access to the port while we
840 * are doing this. This prevents command-completion
841 * from queueing commands for non-polled targets
842 * inbetween our probe steps. We need to do this
843 * because the reset probes can generate severe PHY
844 * and protocol errors and soft-brick the port.
846 if (at
->at_probe
!= ATA_PROBE_FAILED
&&
847 at
->at_probe
!= ATA_PROBE_GOOD
) {
848 ahci_beg_exclusive_access(ap
, at
);
849 if (at
->at_probe
== ATA_PROBE_NEED_INIT
)
850 ahci_pm_port_init(ap
, at
);
851 if (at
->at_probe
== ATA_PROBE_NEED_HARD_RESET
)
852 ahci_port_reset(ap
, at
, 1);
853 if (at
->at_probe
== ATA_PROBE_NEED_SOFT_RESET
)
854 ahci_port_reset(ap
, at
, 0);
855 if (at
->at_probe
== ATA_PROBE_NEED_IDENT
)
856 ahci_cam_probe(ap
, at
);
857 ahci_end_exclusive_access(ap
, at
);
861 * Add or remove from CAM
863 if (at
->at_features
& ATA_PORT_F_RESCAN
) {
864 at
->at_features
&= ~ATA_PORT_F_RESCAN
;
865 if (at
->at_probe
== ATA_PROBE_FAILED
) {
866 ahci_cam_changed(ap
, at
, 0);
867 } else if (at
->at_probe
>= ATA_PROBE_NEED_IDENT
) {
868 ahci_cam_changed(ap
, at
, 1);
871 data
&= ~(1 << target
);
874 kprintf("%s: WARNING (PM): extra bits set in "
875 "EINFO: %08x\n", PORTNAME(ap
), data
);
876 while (target
< AHCI_MAX_PMPORTS
) {
877 ahci_pm_check_good(ap
, target
);
886 * De-initialize and detach a port.
889 ahci_port_free(struct ahci_softc
*sc
, u_int port
)
891 struct ahci_port
*ap
= sc
->sc_ports
[port
];
892 struct ahci_ccb
*ccb
;
896 * Ensure port is disabled and its interrupts are all flushed.
899 ahci_port_stop(ap
, 1);
900 ahci_os_stop_port(ap
);
901 ahci_pwrite(ap
, AHCI_PREG_CMD
, 0);
902 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
903 ahci_pwrite(ap
, AHCI_PREG_IS
, ahci_pread(ap
, AHCI_PREG_IS
));
904 ahci_write(sc
, AHCI_REG_IS
, 1 << port
);
908 while ((ccb
= ahci_get_ccb(ap
)) != NULL
) {
909 if (ccb
->ccb_dmamap
) {
910 bus_dmamap_destroy(sc
->sc_tag_data
,
912 ccb
->ccb_dmamap
= NULL
;
915 if ((ccb
= ap
->ap_err_ccb
) != NULL
) {
916 if (ccb
->ccb_dmamap
) {
917 bus_dmamap_destroy(sc
->sc_tag_data
,
919 ccb
->ccb_dmamap
= NULL
;
921 ap
->ap_err_ccb
= NULL
;
923 kfree(ap
->ap_ccbs
, M_DEVBUF
);
927 if (ap
->ap_dmamem_cmd_list
) {
928 ahci_dmamem_free(sc
, ap
->ap_dmamem_cmd_list
);
929 ap
->ap_dmamem_cmd_list
= NULL
;
931 if (ap
->ap_dmamem_rfis
) {
932 ahci_dmamem_free(sc
, ap
->ap_dmamem_rfis
);
933 ap
->ap_dmamem_rfis
= NULL
;
935 if (ap
->ap_dmamem_cmd_table
) {
936 ahci_dmamem_free(sc
, ap
->ap_dmamem_cmd_table
);
937 ap
->ap_dmamem_cmd_table
= NULL
;
940 for (i
= 0; i
< AHCI_MAX_PMPORTS
; ++i
) {
942 kfree(ap
->ap_ata
[i
], M_DEVBUF
);
943 ap
->ap_ata
[i
] = NULL
;
947 if (ap
->ap_err_scratch
) {
948 kfree(ap
->ap_err_scratch
, M_DEVBUF
);
949 ap
->ap_err_scratch
= NULL
;
952 /* bus_space(9) says we dont free the subregions handle */
955 sc
->sc_ports
[port
] = NULL
;
960 ahci_pactive(struct ahci_port
*ap
)
964 mask
= ahci_pread(ap
, AHCI_PREG_CI
);
965 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SNCQ
)
966 mask
|= ahci_pread(ap
, AHCI_PREG_SACT
);
971 * Start high-level command processing on the port
974 ahci_port_start(struct ahci_port
*ap
)
976 u_int32_t r
, s
, is
, tfd
;
979 * FRE must be turned on before ST. Wait for FR to go active
980 * before turning on ST. The spec doesn't seem to think this
981 * is necessary but waiting here avoids an on-off race in the
982 * ahci_port_stop() code.
984 r
= ahci_pread(ap
, AHCI_PREG_CMD
);
985 if ((r
& AHCI_PREG_CMD_FRE
) == 0) {
986 r
|= AHCI_PREG_CMD_FRE
;
987 ahci_pwrite(ap
, AHCI_PREG_CMD
, r
);
989 if ((ap
->ap_sc
->sc_flags
& AHCI_F_IGN_FR
) == 0) {
990 if (ahci_pwait_set(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_FR
)) {
991 kprintf("%s: Cannot start FIS reception\n",
1000 * Turn on ST, wait for CR to come up.
1002 r
|= AHCI_PREG_CMD_ST
;
1003 ahci_pwrite(ap
, AHCI_PREG_CMD
, r
);
1005 if ((ap
->ap_sc
->sc_flags
& AHCI_F_IGN_CR
) == 0 &&
1006 ahci_pwait_set_to(ap
, 2000, AHCI_PREG_CMD
, AHCI_PREG_CMD_CR
)) {
1007 s
= ahci_pread(ap
, AHCI_PREG_SERR
);
1008 is
= ahci_pread(ap
, AHCI_PREG_IS
);
1009 tfd
= ahci_pread(ap
, AHCI_PREG_TFD
);
1010 kprintf("%s: Cannot start command DMA\n"
1011 "NCMP=%b NSERR=%b\n"
1015 r
, AHCI_PFMT_CMD
, s
, AHCI_PFMT_SERR
,
1017 tfd
, AHCI_PFMT_TFD_STS
);
1021 #ifdef AHCI_COALESCE
1023 * (Re-)enable coalescing on the port.
1025 if (ap
->ap_sc
->sc_ccc_ports
& (1 << ap
->ap_num
)) {
1026 ap
->ap_sc
->sc_ccc_ports_cur
|= (1 << ap
->ap_num
);
1027 ahci_write(ap
->ap_sc
, AHCI_REG_CCC_PORTS
,
1028 ap
->ap_sc
->sc_ccc_ports_cur
);
1036 * Stop high-level command processing on a port
1038 * WARNING! If the port is stopped while CR is still active our saved
1039 * CI/SACT will race any commands completed by the command
1040 * processor prior to being able to stop. Thus we never call
1041 * this function unless we intend to dispose of any remaining
1042 * active commands. In particular, this complicates the timeout
1046 ahci_port_stop(struct ahci_port
*ap
, int stop_fis_rx
)
1050 #ifdef AHCI_COALESCE
1052 * Disable coalescing on the port while it is stopped.
1054 if (ap
->ap_sc
->sc_ccc_ports
& (1 << ap
->ap_num
)) {
1055 ap
->ap_sc
->sc_ccc_ports_cur
&= ~(1 << ap
->ap_num
);
1056 ahci_write(ap
->ap_sc
, AHCI_REG_CCC_PORTS
,
1057 ap
->ap_sc
->sc_ccc_ports_cur
);
1062 * Turn off ST, then wait for CR to go off.
1064 r
= ahci_pread(ap
, AHCI_PREG_CMD
) & ~AHCI_PREG_CMD_ICC
;
1065 r
&= ~AHCI_PREG_CMD_ST
;
1066 ahci_pwrite(ap
, AHCI_PREG_CMD
, r
);
1068 if (ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_CR
)) {
1069 kprintf("%s: Port bricked, unable to stop (ST)\n",
1076 * Turn off FRE, then wait for FR to go off. FRE cannot
1077 * be turned off until CR transitions to 0.
1079 if ((r
& AHCI_PREG_CMD_FR
) == 0) {
1080 kprintf("%s: FR stopped, clear FRE for next start\n",
1086 r
&= ~AHCI_PREG_CMD_FRE
;
1087 ahci_pwrite(ap
, AHCI_PREG_CMD
, r
);
1088 if (ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_FR
)) {
1089 kprintf("%s: Port bricked, unable to stop (FRE)\n",
1098 * AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ}
1101 ahci_port_clo(struct ahci_port
*ap
)
1103 struct ahci_softc
*sc
= ap
->ap_sc
;
1106 /* Only attempt CLO if supported by controller */
1107 if ((ahci_read(sc
, AHCI_REG_CAP
) & AHCI_REG_CAP_SCLO
) == 0)
1111 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
) & ~AHCI_PREG_CMD_ICC
;
1112 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
| AHCI_PREG_CMD_CLO
);
1114 /* Wait for completion */
1115 if (ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_CLO
)) {
1116 kprintf("%s: CLO did not complete\n", PORTNAME(ap
));
1126 * If hard is 0 perform a softreset of the port.
1127 * If hard is 1 perform a hard reset of the port.
1129 * If at is non-NULL an indirect port via a port-multiplier is being
1130 * reset, otherwise a direct port is being reset.
1132 * NOTE: Indirect ports can only be soft-reset.
1135 ahci_port_reset(struct ahci_port
*ap
, struct ata_port
*at
, int hard
)
1141 rc
= ahci_pm_hardreset(ap
, at
->at_target
, hard
);
1143 rc
= ahci_port_hardreset(ap
, hard
);
1146 rc
= ahci_pm_softreset(ap
, at
->at_target
);
1148 rc
= ahci_port_softreset(ap
);
1154 * AHCI soft reset, Section 10.4.1
1156 * (at) will be NULL when soft-resetting a directly-attached device, and
1157 * non-NULL when soft-resetting a device through a port multiplier.
1159 * This function keeps port communications intact and attempts to generate
1160 * a reset to the connected device using device commands.
1163 ahci_port_softreset(struct ahci_port
*ap
)
1165 struct ahci_ccb
*ccb
= NULL
;
1166 struct ahci_cmd_hdr
*cmd_slot
;
1173 kprintf("%s: START SOFTRESET %b\n", PORTNAME(ap
),
1174 ahci_pread(ap
, AHCI_PREG_CMD
), AHCI_PFMT_CMD
);
1177 DPRINTF(AHCI_D_VERBOSE
, "%s: soft reset\n", PORTNAME(ap
));
1180 ap
->ap_flags
|= AP_F_IN_RESET
;
1181 ap
->ap_state
= AP_S_NORMAL
;
1184 * Remember port state in cmd (main to restore start/stop)
1188 if (ahci_port_stop(ap
, 0)) {
1189 kprintf("%s: failed to stop port, cannot softreset\n",
1195 * Request CLO if device appears hung.
1197 if (ahci_pread(ap
, AHCI_PREG_TFD
) &
1198 (AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
1203 * This is an attempt to clear errors so a new signature will
1204 * be latched. It isn't working properly. XXX
1207 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
1210 if (ahci_port_start(ap
)) {
1211 kprintf("%s: failed to start port, cannot softreset\n",
1216 /* Check whether CLO worked */
1217 if (ahci_pwait_clr(ap
, AHCI_PREG_TFD
,
1218 AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
1219 kprintf("%s: CLO %s, need port reset\n",
1221 (ahci_read(ap
->ap_sc
, AHCI_REG_CAP
) & AHCI_REG_CAP_SCLO
)
1222 ? "failed" : "unsupported");
1228 * Prep first D2H command with SRST feature & clear busy/reset flags
1230 * It is unclear which other fields in the FIS are used. Just zero
1233 * NOTE! This CCB is used for both the first and second commands.
1234 * The second command must use CCB slot 1 to properly load
1237 ccb
= ahci_get_err_ccb(ap
);
1238 ccb
->ccb_xa
.complete
= ahci_dummy_done
;
1239 ccb
->ccb_xa
.flags
= ATA_F_POLL
| ATA_F_EXCLUSIVE
;
1240 KKASSERT(ccb
->ccb_slot
== 1);
1241 ccb
->ccb_xa
.at
= NULL
;
1242 cmd_slot
= ccb
->ccb_cmd_hdr
;
1244 fis
= ccb
->ccb_cmd_table
->cfis
;
1245 bzero(fis
, sizeof(ccb
->ccb_cmd_table
->cfis
));
1246 fis
[0] = ATA_FIS_TYPE_H2D
;
1247 fis
[15] = ATA_FIS_CONTROL_SRST
|ATA_FIS_CONTROL_4BIT
;
1249 cmd_slot
->prdtl
= 0;
1250 cmd_slot
->flags
= htole16(5); /* FIS length: 5 DWORDS */
1251 cmd_slot
->flags
|= htole16(AHCI_CMD_LIST_FLAG_C
); /* Clear busy on OK */
1252 cmd_slot
->flags
|= htole16(AHCI_CMD_LIST_FLAG_R
); /* Reset */
1254 ccb
->ccb_xa
.state
= ATA_S_PENDING
;
1256 if (ahci_poll(ccb
, 1000, ahci_quick_timeout
) != ATA_S_COMPLETE
) {
1257 kprintf("%s: First FIS failed\n", PORTNAME(ap
));
1262 * WARNING! TIME SENSITIVE SPACE! WARNING!
1264 * The two FISes are supposed to be back to back. Don't issue other
1265 * commands or even delay if we can help it.
1269 * Prep second D2H command to read status and complete reset sequence
1270 * AHCI 10.4.1 and "Serial ATA Revision 2.6". I can't find the ATA
1271 * Rev 2.6 and it is unclear how the second FIS should be set up
1272 * from the AHCI document.
1274 * It is unclear which other fields in the FIS are used. Just zero
1277 ccb
->ccb_xa
.flags
= ATA_F_POLL
| ATA_F_AUTOSENSE
| ATA_F_EXCLUSIVE
;
1279 bzero(fis
, sizeof(ccb
->ccb_cmd_table
->cfis
));
1280 fis
[0] = ATA_FIS_TYPE_H2D
;
1281 fis
[15] = ATA_FIS_CONTROL_4BIT
;
1283 cmd_slot
->prdtl
= 0;
1284 cmd_slot
->flags
= htole16(5); /* FIS length: 5 DWORDS */
1286 ccb
->ccb_xa
.state
= ATA_S_PENDING
;
1287 if (ahci_poll(ccb
, 1000, ahci_quick_timeout
) != ATA_S_COMPLETE
) {
1288 kprintf("%s: Second FIS failed\n", PORTNAME(ap
));
1292 if (ahci_pwait_clr(ap
, AHCI_PREG_TFD
,
1293 AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
1294 kprintf("%s: device didn't come ready after reset, TFD: 0x%b\n",
1296 ahci_pread(ap
, AHCI_PREG_TFD
), AHCI_PFMT_TFD_STS
);
1302 * If the softreset is trying to clear a BSY condition after a
1303 * normal portreset we assign the port type.
1305 * If the softreset is being run first as part of the ccb error
1306 * processing code then report if the device signature changed
1310 if (ap
->ap_type
== ATA_PORT_T_NONE
) {
1311 ap
->ap_type
= ahci_port_signature_detect(ap
, NULL
);
1313 if (ahci_port_signature_detect(ap
, NULL
) != ap
->ap_type
) {
1314 kprintf("%s: device signature unexpectedly "
1315 "changed\n", PORTNAME(ap
));
1316 error
= EBUSY
; /* XXX */
1324 ahci_put_err_ccb(ccb
);
1327 * If the target is busy use CLO to clear the busy
1328 * condition. The BSY should be cleared on the next
1331 if (ahci_pread(ap
, AHCI_PREG_TFD
) &
1332 (AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
1338 * If we failed to softreset make the port quiescent, otherwise
1339 * make sure the port's start/stop state matches what it was on
1342 * Don't kill the port if the softreset is on a port multiplier
1343 * target, that would kill all the targets!
1346 ahci_port_hardstop(ap
);
1347 /* ap_probe set to failed */
1349 ap
->ap_probe
= ATA_PROBE_NEED_IDENT
;
1351 ahci_port_start(ap
);
1353 ap
->ap_flags
&= ~AP_F_IN_RESET
;
1357 kprintf("%s: END SOFTRESET\n", PORTNAME(ap
));
1363 * Issue just do the core COMRESET and basic device detection on a port.
1365 * NOTE: Only called by ahci_port_hardreset().
1368 ahci_comreset(struct ahci_port
*ap
, int *pmdetectp
)
1377 * Idle the port. We must cycle FRE for certain chips that silently
1378 * clear FR on disconnect. Normally we do not want to cycle FRE
1379 * because other chipsets might react badly to that.
1382 if (ap
->ap_sc
->sc_flags
& AHCI_F_CYCLE_FR
)
1383 ahci_port_stop(ap
, 1);
1385 ahci_port_stop(ap
, 0);
1386 ap
->ap_state
= AP_S_NORMAL
;
1390 * The port may have been quiescent with its SUD bit cleared, so
1391 * set the SUD (spin up device). Also POD (Power up device),
1392 * and issue an ICC_ACTIVE request to bring up communications.
1394 * NOTE: I do not know if SUD is a hardware pin/low-level signal
1395 * or if it is messaged.
1397 r
= ap
->ap_sc
->sc_ipm_disable
;
1398 ahci_pwrite(ap
, AHCI_PREG_SCTL
, r
);
1400 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
) & ~AHCI_PREG_CMD_ICC
;
1401 cmd
|= AHCI_PREG_CMD_SUD
| AHCI_PREG_CMD_POD
;
1402 cmd
|= AHCI_PREG_CMD_ICC_ACTIVE
;
1403 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
1404 ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_ICC
);
1407 * Some parts need FIS reception enabled to be able to COMINIT at
1408 * all, so we can't delay FRE until port-start. Even though that
1409 * isn't what the spec says.
1411 * This is typically the first enablement of FRE, but in most cases
1412 * we never turn it off making this a NOP for later calls.
1414 cmd
|= AHCI_PREG_CMD_FRE
;
1415 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
1416 if ((ap
->ap_sc
->sc_flags
& AHCI_F_IGN_FR
) == 0)
1417 ahci_pwait_set(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_FR
);
1420 * Make sure that all power management is disabled.
1422 * NOTE! AHCI_PREG_SCTL_DET_DISABLE seems to be highly unreliable
1423 * on multiple chipsets and can brick the chipset or even
1424 * the whole PC. Never use it.
1426 ap
->ap_type
= ATA_PORT_T_NONE
;
1430 * Give the new power management state time to settle, then clear
1433 ahci_os_sleep(1000);
1435 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
1438 * Start transmitting COMRESET. The spec says that COMRESET must
1439 * be sent for at least 1ms but in actual fact numerous devices
1440 * appear to take much longer. Delay a whole second here.
1442 * In addition, SATA-3 ports can take longer to train, so even
1443 * SATA-2 devices which would normally detect very quickly may
1444 * take longer when plugged into a SATA-3 port.
1446 r
|= AHCI_PREG_SCTL_DET_INIT
;
1447 switch(AhciForceGen
) {
1449 r
|= AHCI_PREG_SCTL_SPD_ANY
;
1452 r
|= AHCI_PREG_SCTL_SPD_GEN1
;
1455 r
|= AHCI_PREG_SCTL_SPD_GEN2
;
1458 r
|= AHCI_PREG_SCTL_SPD_GEN3
;
1461 r
|= AHCI_PREG_SCTL_SPD_GEN3
;
1464 ahci_pwrite(ap
, AHCI_PREG_SCTL
, r
);
1465 ahci_os_sleep(1000);
1467 ap
->ap_flags
&= ~AP_F_HARSH_REINIT
;
1470 * Only SERR_DIAG_X needs to be cleared for TFD updates, but
1471 * since we are hard-resetting the port we might as well clear
1472 * the whole enchillada. Also be sure to clear any spurious BSY
1473 * prior to clearing INIT.
1475 * Wait 1 whole second after clearing INIT before checking
1476 * the device detection bits in an attempt to work around chipsets
1477 * which do not properly mask PCS/PRCS during low level init.
1480 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
1481 /* ahci_port_clo(ap);*/
1484 r
&= ~AHCI_PREG_SCTL_SPD
;
1485 r
&= ~AHCI_PREG_SCTL_DET_INIT
;
1486 r
|= AHCI_PREG_SCTL_DET_NONE
;
1487 ahci_pwrite(ap
, AHCI_PREG_SCTL
, r
);
1488 ahci_os_sleep(1000);
1491 * Try to determine if there is a device on the port.
1493 * Give the device 3/10 second to at least be detected.
1494 * If we fail clear PRCS (phy detect) since we may cycled
1495 * the phy and probably caused another PRCS interrupt.
1499 r
= ahci_pread(ap
, AHCI_PREG_SSTS
);
1500 if (r
& AHCI_PREG_SSTS_DET
)
1502 loop
-= ahci_os_softsleep();
1505 ahci_pwrite(ap
, AHCI_PREG_IS
, AHCI_PREG_IS_PRCS
);
1507 kprintf("%s: Port appears to be unplugged\n",
1515 * There is something on the port. Regardless of what happens
1516 * after this tell the caller to try to detect a port multiplier.
1518 * Give the device 3 seconds to fully negotiate.
1522 if (ahci_pwait_eq(ap
, 3000, AHCI_PREG_SSTS
,
1523 AHCI_PREG_SSTS_DET
, AHCI_PREG_SSTS_DET_DEV
)) {
1525 kprintf("%s: Device may be powered down\n",
1533 * We got something that definitely looks like a device. Give
1534 * the device time to send us its first D2H FIS. Waiting for
1535 * BSY to clear accomplishes this.
1537 * NOTE: A port multiplier may or may not clear BSY here,
1538 * depending on what is sitting in target 0 behind it.
1540 * NOTE: Intel SSDs seem to have compatibility problems with Intel
1541 * mobo's on cold boots and may leave BSY set. A single
1542 * retry works around the problem. This is definitely a bug
1543 * with the mobo and/or the SSD and does not appear to occur
1544 * with other devices connected to the same port.
1547 if (ahci_pwait_clr_to(ap
, 8000, AHCI_PREG_TFD
,
1548 AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
1549 kprintf("%s: Device BUSY: %b\n",
1551 ahci_pread(ap
, AHCI_PREG_TFD
),
1554 kprintf("%s: Retrying\n", PORTNAME(ap
));
1570 * AHCI port reset, Section 10.4.2
1572 * This function does a hard reset of the port. Note that the device
1573 * connected to the port could still end-up hung.
1576 ahci_port_hardreset(struct ahci_port
*ap
, int hard
)
1583 kprintf("%s: START HARDRESET\n", PORTNAME(ap
));
1584 ap
->ap_flags
|= AP_F_IN_RESET
;
1586 error
= ahci_comreset(ap
, &pmdetect
);
1589 * We may be asked to perform a port multiplier check even if the
1590 * comreset failed. This typically occurs when the PM has nothing
1591 * in slot 0, which can cause BSY to remain set.
1593 * If the PM detection is successful it will override (error),
1594 * otherwise (error) is retained. If an error does occur it
1595 * is possible that a normal device has blown up on us DUE to
1596 * the PM detection code, so re-run the comreset and assume
1600 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SPM
) {
1601 error
= ahci_pm_port_probe(ap
, error
);
1603 error
= ahci_comreset(ap
, &pmdetect
);
1616 * All good, make sure the port is running and set the
1617 * probe state. Ignore the signature junk (it's unreliable)
1618 * until we get to the softreset code.
1620 if (ahci_port_start(ap
)) {
1621 kprintf("%s: failed to start command DMA on port, "
1622 "disabling\n", PORTNAME(ap
));
1626 if (ap
->ap_type
== ATA_PORT_T_PM
)
1627 ap
->ap_probe
= ATA_PROBE_GOOD
;
1629 ap
->ap_probe
= ATA_PROBE_NEED_SOFT_RESET
;
1633 * Normal device probe failure
1635 data
= ahci_pread(ap
, AHCI_PREG_SSTS
);
1637 switch(data
& AHCI_PREG_SSTS_DET
) {
1638 case AHCI_PREG_SSTS_DET_DEV_NE
:
1639 kprintf("%s: Device not communicating\n",
1642 case AHCI_PREG_SSTS_DET_PHYOFFLINE
:
1643 kprintf("%s: PHY offline\n",
1647 kprintf("%s: No device detected\n",
1651 ahci_port_hardstop(ap
);
1655 * Abnormal probe (EBUSY)
1657 kprintf("%s: Device on port is bricked\n",
1659 ahci_port_hardstop(ap
);
1661 rc
= ahci_port_reset(ap
, atx
, 0);
1663 kprintf("%s: Unable unbrick device\n",
1666 kprintf("%s: Successfully unbricked\n",
1676 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
1677 ahci_pwrite(ap
, AHCI_PREG_IS
, AHCI_PREG_IS_PCS
| AHCI_PREG_IS_PRCS
);
1679 ap
->ap_flags
&= ~AP_F_IN_RESET
;
1682 kprintf("%s: END HARDRESET %d\n", PORTNAME(ap
), error
);
1687 * Hard-stop on hot-swap device removal. See 10.10.1
1689 * Place the port in a mode that will allow it to detect hot-swap insertions.
1690 * This is a bit imprecise because just setting-up SCTL to DET_INIT doesn't
1691 * seem to do the job.
1693 * FIS reception is left enabled but command processing is disabled.
1694 * Cycling FIS reception (FRE) can brick ports.
1697 ahci_port_hardstop(struct ahci_port
*ap
)
1699 struct ahci_ccb
*ccb
;
1700 struct ata_port
*at
;
1708 * Stop the port. We can't modify things like SUD if the port
1711 ap
->ap_state
= AP_S_FATAL_ERROR
;
1712 ap
->ap_probe
= ATA_PROBE_FAILED
;
1713 ap
->ap_type
= ATA_PORT_T_NONE
;
1714 ahci_port_stop(ap
, 0);
1715 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
1716 cmd
&= ~(AHCI_PREG_CMD_CLO
| AHCI_PREG_CMD_PMA
| AHCI_PREG_CMD_ICC
);
1717 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
1720 * Clean up AT sub-ports on SATA port.
1722 for (i
= 0; ap
->ap_ata
&& i
< AHCI_MAX_PMPORTS
; ++i
) {
1724 at
->at_type
= ATA_PORT_T_NONE
;
1725 at
->at_probe
= ATA_PROBE_FAILED
;
1729 * 10.10.1 place us in the Listen state.
1731 * 10.10.3 DET must be set to 0 and found to be 0 before
1734 * Deactivating SUD only applies if the controller supports SUD, it
1735 * is a bit unclear what happens w/regards to detecting hotplug
1738 * NOTE: AHCI_PREG_SCTL_SPM_* bits are not implemented by the spec
1741 r
= ap
->ap_sc
->sc_ipm_disable
;
1742 ahci_pwrite(ap
, AHCI_PREG_SCTL
, r
);
1745 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
1746 cmd
&= ~AHCI_PREG_CMD_SUD
;
1747 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
1753 * Transition su to the spin-up state. HBA shall send COMRESET and
1754 * begin initialization sequence (whatever that means). Presumably
1755 * this is edge-triggered. Following the spin-up state the HBA
1756 * will automatically transition to the Normal state.
1758 * This only applies if the controller supports SUD.
1759 * NEVER use AHCI_PREG_DET_DISABLE.
1761 cmd
|= AHCI_PREG_CMD_POD
|
1763 AHCI_PREG_CMD_ICC_ACTIVE
;
1764 ahci_pwrite(ap
, AHCI_PREG_CMD
, cmd
);
1765 ahci_pwait_clr(ap
, AHCI_PREG_CMD
, AHCI_PREG_CMD_ICC
);
1768 * Flush SERR_DIAG_X so the TFD can update.
1773 * Clean out pending ccbs
1776 while (ap
->ap_active
) {
1777 slot
= ffs(ap
->ap_active
) - 1;
1778 ap
->ap_active
&= ~(1 << slot
);
1779 --ap
->ap_active_cnt
;
1780 ccb
= &ap
->ap_ccbs
[slot
];
1781 if (ccb
->ccb_xa
.flags
& ATA_F_TIMEOUT_RUNNING
) {
1782 serial
= ccb
->ccb_xa
.serial
;
1783 callout_stop_sync(&ccb
->ccb_timeout
);
1784 if (serial
!= ccb
->ccb_xa
.serial
) {
1785 kprintf("%s: Warning: timeout race ccb %p\n",
1789 ccb
->ccb_xa
.flags
&= ~ATA_F_TIMEOUT_RUNNING
;
1791 ap
->ap_expired
&= ~(1 << slot
);
1792 ccb
->ccb_xa
.flags
&= ~(ATA_F_TIMEOUT_DESIRED
|
1793 ATA_F_TIMEOUT_EXPIRED
);
1794 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
1796 ccb
->ccb_xa
.complete(&ccb
->ccb_xa
);
1798 while (ap
->ap_sactive
) {
1799 slot
= ffs(ap
->ap_sactive
) - 1;
1800 ap
->ap_sactive
&= ~(1 << slot
);
1801 ccb
= &ap
->ap_ccbs
[slot
];
1802 if (ccb
->ccb_xa
.flags
& ATA_F_TIMEOUT_RUNNING
) {
1803 serial
= ccb
->ccb_xa
.serial
;
1804 callout_stop_sync(&ccb
->ccb_timeout
);
1805 if (serial
!= ccb
->ccb_xa
.serial
) {
1806 kprintf("%s: Warning: timeout race ccb %p\n",
1810 ccb
->ccb_xa
.flags
&= ~ATA_F_TIMEOUT_RUNNING
;
1812 ap
->ap_expired
&= ~(1 << slot
);
1813 ccb
->ccb_xa
.flags
&= ~(ATA_F_TIMEOUT_DESIRED
|
1814 ATA_F_TIMEOUT_EXPIRED
);
1815 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
1817 ccb
->ccb_xa
.complete(&ccb
->ccb_xa
);
1819 KKASSERT(ap
->ap_active_cnt
== 0);
1821 while ((ccb
= TAILQ_FIRST(&ap
->ap_ccb_pending
)) != NULL
) {
1822 TAILQ_REMOVE(&ap
->ap_ccb_pending
, ccb
, ccb_entry
);
1823 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
1824 ccb
->ccb_xa
.flags
&= ~ATA_F_TIMEOUT_DESIRED
;
1826 ccb
->ccb_xa
.complete(&ccb
->ccb_xa
);
1830 * Hot-plug device detection should work at this point. e.g. on
1831 * AMD chipsets Spin-Up/Normal state is sufficient for hot-plug
1832 * detection and entering RESET (continuous COMRESET by setting INIT)
1833 * will actually prevent hot-plug detection from working properly.
1835 * There may be cases where this will fail to work, I have some
1836 * additional code to place the HBA in RESET (send continuous
1837 * COMRESET) and hopefully get DIAG.X or other events when something
1838 * is plugged in. Unfortunately this isn't universal and can
1839 * also prevent events from generating interrupts.
1844 * Transition us to the Reset state. Theoretically we send a
1845 * continuous stream of COMRESETs in this state.
1847 r
|= AHCI_PREG_SCTL_DET_INIT
;
1848 if (AhciForceGen1
& (1 << ap
->ap_num
)) {
1849 kprintf("%s: Force 1.5Gbits\n", PORTNAME(ap
));
1850 r
|= AHCI_PREG_SCTL_SPD_GEN1
;
1852 r
|= AHCI_PREG_SCTL_SPD_ANY
;
1854 ahci_pwrite(ap
, AHCI_PREG_SCTL
, r
);
1858 * Flush SERR_DIAG_X so the TFD can update.
1866 * We can't loop on the X bit, a continuous COMINIT received will make
1867 * it loop forever. Just assume one event has built up and clear X
1868 * so the task file descriptor can update.
1871 ahci_flush_tfd(struct ahci_port
*ap
)
1875 r
= ahci_pread(ap
, AHCI_PREG_SERR
);
1876 if (r
& AHCI_PREG_SERR_DIAG_X
)
1877 ahci_pwrite(ap
, AHCI_PREG_SERR
, AHCI_PREG_SERR_DIAG_X
);
1881 * Figure out what type of device is connected to the port, ATAPI or
1885 ahci_port_signature_detect(struct ahci_port
*ap
, struct ata_port
*at
)
1889 sig
= ahci_pread(ap
, AHCI_PREG_SIG
);
1891 kprintf("%s: sig %08x\n", ATANAME(ap
, at
), sig
);
1892 if ((sig
& 0xffff0000) == (SATA_SIGNATURE_ATAPI
& 0xffff0000)) {
1893 return(ATA_PORT_T_ATAPI
);
1894 } else if ((sig
& 0xffff0000) ==
1895 (SATA_SIGNATURE_PORT_MULTIPLIER
& 0xffff0000)) {
1896 return(ATA_PORT_T_PM
);
1898 return(ATA_PORT_T_DISK
);
1903 * Load the DMA descriptor table for a CCB's buffer.
1906 ahci_load_prdt(struct ahci_ccb
*ccb
)
1908 struct ahci_port
*ap
= ccb
->ccb_port
;
1909 struct ahci_softc
*sc
= ap
->ap_sc
;
1910 struct ata_xfer
*xa
= &ccb
->ccb_xa
;
1911 struct ahci_prdt
*prdt
= ccb
->ccb_cmd_table
->prdt
;
1912 bus_dmamap_t dmap
= ccb
->ccb_dmamap
;
1913 struct ahci_cmd_hdr
*cmd_slot
= ccb
->ccb_cmd_hdr
;
1916 if (xa
->datalen
== 0) {
1917 ccb
->ccb_cmd_hdr
->prdtl
= 0;
1921 error
= bus_dmamap_load(sc
->sc_tag_data
, dmap
,
1922 xa
->data
, xa
->datalen
,
1923 ahci_load_prdt_callback
,
1925 ((xa
->flags
& ATA_F_NOWAIT
) ?
1926 BUS_DMA_NOWAIT
: BUS_DMA_WAITOK
));
1928 kprintf("%s: error %d loading dmamap\n", PORTNAME(ap
), error
);
1932 if (xa
->flags
& ATA_F_PIO
)
1933 prdt
->flags
|= htole32(AHCI_PRDT_FLAG_INTR
);
1936 cmd_slot
->prdtl
= htole16(prdt
- ccb
->ccb_cmd_table
->prdt
+ 1);
1938 if (xa
->flags
& ATA_F_READ
)
1939 bus_dmamap_sync(sc
->sc_tag_data
, dmap
, BUS_DMASYNC_PREREAD
);
1940 if (xa
->flags
& ATA_F_WRITE
)
1941 bus_dmamap_sync(sc
->sc_tag_data
, dmap
, BUS_DMASYNC_PREWRITE
);
1947 * Callback from BUSDMA system to load the segment list. The passed segment
1948 * list is a temporary structure.
1952 ahci_load_prdt_callback(void *info
, bus_dma_segment_t
*segs
, int nsegs
,
1955 struct ahci_prdt
*prd
= *(void **)info
;
1958 KKASSERT(nsegs
<= AHCI_MAX_PRDT
);
1961 addr
= segs
->ds_addr
;
1962 prd
->dba_hi
= htole32((u_int32_t
)(addr
>> 32));
1963 prd
->dba_lo
= htole32((u_int32_t
)addr
);
1964 prd
->flags
= htole32(segs
->ds_len
- 1);
1970 *(void **)info
= prd
; /* return last valid segment */
1974 ahci_unload_prdt(struct ahci_ccb
*ccb
)
1976 struct ahci_port
*ap
= ccb
->ccb_port
;
1977 struct ahci_softc
*sc
= ap
->ap_sc
;
1978 struct ata_xfer
*xa
= &ccb
->ccb_xa
;
1979 bus_dmamap_t dmap
= ccb
->ccb_dmamap
;
1981 if (xa
->datalen
!= 0) {
1982 if (xa
->flags
& ATA_F_READ
) {
1983 bus_dmamap_sync(sc
->sc_tag_data
, dmap
,
1984 BUS_DMASYNC_POSTREAD
);
1986 if (xa
->flags
& ATA_F_WRITE
) {
1987 bus_dmamap_sync(sc
->sc_tag_data
, dmap
,
1988 BUS_DMASYNC_POSTWRITE
);
1990 bus_dmamap_unload(sc
->sc_tag_data
, dmap
);
1993 * prdbc is only updated by hardware for non-NCQ commands.
1995 if (ccb
->ccb_xa
.flags
& ATA_F_NCQ
) {
1998 if (ccb
->ccb_cmd_hdr
->prdbc
== 0 &&
1999 ccb
->ccb_xa
.state
== ATA_S_COMPLETE
) {
2000 kprintf("%s: WARNING! Unload prdbc resid "
2001 "was zero! tag=%d\n",
2002 ATANAME(ap
, xa
->at
), ccb
->ccb_slot
);
2004 xa
->resid
= xa
->datalen
-
2005 le32toh(ccb
->ccb_cmd_hdr
->prdbc
);
2011 * Start a command and poll for completion.
2013 * timeout is in ms and only counts once the command gets on-chip.
2015 * Returns ATA_S_* state, compare against ATA_S_COMPLETE to determine
2016 * that no error occured.
2018 * NOTE: If the caller specifies a NULL timeout function the caller is
2019 * responsible for clearing hardware state on failure, but we will
2020 * deal with removing the ccb from any pending queue.
2022 * NOTE: NCQ should never be used with this function.
2024 * NOTE: If the port is in a failed state and stopped we do not try
2025 * to activate the ccb.
2028 ahci_poll(struct ahci_ccb
*ccb
, int timeout
,
2029 void (*timeout_fn
)(struct ahci_ccb
*))
2031 struct ahci_port
*ap
= ccb
->ccb_port
;
2033 if (ccb
->ccb_port
->ap_state
== AP_S_FATAL_ERROR
) {
2034 ccb
->ccb_xa
.state
= ATA_S_ERROR
;
2035 return(ccb
->ccb_xa
.state
);
2039 kprintf("%s: Start command %02x tag=%d\n",
2040 ATANAME(ccb
->ccb_port
, ccb
->ccb_xa
.at
),
2041 ccb
->ccb_xa
.fis
->command
, ccb
->ccb_slot
);
2046 ahci_port_intr(ap
, 1);
2047 switch(ccb
->ccb_xa
.state
) {
2049 timeout
-= ahci_os_softsleep();
2052 ahci_os_softsleep();
2053 ahci_check_active_timeouts(ap
);
2057 return (ccb
->ccb_xa
.state
);
2059 } while (timeout
> 0);
2061 if ((ccb
->ccb_xa
.flags
& ATA_F_SILENT
) == 0) {
2062 kprintf("%s: Poll timeout slot %d CMD: %b TFD: 0x%b SERR: %b\n",
2063 ATANAME(ap
, ccb
->ccb_xa
.at
), ccb
->ccb_slot
,
2064 ahci_pread(ap
, AHCI_PREG_CMD
), AHCI_PFMT_CMD
,
2065 ahci_pread(ap
, AHCI_PREG_TFD
), AHCI_PFMT_TFD_STS
,
2066 ahci_pread(ap
, AHCI_PREG_SERR
), AHCI_PFMT_SERR
);
2073 return(ccb
->ccb_xa
.state
);
2077 * When polling we have to check if the currently active CCB(s)
2078 * have timed out as the callout will be deadlocked while we
2079 * hold the port lock.
2082 ahci_check_active_timeouts(struct ahci_port
*ap
)
2084 struct ahci_ccb
*ccb
;
2088 mask
= ap
->ap_active
| ap
->ap_sactive
;
2090 tag
= ffs(mask
) - 1;
2091 mask
&= ~(1 << tag
);
2092 ccb
= &ap
->ap_ccbs
[tag
];
2093 if (ccb
->ccb_xa
.flags
& ATA_F_TIMEOUT_EXPIRED
) {
2094 ahci_ata_cmd_timeout(ccb
);
2102 ahci_start_timeout(struct ahci_ccb
*ccb
)
2104 if (ccb
->ccb_xa
.flags
& ATA_F_TIMEOUT_DESIRED
) {
2105 ccb
->ccb_xa
.flags
|= ATA_F_TIMEOUT_RUNNING
;
2106 callout_reset(&ccb
->ccb_timeout
,
2107 (ccb
->ccb_xa
.timeout
* hz
+ 999) / 1000,
2108 ahci_ata_cmd_timeout_unserialized
, ccb
);
2113 ahci_start(struct ahci_ccb
*ccb
)
2115 struct ahci_port
*ap
= ccb
->ccb_port
;
2116 struct ahci_softc
*sc
= ap
->ap_sc
;
2118 KKASSERT(ccb
->ccb_xa
.state
== ATA_S_PENDING
);
2120 /* Zero transferred byte count before transfer */
2121 ccb
->ccb_cmd_hdr
->prdbc
= 0;
2123 /* Sync command list entry and corresponding command table entry */
2124 bus_dmamap_sync(sc
->sc_tag_cmdh
,
2125 AHCI_DMA_MAP(ap
->ap_dmamem_cmd_list
),
2126 BUS_DMASYNC_PREWRITE
);
2127 bus_dmamap_sync(sc
->sc_tag_cmdt
,
2128 AHCI_DMA_MAP(ap
->ap_dmamem_cmd_table
),
2129 BUS_DMASYNC_PREWRITE
);
2131 /* Prepare RFIS area for write by controller */
2132 bus_dmamap_sync(sc
->sc_tag_rfis
,
2133 AHCI_DMA_MAP(ap
->ap_dmamem_rfis
),
2134 BUS_DMASYNC_PREREAD
);
2137 * There's no point trying to optimize this, it only shaves a few
2138 * nanoseconds so just queue the command and call our generic issue.
2140 ahci_issue_pending_commands(ap
, ccb
);
2144 * While holding the port lock acquire exclusive access to the port.
2146 * This is used when running the state machine to initialize and identify
2147 * targets over a port multiplier. Setting exclusive access prevents
2148 * ahci_port_intr() from activating any requests sitting on the pending
2152 ahci_beg_exclusive_access(struct ahci_port
*ap
, struct ata_port
*at
)
2154 KKASSERT((ap
->ap_flags
& AP_F_EXCLUSIVE_ACCESS
) == 0);
2155 ap
->ap_flags
|= AP_F_EXCLUSIVE_ACCESS
;
2156 while (ap
->ap_active
|| ap
->ap_sactive
) {
2157 ahci_port_intr(ap
, 1);
2158 ahci_os_softsleep();
2163 ahci_end_exclusive_access(struct ahci_port
*ap
, struct ata_port
*at
)
2165 KKASSERT((ap
->ap_flags
& AP_F_EXCLUSIVE_ACCESS
) != 0);
2166 ap
->ap_flags
&= ~AP_F_EXCLUSIVE_ACCESS
;
2167 ahci_issue_pending_commands(ap
, NULL
);
2171 * If ccb is not NULL enqueue and/or issue it.
2173 * If ccb is NULL issue whatever we can from the queue. However, nothing
2174 * new is issued if the exclusive access flag is set or expired ccb's are
2177 * If existing commands are still active (ap_active/ap_sactive) we can only
2178 * issue matching new commands.
2181 ahci_issue_pending_commands(struct ahci_port
*ap
, struct ahci_ccb
*ccb
)
2189 * If just running the queue and in exclusive access mode we
2190 * just return. Also in this case if there are any expired ccb's
2191 * we want to clear the queue so the port can be safely stopped.
2194 TAILQ_INSERT_TAIL(&ap
->ap_ccb_pending
, ccb
, ccb_entry
);
2195 } else if ((ap
->ap_flags
& AP_F_EXCLUSIVE_ACCESS
) || ap
->ap_expired
) {
2200 * Pull the next ccb off the queue and run it if possible.
2202 * The error CCB supercedes all normal queue operations and
2203 * implies exclusive access while the error CCB is active.
2205 if (ccb
!= ap
->ap_err_ccb
) {
2206 if ((ccb
= TAILQ_FIRST(&ap
->ap_ccb_pending
)) == NULL
)
2208 if (ap
->ap_flags
& AP_F_ERR_CCB_RESERVED
) {
2209 kprintf("DELAY CCB slot %d\n", ccb
->ccb_slot
);
2215 * Handle exclusivity requirements.
2217 * ATA_F_EXCLUSIVE is used when we want to be the only command
2220 * ATA_F_AUTOSENSE is used when we want the D2H rfis loaded
2221 * back into the ccb on a normal (non-errored) command completion.
2222 * For example, for PM requests to target 15. Because the AHCI
2223 * spec does not stop the command processor and has only one rfis
2224 * area (for non-FBSS anyway), AUTOSENSE currently implies EXCLUSIVE.
2225 * Otherwise multiple completions can destroy the rfis data before
2226 * we have a chance to copy it.
2228 if (ap
->ap_active
& ~ap
->ap_expired
) {
2230 * There may be multiple ccb's already running,
2231 * if any are running and ap_run_flags sets
2232 * one of these flags then we know only one is
2235 * XXX Current AUTOSENSE code forces exclusivity
2236 * to simplify the code.
2238 if (ap
->ap_run_flags
&
2239 (ATA_F_EXCLUSIVE
| ATA_F_AUTOSENSE
)) {
2243 if (ccb
->ccb_xa
.flags
&
2244 (ATA_F_EXCLUSIVE
| ATA_F_AUTOSENSE
)) {
2249 if (ccb
->ccb_xa
.flags
& ATA_F_NCQ
) {
2251 * The next command is a NCQ command and can be issued as
2252 * long as currently active commands are not standard.
2254 if (ap
->ap_active
) {
2255 KKASSERT(ap
->ap_active_cnt
> 0);
2258 KKASSERT(ap
->ap_active_cnt
== 0);
2262 TAILQ_REMOVE(&ap
->ap_ccb_pending
, ccb
, ccb_entry
);
2263 KKASSERT((mask
& (1 << ccb
->ccb_slot
)) == 0);
2264 mask
|= 1 << ccb
->ccb_slot
;
2265 KKASSERT(ccb
->ccb_xa
.state
== ATA_S_PENDING
);
2266 KKASSERT(ccb
== &ap
->ap_ccbs
[ccb
->ccb_slot
]);
2267 ccb
->ccb_xa
.state
= ATA_S_ONCHIP
;
2268 ahci_start_timeout(ccb
);
2269 ap
->ap_run_flags
= ccb
->ccb_xa
.flags
;
2270 ccb
= TAILQ_FIRST(&ap
->ap_ccb_pending
);
2271 } while (ccb
&& (ccb
->ccb_xa
.flags
& ATA_F_NCQ
) &&
2273 (ATA_F_EXCLUSIVE
| ATA_F_AUTOSENSE
)) == 0);
2275 KKASSERT(((ap
->ap_active
| ap
->ap_sactive
) & mask
) == 0);
2277 ap
->ap_sactive
|= mask
;
2278 ahci_pwrite(ap
, AHCI_PREG_SACT
, mask
);
2279 ahci_pwrite(ap
, AHCI_PREG_CI
, mask
);
2282 * The next command is a standard command and can be issued
2283 * as long as currently active commands are not NCQ.
2285 * We limit ourself to 1 command if we have a port multiplier,
2286 * (at least without FBSS support), otherwise timeouts on
2287 * one port can race completions on other ports (see
2288 * ahci_ata_cmd_timeout() for more information).
2290 * If not on a port multiplier generally allow up to 4
2291 * standard commands to be enqueued. Remember that the
2292 * command processor will still process them sequentially.
2296 if (ap
->ap_type
== ATA_PORT_T_PM
)
2298 else if (ap
->ap_sc
->sc_ncmds
> 4)
2303 while (ap
->ap_active_cnt
< limit
&& ccb
&&
2304 (ccb
->ccb_xa
.flags
& ATA_F_NCQ
) == 0) {
2305 TAILQ_REMOVE(&ap
->ap_ccb_pending
, ccb
, ccb_entry
);
2306 KKASSERT(((ap
->ap_active
| ap
->ap_sactive
) &
2307 (1 << ccb
->ccb_slot
)) == 0);
2308 ap
->ap_active
|= 1 << ccb
->ccb_slot
;
2309 ap
->ap_active_cnt
++;
2310 ap
->ap_run_flags
= ccb
->ccb_xa
.flags
;
2311 ccb
->ccb_xa
.state
= ATA_S_ONCHIP
;
2312 ahci_start_timeout(ccb
);
2313 ahci_pwrite(ap
, AHCI_PREG_CI
, 1 << ccb
->ccb_slot
);
2314 if ((ap
->ap_run_flags
&
2315 (ATA_F_EXCLUSIVE
| ATA_F_AUTOSENSE
)) == 0) {
2318 ccb
= TAILQ_FIRST(&ap
->ap_ccb_pending
);
2319 if (ccb
&& (ccb
->ccb_xa
.flags
&
2320 (ATA_F_EXCLUSIVE
| ATA_F_AUTOSENSE
))) {
2328 ahci_intr(void *arg
)
2330 struct ahci_softc
*sc
= arg
;
2331 struct ahci_port
*ap
;
2337 * Check if the master enable is up, and whether any interrupts are
2340 if ((sc
->sc_flags
& AHCI_F_INT_GOOD
) == 0)
2342 is
= ahci_read(sc
, AHCI_REG_IS
);
2343 if (is
== 0 || is
== 0xffffffff) {
2346 is
&= sc
->sc_portmask
;
2348 #ifdef AHCI_COALESCE
2349 /* Check coalescing interrupt first */
2350 if (is
& sc
->sc_ccc_mask
) {
2351 DPRINTF(AHCI_D_INTR
, "%s: command coalescing interrupt\n",
2353 is
&= ~sc
->sc_ccc_mask
;
2354 is
|= sc
->sc_ccc_ports_cur
;
2359 * Process interrupts for each port in a non-blocking fashion.
2361 * The global IS bit is supposed to be forced on if any unmasked
2362 * port interrupt is pending, even if we clear it.
2364 * However it would appear that it is simply latched on some parts,
2365 * which means we have to clear it BEFORE processing the status bits
2368 ahci_write(sc
, AHCI_REG_IS
, is
);
2369 for (ack
= 0; is
; is
&= ~(1 << port
)) {
2373 ap
= sc
->sc_ports
[port
];
2377 if (ahci_os_lock_port_nb(ap
) == 0) {
2378 ahci_port_intr(ap
, 0);
2379 ahci_os_unlock_port(ap
);
2381 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
2382 ahci_os_signal_port_thread(ap
, AP_SIGF_PORTINT
);
2388 * Core called from helper thread.
2391 ahci_port_thread_core(struct ahci_port
*ap
, int mask
)
2394 * Process any expired timedouts.
2396 ahci_os_lock_port(ap
);
2397 if (mask
& AP_SIGF_TIMEOUT
) {
2398 ahci_check_active_timeouts(ap
);
2402 * Process port interrupts which require a higher level of
2405 if (mask
& AP_SIGF_PORTINT
) {
2406 ahci_port_intr(ap
, 1);
2407 ahci_port_interrupt_enable(ap
);
2408 } else if (ap
->ap_probe
!= ATA_PROBE_FAILED
) {
2409 ahci_port_intr(ap
, 1);
2410 ahci_port_interrupt_enable(ap
);
2412 ahci_os_unlock_port(ap
);
2416 * Core per-port interrupt handler.
2418 * If blockable is 0 we cannot call ahci_os_sleep() at all and we can only
2419 * deal with normal command completions which do not require blocking.
2422 ahci_port_intr(struct ahci_port
*ap
, int blockable
)
2424 struct ahci_softc
*sc
= ap
->ap_sc
;
2425 u_int32_t is
, ci_saved
, ci_masked
;
2428 struct ahci_ccb
*ccb
= NULL
;
2429 struct ata_port
*ccb_at
= NULL
;
2430 volatile u_int32_t
*active
;
2431 const u_int32_t blockable_mask
= AHCI_PREG_IS_TFES
|
2439 enum { NEED_NOTHING
, NEED_REINIT
, NEED_RESTART
,
2440 NEED_HOTPLUG_INSERT
, NEED_HOTPLUG_REMOVE
} need
= NEED_NOTHING
;
2443 * All basic command completions are always processed.
2445 is
= ahci_pread(ap
, AHCI_PREG_IS
);
2446 if (is
& AHCI_PREG_IS_DPS
)
2447 ahci_pwrite(ap
, AHCI_PREG_IS
, is
& AHCI_PREG_IS_DPS
);
2450 * If we can't block then we can't handle these here. Disable
2451 * the interrupts in question so we don't live-lock, the helper
2452 * thread will re-enable them.
2454 * If the port is in a completely failed state we do not want
2455 * to drop through to failed-command-processing if blockable is 0,
2456 * just let the thread deal with it all.
2458 * Otherwise we fall through and still handle DHRS and any commands
2459 * which completed normally. Even if we are errored we haven't
2460 * stopped the port yet so CI/SACT are still good.
2462 if (blockable
== 0) {
2463 if (ap
->ap_state
== AP_S_FATAL_ERROR
) {
2464 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
2465 ahci_os_signal_port_thread(ap
, AP_SIGF_PORTINT
);
2468 if (is
& blockable_mask
) {
2469 ahci_pwrite(ap
, AHCI_PREG_IE
, 0);
2470 ahci_os_signal_port_thread(ap
, AP_SIGF_PORTINT
);
2476 * Either NCQ or non-NCQ commands will be active, never both.
2478 if (ap
->ap_sactive
) {
2479 KKASSERT(ap
->ap_active
== 0);
2480 KKASSERT(ap
->ap_active_cnt
== 0);
2481 ci_saved
= ahci_pread(ap
, AHCI_PREG_SACT
);
2482 active
= &ap
->ap_sactive
;
2484 ci_saved
= ahci_pread(ap
, AHCI_PREG_CI
);
2485 active
= &ap
->ap_active
;
2487 KKASSERT(!(ap
->ap_sactive
&& ap
->ap_active
));
2488 KKASSERT((ci_saved
& (ap
->ap_sactive
| ap
->ap_active
)) == ci_saved
);
2490 kprintf("CHECK act=%08x/%08x sact=%08x/%08x\n",
2491 ap
->ap_active
, ahci_pread(ap
, AHCI_PREG_CI
),
2492 ap
->ap_sactive
, ahci_pread(ap
, AHCI_PREG_SACT
));
2496 * Ignore AHCI_PREG_IS_PRCS when link power management is on
2498 if (ap
->link_pwr_mgmt
!= AHCI_LINK_PWR_MGMT_NONE
) {
2499 is
&= ~AHCI_PREG_IS_PRCS
;
2500 ahci_pwrite(ap
, AHCI_PREG_SERR
,
2501 AHCI_PREG_SERR_DIAG_N
| AHCI_PREG_SERR_DIAG_W
);
2505 * Command failed (blockable).
2507 * See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2.
2509 * This stops command processing.
2511 if (is
& AHCI_PREG_IS_TFES
) {
2512 u_int32_t tfd
, serr
;
2516 tfd
= ahci_pread(ap
, AHCI_PREG_TFD
);
2517 serr
= ahci_pread(ap
, AHCI_PREG_SERR
);
2520 * Load the error slot and restart command processing.
2521 * CLO if we need to. The error slot may not be valid.
2522 * MUST BE DONE BEFORE CLEARING ST!
2526 * It is unclear but we may have to clear SERR to reenable
2529 err_slot
= AHCI_PREG_CMD_CCS(ahci_pread(ap
, AHCI_PREG_CMD
));
2530 ahci_pwrite(ap
, AHCI_PREG_IS
, AHCI_PREG_IS_TFES
|
2534 is
&= ~(AHCI_PREG_IS_TFES
| AHCI_PREG_IS_PSS
|
2535 AHCI_PREG_IS_DHRS
| AHCI_PREG_IS_SDBS
);
2536 ahci_pwrite(ap
, AHCI_PREG_SERR
, serr
);
2537 ahci_port_stop(ap
, 0);
2538 ahci_os_hardsleep(10);
2539 if (tfd
& (AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
2540 kprintf("%s: Issuing CLO\n", PORTNAME(ap
));
2545 * We are now stopped and need a restart. If we have to
2546 * process a NCQ error we will temporarily start and then
2547 * stop the port again, so this condition holds.
2550 need
= NEED_RESTART
;
2553 * ATAPI errors are fairly common from probing, just
2554 * report disk errors or if bootverbose is on.
2556 if (bootverbose
|| ap
->ap_type
!= ATA_PORT_T_ATAPI
) {
2557 kprintf("%s: TFES slot %d ci_saved = %08x\n",
2558 PORTNAME(ap
), err_slot
, ci_saved
);
2562 * If we got an error on an error CCB just complete it
2563 * with an error. ci_saved has the mask to restart
2564 * (the err_ccb will be removed from it by finish_error).
2566 if (ap
->ap_flags
& AP_F_ERR_CCB_RESERVED
) {
2567 err_slot
= ap
->ap_err_ccb
->ccb_slot
;
2572 * If NCQ commands were active get the error slot from
2573 * the log page. NCQ is not supported for PM's so this
2574 * is a direct-attached target.
2576 * Otherwise if no commands were active we have a problem.
2578 * Otherwise if the error slot is bad we have a problem.
2580 * Otherwise process the error for the slot.
2582 if (ap
->ap_sactive
) {
2583 ahci_port_start(ap
);
2584 err_slot
= ahci_port_read_ncq_error(ap
, 0);
2585 ahci_port_stop(ap
, 0);
2586 } else if (ap
->ap_active
== 0) {
2587 kprintf("%s: TFES with no commands pending\n",
2590 } else if (err_slot
< 0 || err_slot
>= ap
->ap_sc
->sc_ncmds
) {
2591 kprintf("%s: bad error slot %d\n",
2592 PORTNAME(ap
), err_slot
);
2595 ccb
= &ap
->ap_ccbs
[err_slot
];
2598 * Validate the errored ccb. Note that ccb_at can
2599 * be NULL for direct-attached ccb's.
2601 * Copy received taskfile data from the RFIS.
2603 if (ccb
->ccb_xa
.state
== ATA_S_ONCHIP
) {
2604 ccb_at
= ccb
->ccb_xa
.at
;
2605 memcpy(&ccb
->ccb_xa
.rfis
, ap
->ap_rfis
->rfis
,
2606 sizeof(struct ata_fis_d2h
));
2608 kprintf("%s: Copying rfis slot %d\n",
2609 ATANAME(ap
, ccb_at
), err_slot
);
2612 kprintf("%s: Cannot copy rfis, CCB slot "
2613 "%d is not on-chip (state=%d)\n",
2614 ATANAME(ap
, ccb
->ccb_xa
.at
),
2615 err_slot
, ccb
->ccb_xa
.state
);
2621 * If we could not determine the errored slot then
2625 kprintf("%s: TFES: Unable to determine errored slot\n",
2627 if (ap
->ap_flags
& AP_F_IN_RESET
)
2633 * Finish error on slot. We will restart ci_saved
2634 * commands except the errored slot which we generate
2638 ccb
= &ap
->ap_ccbs
[err_slot
];
2639 ci_saved
&= ~(1 << err_slot
);
2640 KKASSERT(ccb
->ccb_xa
.state
== ATA_S_ONCHIP
);
2641 ccb
->ccb_xa
.state
= ATA_S_ERROR
;
2642 } else if (is
& AHCI_PREG_IS_DHRS
) {
2644 * Command posted D2H register FIS to the rfis (non-blocking).
2646 * A normal completion with an error may set DHRS instead
2647 * of TFES. The CCS bits are only valid if ERR was set.
2648 * If ERR is set command processing was probably stopped.
2650 * If ERR was not set we can only copy-back data for
2651 * exclusive-mode commands because otherwise we won't know
2652 * which tag the rfis belonged to.
2654 * err_slot must be read from the CCS before any other port
2655 * action, such as stopping the port.
2657 * WARNING! This is not well documented in the AHCI spec.
2658 * It can be found in the state machine tables
2659 * but not in the explanations.
2665 tfd
= ahci_pread(ap
, AHCI_PREG_TFD
);
2666 cmd
= ahci_pread(ap
, AHCI_PREG_CMD
);
2668 ahci_pwrite(ap
, AHCI_PREG_IS
, AHCI_PREG_IS_DHRS
);
2671 * If command processing is turned off we can process the
2672 * error immediately. Use the ST bit here instead of the
2673 * CR bit in case the CR bit is not implemented via the
2676 if ((tfd
& AHCI_PREG_TFD_STS_ERR
) &&
2677 (cmd
& AHCI_PREG_CMD_ST
) == 0) {
2678 err_slot
= AHCI_PREG_CMD_CCS(
2679 ahci_pread(ap
, AHCI_PREG_CMD
));
2680 ccb
= &ap
->ap_ccbs
[err_slot
];
2681 kprintf("%s: DHRS tfd=%b err_slot=%d cmd=%02x\n",
2683 tfd
, AHCI_PFMT_TFD_STS
,
2684 err_slot
, ccb
->ccb_xa
.fis
->command
);
2688 * NO ELSE... copy back is in the normal command completion
2689 * code and only if no error occured and ATA_F_AUTOSENSE
2695 * Device notification to us (non-blocking)
2697 * NOTE! On some parts notification bits can cause an IPMS
2698 * interrupt instead of a SDBS interrupt.
2700 * NOTE! On some parts (e.g. VBOX, probably intel ICHx),
2701 * SDBS notifies us of the completion of a NCQ command
2704 if (is
& (AHCI_PREG_IS_SDBS
| AHCI_PREG_IS_IPMS
)) {
2707 ahci_pwrite(ap
, AHCI_PREG_IS
,
2708 AHCI_PREG_IS_SDBS
| AHCI_PREG_IS_IPMS
);
2709 if (sc
->sc_cap
& AHCI_REG_CAP_SSNTF
) {
2710 data
= ahci_pread(ap
, AHCI_PREG_SNTF
);
2712 ahci_pwrite(ap
, AHCI_PREG_IS
,
2714 kprintf("%s: NOTIFY %08x\n",
2715 PORTNAME(ap
), data
);
2716 ahci_pwrite(ap
, AHCI_PREG_SERR
,
2717 AHCI_PREG_SERR_DIAG_N
);
2718 ahci_pwrite(ap
, AHCI_PREG_SNTF
, data
);
2719 ahci_cam_changed(ap
, NULL
, -1);
2722 is
&= ~(AHCI_PREG_IS_SDBS
| AHCI_PREG_IS_IPMS
);
2726 * Spurious IFS errors (blockable) - when AP_F_IGNORE_IFS is set.
2728 * Spurious IFS errors can occur while we are doing a reset
2729 * sequence through a PM, probably due to an unexpected FIS
2730 * being received during the PM target reset sequence. Chipsets
2731 * are supposed to mask these events but some do not.
2733 * Try to recover from the condition.
2735 if ((is
& AHCI_PREG_IS_IFS
) && (ap
->ap_flags
& AP_F_IGNORE_IFS
)) {
2736 u_int32_t serr
= ahci_pread(ap
, AHCI_PREG_SERR
);
2737 if ((ap
->ap_flags
& AP_F_IFS_IGNORED
) == 0) {
2738 kprintf("%s: IFS during PM probe (ignored) "
2742 serr
, AHCI_PFMT_SERR
);
2743 ap
->ap_flags
|= AP_F_IFS_IGNORED
;
2747 * Try to clear the error condition. The IFS error killed
2748 * the port so stop it so we can restart it.
2750 ahci_pwrite(ap
, AHCI_PREG_IS
, AHCI_PREG_IS_IFS
);
2751 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
2752 is
&= ~AHCI_PREG_IS_IFS
;
2753 need
= NEED_RESTART
;
2758 * Port change (hot-plug) (blockable).
2760 * A PRCS interrupt can occur:
2761 * (1) On hot-unplug / normal-unplug (phy lost)
2762 * (2) Sometimes on hot-plug too.
2764 * A PCS interrupt can occur in a number of situations:
2765 * (1) On hot-plug once communication is established
2766 * (2) On hot-unplug sometimes.
2767 * (3) For chipsets with badly written firmware it can occur
2768 * during INIT/RESET sequences due to the device reset.
2769 * (4) For chipsets with badly written firmware it can occur
2770 * when it thinks an unsolicited COMRESET is received
2771 * during a INIT/RESET sequence, even though we actually
2774 * XXX We can then check the CPS (Cold Presence State) bit, if
2775 * supported, to determine if a device is plugged in or not and do
2778 * PCS interrupts are cleared by clearing DIAG_X. If this occurs
2779 * command processing is automatically stopped (CR goes inactive)
2780 * and the port must be stopped and restarted.
2782 * WARNING: AMD parts (e.g. 880G chipset, probably others) can
2783 * generate PCS on initialization even when device is
2784 * already connected up. It is unclear why this happens.
2785 * Depending on the state of the device detect this can
2786 * cause us to go into harsh reinit or hot-plug insertion
2789 * WARNING: PCS errors can be repetitive (e.g. unsolicited COMRESET
2790 * continues to flow in from the device), we must clear the
2791 * interrupt in all cases and enforce a delay to prevent
2792 * a livelock and give the port time to settle down.
2793 * Only print something if we aren't in INIT/HARD-RESET.
2795 if (is
& (AHCI_PREG_IS_PCS
| AHCI_PREG_IS_PRCS
)) {
2796 ahci_pwrite(ap
, AHCI_PREG_IS
,
2797 is
& (AHCI_PREG_IS_PCS
| AHCI_PREG_IS_PRCS
));
2799 * Try to clear the error. Because of the repetitiveness
2800 * of this interrupt avoid any harsh action if the port is
2801 * already in the init or hard-reset probe state.
2803 ahci_pwrite(ap
, AHCI_PREG_SERR
, -1);
2804 /* (AHCI_PREG_SERR_DIAG_N | AHCI_PREG_SERR_DIAG_X) */
2807 * Ignore PCS/PRCS errors during probes (but still clear the
2808 * interrupt to avoid a livelock). The AMD 880/890/SB850
2809 * chipsets do not mask PCS/PRCS internally during reset
2812 if (ap
->ap_flags
& AP_F_IN_RESET
)
2815 if (ap
->ap_probe
== ATA_PROBE_NEED_INIT
||
2816 ap
->ap_probe
== ATA_PROBE_NEED_HARD_RESET
) {
2817 is
&= ~(AHCI_PREG_IS_PCS
| AHCI_PREG_IS_PRCS
);
2818 need
= NEED_NOTHING
;
2819 ahci_os_sleep(1000);
2822 kprintf("%s: Transient Errors: %b (%d)\n",
2823 PORTNAME(ap
), is
, AHCI_PFMT_IS
, ap
->ap_probe
);
2824 is
&= ~(AHCI_PREG_IS_PCS
| AHCI_PREG_IS_PRCS
);
2828 * Stop the port and figure out what to do next.
2830 ahci_port_stop(ap
, 0);
2833 switch (ahci_pread(ap
, AHCI_PREG_SSTS
) & AHCI_PREG_SSTS_DET
) {
2834 case AHCI_PREG_SSTS_DET_DEV
:
2838 if (ap
->ap_probe
== ATA_PROBE_FAILED
) {
2839 need
= NEED_HOTPLUG_INSERT
;
2842 need
= NEED_RESTART
;
2844 case AHCI_PREG_SSTS_DET_DEV_NE
:
2846 * Device not communicating. AMD parts seem to
2847 * like to throw this error on initialization
2848 * for no reason that I can fathom.
2850 kprintf("%s: Device present but not communicating, "
2851 "attempting port restart\n",
2856 if (ap
->ap_probe
!= ATA_PROBE_FAILED
) {
2857 need
= NEED_HOTPLUG_REMOVE
;
2860 need
= NEED_RESTART
;
2868 * Check for remaining errors - they are fatal. (blockable)
2870 if (is
& (AHCI_PREG_IS_TFES
| AHCI_PREG_IS_HBFS
| AHCI_PREG_IS_IFS
|
2871 AHCI_PREG_IS_OFS
| AHCI_PREG_IS_UFS
)) {
2874 ahci_pwrite(ap
, AHCI_PREG_IS
,
2875 is
& (AHCI_PREG_IS_TFES
| AHCI_PREG_IS_HBFS
|
2876 AHCI_PREG_IS_IFS
| AHCI_PREG_IS_OFS
|
2878 serr
= ahci_pread(ap
, AHCI_PREG_SERR
);
2879 kprintf("%s: Unrecoverable errors (IS: %b, SERR: %b), "
2880 "disabling port.\n",
2883 serr
, AHCI_PFMT_SERR
2885 is
&= ~(AHCI_PREG_IS_TFES
| AHCI_PREG_IS_HBFS
|
2886 AHCI_PREG_IS_IFS
| AHCI_PREG_IS_OFS
|
2890 * Fail all commands but then what? For now try to
2891 * reinitialize the port.
2898 * Fail all outstanding commands if we know the port won't recover.
2900 * We may have a ccb_at if the failed command is known and was
2901 * being sent to a device over a port multiplier (PM). In this
2902 * case if the port itself has not completely failed we fail just
2903 * the commands related to that target.
2905 * ci_saved contains the mask of active commands as of when the
2906 * error occured, prior to any port stops.
2908 if (ap
->ap_state
== AP_S_FATAL_ERROR
) {
2910 ap
->ap_state
= AP_S_FATAL_ERROR
;
2912 ahci_port_stop(ap
, 0);
2916 * Error all the active slots not already errored.
2918 ci_masked
= ci_saved
& *active
& ~ap
->ap_expired
;
2920 kprintf("%s: Failing all commands: %08x\n",
2921 PORTNAME(ap
), ci_masked
);
2925 slot
= ffs(ci_masked
) - 1;
2926 ccb
= &ap
->ap_ccbs
[slot
];
2927 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
2928 ap
->ap_expired
|= 1 << slot
;
2929 ci_saved
&= ~(1 << slot
);
2930 ci_masked
&= ~(1 << slot
);
2934 * Clear bits in ci_saved (cause completions to be run)
2935 * for all slots which are not active.
2937 ci_saved
&= ~*active
;
2940 * Don't restart the port if our problems were deemed fatal.
2942 * Also acknowlege all fatal interrupt sources to prevent
2945 if (ap
->ap_state
== AP_S_FATAL_ERROR
) {
2946 if (need
== NEED_RESTART
)
2947 need
= NEED_NOTHING
;
2948 ahci_pwrite(ap
, AHCI_PREG_IS
,
2949 AHCI_PREG_IS_TFES
| AHCI_PREG_IS_HBFS
|
2950 AHCI_PREG_IS_IFS
| AHCI_PREG_IS_OFS
|
2956 * If we are stopped the AHCI chipset is supposed to have cleared
2957 * CI and SACT. Did it? If it didn't we try very hard to clear
2958 * the fields otherwise we may end up completing CCBs which are
2959 * actually still active.
2961 * IFS errors on (at least) AMD chipsets create this confusion.
2965 if ((mask
= ahci_pactive(ap
)) != 0) {
2966 kprintf("%s: chipset failed to clear "
2967 "active cmds %08x\n",
2968 PORTNAME(ap
), mask
);
2969 ahci_port_start(ap
);
2970 ahci_port_stop(ap
, 0);
2971 if ((mask
= ahci_pactive(ap
)) != 0) {
2972 kprintf("%s: unable to prod the chip into "
2973 "clearing active cmds %08x\n",
2974 PORTNAME(ap
), mask
);
2975 /* what do we do now? */
2981 * CCB completion (non blocking).
2983 * CCB completion is detected by noticing its slot's bit in CI has
2984 * changed to zero some time after we activated it.
2985 * If we are polling, we may only be interested in particular slot(s).
2987 * Any active bits not saved are completed within the restrictions
2988 * imposed by the caller.
2990 ci_masked
= ~ci_saved
& *active
;
2992 slot
= ffs(ci_masked
) - 1;
2993 ccb
= &ap
->ap_ccbs
[slot
];
2994 ci_masked
&= ~(1 << slot
);
2996 DPRINTF(AHCI_D_INTR
, "%s: slot %d is complete%s\n",
2997 PORTNAME(ap
), slot
, ccb
->ccb_xa
.state
== ATA_S_ERROR
?
3000 bus_dmamap_sync(sc
->sc_tag_cmdh
,
3001 AHCI_DMA_MAP(ap
->ap_dmamem_cmd_list
),
3002 BUS_DMASYNC_POSTWRITE
);
3004 bus_dmamap_sync(sc
->sc_tag_cmdt
,
3005 AHCI_DMA_MAP(ap
->ap_dmamem_cmd_table
),
3006 BUS_DMASYNC_POSTWRITE
);
3008 bus_dmamap_sync(sc
->sc_tag_rfis
,
3009 AHCI_DMA_MAP(ap
->ap_dmamem_rfis
),
3010 BUS_DMASYNC_POSTREAD
);
3012 *active
&= ~(1 << ccb
->ccb_slot
);
3013 if (active
== &ap
->ap_active
) {
3014 KKASSERT(ap
->ap_active_cnt
> 0);
3015 --ap
->ap_active_cnt
;
3019 * Complete the ccb. If the ccb was marked expired it
3020 * was probably already removed from the command processor,
3021 * so don't take the clear ci_saved bit as meaning the
3022 * command actually succeeded, it didn't.
3024 if (ap
->ap_expired
& (1 << ccb
->ccb_slot
)) {
3025 ap
->ap_expired
&= ~(1 << ccb
->ccb_slot
);
3026 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
3028 ccb
->ccb_xa
.complete(&ccb
->ccb_xa
);
3030 if (ccb
->ccb_xa
.state
== ATA_S_ONCHIP
) {
3031 ccb
->ccb_xa
.state
= ATA_S_COMPLETE
;
3032 if (ccb
->ccb_xa
.flags
& ATA_F_AUTOSENSE
) {
3033 memcpy(&ccb
->ccb_xa
.rfis
,
3035 sizeof(struct ata_fis_d2h
));
3036 if (ccb
->ccb_xa
.state
== ATA_S_TIMEOUT
)
3037 ccb
->ccb_xa
.state
= ATA_S_ERROR
;
3045 * Cleanup. Will not be set if non-blocking.
3050 * If operating normally and not stopped the interrupt was
3051 * probably just a normal completion and we may be able to
3052 * issue more commands.
3054 if (stopped
== 0 && ap
->ap_state
!= AP_S_FATAL_ERROR
)
3055 ahci_issue_pending_commands(ap
, NULL
);
3059 * A recoverable error occured and we can restart outstanding
3060 * commands on the port.
3062 ci_saved
&= ~ap
->ap_expired
;
3064 kprintf("%s: Restart %08x\n", PORTNAME(ap
), ci_saved
);
3065 ahci_issue_saved_commands(ap
, ci_saved
);
3069 * Potentially issue new commands if not in a failed
3072 if (ap
->ap_state
!= AP_S_FATAL_ERROR
) {
3073 ahci_port_start(ap
);
3074 ahci_issue_pending_commands(ap
, NULL
);
3079 * Something horrible happened to the port and we
3080 * need to reinitialize it.
3082 kprintf("%s: REINIT - Attempting to reinitialize the port "
3083 "after it had a horrible accident\n",
3085 ap
->ap_flags
|= AP_F_IN_RESET
;
3086 ap
->ap_flags
|= AP_F_HARSH_REINIT
;
3087 ap
->ap_probe
= ATA_PROBE_NEED_INIT
;
3088 ahci_cam_changed(ap
, NULL
, -1);
3090 case NEED_HOTPLUG_INSERT
:
3092 * A hot-plug insertion event has occured and all
3093 * outstanding commands have already been revoked.
3095 * Don't recurse if this occurs while we are
3096 * resetting the port.
3098 if ((ap
->ap_flags
& AP_F_IN_RESET
) == 0) {
3099 kprintf("%s: HOTPLUG - Device inserted\n",
3101 ap
->ap_probe
= ATA_PROBE_NEED_INIT
;
3102 ahci_cam_changed(ap
, NULL
, -1);
3105 case NEED_HOTPLUG_REMOVE
:
3107 * A hot-plug removal event has occured and all
3108 * outstanding commands have already been revoked.
3110 * Don't recurse if this occurs while we are
3111 * resetting the port.
3113 if ((ap
->ap_flags
& AP_F_IN_RESET
) == 0) {
3114 kprintf("%s: HOTPLUG - Device removed\n",
3116 ahci_port_hardstop(ap
);
3117 /* ap_probe set to failed */
3118 ahci_cam_changed(ap
, NULL
, -1);
3127 ahci_get_ccb(struct ahci_port
*ap
)
3129 struct ahci_ccb
*ccb
;
3131 lockmgr(&ap
->ap_ccb_lock
, LK_EXCLUSIVE
);
3132 ccb
= TAILQ_FIRST(&ap
->ap_ccb_free
);
3134 KKASSERT((ap
->ap_sactive
& (1 << ccb
->ccb_slot
)) == 0);
3135 KKASSERT(ccb
->ccb_xa
.state
== ATA_S_PUT
);
3136 TAILQ_REMOVE(&ap
->ap_ccb_free
, ccb
, ccb_entry
);
3137 ccb
->ccb_xa
.state
= ATA_S_SETUP
;
3138 ccb
->ccb_xa
.flags
= 0;
3139 ccb
->ccb_xa
.at
= NULL
;
3141 lockmgr(&ap
->ap_ccb_lock
, LK_RELEASE
);
3147 ahci_put_ccb(struct ahci_ccb
*ccb
)
3149 struct ahci_port
*ap
= ccb
->ccb_port
;
3151 KKASSERT(ccb
->ccb_xa
.state
!= ATA_S_PUT
);
3152 KKASSERT((ap
->ap_sactive
& (1 << ccb
->ccb_slot
)) == 0);
3153 lockmgr(&ap
->ap_ccb_lock
, LK_EXCLUSIVE
);
3154 ccb
->ccb_xa
.state
= ATA_S_PUT
;
3155 ++ccb
->ccb_xa
.serial
;
3156 TAILQ_INSERT_TAIL(&ap
->ap_ccb_free
, ccb
, ccb_entry
);
3157 lockmgr(&ap
->ap_ccb_lock
, LK_RELEASE
);
3161 ahci_get_err_ccb(struct ahci_port
*ap
)
3163 struct ahci_ccb
*err_ccb
;
3167 /* No commands may be active on the chip. */
3169 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SNCQ
) {
3170 sact
= ahci_pread(ap
, AHCI_PREG_SACT
);
3172 kprintf("%s: ahci_get_err_ccb but SACT %08x != 0?\n",
3173 PORTNAME(ap
), sact
);
3176 ci
= ahci_pread(ap
, AHCI_PREG_CI
);
3178 kprintf("%s: ahci_get_err_ccb: ci not 0 (%08x)\n",
3182 KKASSERT((ap
->ap_flags
& AP_F_ERR_CCB_RESERVED
) == 0);
3183 ap
->ap_flags
|= AP_F_ERR_CCB_RESERVED
;
3185 /* Save outstanding command state. */
3186 ap
->ap_err_saved_active
= ap
->ap_active
;
3187 ap
->ap_err_saved_active_cnt
= ap
->ap_active_cnt
;
3188 ap
->ap_err_saved_sactive
= ap
->ap_sactive
;
3191 * Pretend we have no commands outstanding, so that completions won't
3194 ap
->ap_active
= ap
->ap_active_cnt
= ap
->ap_sactive
= 0;
3197 * Grab a CCB to use for error recovery. This should never fail, as
3198 * we ask atascsi to reserve one for us at init time.
3200 err_ccb
= ap
->ap_err_ccb
;
3201 KKASSERT(err_ccb
!= NULL
);
3202 err_ccb
->ccb_xa
.flags
= 0;
3203 err_ccb
->ccb_done
= ahci_empty_done
;
3209 ahci_put_err_ccb(struct ahci_ccb
*ccb
)
3211 struct ahci_port
*ap
= ccb
->ccb_port
;
3215 KKASSERT((ap
->ap_flags
& AP_F_ERR_CCB_RESERVED
) != 0);
3218 * No commands may be active on the chip
3220 if (ap
->ap_sc
->sc_cap
& AHCI_REG_CAP_SNCQ
) {
3221 sact
= ahci_pread(ap
, AHCI_PREG_SACT
);
3223 panic("ahci_port_err_ccb(%d) but SACT %08x != 0",
3224 ccb
->ccb_slot
, sact
);
3227 ci
= ahci_pread(ap
, AHCI_PREG_CI
);
3229 panic("ahci_put_err_ccb(%d) but CI %08x != 0 "
3230 "(act=%08x sact=%08x)\n",
3232 ap
->ap_active
, ap
->ap_sactive
);
3235 KKASSERT(ccb
== ap
->ap_err_ccb
);
3237 /* Restore outstanding command state */
3238 ap
->ap_sactive
= ap
->ap_err_saved_sactive
;
3239 ap
->ap_active_cnt
= ap
->ap_err_saved_active_cnt
;
3240 ap
->ap_active
= ap
->ap_err_saved_active
;
3242 ap
->ap_flags
&= ~AP_F_ERR_CCB_RESERVED
;
3246 * Read log page to get NCQ error.
3248 * NOTE: NCQ not currently supported on port multipliers. XXX
3251 ahci_port_read_ncq_error(struct ahci_port
*ap
, int target
)
3253 struct ata_log_page_10h
*log
;
3254 struct ahci_ccb
*ccb
;
3255 struct ahci_ccb
*ccb2
;
3256 struct ahci_cmd_hdr
*cmd_slot
;
3257 struct ata_fis_h2d
*fis
;
3261 kprintf("%s: READ LOG PAGE target %d\n", PORTNAME(ap
),
3266 * Prep error CCB for READ LOG EXT, page 10h, 1 sector.
3268 * Getting err_ccb clears active/sactive/active_cnt, putting
3269 * it back restores the fields.
3271 ccb
= ahci_get_err_ccb(ap
);
3272 ccb
->ccb_xa
.flags
= ATA_F_READ
| ATA_F_POLL
;
3273 ccb
->ccb_xa
.data
= ap
->ap_err_scratch
;
3274 ccb
->ccb_xa
.datalen
= 512;
3275 ccb
->ccb_xa
.complete
= ahci_dummy_done
;
3276 ccb
->ccb_xa
.at
= ap
->ap_ata
[target
];
3278 fis
= (struct ata_fis_h2d
*)ccb
->ccb_cmd_table
->cfis
;
3279 bzero(fis
, sizeof(*fis
));
3280 fis
->type
= ATA_FIS_TYPE_H2D
;
3281 fis
->flags
= ATA_H2D_FLAGS_CMD
| target
;
3282 fis
->command
= ATA_C_READ_LOG_EXT
;
3283 fis
->lba_low
= 0x10; /* queued error log page (10h) */
3284 fis
->sector_count
= 1; /* number of sectors (1) */
3285 fis
->sector_count_exp
= 0;
3286 fis
->lba_mid
= 0; /* starting offset */
3287 fis
->lba_mid_exp
= 0;
3290 cmd_slot
= ccb
->ccb_cmd_hdr
;
3291 cmd_slot
->flags
= htole16(5); /* FIS length: 5 DWORDS */
3293 if (ahci_load_prdt(ccb
) != 0) {
3298 ccb
->ccb_xa
.state
= ATA_S_PENDING
;
3299 if (ahci_poll(ccb
, 1000, ahci_quick_timeout
) != ATA_S_COMPLETE
) {
3301 ahci_unload_prdt(ccb
);
3304 ahci_unload_prdt(ccb
);
3307 * Success, extract failed register set and tags from the scratch
3310 log
= (struct ata_log_page_10h
*)ap
->ap_err_scratch
;
3311 if (log
->err_regs
.type
& ATA_LOG_10H_TYPE_NOTQUEUED
) {
3312 /* Not queued bit was set - wasn't an NCQ error? */
3313 kprintf("%s: read NCQ error page, but not an NCQ error?\n",
3317 /* Copy back the log record as a D2H register FIS. */
3318 err_slot
= log
->err_regs
.type
& ATA_LOG_10H_TYPE_TAG_MASK
;
3320 ccb2
= &ap
->ap_ccbs
[err_slot
];
3321 if (ccb2
->ccb_xa
.state
== ATA_S_ONCHIP
) {
3322 kprintf("%s: read NCQ error page slot=%d\n",
3323 ATANAME(ap
, ccb2
->ccb_xa
.at
),
3325 memcpy(&ccb2
->ccb_xa
.rfis
, &log
->err_regs
,
3326 sizeof(struct ata_fis_d2h
));
3327 ccb2
->ccb_xa
.rfis
.type
= ATA_FIS_TYPE_D2H
;
3328 ccb2
->ccb_xa
.rfis
.flags
= 0;
3330 kprintf("%s: read NCQ error page slot=%d, "
3331 "slot does not match any cmds\n",
3332 ATANAME(ccb2
->ccb_port
, ccb2
->ccb_xa
.at
),
3338 ahci_put_err_ccb(ccb
);
3339 kprintf("%s: DONE log page target %d err_slot=%d\n",
3340 PORTNAME(ap
), target
, err_slot
);
3345 * Allocate memory for various structures DMAd by hardware. The maximum
3346 * number of segments for these tags is 1 so the DMA memory will have a
3347 * single physical base address.
3349 struct ahci_dmamem
*
3350 ahci_dmamem_alloc(struct ahci_softc
*sc
, bus_dma_tag_t tag
)
3352 struct ahci_dmamem
*adm
;
3355 adm
= kmalloc(sizeof(*adm
), M_DEVBUF
, M_INTWAIT
| M_ZERO
);
3357 error
= bus_dmamem_alloc(tag
, (void **)&adm
->adm_kva
,
3358 BUS_DMA_ZERO
, &adm
->adm_map
);
3361 error
= bus_dmamap_load(tag
, adm
->adm_map
,
3363 bus_dma_tag_getmaxsize(tag
),
3364 ahci_dmamem_saveseg
, &adm
->adm_busaddr
,
3369 bus_dmamap_destroy(tag
, adm
->adm_map
);
3370 adm
->adm_map
= NULL
;
3371 adm
->adm_tag
= NULL
;
3372 adm
->adm_kva
= NULL
;
3374 kfree(adm
, M_DEVBUF
);
3382 ahci_dmamem_saveseg(void *info
, bus_dma_segment_t
*segs
, int nsegs
, int error
)
3384 KKASSERT(error
== 0);
3385 KKASSERT(nsegs
== 1);
3386 *(bus_addr_t
*)info
= segs
->ds_addr
;
3391 ahci_dmamem_free(struct ahci_softc
*sc
, struct ahci_dmamem
*adm
)
3394 bus_dmamap_unload(adm
->adm_tag
, adm
->adm_map
);
3395 bus_dmamap_destroy(adm
->adm_tag
, adm
->adm_map
);
3396 adm
->adm_map
= NULL
;
3397 adm
->adm_tag
= NULL
;
3398 adm
->adm_kva
= NULL
;
3400 kfree(adm
, M_DEVBUF
);
3404 ahci_read(struct ahci_softc
*sc
, bus_size_t r
)
3406 bus_space_barrier(sc
->sc_iot
, sc
->sc_ioh
, r
, 4,
3407 BUS_SPACE_BARRIER_READ
);
3408 return (bus_space_read_4(sc
->sc_iot
, sc
->sc_ioh
, r
));
3412 ahci_write(struct ahci_softc
*sc
, bus_size_t r
, u_int32_t v
)
3414 bus_space_write_4(sc
->sc_iot
, sc
->sc_ioh
, r
, v
);
3415 bus_space_barrier(sc
->sc_iot
, sc
->sc_ioh
, r
, 4,
3416 BUS_SPACE_BARRIER_WRITE
);
3420 ahci_pread(struct ahci_port
*ap
, bus_size_t r
)
3422 bus_space_barrier(ap
->ap_sc
->sc_iot
, ap
->ap_ioh
, r
, 4,
3423 BUS_SPACE_BARRIER_READ
);
3424 return (bus_space_read_4(ap
->ap_sc
->sc_iot
, ap
->ap_ioh
, r
));
3428 ahci_pwrite(struct ahci_port
*ap
, bus_size_t r
, u_int32_t v
)
3430 bus_space_write_4(ap
->ap_sc
->sc_iot
, ap
->ap_ioh
, r
, v
);
3431 bus_space_barrier(ap
->ap_sc
->sc_iot
, ap
->ap_ioh
, r
, 4,
3432 BUS_SPACE_BARRIER_WRITE
);
3436 * Wait up to (timeout) milliseconds for the masked port register to
3439 * Timeout is in milliseconds.
3442 ahci_pwait_eq(struct ahci_port
*ap
, int timeout
,
3443 bus_size_t r
, u_int32_t mask
, u_int32_t target
)
3448 * Loop hard up to 100uS
3450 for (t
= 0; t
< 100; ++t
) {
3451 if ((ahci_pread(ap
, r
) & mask
) == target
)
3453 ahci_os_hardsleep(1); /* us */
3457 timeout
-= ahci_os_softsleep();
3458 if ((ahci_pread(ap
, r
) & mask
) == target
)
3460 } while (timeout
> 0);
3465 ahci_wait_ne(struct ahci_softc
*sc
, bus_size_t r
, u_int32_t mask
,
3471 * Loop hard up to 100uS
3473 for (t
= 0; t
< 100; ++t
) {
3474 if ((ahci_read(sc
, r
) & mask
) != target
)
3476 ahci_os_hardsleep(1); /* us */
3480 * And one millisecond the slow way
3484 t
-= ahci_os_softsleep();
3485 if ((ahci_read(sc
, r
) & mask
) != target
)
3494 * Acquire an ata transfer.
3496 * Pass a NULL at for direct-attached transfers, and a non-NULL at for
3497 * targets that go through the port multiplier.
3500 ahci_ata_get_xfer(struct ahci_port
*ap
, struct ata_port
*at
)
3502 struct ahci_ccb
*ccb
;
3504 ccb
= ahci_get_ccb(ap
);
3506 DPRINTF(AHCI_D_XFER
, "%s: ahci_ata_get_xfer: NULL ccb\n",
3511 DPRINTF(AHCI_D_XFER
, "%s: ahci_ata_get_xfer got slot %d\n",
3512 PORTNAME(ap
), ccb
->ccb_slot
);
3514 bzero(ccb
->ccb_xa
.fis
, sizeof(*ccb
->ccb_xa
.fis
));
3515 ccb
->ccb_xa
.at
= at
;
3516 ccb
->ccb_xa
.fis
->type
= ATA_FIS_TYPE_H2D
;
3518 return (&ccb
->ccb_xa
);
3522 ahci_ata_put_xfer(struct ata_xfer
*xa
)
3524 struct ahci_ccb
*ccb
= (struct ahci_ccb
*)xa
;
3526 DPRINTF(AHCI_D_XFER
, "ahci_ata_put_xfer slot %d\n", ccb
->ccb_slot
);
3532 ahci_ata_cmd(struct ata_xfer
*xa
)
3534 struct ahci_ccb
*ccb
= (struct ahci_ccb
*)xa
;
3535 struct ahci_cmd_hdr
*cmd_slot
;
3537 KKASSERT(xa
->state
== ATA_S_SETUP
);
3539 if (ccb
->ccb_port
->ap_state
== AP_S_FATAL_ERROR
)
3541 ccb
->ccb_done
= ahci_ata_cmd_done
;
3543 cmd_slot
= ccb
->ccb_cmd_hdr
;
3544 cmd_slot
->flags
= htole16(5); /* FIS length (in DWORDs) */
3545 if (ccb
->ccb_xa
.at
) {
3546 cmd_slot
->flags
|= htole16(ccb
->ccb_xa
.at
->at_target
<<
3547 AHCI_CMD_LIST_FLAG_PMP_SHIFT
);
3550 if (xa
->flags
& ATA_F_WRITE
)
3551 cmd_slot
->flags
|= htole16(AHCI_CMD_LIST_FLAG_W
);
3553 if (xa
->flags
& ATA_F_PACKET
)
3554 cmd_slot
->flags
|= htole16(AHCI_CMD_LIST_FLAG_A
);
3556 if (ahci_load_prdt(ccb
) != 0)
3559 xa
->state
= ATA_S_PENDING
;
3561 if (xa
->flags
& ATA_F_POLL
)
3562 return (ahci_poll(ccb
, xa
->timeout
, ahci_ata_cmd_timeout
));
3565 KKASSERT((xa
->flags
& ATA_F_TIMEOUT_EXPIRED
) == 0);
3566 xa
->flags
|= ATA_F_TIMEOUT_DESIRED
;
3573 xa
->state
= ATA_S_ERROR
;
3576 return (ATA_S_ERROR
);
3580 ahci_ata_cmd_done(struct ahci_ccb
*ccb
)
3582 struct ata_xfer
*xa
= &ccb
->ccb_xa
;
3586 * NOTE: Callout does not lock port and may race us modifying
3587 * the flags, so make sure its stopped.
3589 * A callout race can clean up the ccb. A change in the
3590 * serial number should catch this condition.
3592 if (xa
->flags
& ATA_F_TIMEOUT_RUNNING
) {
3593 serial
= ccb
->ccb_xa
.serial
;
3594 callout_stop_sync(&ccb
->ccb_timeout
);
3595 if (serial
!= ccb
->ccb_xa
.serial
) {
3596 kprintf("%s: Warning: timeout race ccb %p\n",
3597 PORTNAME(ccb
->ccb_port
), ccb
);
3600 xa
->flags
&= ~ATA_F_TIMEOUT_RUNNING
;
3602 xa
->flags
&= ~(ATA_F_TIMEOUT_DESIRED
| ATA_F_TIMEOUT_EXPIRED
);
3603 ccb
->ccb_port
->ap_expired
&= ~(1 << ccb
->ccb_slot
);
3605 KKASSERT(xa
->state
!= ATA_S_ONCHIP
&& xa
->state
!= ATA_S_PUT
);
3606 ahci_unload_prdt(ccb
);
3608 if (xa
->state
!= ATA_S_TIMEOUT
)
3613 * Timeout from callout, MPSAFE - nothing can mess with the CCB's flags
3614 * while the callout is runing.
3616 * We can't safely get the port lock here or delay, we could block
3617 * the callout thread.
3620 ahci_ata_cmd_timeout_unserialized(void *arg
)
3622 struct ahci_ccb
*ccb
= arg
;
3623 struct ahci_port
*ap
= ccb
->ccb_port
;
3625 KKASSERT(ccb
->ccb_xa
.flags
& ATA_F_TIMEOUT_RUNNING
);
3626 ccb
->ccb_xa
.flags
&= ~ATA_F_TIMEOUT_RUNNING
;
3627 ccb
->ccb_xa
.flags
|= ATA_F_TIMEOUT_EXPIRED
;
3628 ahci_os_signal_port_thread(ap
, AP_SIGF_TIMEOUT
);
3632 * Timeout code, typically called when the port command processor is running.
3634 * We have to be very very careful here. We cannot stop the port unless
3635 * CR is already clear or the only active commands remaining are timed-out
3636 * ones. Otherwise stopping the port will race the command processor and
3637 * we can lose events. While we can theoretically just restart everything
3638 * that could result in a double-issue which will not work for ATAPI commands.
3641 ahci_ata_cmd_timeout(struct ahci_ccb
*ccb
)
3643 struct ata_xfer
*xa
= &ccb
->ccb_xa
;
3644 struct ahci_port
*ap
= ccb
->ccb_port
;
3645 struct ata_port
*at
;
3650 at
= ccb
->ccb_xa
.at
;
3652 kprintf("%s: CMD TIMEOUT state=%d slot=%d\n"
3653 "\tglb-status 0x%08x\n"
3655 "\tport_status 0x%b\n"
3656 "\tsactive=%08x active=%08x expired=%08x\n"
3657 "\t sact=%08x ci=%08x\n"
3660 ccb
->ccb_xa
.state
, ccb
->ccb_slot
,
3661 ahci_read(ap
->ap_sc
, AHCI_REG_IS
),
3662 ahci_pread(ap
, AHCI_PREG_CMD
), AHCI_PFMT_CMD
,
3663 ahci_pread(ap
, AHCI_PREG_IS
), AHCI_PFMT_IS
,
3664 ap
->ap_sactive
, ap
->ap_active
, ap
->ap_expired
,
3665 ahci_pread(ap
, AHCI_PREG_SACT
),
3666 ahci_pread(ap
, AHCI_PREG_CI
),
3667 ahci_pread(ap
, AHCI_PREG_TFD
), AHCI_PFMT_TFD_STS
3672 * NOTE: Timeout will not be running if the command was polled.
3673 * If we got here at least one of these flags should be set.
3675 KKASSERT(xa
->flags
& (ATA_F_POLL
| ATA_F_TIMEOUT_DESIRED
|
3676 ATA_F_TIMEOUT_RUNNING
));
3677 xa
->flags
&= ~(ATA_F_TIMEOUT_RUNNING
| ATA_F_TIMEOUT_EXPIRED
);
3679 if (ccb
->ccb_xa
.state
== ATA_S_PENDING
) {
3680 TAILQ_REMOVE(&ap
->ap_ccb_pending
, ccb
, ccb_entry
);
3681 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
3684 ahci_issue_pending_commands(ap
, NULL
);
3687 if (ccb
->ccb_xa
.state
!= ATA_S_ONCHIP
) {
3688 kprintf("%s: Unexpected state during timeout: %d\n",
3689 ATANAME(ap
, at
), ccb
->ccb_xa
.state
);
3694 * Ok, we can only get this command off the chip if CR is inactive
3695 * or if the only commands running on the chip are all expired.
3696 * Otherwise we have to wait until the port is in a safe state.
3697 * Use the ST bit here instead of the CR bit in case the CR bit is
3698 * not implemented via the F_IGN_CR quirk.
3700 * Do not set state here, it will cause polls to return when the
3701 * ccb is not yet off the chip.
3703 ap
->ap_expired
|= 1 << ccb
->ccb_slot
;
3705 if ((ahci_pread(ap
, AHCI_PREG_CMD
) & AHCI_PREG_CMD_ST
) &&
3706 (ap
->ap_active
| ap
->ap_sactive
) != ap
->ap_expired
) {
3708 * If using FBSS or NCQ we can't safely stop the port
3711 kprintf("%s: Deferred timeout until its safe, slot %d\n",
3712 ATANAME(ap
, at
), ccb
->ccb_slot
);
3717 * We can safely stop the port and process all expired ccb's,
3718 * which will include our current ccb.
3720 ci_saved
= (ap
->ap_sactive
) ? ahci_pread(ap
, AHCI_PREG_SACT
) :
3721 ahci_pread(ap
, AHCI_PREG_CI
);
3722 ahci_port_stop(ap
, 0);
3724 while (ap
->ap_expired
) {
3725 slot
= ffs(ap
->ap_expired
) - 1;
3726 ap
->ap_expired
&= ~(1 << slot
);
3727 ci_saved
&= ~(1 << slot
);
3728 ccb
= &ap
->ap_ccbs
[slot
];
3729 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
3730 if (ccb
->ccb_xa
.flags
& ATA_F_NCQ
) {
3731 KKASSERT(ap
->ap_sactive
& (1 << slot
));
3732 ap
->ap_sactive
&= ~(1 << slot
);
3734 KKASSERT(ap
->ap_active
& (1 << slot
));
3735 ap
->ap_active
&= ~(1 << slot
);
3736 --ap
->ap_active_cnt
;
3739 ccb
->ccb_xa
.complete(&ccb
->ccb_xa
);
3741 /* ccb invalid now */
3744 * We can safely CLO the port to clear any BSY/DRQ, a case which
3745 * can occur with port multipliers. This will unbrick the port
3746 * and allow commands to other targets behind the PM continue.
3749 * Finally, once the port has been restarted we can issue any
3750 * previously saved pending commands, and run the port interrupt
3751 * code to handle any completions which may have occured when
3754 if (ahci_pread(ap
, AHCI_PREG_TFD
) &
3755 (AHCI_PREG_TFD_STS_BSY
| AHCI_PREG_TFD_STS_DRQ
)) {
3756 kprintf("%s: Warning, issuing CLO after timeout\n",
3760 ahci_port_start(ap
);
3763 * We absolutely must make sure the chipset cleared activity on
3764 * all slots. This sometimes might not happen due to races with
3765 * a chipset interrupt which stops the port before we can manage
3766 * to. For some reason some chipsets don't clear the active
3767 * commands when we turn off CMD_ST after the chip has stopped
3768 * operations itself.
3770 if (ahci_pactive(ap
) != 0) {
3771 ahci_port_stop(ap
, 0);
3772 ahci_port_start(ap
);
3773 if ((mask
= ahci_pactive(ap
)) != 0) {
3774 kprintf("%s: quick-timeout: chipset failed "
3775 "to clear active cmds %08x\n",
3776 PORTNAME(ap
), mask
);
3779 ahci_issue_saved_commands(ap
, ci_saved
& ~ap
->ap_expired
);
3780 ahci_issue_pending_commands(ap
, NULL
);
3781 ahci_port_intr(ap
, 0);
3785 * Issue a previously saved set of commands
3788 ahci_issue_saved_commands(struct ahci_port
*ap
, u_int32_t ci_saved
)
3791 KKASSERT(!((ap
->ap_active
& ci_saved
) &&
3792 (ap
->ap_sactive
& ci_saved
)));
3793 KKASSERT((ci_saved
& ap
->ap_expired
) == 0);
3794 if (ap
->ap_sactive
& ci_saved
)
3795 ahci_pwrite(ap
, AHCI_PREG_SACT
, ci_saved
);
3796 ahci_pwrite(ap
, AHCI_PREG_CI
, ci_saved
);
3801 * Used by the softreset, pmprobe, and read_ncq_error only, in very
3802 * specialized, controlled circumstances.
3804 * Only one command may be pending.
3807 ahci_quick_timeout(struct ahci_ccb
*ccb
)
3809 struct ahci_port
*ap
= ccb
->ccb_port
;
3812 switch (ccb
->ccb_xa
.state
) {
3814 TAILQ_REMOVE(&ap
->ap_ccb_pending
, ccb
, ccb_entry
);
3815 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
3819 * We have to clear the command on-chip.
3821 KKASSERT(ap
->ap_active
== (1 << ccb
->ccb_slot
) &&
3822 ap
->ap_sactive
== 0);
3823 ahci_port_stop(ap
, 0);
3824 ahci_port_start(ap
);
3825 if (ahci_pactive(ap
) != 0) {
3826 ahci_port_stop(ap
, 0);
3827 ahci_port_start(ap
);
3828 if ((mask
= ahci_pactive(ap
)) != 0) {
3829 kprintf("%s: quick-timeout: chipset failed "
3830 "to clear active cmds %08x\n",
3831 PORTNAME(ap
), mask
);
3835 ccb
->ccb_xa
.state
= ATA_S_TIMEOUT
;
3836 ap
->ap_active
&= ~(1 << ccb
->ccb_slot
);
3837 KKASSERT(ap
->ap_active_cnt
> 0);
3838 --ap
->ap_active_cnt
;
3841 panic("%s: ahci_quick_timeout: ccb in bad state %d",
3842 ATANAME(ap
, ccb
->ccb_xa
.at
), ccb
->ccb_xa
.state
);
3847 ahci_dummy_done(struct ata_xfer
*xa
)
3852 ahci_empty_done(struct ahci_ccb
*ccb
)
3857 ahci_set_feature(struct ahci_port
*ap
, struct ata_port
*atx
,
3858 int feature
, int enable
)
3860 struct ata_port
*at
;
3861 struct ata_xfer
*xa
;
3864 at
= atx
? atx
: ap
->ap_ata
[0];
3866 xa
= ahci_ata_get_xfer(ap
, atx
);
3868 xa
->fis
->type
= ATA_FIS_TYPE_H2D
;
3869 xa
->fis
->flags
= ATA_H2D_FLAGS_CMD
| at
->at_target
;
3870 xa
->fis
->command
= ATA_C_SET_FEATURES
;
3871 xa
->fis
->features
= enable
? ATA_SF_SATAFT_ENA
: ATA_SF_SATAFT_DIS
;
3872 xa
->fis
->sector_count
= feature
;
3873 xa
->fis
->control
= ATA_FIS_CONTROL_4BIT
;
3875 xa
->complete
= ahci_dummy_done
;
3877 xa
->flags
= ATA_F_POLL
;
3880 if (ahci_ata_cmd(xa
) == ATA_S_COMPLETE
)
3884 ahci_ata_put_xfer(xa
);