2 * libahci.c - Common AHCI SATA low-level routines
4 * Maintained by: Tejun Heo <tj@kernel.org>
5 * Please ALWAYS copy linux-ide@vger.kernel.org
8 * Copyright 2004-2005 Red Hat, Inc.
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2, or (at your option)
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program; see the file COPYING. If not, write to
23 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
26 * libata documentation is available via 'make {ps|pdf}docs',
27 * as Documentation/DocBook/libata.*
29 * AHCI hardware documentation:
30 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
31 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
35 #include <linux/kernel.h>
36 #include <linux/gfp.h>
37 #include <linux/module.h>
38 #include <linux/blkdev.h>
39 #include <linux/delay.h>
40 #include <linux/interrupt.h>
41 #include <linux/dma-mapping.h>
42 #include <linux/device.h>
43 #include <scsi/scsi_host.h>
44 #include <scsi/scsi_cmnd.h>
45 #include <linux/libata.h>
46 #include <linux/pci.h>
50 static int ahci_skip_host_reset
;
52 EXPORT_SYMBOL_GPL(ahci_ignore_sss
);
54 module_param_named(skip_host_reset
, ahci_skip_host_reset
, int, 0444);
55 MODULE_PARM_DESC(skip_host_reset
, "skip global host reset (0=don't skip, 1=skip)");
57 module_param_named(ignore_sss
, ahci_ignore_sss
, int, 0444);
58 MODULE_PARM_DESC(ignore_sss
, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");
60 static int ahci_set_lpm(struct ata_link
*link
, enum ata_lpm_policy policy
,
62 static ssize_t
ahci_led_show(struct ata_port
*ap
, char *buf
);
63 static ssize_t
ahci_led_store(struct ata_port
*ap
, const char *buf
,
65 static ssize_t
ahci_transmit_led_message(struct ata_port
*ap
, u32 state
,
70 static int ahci_scr_read(struct ata_link
*link
, unsigned int sc_reg
, u32
*val
);
71 static int ahci_scr_write(struct ata_link
*link
, unsigned int sc_reg
, u32 val
);
72 static bool ahci_qc_fill_rtf(struct ata_queued_cmd
*qc
);
73 static int ahci_port_start(struct ata_port
*ap
);
74 static void ahci_port_stop(struct ata_port
*ap
);
75 static void ahci_qc_prep(struct ata_queued_cmd
*qc
);
76 static int ahci_pmp_qc_defer(struct ata_queued_cmd
*qc
);
77 static void ahci_freeze(struct ata_port
*ap
);
78 static void ahci_thaw(struct ata_port
*ap
);
79 static void ahci_set_aggressive_devslp(struct ata_port
*ap
, bool sleep
);
80 static void ahci_enable_fbs(struct ata_port
*ap
);
81 static void ahci_disable_fbs(struct ata_port
*ap
);
82 static void ahci_pmp_attach(struct ata_port
*ap
);
83 static void ahci_pmp_detach(struct ata_port
*ap
);
84 static int ahci_softreset(struct ata_link
*link
, unsigned int *class,
85 unsigned long deadline
);
86 static int ahci_pmp_retry_softreset(struct ata_link
*link
, unsigned int *class,
87 unsigned long deadline
);
88 static int ahci_hardreset(struct ata_link
*link
, unsigned int *class,
89 unsigned long deadline
);
90 static void ahci_postreset(struct ata_link
*link
, unsigned int *class);
91 static void ahci_post_internal_cmd(struct ata_queued_cmd
*qc
);
92 static void ahci_dev_config(struct ata_device
*dev
);
94 static int ahci_port_suspend(struct ata_port
*ap
, pm_message_t mesg
);
96 static ssize_t
ahci_activity_show(struct ata_device
*dev
, char *buf
);
97 static ssize_t
ahci_activity_store(struct ata_device
*dev
,
98 enum sw_activity val
);
99 static void ahci_init_sw_activity(struct ata_link
*link
);
101 static ssize_t
ahci_show_host_caps(struct device
*dev
,
102 struct device_attribute
*attr
, char *buf
);
103 static ssize_t
ahci_show_host_cap2(struct device
*dev
,
104 struct device_attribute
*attr
, char *buf
);
105 static ssize_t
ahci_show_host_version(struct device
*dev
,
106 struct device_attribute
*attr
, char *buf
);
107 static ssize_t
ahci_show_port_cmd(struct device
*dev
,
108 struct device_attribute
*attr
, char *buf
);
109 static ssize_t
ahci_read_em_buffer(struct device
*dev
,
110 struct device_attribute
*attr
, char *buf
);
111 static ssize_t
ahci_store_em_buffer(struct device
*dev
,
112 struct device_attribute
*attr
,
113 const char *buf
, size_t size
);
114 static ssize_t
ahci_show_em_supported(struct device
*dev
,
115 struct device_attribute
*attr
, char *buf
);
117 static DEVICE_ATTR(ahci_host_caps
, S_IRUGO
, ahci_show_host_caps
, NULL
);
118 static DEVICE_ATTR(ahci_host_cap2
, S_IRUGO
, ahci_show_host_cap2
, NULL
);
119 static DEVICE_ATTR(ahci_host_version
, S_IRUGO
, ahci_show_host_version
, NULL
);
120 static DEVICE_ATTR(ahci_port_cmd
, S_IRUGO
, ahci_show_port_cmd
, NULL
);
121 static DEVICE_ATTR(em_buffer
, S_IWUSR
| S_IRUGO
,
122 ahci_read_em_buffer
, ahci_store_em_buffer
);
123 static DEVICE_ATTR(em_message_supported
, S_IRUGO
, ahci_show_em_supported
, NULL
);
125 struct device_attribute
*ahci_shost_attrs
[] = {
126 &dev_attr_link_power_management_policy
,
127 &dev_attr_em_message_type
,
128 &dev_attr_em_message
,
129 &dev_attr_ahci_host_caps
,
130 &dev_attr_ahci_host_cap2
,
131 &dev_attr_ahci_host_version
,
132 &dev_attr_ahci_port_cmd
,
134 &dev_attr_em_message_supported
,
137 EXPORT_SYMBOL_GPL(ahci_shost_attrs
);
139 struct device_attribute
*ahci_sdev_attrs
[] = {
140 &dev_attr_sw_activity
,
141 &dev_attr_unload_heads
,
144 EXPORT_SYMBOL_GPL(ahci_sdev_attrs
);
146 struct ata_port_operations ahci_ops
= {
147 .inherits
= &sata_pmp_port_ops
,
149 .qc_defer
= ahci_pmp_qc_defer
,
150 .qc_prep
= ahci_qc_prep
,
151 .qc_issue
= ahci_qc_issue
,
152 .qc_fill_rtf
= ahci_qc_fill_rtf
,
154 .freeze
= ahci_freeze
,
156 .softreset
= ahci_softreset
,
157 .hardreset
= ahci_hardreset
,
158 .postreset
= ahci_postreset
,
159 .pmp_softreset
= ahci_softreset
,
160 .error_handler
= ahci_error_handler
,
161 .post_internal_cmd
= ahci_post_internal_cmd
,
162 .dev_config
= ahci_dev_config
,
164 .scr_read
= ahci_scr_read
,
165 .scr_write
= ahci_scr_write
,
166 .pmp_attach
= ahci_pmp_attach
,
167 .pmp_detach
= ahci_pmp_detach
,
169 .set_lpm
= ahci_set_lpm
,
170 .em_show
= ahci_led_show
,
171 .em_store
= ahci_led_store
,
172 .sw_activity_show
= ahci_activity_show
,
173 .sw_activity_store
= ahci_activity_store
,
174 .transmit_led_message
= ahci_transmit_led_message
,
176 .port_suspend
= ahci_port_suspend
,
177 .port_resume
= ahci_port_resume
,
179 .port_start
= ahci_port_start
,
180 .port_stop
= ahci_port_stop
,
182 EXPORT_SYMBOL_GPL(ahci_ops
);
184 struct ata_port_operations ahci_pmp_retry_srst_ops
= {
185 .inherits
= &ahci_ops
,
186 .softreset
= ahci_pmp_retry_softreset
,
188 EXPORT_SYMBOL_GPL(ahci_pmp_retry_srst_ops
);
190 static bool ahci_em_messages __read_mostly
= true;
191 EXPORT_SYMBOL_GPL(ahci_em_messages
);
192 module_param(ahci_em_messages
, bool, 0444);
193 /* add other LED protocol types when they become supported */
194 MODULE_PARM_DESC(ahci_em_messages
,
195 "AHCI Enclosure Management Message control (0 = off, 1 = on)");
197 /* device sleep idle timeout in ms */
198 static int devslp_idle_timeout __read_mostly
= 1000;
199 module_param(devslp_idle_timeout
, int, 0644);
200 MODULE_PARM_DESC(devslp_idle_timeout
, "device sleep idle timeout");
202 static void ahci_enable_ahci(void __iomem
*mmio
)
207 /* turn on AHCI_EN */
208 tmp
= readl(mmio
+ HOST_CTL
);
209 if (tmp
& HOST_AHCI_EN
)
212 /* Some controllers need AHCI_EN to be written multiple times.
213 * Try a few times before giving up.
215 for (i
= 0; i
< 5; i
++) {
217 writel(tmp
, mmio
+ HOST_CTL
);
218 tmp
= readl(mmio
+ HOST_CTL
); /* flush && sanity check */
219 if (tmp
& HOST_AHCI_EN
)
227 static ssize_t
ahci_show_host_caps(struct device
*dev
,
228 struct device_attribute
*attr
, char *buf
)
230 struct Scsi_Host
*shost
= class_to_shost(dev
);
231 struct ata_port
*ap
= ata_shost_to_port(shost
);
232 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
234 return sprintf(buf
, "%x\n", hpriv
->cap
);
237 static ssize_t
ahci_show_host_cap2(struct device
*dev
,
238 struct device_attribute
*attr
, char *buf
)
240 struct Scsi_Host
*shost
= class_to_shost(dev
);
241 struct ata_port
*ap
= ata_shost_to_port(shost
);
242 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
244 return sprintf(buf
, "%x\n", hpriv
->cap2
);
247 static ssize_t
ahci_show_host_version(struct device
*dev
,
248 struct device_attribute
*attr
, char *buf
)
250 struct Scsi_Host
*shost
= class_to_shost(dev
);
251 struct ata_port
*ap
= ata_shost_to_port(shost
);
252 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
253 void __iomem
*mmio
= hpriv
->mmio
;
255 return sprintf(buf
, "%x\n", readl(mmio
+ HOST_VERSION
));
258 static ssize_t
ahci_show_port_cmd(struct device
*dev
,
259 struct device_attribute
*attr
, char *buf
)
261 struct Scsi_Host
*shost
= class_to_shost(dev
);
262 struct ata_port
*ap
= ata_shost_to_port(shost
);
263 void __iomem
*port_mmio
= ahci_port_base(ap
);
265 return sprintf(buf
, "%x\n", readl(port_mmio
+ PORT_CMD
));
268 static ssize_t
ahci_read_em_buffer(struct device
*dev
,
269 struct device_attribute
*attr
, char *buf
)
271 struct Scsi_Host
*shost
= class_to_shost(dev
);
272 struct ata_port
*ap
= ata_shost_to_port(shost
);
273 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
274 void __iomem
*mmio
= hpriv
->mmio
;
275 void __iomem
*em_mmio
= mmio
+ hpriv
->em_loc
;
281 spin_lock_irqsave(ap
->lock
, flags
);
283 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
284 if (!(ap
->flags
& ATA_FLAG_EM
) || em_ctl
& EM_CTL_XMT
||
285 !(hpriv
->em_msg_type
& EM_MSG_TYPE_SGPIO
)) {
286 spin_unlock_irqrestore(ap
->lock
, flags
);
290 if (!(em_ctl
& EM_CTL_MR
)) {
291 spin_unlock_irqrestore(ap
->lock
, flags
);
295 if (!(em_ctl
& EM_CTL_SMB
))
296 em_mmio
+= hpriv
->em_buf_sz
;
298 count
= hpriv
->em_buf_sz
;
300 /* the count should not be larger than PAGE_SIZE */
301 if (count
> PAGE_SIZE
) {
302 if (printk_ratelimit())
304 "EM read buffer size too large: "
305 "buffer size %u, page size %lu\n",
306 hpriv
->em_buf_sz
, PAGE_SIZE
);
310 for (i
= 0; i
< count
; i
+= 4) {
311 msg
= readl(em_mmio
+ i
);
313 buf
[i
+ 1] = (msg
>> 8) & 0xff;
314 buf
[i
+ 2] = (msg
>> 16) & 0xff;
315 buf
[i
+ 3] = (msg
>> 24) & 0xff;
318 spin_unlock_irqrestore(ap
->lock
, flags
);
323 static ssize_t
ahci_store_em_buffer(struct device
*dev
,
324 struct device_attribute
*attr
,
325 const char *buf
, size_t size
)
327 struct Scsi_Host
*shost
= class_to_shost(dev
);
328 struct ata_port
*ap
= ata_shost_to_port(shost
);
329 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
330 void __iomem
*mmio
= hpriv
->mmio
;
331 void __iomem
*em_mmio
= mmio
+ hpriv
->em_loc
;
332 const unsigned char *msg_buf
= buf
;
337 /* check size validity */
338 if (!(ap
->flags
& ATA_FLAG_EM
) ||
339 !(hpriv
->em_msg_type
& EM_MSG_TYPE_SGPIO
) ||
340 size
% 4 || size
> hpriv
->em_buf_sz
)
343 spin_lock_irqsave(ap
->lock
, flags
);
345 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
346 if (em_ctl
& EM_CTL_TM
) {
347 spin_unlock_irqrestore(ap
->lock
, flags
);
351 for (i
= 0; i
< size
; i
+= 4) {
352 msg
= msg_buf
[i
] | msg_buf
[i
+ 1] << 8 |
353 msg_buf
[i
+ 2] << 16 | msg_buf
[i
+ 3] << 24;
354 writel(msg
, em_mmio
+ i
);
357 writel(em_ctl
| EM_CTL_TM
, mmio
+ HOST_EM_CTL
);
359 spin_unlock_irqrestore(ap
->lock
, flags
);
364 static ssize_t
ahci_show_em_supported(struct device
*dev
,
365 struct device_attribute
*attr
, char *buf
)
367 struct Scsi_Host
*shost
= class_to_shost(dev
);
368 struct ata_port
*ap
= ata_shost_to_port(shost
);
369 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
370 void __iomem
*mmio
= hpriv
->mmio
;
373 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
375 return sprintf(buf
, "%s%s%s%s\n",
376 em_ctl
& EM_CTL_LED
? "led " : "",
377 em_ctl
& EM_CTL_SAFTE
? "saf-te " : "",
378 em_ctl
& EM_CTL_SES
? "ses-2 " : "",
379 em_ctl
& EM_CTL_SGPIO
? "sgpio " : "");
383 * ahci_save_initial_config - Save and fixup initial config values
384 * @dev: target AHCI device
385 * @hpriv: host private area to store config values
387 * Some registers containing configuration info might be setup by
388 * BIOS and might be cleared on reset. This function saves the
389 * initial values of those registers into @hpriv such that they
390 * can be restored after controller reset.
392 * If inconsistent, config values are fixed up by this function.
394 * If it is not set already this function sets hpriv->start_engine to
400 void ahci_save_initial_config(struct device
*dev
, struct ahci_host_priv
*hpriv
)
402 void __iomem
*mmio
= hpriv
->mmio
;
403 u32 cap
, cap2
, vers
, port_map
;
406 /* make sure AHCI mode is enabled before accessing CAP */
407 ahci_enable_ahci(mmio
);
409 /* Values prefixed with saved_ are written back to host after
410 * reset. Values without are used for driver operation.
412 hpriv
->saved_cap
= cap
= readl(mmio
+ HOST_CAP
);
413 hpriv
->saved_port_map
= port_map
= readl(mmio
+ HOST_PORTS_IMPL
);
415 /* CAP2 register is only defined for AHCI 1.2 and later */
416 vers
= readl(mmio
+ HOST_VERSION
);
417 if ((vers
>> 16) > 1 ||
418 ((vers
>> 16) == 1 && (vers
& 0xFFFF) >= 0x200))
419 hpriv
->saved_cap2
= cap2
= readl(mmio
+ HOST_CAP2
);
421 hpriv
->saved_cap2
= cap2
= 0;
423 /* some chips have errata preventing 64bit use */
424 if ((cap
& HOST_CAP_64
) && (hpriv
->flags
& AHCI_HFLAG_32BIT_ONLY
)) {
425 dev_info(dev
, "controller can't do 64bit DMA, forcing 32bit\n");
429 if ((cap
& HOST_CAP_NCQ
) && (hpriv
->flags
& AHCI_HFLAG_NO_NCQ
)) {
430 dev_info(dev
, "controller can't do NCQ, turning off CAP_NCQ\n");
431 cap
&= ~HOST_CAP_NCQ
;
434 if (!(cap
& HOST_CAP_NCQ
) && (hpriv
->flags
& AHCI_HFLAG_YES_NCQ
)) {
435 dev_info(dev
, "controller can do NCQ, turning on CAP_NCQ\n");
439 if ((cap
& HOST_CAP_PMP
) && (hpriv
->flags
& AHCI_HFLAG_NO_PMP
)) {
440 dev_info(dev
, "controller can't do PMP, turning off CAP_PMP\n");
441 cap
&= ~HOST_CAP_PMP
;
444 if ((cap
& HOST_CAP_SNTF
) && (hpriv
->flags
& AHCI_HFLAG_NO_SNTF
)) {
446 "controller can't do SNTF, turning off CAP_SNTF\n");
447 cap
&= ~HOST_CAP_SNTF
;
450 if ((cap2
& HOST_CAP2_SDS
) && (hpriv
->flags
& AHCI_HFLAG_NO_DEVSLP
)) {
452 "controller can't do DEVSLP, turning off\n");
453 cap2
&= ~HOST_CAP2_SDS
;
454 cap2
&= ~HOST_CAP2_SADM
;
457 if (!(cap
& HOST_CAP_FBS
) && (hpriv
->flags
& AHCI_HFLAG_YES_FBS
)) {
458 dev_info(dev
, "controller can do FBS, turning on CAP_FBS\n");
462 if ((cap
& HOST_CAP_FBS
) && (hpriv
->flags
& AHCI_HFLAG_NO_FBS
)) {
463 dev_info(dev
, "controller can't do FBS, turning off CAP_FBS\n");
464 cap
&= ~HOST_CAP_FBS
;
467 if (hpriv
->force_port_map
&& port_map
!= hpriv
->force_port_map
) {
468 dev_info(dev
, "forcing port_map 0x%x -> 0x%x\n",
469 port_map
, hpriv
->force_port_map
);
470 port_map
= hpriv
->force_port_map
;
473 if (hpriv
->mask_port_map
) {
474 dev_warn(dev
, "masking port_map 0x%x -> 0x%x\n",
476 port_map
& hpriv
->mask_port_map
);
477 port_map
&= hpriv
->mask_port_map
;
480 /* cross check port_map and cap.n_ports */
484 for (i
= 0; i
< AHCI_MAX_PORTS
; i
++)
485 if (port_map
& (1 << i
))
488 /* If PI has more ports than n_ports, whine, clear
489 * port_map and let it be generated from n_ports.
491 if (map_ports
> ahci_nr_ports(cap
)) {
493 "implemented port map (0x%x) contains more ports than nr_ports (%u), using nr_ports\n",
494 port_map
, ahci_nr_ports(cap
));
499 /* fabricate port_map from cap.nr_ports */
501 port_map
= (1 << ahci_nr_ports(cap
)) - 1;
502 dev_warn(dev
, "forcing PORTS_IMPL to 0x%x\n", port_map
);
504 /* write the fixed up value to the PI register */
505 hpriv
->saved_port_map
= port_map
;
508 /* record values to use during operation */
511 hpriv
->port_map
= port_map
;
513 if (!hpriv
->start_engine
)
514 hpriv
->start_engine
= ahci_start_engine
;
516 EXPORT_SYMBOL_GPL(ahci_save_initial_config
);
519 * ahci_restore_initial_config - Restore initial config
520 * @host: target ATA host
522 * Restore initial config stored by ahci_save_initial_config().
527 static void ahci_restore_initial_config(struct ata_host
*host
)
529 struct ahci_host_priv
*hpriv
= host
->private_data
;
530 void __iomem
*mmio
= hpriv
->mmio
;
532 writel(hpriv
->saved_cap
, mmio
+ HOST_CAP
);
533 if (hpriv
->saved_cap2
)
534 writel(hpriv
->saved_cap2
, mmio
+ HOST_CAP2
);
535 writel(hpriv
->saved_port_map
, mmio
+ HOST_PORTS_IMPL
);
536 (void) readl(mmio
+ HOST_PORTS_IMPL
); /* flush */
539 static unsigned ahci_scr_offset(struct ata_port
*ap
, unsigned int sc_reg
)
541 static const int offset
[] = {
542 [SCR_STATUS
] = PORT_SCR_STAT
,
543 [SCR_CONTROL
] = PORT_SCR_CTL
,
544 [SCR_ERROR
] = PORT_SCR_ERR
,
545 [SCR_ACTIVE
] = PORT_SCR_ACT
,
546 [SCR_NOTIFICATION
] = PORT_SCR_NTF
,
548 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
550 if (sc_reg
< ARRAY_SIZE(offset
) &&
551 (sc_reg
!= SCR_NOTIFICATION
|| (hpriv
->cap
& HOST_CAP_SNTF
)))
552 return offset
[sc_reg
];
556 static int ahci_scr_read(struct ata_link
*link
, unsigned int sc_reg
, u32
*val
)
558 void __iomem
*port_mmio
= ahci_port_base(link
->ap
);
559 int offset
= ahci_scr_offset(link
->ap
, sc_reg
);
562 *val
= readl(port_mmio
+ offset
);
568 static int ahci_scr_write(struct ata_link
*link
, unsigned int sc_reg
, u32 val
)
570 void __iomem
*port_mmio
= ahci_port_base(link
->ap
);
571 int offset
= ahci_scr_offset(link
->ap
, sc_reg
);
574 writel(val
, port_mmio
+ offset
);
580 void ahci_start_engine(struct ata_port
*ap
)
582 void __iomem
*port_mmio
= ahci_port_base(ap
);
586 tmp
= readl(port_mmio
+ PORT_CMD
);
587 tmp
|= PORT_CMD_START
;
588 writel(tmp
, port_mmio
+ PORT_CMD
);
589 readl(port_mmio
+ PORT_CMD
); /* flush */
591 EXPORT_SYMBOL_GPL(ahci_start_engine
);
593 int ahci_stop_engine(struct ata_port
*ap
)
595 void __iomem
*port_mmio
= ahci_port_base(ap
);
598 tmp
= readl(port_mmio
+ PORT_CMD
);
600 /* check if the HBA is idle */
601 if ((tmp
& (PORT_CMD_START
| PORT_CMD_LIST_ON
)) == 0)
604 /* setting HBA to idle */
605 tmp
&= ~PORT_CMD_START
;
606 writel(tmp
, port_mmio
+ PORT_CMD
);
608 /* wait for engine to stop. This could be as long as 500 msec */
609 tmp
= ata_wait_register(ap
, port_mmio
+ PORT_CMD
,
610 PORT_CMD_LIST_ON
, PORT_CMD_LIST_ON
, 1, 500);
611 if (tmp
& PORT_CMD_LIST_ON
)
616 EXPORT_SYMBOL_GPL(ahci_stop_engine
);
618 void ahci_start_fis_rx(struct ata_port
*ap
)
620 void __iomem
*port_mmio
= ahci_port_base(ap
);
621 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
622 struct ahci_port_priv
*pp
= ap
->private_data
;
625 /* set FIS registers */
626 if (hpriv
->cap
& HOST_CAP_64
)
627 writel((pp
->cmd_slot_dma
>> 16) >> 16,
628 port_mmio
+ PORT_LST_ADDR_HI
);
629 writel(pp
->cmd_slot_dma
& 0xffffffff, port_mmio
+ PORT_LST_ADDR
);
631 if (hpriv
->cap
& HOST_CAP_64
)
632 writel((pp
->rx_fis_dma
>> 16) >> 16,
633 port_mmio
+ PORT_FIS_ADDR_HI
);
634 writel(pp
->rx_fis_dma
& 0xffffffff, port_mmio
+ PORT_FIS_ADDR
);
636 /* enable FIS reception */
637 tmp
= readl(port_mmio
+ PORT_CMD
);
638 tmp
|= PORT_CMD_FIS_RX
;
639 writel(tmp
, port_mmio
+ PORT_CMD
);
642 readl(port_mmio
+ PORT_CMD
);
644 EXPORT_SYMBOL_GPL(ahci_start_fis_rx
);
646 static int ahci_stop_fis_rx(struct ata_port
*ap
)
648 void __iomem
*port_mmio
= ahci_port_base(ap
);
651 /* disable FIS reception */
652 tmp
= readl(port_mmio
+ PORT_CMD
);
653 tmp
&= ~PORT_CMD_FIS_RX
;
654 writel(tmp
, port_mmio
+ PORT_CMD
);
656 /* wait for completion, spec says 500ms, give it 1000 */
657 tmp
= ata_wait_register(ap
, port_mmio
+ PORT_CMD
, PORT_CMD_FIS_ON
,
658 PORT_CMD_FIS_ON
, 10, 1000);
659 if (tmp
& PORT_CMD_FIS_ON
)
665 static void ahci_power_up(struct ata_port
*ap
)
667 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
668 void __iomem
*port_mmio
= ahci_port_base(ap
);
671 cmd
= readl(port_mmio
+ PORT_CMD
) & ~PORT_CMD_ICC_MASK
;
674 if (hpriv
->cap
& HOST_CAP_SSS
) {
675 cmd
|= PORT_CMD_SPIN_UP
;
676 writel(cmd
, port_mmio
+ PORT_CMD
);
680 writel(cmd
| PORT_CMD_ICC_ACTIVE
, port_mmio
+ PORT_CMD
);
683 static int ahci_set_lpm(struct ata_link
*link
, enum ata_lpm_policy policy
,
686 struct ata_port
*ap
= link
->ap
;
687 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
688 struct ahci_port_priv
*pp
= ap
->private_data
;
689 void __iomem
*port_mmio
= ahci_port_base(ap
);
691 if (policy
!= ATA_LPM_MAX_POWER
) {
693 * Disable interrupts on Phy Ready. This keeps us from
694 * getting woken up due to spurious phy ready
697 pp
->intr_mask
&= ~PORT_IRQ_PHYRDY
;
698 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
700 sata_link_scr_lpm(link
, policy
, false);
703 if (hpriv
->cap
& HOST_CAP_ALPM
) {
704 u32 cmd
= readl(port_mmio
+ PORT_CMD
);
706 if (policy
== ATA_LPM_MAX_POWER
|| !(hints
& ATA_LPM_HIPM
)) {
707 cmd
&= ~(PORT_CMD_ASP
| PORT_CMD_ALPE
);
708 cmd
|= PORT_CMD_ICC_ACTIVE
;
710 writel(cmd
, port_mmio
+ PORT_CMD
);
711 readl(port_mmio
+ PORT_CMD
);
713 /* wait 10ms to be sure we've come out of LPM state */
716 cmd
|= PORT_CMD_ALPE
;
717 if (policy
== ATA_LPM_MIN_POWER
)
720 /* write out new cmd value */
721 writel(cmd
, port_mmio
+ PORT_CMD
);
725 /* set aggressive device sleep */
726 if ((hpriv
->cap2
& HOST_CAP2_SDS
) &&
727 (hpriv
->cap2
& HOST_CAP2_SADM
) &&
728 (link
->device
->flags
& ATA_DFLAG_DEVSLP
)) {
729 if (policy
== ATA_LPM_MIN_POWER
)
730 ahci_set_aggressive_devslp(ap
, true);
732 ahci_set_aggressive_devslp(ap
, false);
735 if (policy
== ATA_LPM_MAX_POWER
) {
736 sata_link_scr_lpm(link
, policy
, false);
738 /* turn PHYRDY IRQ back on */
739 pp
->intr_mask
|= PORT_IRQ_PHYRDY
;
740 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
747 static void ahci_power_down(struct ata_port
*ap
)
749 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
750 void __iomem
*port_mmio
= ahci_port_base(ap
);
753 if (!(hpriv
->cap
& HOST_CAP_SSS
))
756 /* put device into listen mode, first set PxSCTL.DET to 0 */
757 scontrol
= readl(port_mmio
+ PORT_SCR_CTL
);
759 writel(scontrol
, port_mmio
+ PORT_SCR_CTL
);
761 /* then set PxCMD.SUD to 0 */
762 cmd
= readl(port_mmio
+ PORT_CMD
) & ~PORT_CMD_ICC_MASK
;
763 cmd
&= ~PORT_CMD_SPIN_UP
;
764 writel(cmd
, port_mmio
+ PORT_CMD
);
768 static void ahci_start_port(struct ata_port
*ap
)
770 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
771 struct ahci_port_priv
*pp
= ap
->private_data
;
772 struct ata_link
*link
;
773 struct ahci_em_priv
*emp
;
777 /* enable FIS reception */
778 ahci_start_fis_rx(ap
);
781 if (!(hpriv
->flags
& AHCI_HFLAG_DELAY_ENGINE
))
782 hpriv
->start_engine(ap
);
785 if (ap
->flags
& ATA_FLAG_EM
) {
786 ata_for_each_link(link
, ap
, EDGE
) {
787 emp
= &pp
->em_priv
[link
->pmp
];
789 /* EM Transmit bit maybe busy during init */
790 for (i
= 0; i
< EM_MAX_RETRY
; i
++) {
791 rc
= ap
->ops
->transmit_led_message(ap
,
795 * If busy, give a breather but do not
796 * release EH ownership by using msleep()
797 * instead of ata_msleep(). EM Transmit
798 * bit is busy for the whole host and
799 * releasing ownership will cause other
800 * ports to fail the same way.
810 if (ap
->flags
& ATA_FLAG_SW_ACTIVITY
)
811 ata_for_each_link(link
, ap
, EDGE
)
812 ahci_init_sw_activity(link
);
816 static int ahci_deinit_port(struct ata_port
*ap
, const char **emsg
)
821 rc
= ahci_stop_engine(ap
);
823 *emsg
= "failed to stop engine";
827 /* disable FIS reception */
828 rc
= ahci_stop_fis_rx(ap
);
830 *emsg
= "failed stop FIS RX";
837 int ahci_reset_controller(struct ata_host
*host
)
839 struct ahci_host_priv
*hpriv
= host
->private_data
;
840 void __iomem
*mmio
= hpriv
->mmio
;
843 /* we must be in AHCI mode, before using anything
844 * AHCI-specific, such as HOST_RESET.
846 ahci_enable_ahci(mmio
);
848 /* global controller reset */
849 if (!ahci_skip_host_reset
) {
850 tmp
= readl(mmio
+ HOST_CTL
);
851 if ((tmp
& HOST_RESET
) == 0) {
852 writel(tmp
| HOST_RESET
, mmio
+ HOST_CTL
);
853 readl(mmio
+ HOST_CTL
); /* flush */
857 * to perform host reset, OS should set HOST_RESET
858 * and poll until this bit is read to be "0".
859 * reset must complete within 1 second, or
860 * the hardware should be considered fried.
862 tmp
= ata_wait_register(NULL
, mmio
+ HOST_CTL
, HOST_RESET
,
863 HOST_RESET
, 10, 1000);
865 if (tmp
& HOST_RESET
) {
866 dev_err(host
->dev
, "controller reset failed (0x%x)\n",
871 /* turn on AHCI mode */
872 ahci_enable_ahci(mmio
);
874 /* Some registers might be cleared on reset. Restore
877 ahci_restore_initial_config(host
);
879 dev_info(host
->dev
, "skipping global host reset\n");
883 EXPORT_SYMBOL_GPL(ahci_reset_controller
);
885 static void ahci_sw_activity(struct ata_link
*link
)
887 struct ata_port
*ap
= link
->ap
;
888 struct ahci_port_priv
*pp
= ap
->private_data
;
889 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
891 if (!(link
->flags
& ATA_LFLAG_SW_ACTIVITY
))
895 if (!timer_pending(&emp
->timer
))
896 mod_timer(&emp
->timer
, jiffies
+ msecs_to_jiffies(10));
899 static void ahci_sw_activity_blink(unsigned long arg
)
901 struct ata_link
*link
= (struct ata_link
*)arg
;
902 struct ata_port
*ap
= link
->ap
;
903 struct ahci_port_priv
*pp
= ap
->private_data
;
904 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
905 unsigned long led_message
= emp
->led_state
;
906 u32 activity_led_state
;
909 led_message
&= EM_MSG_LED_VALUE
;
910 led_message
|= ap
->port_no
| (link
->pmp
<< 8);
912 /* check to see if we've had activity. If so,
913 * toggle state of LED and reset timer. If not,
914 * turn LED to desired idle state.
916 spin_lock_irqsave(ap
->lock
, flags
);
917 if (emp
->saved_activity
!= emp
->activity
) {
918 emp
->saved_activity
= emp
->activity
;
919 /* get the current LED state */
920 activity_led_state
= led_message
& EM_MSG_LED_VALUE_ON
;
922 if (activity_led_state
)
923 activity_led_state
= 0;
925 activity_led_state
= 1;
927 /* clear old state */
928 led_message
&= ~EM_MSG_LED_VALUE_ACTIVITY
;
931 led_message
|= (activity_led_state
<< 16);
932 mod_timer(&emp
->timer
, jiffies
+ msecs_to_jiffies(100));
935 led_message
&= ~EM_MSG_LED_VALUE_ACTIVITY
;
936 if (emp
->blink_policy
== BLINK_OFF
)
937 led_message
|= (1 << 16);
939 spin_unlock_irqrestore(ap
->lock
, flags
);
940 ap
->ops
->transmit_led_message(ap
, led_message
, 4);
943 static void ahci_init_sw_activity(struct ata_link
*link
)
945 struct ata_port
*ap
= link
->ap
;
946 struct ahci_port_priv
*pp
= ap
->private_data
;
947 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
949 /* init activity stats, setup timer */
950 emp
->saved_activity
= emp
->activity
= 0;
951 setup_timer(&emp
->timer
, ahci_sw_activity_blink
, (unsigned long)link
);
953 /* check our blink policy and set flag for link if it's enabled */
954 if (emp
->blink_policy
)
955 link
->flags
|= ATA_LFLAG_SW_ACTIVITY
;
958 int ahci_reset_em(struct ata_host
*host
)
960 struct ahci_host_priv
*hpriv
= host
->private_data
;
961 void __iomem
*mmio
= hpriv
->mmio
;
964 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
965 if ((em_ctl
& EM_CTL_TM
) || (em_ctl
& EM_CTL_RST
))
968 writel(em_ctl
| EM_CTL_RST
, mmio
+ HOST_EM_CTL
);
971 EXPORT_SYMBOL_GPL(ahci_reset_em
);
973 static ssize_t
ahci_transmit_led_message(struct ata_port
*ap
, u32 state
,
976 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
977 struct ahci_port_priv
*pp
= ap
->private_data
;
978 void __iomem
*mmio
= hpriv
->mmio
;
980 u32 message
[] = {0, 0};
983 struct ahci_em_priv
*emp
;
985 /* get the slot number from the message */
986 pmp
= (state
& EM_MSG_LED_PMP_SLOT
) >> 8;
987 if (pmp
< EM_MAX_SLOTS
)
988 emp
= &pp
->em_priv
[pmp
];
992 spin_lock_irqsave(ap
->lock
, flags
);
995 * if we are still busy transmitting a previous message,
998 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
999 if (em_ctl
& EM_CTL_TM
) {
1000 spin_unlock_irqrestore(ap
->lock
, flags
);
1004 if (hpriv
->em_msg_type
& EM_MSG_TYPE_LED
) {
1006 * create message header - this is all zero except for
1007 * the message size, which is 4 bytes.
1009 message
[0] |= (4 << 8);
1011 /* ignore 0:4 of byte zero, fill in port info yourself */
1012 message
[1] = ((state
& ~EM_MSG_LED_HBA_PORT
) | ap
->port_no
);
1014 /* write message to EM_LOC */
1015 writel(message
[0], mmio
+ hpriv
->em_loc
);
1016 writel(message
[1], mmio
+ hpriv
->em_loc
+4);
1019 * tell hardware to transmit the message
1021 writel(em_ctl
| EM_CTL_TM
, mmio
+ HOST_EM_CTL
);
1024 /* save off new led state for port/slot */
1025 emp
->led_state
= state
;
1027 spin_unlock_irqrestore(ap
->lock
, flags
);
1031 static ssize_t
ahci_led_show(struct ata_port
*ap
, char *buf
)
1033 struct ahci_port_priv
*pp
= ap
->private_data
;
1034 struct ata_link
*link
;
1035 struct ahci_em_priv
*emp
;
1038 ata_for_each_link(link
, ap
, EDGE
) {
1039 emp
= &pp
->em_priv
[link
->pmp
];
1040 rc
+= sprintf(buf
, "%lx\n", emp
->led_state
);
1045 static ssize_t
ahci_led_store(struct ata_port
*ap
, const char *buf
,
1050 struct ahci_port_priv
*pp
= ap
->private_data
;
1051 struct ahci_em_priv
*emp
;
1053 if (kstrtouint(buf
, 0, &state
) < 0)
1056 /* get the slot number from the message */
1057 pmp
= (state
& EM_MSG_LED_PMP_SLOT
) >> 8;
1058 if (pmp
< EM_MAX_SLOTS
)
1059 emp
= &pp
->em_priv
[pmp
];
1063 /* mask off the activity bits if we are in sw_activity
1064 * mode, user should turn off sw_activity before setting
1065 * activity led through em_message
1067 if (emp
->blink_policy
)
1068 state
&= ~EM_MSG_LED_VALUE_ACTIVITY
;
1070 return ap
->ops
->transmit_led_message(ap
, state
, size
);
1073 static ssize_t
ahci_activity_store(struct ata_device
*dev
, enum sw_activity val
)
1075 struct ata_link
*link
= dev
->link
;
1076 struct ata_port
*ap
= link
->ap
;
1077 struct ahci_port_priv
*pp
= ap
->private_data
;
1078 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
1079 u32 port_led_state
= emp
->led_state
;
1081 /* save the desired Activity LED behavior */
1084 link
->flags
&= ~(ATA_LFLAG_SW_ACTIVITY
);
1086 /* set the LED to OFF */
1087 port_led_state
&= EM_MSG_LED_VALUE_OFF
;
1088 port_led_state
|= (ap
->port_no
| (link
->pmp
<< 8));
1089 ap
->ops
->transmit_led_message(ap
, port_led_state
, 4);
1091 link
->flags
|= ATA_LFLAG_SW_ACTIVITY
;
1092 if (val
== BLINK_OFF
) {
1093 /* set LED to ON for idle */
1094 port_led_state
&= EM_MSG_LED_VALUE_OFF
;
1095 port_led_state
|= (ap
->port_no
| (link
->pmp
<< 8));
1096 port_led_state
|= EM_MSG_LED_VALUE_ON
; /* check this */
1097 ap
->ops
->transmit_led_message(ap
, port_led_state
, 4);
1100 emp
->blink_policy
= val
;
1104 static ssize_t
ahci_activity_show(struct ata_device
*dev
, char *buf
)
1106 struct ata_link
*link
= dev
->link
;
1107 struct ata_port
*ap
= link
->ap
;
1108 struct ahci_port_priv
*pp
= ap
->private_data
;
1109 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
1111 /* display the saved value of activity behavior for this
1114 return sprintf(buf
, "%d\n", emp
->blink_policy
);
1117 static void ahci_port_init(struct device
*dev
, struct ata_port
*ap
,
1118 int port_no
, void __iomem
*mmio
,
1119 void __iomem
*port_mmio
)
1121 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1122 const char *emsg
= NULL
;
1126 /* make sure port is not active */
1127 rc
= ahci_deinit_port(ap
, &emsg
);
1129 dev_warn(dev
, "%s (%d)\n", emsg
, rc
);
1132 tmp
= readl(port_mmio
+ PORT_SCR_ERR
);
1133 VPRINTK("PORT_SCR_ERR 0x%x\n", tmp
);
1134 writel(tmp
, port_mmio
+ PORT_SCR_ERR
);
1136 /* clear port IRQ */
1137 tmp
= readl(port_mmio
+ PORT_IRQ_STAT
);
1138 VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp
);
1140 writel(tmp
, port_mmio
+ PORT_IRQ_STAT
);
1142 writel(1 << port_no
, mmio
+ HOST_IRQ_STAT
);
1144 /* mark esata ports */
1145 tmp
= readl(port_mmio
+ PORT_CMD
);
1146 if ((tmp
& PORT_CMD_HPCP
) ||
1147 ((tmp
& PORT_CMD_ESP
) && (hpriv
->cap
& HOST_CAP_SXS
)))
1148 ap
->pflags
|= ATA_PFLAG_EXTERNAL
;
1151 void ahci_init_controller(struct ata_host
*host
)
1153 struct ahci_host_priv
*hpriv
= host
->private_data
;
1154 void __iomem
*mmio
= hpriv
->mmio
;
1156 void __iomem
*port_mmio
;
1159 for (i
= 0; i
< host
->n_ports
; i
++) {
1160 struct ata_port
*ap
= host
->ports
[i
];
1162 port_mmio
= ahci_port_base(ap
);
1163 if (ata_port_is_dummy(ap
))
1166 ahci_port_init(host
->dev
, ap
, i
, mmio
, port_mmio
);
1169 tmp
= readl(mmio
+ HOST_CTL
);
1170 VPRINTK("HOST_CTL 0x%x\n", tmp
);
1171 writel(tmp
| HOST_IRQ_EN
, mmio
+ HOST_CTL
);
1172 tmp
= readl(mmio
+ HOST_CTL
);
1173 VPRINTK("HOST_CTL 0x%x\n", tmp
);
1175 EXPORT_SYMBOL_GPL(ahci_init_controller
);
1177 static void ahci_dev_config(struct ata_device
*dev
)
1179 struct ahci_host_priv
*hpriv
= dev
->link
->ap
->host
->private_data
;
1181 if (hpriv
->flags
& AHCI_HFLAG_SECT255
) {
1182 dev
->max_sectors
= 255;
1184 "SB600 AHCI: limiting to 255 sectors per cmd\n");
1188 unsigned int ahci_dev_classify(struct ata_port
*ap
)
1190 void __iomem
*port_mmio
= ahci_port_base(ap
);
1191 struct ata_taskfile tf
;
1194 tmp
= readl(port_mmio
+ PORT_SIG
);
1195 tf
.lbah
= (tmp
>> 24) & 0xff;
1196 tf
.lbam
= (tmp
>> 16) & 0xff;
1197 tf
.lbal
= (tmp
>> 8) & 0xff;
1198 tf
.nsect
= (tmp
) & 0xff;
1200 return ata_dev_classify(&tf
);
1202 EXPORT_SYMBOL_GPL(ahci_dev_classify
);
1204 void ahci_fill_cmd_slot(struct ahci_port_priv
*pp
, unsigned int tag
,
1207 dma_addr_t cmd_tbl_dma
;
1209 cmd_tbl_dma
= pp
->cmd_tbl_dma
+ tag
* AHCI_CMD_TBL_SZ
;
1211 pp
->cmd_slot
[tag
].opts
= cpu_to_le32(opts
);
1212 pp
->cmd_slot
[tag
].status
= 0;
1213 pp
->cmd_slot
[tag
].tbl_addr
= cpu_to_le32(cmd_tbl_dma
& 0xffffffff);
1214 pp
->cmd_slot
[tag
].tbl_addr_hi
= cpu_to_le32((cmd_tbl_dma
>> 16) >> 16);
1216 EXPORT_SYMBOL_GPL(ahci_fill_cmd_slot
);
1218 int ahci_kick_engine(struct ata_port
*ap
)
1220 void __iomem
*port_mmio
= ahci_port_base(ap
);
1221 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1222 u8 status
= readl(port_mmio
+ PORT_TFDATA
) & 0xFF;
1227 rc
= ahci_stop_engine(ap
);
1232 * always do CLO if PMP is attached (AHCI-1.3 9.2)
1234 busy
= status
& (ATA_BUSY
| ATA_DRQ
);
1235 if (!busy
&& !sata_pmp_attached(ap
)) {
1240 if (!(hpriv
->cap
& HOST_CAP_CLO
)) {
1246 tmp
= readl(port_mmio
+ PORT_CMD
);
1247 tmp
|= PORT_CMD_CLO
;
1248 writel(tmp
, port_mmio
+ PORT_CMD
);
1251 tmp
= ata_wait_register(ap
, port_mmio
+ PORT_CMD
,
1252 PORT_CMD_CLO
, PORT_CMD_CLO
, 1, 500);
1253 if (tmp
& PORT_CMD_CLO
)
1256 /* restart engine */
1258 hpriv
->start_engine(ap
);
1261 EXPORT_SYMBOL_GPL(ahci_kick_engine
);
1263 static int ahci_exec_polled_cmd(struct ata_port
*ap
, int pmp
,
1264 struct ata_taskfile
*tf
, int is_cmd
, u16 flags
,
1265 unsigned long timeout_msec
)
1267 const u32 cmd_fis_len
= 5; /* five dwords */
1268 struct ahci_port_priv
*pp
= ap
->private_data
;
1269 void __iomem
*port_mmio
= ahci_port_base(ap
);
1270 u8
*fis
= pp
->cmd_tbl
;
1273 /* prep the command */
1274 ata_tf_to_fis(tf
, pmp
, is_cmd
, fis
);
1275 ahci_fill_cmd_slot(pp
, 0, cmd_fis_len
| flags
| (pmp
<< 12));
1277 /* set port value for softreset of Port Multiplier */
1278 if (pp
->fbs_enabled
&& pp
->fbs_last_dev
!= pmp
) {
1279 tmp
= readl(port_mmio
+ PORT_FBS
);
1280 tmp
&= ~(PORT_FBS_DEV_MASK
| PORT_FBS_DEC
);
1281 tmp
|= pmp
<< PORT_FBS_DEV_OFFSET
;
1282 writel(tmp
, port_mmio
+ PORT_FBS
);
1283 pp
->fbs_last_dev
= pmp
;
1287 writel(1, port_mmio
+ PORT_CMD_ISSUE
);
1290 tmp
= ata_wait_register(ap
, port_mmio
+ PORT_CMD_ISSUE
,
1291 0x1, 0x1, 1, timeout_msec
);
1293 ahci_kick_engine(ap
);
1297 readl(port_mmio
+ PORT_CMD_ISSUE
); /* flush */
1302 int ahci_do_softreset(struct ata_link
*link
, unsigned int *class,
1303 int pmp
, unsigned long deadline
,
1304 int (*check_ready
)(struct ata_link
*link
))
1306 struct ata_port
*ap
= link
->ap
;
1307 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1308 struct ahci_port_priv
*pp
= ap
->private_data
;
1309 const char *reason
= NULL
;
1310 unsigned long now
, msecs
;
1311 struct ata_taskfile tf
;
1312 bool fbs_disabled
= false;
1317 /* prepare for SRST (AHCI-1.1 10.4.1) */
1318 rc
= ahci_kick_engine(ap
);
1319 if (rc
&& rc
!= -EOPNOTSUPP
)
1320 ata_link_warn(link
, "failed to reset engine (errno=%d)\n", rc
);
1323 * According to AHCI-1.2 9.3.9: if FBS is enable, software shall
1324 * clear PxFBS.EN to '0' prior to issuing software reset to devices
1325 * that is attached to port multiplier.
1327 if (!ata_is_host_link(link
) && pp
->fbs_enabled
) {
1328 ahci_disable_fbs(ap
);
1329 fbs_disabled
= true;
1332 ata_tf_init(link
->device
, &tf
);
1334 /* issue the first D2H Register FIS */
1337 if (time_after(deadline
, now
))
1338 msecs
= jiffies_to_msecs(deadline
- now
);
1341 if (ahci_exec_polled_cmd(ap
, pmp
, &tf
, 0,
1342 AHCI_CMD_RESET
| AHCI_CMD_CLR_BUSY
, msecs
)) {
1344 reason
= "1st FIS failed";
1348 /* spec says at least 5us, but be generous and sleep for 1ms */
1351 /* issue the second D2H Register FIS */
1352 tf
.ctl
&= ~ATA_SRST
;
1353 ahci_exec_polled_cmd(ap
, pmp
, &tf
, 0, 0, 0);
1355 /* wait for link to become ready */
1356 rc
= ata_wait_after_reset(link
, deadline
, check_ready
);
1357 if (rc
== -EBUSY
&& hpriv
->flags
& AHCI_HFLAG_SRST_TOUT_IS_OFFLINE
) {
1359 * Workaround for cases where link online status can't
1360 * be trusted. Treat device readiness timeout as link
1363 ata_link_info(link
, "device not ready, treating as offline\n");
1364 *class = ATA_DEV_NONE
;
1366 /* link occupied, -ENODEV too is an error */
1367 reason
= "device not ready";
1370 *class = ahci_dev_classify(ap
);
1372 /* re-enable FBS if disabled before */
1374 ahci_enable_fbs(ap
);
1376 DPRINTK("EXIT, class=%u\n", *class);
1380 ata_link_err(link
, "softreset failed (%s)\n", reason
);
1384 int ahci_check_ready(struct ata_link
*link
)
1386 void __iomem
*port_mmio
= ahci_port_base(link
->ap
);
1387 u8 status
= readl(port_mmio
+ PORT_TFDATA
) & 0xFF;
1389 return ata_check_ready(status
);
1391 EXPORT_SYMBOL_GPL(ahci_check_ready
);
1393 static int ahci_softreset(struct ata_link
*link
, unsigned int *class,
1394 unsigned long deadline
)
1396 int pmp
= sata_srst_pmp(link
);
1400 return ahci_do_softreset(link
, class, pmp
, deadline
, ahci_check_ready
);
1402 EXPORT_SYMBOL_GPL(ahci_do_softreset
);
1404 static int ahci_bad_pmp_check_ready(struct ata_link
*link
)
1406 void __iomem
*port_mmio
= ahci_port_base(link
->ap
);
1407 u8 status
= readl(port_mmio
+ PORT_TFDATA
) & 0xFF;
1408 u32 irq_status
= readl(port_mmio
+ PORT_IRQ_STAT
);
1411 * There is no need to check TFDATA if BAD PMP is found due to HW bug,
1412 * which can save timeout delay.
1414 if (irq_status
& PORT_IRQ_BAD_PMP
)
1417 return ata_check_ready(status
);
1420 static int ahci_pmp_retry_softreset(struct ata_link
*link
, unsigned int *class,
1421 unsigned long deadline
)
1423 struct ata_port
*ap
= link
->ap
;
1424 void __iomem
*port_mmio
= ahci_port_base(ap
);
1425 int pmp
= sata_srst_pmp(link
);
1431 rc
= ahci_do_softreset(link
, class, pmp
, deadline
,
1432 ahci_bad_pmp_check_ready
);
1435 * Soft reset fails with IPMS set when PMP is enabled but
1436 * SATA HDD/ODD is connected to SATA port, do soft reset
1440 irq_sts
= readl(port_mmio
+ PORT_IRQ_STAT
);
1441 if (irq_sts
& PORT_IRQ_BAD_PMP
) {
1443 "applying PMP SRST workaround "
1445 rc
= ahci_do_softreset(link
, class, 0, deadline
,
1453 static int ahci_hardreset(struct ata_link
*link
, unsigned int *class,
1454 unsigned long deadline
)
1456 const unsigned long *timing
= sata_ehc_deb_timing(&link
->eh_context
);
1457 struct ata_port
*ap
= link
->ap
;
1458 struct ahci_port_priv
*pp
= ap
->private_data
;
1459 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1460 u8
*d2h_fis
= pp
->rx_fis
+ RX_FIS_D2H_REG
;
1461 struct ata_taskfile tf
;
1467 ahci_stop_engine(ap
);
1469 /* clear D2H reception area to properly wait for D2H FIS */
1470 ata_tf_init(link
->device
, &tf
);
1471 tf
.command
= ATA_BUSY
;
1472 ata_tf_to_fis(&tf
, 0, 0, d2h_fis
);
1474 rc
= sata_link_hardreset(link
, timing
, deadline
, &online
,
1477 hpriv
->start_engine(ap
);
1480 *class = ahci_dev_classify(ap
);
1482 DPRINTK("EXIT, rc=%d, class=%u\n", rc
, *class);
1486 static void ahci_postreset(struct ata_link
*link
, unsigned int *class)
1488 struct ata_port
*ap
= link
->ap
;
1489 void __iomem
*port_mmio
= ahci_port_base(ap
);
1492 ata_std_postreset(link
, class);
1494 /* Make sure port's ATAPI bit is set appropriately */
1495 new_tmp
= tmp
= readl(port_mmio
+ PORT_CMD
);
1496 if (*class == ATA_DEV_ATAPI
)
1497 new_tmp
|= PORT_CMD_ATAPI
;
1499 new_tmp
&= ~PORT_CMD_ATAPI
;
1500 if (new_tmp
!= tmp
) {
1501 writel(new_tmp
, port_mmio
+ PORT_CMD
);
1502 readl(port_mmio
+ PORT_CMD
); /* flush */
1506 static unsigned int ahci_fill_sg(struct ata_queued_cmd
*qc
, void *cmd_tbl
)
1508 struct scatterlist
*sg
;
1509 struct ahci_sg
*ahci_sg
= cmd_tbl
+ AHCI_CMD_TBL_HDR_SZ
;
1515 * Next, the S/G list.
1517 for_each_sg(qc
->sg
, sg
, qc
->n_elem
, si
) {
1518 dma_addr_t addr
= sg_dma_address(sg
);
1519 u32 sg_len
= sg_dma_len(sg
);
1521 ahci_sg
[si
].addr
= cpu_to_le32(addr
& 0xffffffff);
1522 ahci_sg
[si
].addr_hi
= cpu_to_le32((addr
>> 16) >> 16);
1523 ahci_sg
[si
].flags_size
= cpu_to_le32(sg_len
- 1);
1529 static int ahci_pmp_qc_defer(struct ata_queued_cmd
*qc
)
1531 struct ata_port
*ap
= qc
->ap
;
1532 struct ahci_port_priv
*pp
= ap
->private_data
;
1534 if (!sata_pmp_attached(ap
) || pp
->fbs_enabled
)
1535 return ata_std_qc_defer(qc
);
1537 return sata_pmp_qc_defer_cmd_switch(qc
);
1540 static void ahci_qc_prep(struct ata_queued_cmd
*qc
)
1542 struct ata_port
*ap
= qc
->ap
;
1543 struct ahci_port_priv
*pp
= ap
->private_data
;
1544 int is_atapi
= ata_is_atapi(qc
->tf
.protocol
);
1547 const u32 cmd_fis_len
= 5; /* five dwords */
1548 unsigned int n_elem
;
1551 * Fill in command table information. First, the header,
1552 * a SATA Register - Host to Device command FIS.
1554 cmd_tbl
= pp
->cmd_tbl
+ qc
->tag
* AHCI_CMD_TBL_SZ
;
1556 ata_tf_to_fis(&qc
->tf
, qc
->dev
->link
->pmp
, 1, cmd_tbl
);
1558 memset(cmd_tbl
+ AHCI_CMD_TBL_CDB
, 0, 32);
1559 memcpy(cmd_tbl
+ AHCI_CMD_TBL_CDB
, qc
->cdb
, qc
->dev
->cdb_len
);
1563 if (qc
->flags
& ATA_QCFLAG_DMAMAP
)
1564 n_elem
= ahci_fill_sg(qc
, cmd_tbl
);
1567 * Fill in command slot information.
1569 opts
= cmd_fis_len
| n_elem
<< 16 | (qc
->dev
->link
->pmp
<< 12);
1570 if (qc
->tf
.flags
& ATA_TFLAG_WRITE
)
1571 opts
|= AHCI_CMD_WRITE
;
1573 opts
|= AHCI_CMD_ATAPI
| AHCI_CMD_PREFETCH
;
1575 ahci_fill_cmd_slot(pp
, qc
->tag
, opts
);
1578 static void ahci_fbs_dec_intr(struct ata_port
*ap
)
1580 struct ahci_port_priv
*pp
= ap
->private_data
;
1581 void __iomem
*port_mmio
= ahci_port_base(ap
);
1582 u32 fbs
= readl(port_mmio
+ PORT_FBS
);
1586 BUG_ON(!pp
->fbs_enabled
);
1588 /* time to wait for DEC is not specified by AHCI spec,
1589 * add a retry loop for safety.
1591 writel(fbs
| PORT_FBS_DEC
, port_mmio
+ PORT_FBS
);
1592 fbs
= readl(port_mmio
+ PORT_FBS
);
1593 while ((fbs
& PORT_FBS_DEC
) && retries
--) {
1595 fbs
= readl(port_mmio
+ PORT_FBS
);
1598 if (fbs
& PORT_FBS_DEC
)
1599 dev_err(ap
->host
->dev
, "failed to clear device error\n");
1602 static void ahci_error_intr(struct ata_port
*ap
, u32 irq_stat
)
1604 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1605 struct ahci_port_priv
*pp
= ap
->private_data
;
1606 struct ata_eh_info
*host_ehi
= &ap
->link
.eh_info
;
1607 struct ata_link
*link
= NULL
;
1608 struct ata_queued_cmd
*active_qc
;
1609 struct ata_eh_info
*active_ehi
;
1610 bool fbs_need_dec
= false;
1613 /* determine active link with error */
1614 if (pp
->fbs_enabled
) {
1615 void __iomem
*port_mmio
= ahci_port_base(ap
);
1616 u32 fbs
= readl(port_mmio
+ PORT_FBS
);
1617 int pmp
= fbs
>> PORT_FBS_DWE_OFFSET
;
1619 if ((fbs
& PORT_FBS_SDE
) && (pmp
< ap
->nr_pmp_links
)) {
1620 link
= &ap
->pmp_link
[pmp
];
1621 fbs_need_dec
= true;
1625 ata_for_each_link(link
, ap
, EDGE
)
1626 if (ata_link_active(link
))
1632 active_qc
= ata_qc_from_tag(ap
, link
->active_tag
);
1633 active_ehi
= &link
->eh_info
;
1635 /* record irq stat */
1636 ata_ehi_clear_desc(host_ehi
);
1637 ata_ehi_push_desc(host_ehi
, "irq_stat 0x%08x", irq_stat
);
1639 /* AHCI needs SError cleared; otherwise, it might lock up */
1640 ahci_scr_read(&ap
->link
, SCR_ERROR
, &serror
);
1641 ahci_scr_write(&ap
->link
, SCR_ERROR
, serror
);
1642 host_ehi
->serror
|= serror
;
1644 /* some controllers set IRQ_IF_ERR on device errors, ignore it */
1645 if (hpriv
->flags
& AHCI_HFLAG_IGN_IRQ_IF_ERR
)
1646 irq_stat
&= ~PORT_IRQ_IF_ERR
;
1648 if (irq_stat
& PORT_IRQ_TF_ERR
) {
1649 /* If qc is active, charge it; otherwise, the active
1650 * link. There's no active qc on NCQ errors. It will
1651 * be determined by EH by reading log page 10h.
1654 active_qc
->err_mask
|= AC_ERR_DEV
;
1656 active_ehi
->err_mask
|= AC_ERR_DEV
;
1658 if (hpriv
->flags
& AHCI_HFLAG_IGN_SERR_INTERNAL
)
1659 host_ehi
->serror
&= ~SERR_INTERNAL
;
1662 if (irq_stat
& PORT_IRQ_UNK_FIS
) {
1663 u32
*unk
= pp
->rx_fis
+ RX_FIS_UNK
;
1665 active_ehi
->err_mask
|= AC_ERR_HSM
;
1666 active_ehi
->action
|= ATA_EH_RESET
;
1667 ata_ehi_push_desc(active_ehi
,
1668 "unknown FIS %08x %08x %08x %08x" ,
1669 unk
[0], unk
[1], unk
[2], unk
[3]);
1672 if (sata_pmp_attached(ap
) && (irq_stat
& PORT_IRQ_BAD_PMP
)) {
1673 active_ehi
->err_mask
|= AC_ERR_HSM
;
1674 active_ehi
->action
|= ATA_EH_RESET
;
1675 ata_ehi_push_desc(active_ehi
, "incorrect PMP");
1678 if (irq_stat
& (PORT_IRQ_HBUS_ERR
| PORT_IRQ_HBUS_DATA_ERR
)) {
1679 host_ehi
->err_mask
|= AC_ERR_HOST_BUS
;
1680 host_ehi
->action
|= ATA_EH_RESET
;
1681 ata_ehi_push_desc(host_ehi
, "host bus error");
1684 if (irq_stat
& PORT_IRQ_IF_ERR
) {
1686 active_ehi
->err_mask
|= AC_ERR_DEV
;
1688 host_ehi
->err_mask
|= AC_ERR_ATA_BUS
;
1689 host_ehi
->action
|= ATA_EH_RESET
;
1692 ata_ehi_push_desc(host_ehi
, "interface fatal error");
1695 if (irq_stat
& (PORT_IRQ_CONNECT
| PORT_IRQ_PHYRDY
)) {
1696 ata_ehi_hotplugged(host_ehi
);
1697 ata_ehi_push_desc(host_ehi
, "%s",
1698 irq_stat
& PORT_IRQ_CONNECT
?
1699 "connection status changed" : "PHY RDY changed");
1702 /* okay, let's hand over to EH */
1704 if (irq_stat
& PORT_IRQ_FREEZE
)
1705 ata_port_freeze(ap
);
1706 else if (fbs_need_dec
) {
1707 ata_link_abort(link
);
1708 ahci_fbs_dec_intr(ap
);
1713 static void ahci_handle_port_interrupt(struct ata_port
*ap
,
1714 void __iomem
*port_mmio
, u32 status
)
1716 struct ata_eh_info
*ehi
= &ap
->link
.eh_info
;
1717 struct ahci_port_priv
*pp
= ap
->private_data
;
1718 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1719 int resetting
= !!(ap
->pflags
& ATA_PFLAG_RESETTING
);
1723 /* ignore BAD_PMP while resetting */
1724 if (unlikely(resetting
))
1725 status
&= ~PORT_IRQ_BAD_PMP
;
1727 if (sata_lpm_ignore_phy_events(&ap
->link
)) {
1728 status
&= ~PORT_IRQ_PHYRDY
;
1729 ahci_scr_write(&ap
->link
, SCR_ERROR
, SERR_PHYRDY_CHG
);
1732 if (unlikely(status
& PORT_IRQ_ERROR
)) {
1733 ahci_error_intr(ap
, status
);
1737 if (status
& PORT_IRQ_SDB_FIS
) {
1738 /* If SNotification is available, leave notification
1739 * handling to sata_async_notification(). If not,
1740 * emulate it by snooping SDB FIS RX area.
1742 * Snooping FIS RX area is probably cheaper than
1743 * poking SNotification but some constrollers which
1744 * implement SNotification, ICH9 for example, don't
1745 * store AN SDB FIS into receive area.
1747 if (hpriv
->cap
& HOST_CAP_SNTF
)
1748 sata_async_notification(ap
);
1750 /* If the 'N' bit in word 0 of the FIS is set,
1751 * we just received asynchronous notification.
1752 * Tell libata about it.
1754 * Lack of SNotification should not appear in
1755 * ahci 1.2, so the workaround is unnecessary
1756 * when FBS is enabled.
1758 if (pp
->fbs_enabled
)
1761 const __le32
*f
= pp
->rx_fis
+ RX_FIS_SDB
;
1762 u32 f0
= le32_to_cpu(f
[0]);
1764 sata_async_notification(ap
);
1769 /* pp->active_link is not reliable once FBS is enabled, both
1770 * PORT_SCR_ACT and PORT_CMD_ISSUE should be checked because
1771 * NCQ and non-NCQ commands may be in flight at the same time.
1773 if (pp
->fbs_enabled
) {
1774 if (ap
->qc_active
) {
1775 qc_active
= readl(port_mmio
+ PORT_SCR_ACT
);
1776 qc_active
|= readl(port_mmio
+ PORT_CMD_ISSUE
);
1779 /* pp->active_link is valid iff any command is in flight */
1780 if (ap
->qc_active
&& pp
->active_link
->sactive
)
1781 qc_active
= readl(port_mmio
+ PORT_SCR_ACT
);
1783 qc_active
= readl(port_mmio
+ PORT_CMD_ISSUE
);
1787 rc
= ata_qc_complete_multiple(ap
, qc_active
);
1789 /* while resetting, invalid completions are expected */
1790 if (unlikely(rc
< 0 && !resetting
)) {
1791 ehi
->err_mask
|= AC_ERR_HSM
;
1792 ehi
->action
|= ATA_EH_RESET
;
1793 ata_port_freeze(ap
);
1797 static void ahci_port_intr(struct ata_port
*ap
)
1799 void __iomem
*port_mmio
= ahci_port_base(ap
);
1802 status
= readl(port_mmio
+ PORT_IRQ_STAT
);
1803 writel(status
, port_mmio
+ PORT_IRQ_STAT
);
1805 ahci_handle_port_interrupt(ap
, port_mmio
, status
);
1808 static irqreturn_t
ahci_multi_irqs_intr_hard(int irq
, void *dev_instance
)
1810 struct ata_port
*ap
= dev_instance
;
1811 void __iomem
*port_mmio
= ahci_port_base(ap
);
1816 status
= readl(port_mmio
+ PORT_IRQ_STAT
);
1817 writel(status
, port_mmio
+ PORT_IRQ_STAT
);
1819 spin_lock(ap
->lock
);
1820 ahci_handle_port_interrupt(ap
, port_mmio
, status
);
1821 spin_unlock(ap
->lock
);
1828 static u32
ahci_handle_port_intr(struct ata_host
*host
, u32 irq_masked
)
1830 unsigned int i
, handled
= 0;
1832 for (i
= 0; i
< host
->n_ports
; i
++) {
1833 struct ata_port
*ap
;
1835 if (!(irq_masked
& (1 << i
)))
1838 ap
= host
->ports
[i
];
1841 VPRINTK("port %u\n", i
);
1843 VPRINTK("port %u (no irq)\n", i
);
1844 if (ata_ratelimit())
1846 "interrupt on disabled port %u\n", i
);
1855 static irqreturn_t
ahci_single_edge_irq_intr(int irq
, void *dev_instance
)
1857 struct ata_host
*host
= dev_instance
;
1858 struct ahci_host_priv
*hpriv
;
1859 unsigned int rc
= 0;
1861 u32 irq_stat
, irq_masked
;
1865 hpriv
= host
->private_data
;
1868 /* sigh. 0xffffffff is a valid return from h/w */
1869 irq_stat
= readl(mmio
+ HOST_IRQ_STAT
);
1873 irq_masked
= irq_stat
& hpriv
->port_map
;
1875 spin_lock(&host
->lock
);
1878 * HOST_IRQ_STAT behaves as edge triggered latch meaning that
1879 * it should be cleared before all the port events are cleared.
1881 writel(irq_stat
, mmio
+ HOST_IRQ_STAT
);
1883 rc
= ahci_handle_port_intr(host
, irq_masked
);
1885 spin_unlock(&host
->lock
);
1889 return IRQ_RETVAL(rc
);
1892 static irqreturn_t
ahci_single_level_irq_intr(int irq
, void *dev_instance
)
1894 struct ata_host
*host
= dev_instance
;
1895 struct ahci_host_priv
*hpriv
;
1896 unsigned int rc
= 0;
1898 u32 irq_stat
, irq_masked
;
1902 hpriv
= host
->private_data
;
1905 /* sigh. 0xffffffff is a valid return from h/w */
1906 irq_stat
= readl(mmio
+ HOST_IRQ_STAT
);
1910 irq_masked
= irq_stat
& hpriv
->port_map
;
1912 spin_lock(&host
->lock
);
1914 rc
= ahci_handle_port_intr(host
, irq_masked
);
1916 /* HOST_IRQ_STAT behaves as level triggered latch meaning that
1917 * it should be cleared after all the port events are cleared;
1918 * otherwise, it will raise a spurious interrupt after each
1919 * valid one. Please read section 10.6.2 of ahci 1.1 for more
1922 * Also, use the unmasked value to clear interrupt as spurious
1923 * pending event on a dummy port might cause screaming IRQ.
1925 writel(irq_stat
, mmio
+ HOST_IRQ_STAT
);
1927 spin_unlock(&host
->lock
);
1931 return IRQ_RETVAL(rc
);
1934 unsigned int ahci_qc_issue(struct ata_queued_cmd
*qc
)
1936 struct ata_port
*ap
= qc
->ap
;
1937 void __iomem
*port_mmio
= ahci_port_base(ap
);
1938 struct ahci_port_priv
*pp
= ap
->private_data
;
1940 /* Keep track of the currently active link. It will be used
1941 * in completion path to determine whether NCQ phase is in
1944 pp
->active_link
= qc
->dev
->link
;
1946 if (qc
->tf
.protocol
== ATA_PROT_NCQ
)
1947 writel(1 << qc
->tag
, port_mmio
+ PORT_SCR_ACT
);
1949 if (pp
->fbs_enabled
&& pp
->fbs_last_dev
!= qc
->dev
->link
->pmp
) {
1950 u32 fbs
= readl(port_mmio
+ PORT_FBS
);
1951 fbs
&= ~(PORT_FBS_DEV_MASK
| PORT_FBS_DEC
);
1952 fbs
|= qc
->dev
->link
->pmp
<< PORT_FBS_DEV_OFFSET
;
1953 writel(fbs
, port_mmio
+ PORT_FBS
);
1954 pp
->fbs_last_dev
= qc
->dev
->link
->pmp
;
1957 writel(1 << qc
->tag
, port_mmio
+ PORT_CMD_ISSUE
);
1959 ahci_sw_activity(qc
->dev
->link
);
1963 EXPORT_SYMBOL_GPL(ahci_qc_issue
);
1965 static bool ahci_qc_fill_rtf(struct ata_queued_cmd
*qc
)
1967 struct ahci_port_priv
*pp
= qc
->ap
->private_data
;
1968 u8
*rx_fis
= pp
->rx_fis
;
1970 if (pp
->fbs_enabled
)
1971 rx_fis
+= qc
->dev
->link
->pmp
* AHCI_RX_FIS_SZ
;
1974 * After a successful execution of an ATA PIO data-in command,
1975 * the device doesn't send D2H Reg FIS to update the TF and
1976 * the host should take TF and E_Status from the preceding PIO
1979 if (qc
->tf
.protocol
== ATA_PROT_PIO
&& qc
->dma_dir
== DMA_FROM_DEVICE
&&
1980 !(qc
->flags
& ATA_QCFLAG_FAILED
)) {
1981 ata_tf_from_fis(rx_fis
+ RX_FIS_PIO_SETUP
, &qc
->result_tf
);
1982 qc
->result_tf
.command
= (rx_fis
+ RX_FIS_PIO_SETUP
)[15];
1984 ata_tf_from_fis(rx_fis
+ RX_FIS_D2H_REG
, &qc
->result_tf
);
1989 static void ahci_freeze(struct ata_port
*ap
)
1991 void __iomem
*port_mmio
= ahci_port_base(ap
);
1994 writel(0, port_mmio
+ PORT_IRQ_MASK
);
1997 static void ahci_thaw(struct ata_port
*ap
)
1999 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2000 void __iomem
*mmio
= hpriv
->mmio
;
2001 void __iomem
*port_mmio
= ahci_port_base(ap
);
2003 struct ahci_port_priv
*pp
= ap
->private_data
;
2006 tmp
= readl(port_mmio
+ PORT_IRQ_STAT
);
2007 writel(tmp
, port_mmio
+ PORT_IRQ_STAT
);
2008 writel(1 << ap
->port_no
, mmio
+ HOST_IRQ_STAT
);
2010 /* turn IRQ back on */
2011 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
2014 void ahci_error_handler(struct ata_port
*ap
)
2016 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2018 if (!(ap
->pflags
& ATA_PFLAG_FROZEN
)) {
2019 /* restart engine */
2020 ahci_stop_engine(ap
);
2021 hpriv
->start_engine(ap
);
2024 sata_pmp_error_handler(ap
);
2026 if (!ata_dev_enabled(ap
->link
.device
))
2027 ahci_stop_engine(ap
);
2029 EXPORT_SYMBOL_GPL(ahci_error_handler
);
2031 static void ahci_post_internal_cmd(struct ata_queued_cmd
*qc
)
2033 struct ata_port
*ap
= qc
->ap
;
2035 /* make DMA engine forget about the failed command */
2036 if (qc
->flags
& ATA_QCFLAG_FAILED
)
2037 ahci_kick_engine(ap
);
2040 static void ahci_set_aggressive_devslp(struct ata_port
*ap
, bool sleep
)
2042 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2043 void __iomem
*port_mmio
= ahci_port_base(ap
);
2044 struct ata_device
*dev
= ap
->link
.device
;
2045 u32 devslp
, dm
, dito
, mdat
, deto
;
2047 unsigned int err_mask
;
2049 devslp
= readl(port_mmio
+ PORT_DEVSLP
);
2050 if (!(devslp
& PORT_DEVSLP_DSP
)) {
2051 dev_info(ap
->host
->dev
, "port does not support device sleep\n");
2055 /* disable device sleep */
2057 if (devslp
& PORT_DEVSLP_ADSE
) {
2058 writel(devslp
& ~PORT_DEVSLP_ADSE
,
2059 port_mmio
+ PORT_DEVSLP
);
2060 err_mask
= ata_dev_set_feature(dev
,
2061 SETFEATURES_SATA_DISABLE
,
2063 if (err_mask
&& err_mask
!= AC_ERR_DEV
)
2064 ata_dev_warn(dev
, "failed to disable DEVSLP\n");
2069 /* device sleep was already enabled */
2070 if (devslp
& PORT_DEVSLP_ADSE
)
2073 /* set DITO, MDAT, DETO and enable DevSlp, need to stop engine first */
2074 rc
= ahci_stop_engine(ap
);
2078 dm
= (devslp
& PORT_DEVSLP_DM_MASK
) >> PORT_DEVSLP_DM_OFFSET
;
2079 dito
= devslp_idle_timeout
/ (dm
+ 1);
2083 /* Use the nominal value 10 ms if the read MDAT is zero,
2084 * the nominal value of DETO is 20 ms.
2086 if (dev
->devslp_timing
[ATA_LOG_DEVSLP_VALID
] &
2087 ATA_LOG_DEVSLP_VALID_MASK
) {
2088 mdat
= dev
->devslp_timing
[ATA_LOG_DEVSLP_MDAT
] &
2089 ATA_LOG_DEVSLP_MDAT_MASK
;
2092 deto
= dev
->devslp_timing
[ATA_LOG_DEVSLP_DETO
];
2100 devslp
|= ((dito
<< PORT_DEVSLP_DITO_OFFSET
) |
2101 (mdat
<< PORT_DEVSLP_MDAT_OFFSET
) |
2102 (deto
<< PORT_DEVSLP_DETO_OFFSET
) |
2104 writel(devslp
, port_mmio
+ PORT_DEVSLP
);
2106 hpriv
->start_engine(ap
);
2108 /* enable device sleep feature for the drive */
2109 err_mask
= ata_dev_set_feature(dev
,
2110 SETFEATURES_SATA_ENABLE
,
2112 if (err_mask
&& err_mask
!= AC_ERR_DEV
)
2113 ata_dev_warn(dev
, "failed to enable DEVSLP\n");
2116 static void ahci_enable_fbs(struct ata_port
*ap
)
2118 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2119 struct ahci_port_priv
*pp
= ap
->private_data
;
2120 void __iomem
*port_mmio
= ahci_port_base(ap
);
2124 if (!pp
->fbs_supported
)
2127 fbs
= readl(port_mmio
+ PORT_FBS
);
2128 if (fbs
& PORT_FBS_EN
) {
2129 pp
->fbs_enabled
= true;
2130 pp
->fbs_last_dev
= -1; /* initialization */
2134 rc
= ahci_stop_engine(ap
);
2138 writel(fbs
| PORT_FBS_EN
, port_mmio
+ PORT_FBS
);
2139 fbs
= readl(port_mmio
+ PORT_FBS
);
2140 if (fbs
& PORT_FBS_EN
) {
2141 dev_info(ap
->host
->dev
, "FBS is enabled\n");
2142 pp
->fbs_enabled
= true;
2143 pp
->fbs_last_dev
= -1; /* initialization */
2145 dev_err(ap
->host
->dev
, "Failed to enable FBS\n");
2147 hpriv
->start_engine(ap
);
2150 static void ahci_disable_fbs(struct ata_port
*ap
)
2152 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2153 struct ahci_port_priv
*pp
= ap
->private_data
;
2154 void __iomem
*port_mmio
= ahci_port_base(ap
);
2158 if (!pp
->fbs_supported
)
2161 fbs
= readl(port_mmio
+ PORT_FBS
);
2162 if ((fbs
& PORT_FBS_EN
) == 0) {
2163 pp
->fbs_enabled
= false;
2167 rc
= ahci_stop_engine(ap
);
2171 writel(fbs
& ~PORT_FBS_EN
, port_mmio
+ PORT_FBS
);
2172 fbs
= readl(port_mmio
+ PORT_FBS
);
2173 if (fbs
& PORT_FBS_EN
)
2174 dev_err(ap
->host
->dev
, "Failed to disable FBS\n");
2176 dev_info(ap
->host
->dev
, "FBS is disabled\n");
2177 pp
->fbs_enabled
= false;
2180 hpriv
->start_engine(ap
);
2183 static void ahci_pmp_attach(struct ata_port
*ap
)
2185 void __iomem
*port_mmio
= ahci_port_base(ap
);
2186 struct ahci_port_priv
*pp
= ap
->private_data
;
2189 cmd
= readl(port_mmio
+ PORT_CMD
);
2190 cmd
|= PORT_CMD_PMP
;
2191 writel(cmd
, port_mmio
+ PORT_CMD
);
2193 ahci_enable_fbs(ap
);
2195 pp
->intr_mask
|= PORT_IRQ_BAD_PMP
;
2198 * We must not change the port interrupt mask register if the
2199 * port is marked frozen, the value in pp->intr_mask will be
2200 * restored later when the port is thawed.
2202 * Note that during initialization, the port is marked as
2203 * frozen since the irq handler is not yet registered.
2205 if (!(ap
->pflags
& ATA_PFLAG_FROZEN
))
2206 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
2209 static void ahci_pmp_detach(struct ata_port
*ap
)
2211 void __iomem
*port_mmio
= ahci_port_base(ap
);
2212 struct ahci_port_priv
*pp
= ap
->private_data
;
2215 ahci_disable_fbs(ap
);
2217 cmd
= readl(port_mmio
+ PORT_CMD
);
2218 cmd
&= ~PORT_CMD_PMP
;
2219 writel(cmd
, port_mmio
+ PORT_CMD
);
2221 pp
->intr_mask
&= ~PORT_IRQ_BAD_PMP
;
2223 /* see comment above in ahci_pmp_attach() */
2224 if (!(ap
->pflags
& ATA_PFLAG_FROZEN
))
2225 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
2228 int ahci_port_resume(struct ata_port
*ap
)
2231 ahci_start_port(ap
);
2233 if (sata_pmp_attached(ap
))
2234 ahci_pmp_attach(ap
);
2236 ahci_pmp_detach(ap
);
2240 EXPORT_SYMBOL_GPL(ahci_port_resume
);
2243 static int ahci_port_suspend(struct ata_port
*ap
, pm_message_t mesg
)
2245 const char *emsg
= NULL
;
2248 rc
= ahci_deinit_port(ap
, &emsg
);
2250 ahci_power_down(ap
);
2252 ata_port_err(ap
, "%s (%d)\n", emsg
, rc
);
2253 ata_port_freeze(ap
);
2260 static int ahci_port_start(struct ata_port
*ap
)
2262 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2263 struct device
*dev
= ap
->host
->dev
;
2264 struct ahci_port_priv
*pp
;
2267 size_t dma_sz
, rx_fis_sz
;
2269 pp
= devm_kzalloc(dev
, sizeof(*pp
), GFP_KERNEL
);
2273 if (ap
->host
->n_ports
> 1) {
2274 pp
->irq_desc
= devm_kzalloc(dev
, 8, GFP_KERNEL
);
2275 if (!pp
->irq_desc
) {
2276 devm_kfree(dev
, pp
);
2279 snprintf(pp
->irq_desc
, 8,
2280 "%s%d", dev_driver_string(dev
), ap
->port_no
);
2283 /* check FBS capability */
2284 if ((hpriv
->cap
& HOST_CAP_FBS
) && sata_pmp_supported(ap
)) {
2285 void __iomem
*port_mmio
= ahci_port_base(ap
);
2286 u32 cmd
= readl(port_mmio
+ PORT_CMD
);
2287 if (cmd
& PORT_CMD_FBSCP
)
2288 pp
->fbs_supported
= true;
2289 else if (hpriv
->flags
& AHCI_HFLAG_YES_FBS
) {
2290 dev_info(dev
, "port %d can do FBS, forcing FBSCP\n",
2292 pp
->fbs_supported
= true;
2294 dev_warn(dev
, "port %d is not capable of FBS\n",
2298 if (pp
->fbs_supported
) {
2299 dma_sz
= AHCI_PORT_PRIV_FBS_DMA_SZ
;
2300 rx_fis_sz
= AHCI_RX_FIS_SZ
* 16;
2302 dma_sz
= AHCI_PORT_PRIV_DMA_SZ
;
2303 rx_fis_sz
= AHCI_RX_FIS_SZ
;
2306 mem
= dmam_alloc_coherent(dev
, dma_sz
, &mem_dma
, GFP_KERNEL
);
2309 memset(mem
, 0, dma_sz
);
2312 * First item in chunk of DMA memory: 32-slot command table,
2313 * 32 bytes each in size
2316 pp
->cmd_slot_dma
= mem_dma
;
2318 mem
+= AHCI_CMD_SLOT_SZ
;
2319 mem_dma
+= AHCI_CMD_SLOT_SZ
;
2322 * Second item: Received-FIS area
2325 pp
->rx_fis_dma
= mem_dma
;
2328 mem_dma
+= rx_fis_sz
;
2331 * Third item: data area for storing a single command
2332 * and its scatter-gather table
2335 pp
->cmd_tbl_dma
= mem_dma
;
2338 * Save off initial list of interrupts to be enabled.
2339 * This could be changed later
2341 pp
->intr_mask
= DEF_PORT_IRQ
;
2344 * Switch to per-port locking in case each port has its own MSI vector.
2346 if (hpriv
->flags
& AHCI_HFLAG_MULTI_MSI
) {
2347 spin_lock_init(&pp
->lock
);
2348 ap
->lock
= &pp
->lock
;
2351 ap
->private_data
= pp
;
2353 /* engage engines, captain */
2354 return ahci_port_resume(ap
);
2357 static void ahci_port_stop(struct ata_port
*ap
)
2359 const char *emsg
= NULL
;
2362 /* de-initialize port */
2363 rc
= ahci_deinit_port(ap
, &emsg
);
2365 ata_port_warn(ap
, "%s (%d)\n", emsg
, rc
);
2368 void ahci_print_info(struct ata_host
*host
, const char *scc_s
)
2370 struct ahci_host_priv
*hpriv
= host
->private_data
;
2371 void __iomem
*mmio
= hpriv
->mmio
;
2372 u32 vers
, cap
, cap2
, impl
, speed
;
2373 const char *speed_s
;
2375 vers
= readl(mmio
+ HOST_VERSION
);
2378 impl
= hpriv
->port_map
;
2380 speed
= (cap
>> 20) & 0xf;
2383 else if (speed
== 2)
2385 else if (speed
== 3)
2391 "AHCI %02x%02x.%02x%02x "
2392 "%u slots %u ports %s Gbps 0x%x impl %s mode\n"
2395 (vers
>> 24) & 0xff,
2396 (vers
>> 16) & 0xff,
2400 ((cap
>> 8) & 0x1f) + 1,
2414 cap
& HOST_CAP_64
? "64bit " : "",
2415 cap
& HOST_CAP_NCQ
? "ncq " : "",
2416 cap
& HOST_CAP_SNTF
? "sntf " : "",
2417 cap
& HOST_CAP_MPS
? "ilck " : "",
2418 cap
& HOST_CAP_SSS
? "stag " : "",
2419 cap
& HOST_CAP_ALPM
? "pm " : "",
2420 cap
& HOST_CAP_LED
? "led " : "",
2421 cap
& HOST_CAP_CLO
? "clo " : "",
2422 cap
& HOST_CAP_ONLY
? "only " : "",
2423 cap
& HOST_CAP_PMP
? "pmp " : "",
2424 cap
& HOST_CAP_FBS
? "fbs " : "",
2425 cap
& HOST_CAP_PIO_MULTI
? "pio " : "",
2426 cap
& HOST_CAP_SSC
? "slum " : "",
2427 cap
& HOST_CAP_PART
? "part " : "",
2428 cap
& HOST_CAP_CCC
? "ccc " : "",
2429 cap
& HOST_CAP_EMS
? "ems " : "",
2430 cap
& HOST_CAP_SXS
? "sxs " : "",
2431 cap2
& HOST_CAP2_DESO
? "deso " : "",
2432 cap2
& HOST_CAP2_SADM
? "sadm " : "",
2433 cap2
& HOST_CAP2_SDS
? "sds " : "",
2434 cap2
& HOST_CAP2_APST
? "apst " : "",
2435 cap2
& HOST_CAP2_NVMHCI
? "nvmp " : "",
2436 cap2
& HOST_CAP2_BOH
? "boh " : ""
2439 EXPORT_SYMBOL_GPL(ahci_print_info
);
2441 void ahci_set_em_messages(struct ahci_host_priv
*hpriv
,
2442 struct ata_port_info
*pi
)
2445 void __iomem
*mmio
= hpriv
->mmio
;
2446 u32 em_loc
= readl(mmio
+ HOST_EM_LOC
);
2447 u32 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
2449 if (!ahci_em_messages
|| !(hpriv
->cap
& HOST_CAP_EMS
))
2452 messages
= (em_ctl
& EM_CTRL_MSG_TYPE
) >> 16;
2456 hpriv
->em_loc
= ((em_loc
>> 16) * 4);
2457 hpriv
->em_buf_sz
= ((em_loc
& 0xff) * 4);
2458 hpriv
->em_msg_type
= messages
;
2459 pi
->flags
|= ATA_FLAG_EM
;
2460 if (!(em_ctl
& EM_CTL_ALHD
))
2461 pi
->flags
|= ATA_FLAG_SW_ACTIVITY
;
2464 EXPORT_SYMBOL_GPL(ahci_set_em_messages
);
2466 static int ahci_host_activate_multi_irqs(struct ata_host
*host
,
2467 struct scsi_host_template
*sht
)
2469 struct ahci_host_priv
*hpriv
= host
->private_data
;
2472 rc
= ata_host_start(host
);
2476 * Requests IRQs according to AHCI-1.1 when multiple MSIs were
2477 * allocated. That is one MSI per port, starting from @irq.
2479 for (i
= 0; i
< host
->n_ports
; i
++) {
2480 struct ahci_port_priv
*pp
= host
->ports
[i
]->private_data
;
2481 int irq
= ahci_irq_vector(hpriv
, i
);
2483 /* Do not receive interrupts sent by dummy ports */
2485 disable_irq(irq
+ i
);
2489 rc
= devm_request_irq(host
->dev
, irq
, ahci_multi_irqs_intr_hard
,
2490 0, pp
->irq_desc
, host
->ports
[i
]);
2494 ata_port_desc(host
->ports
[i
], "irq %d", irq
);
2497 return ata_host_register(host
, sht
);
2501 * ahci_host_activate - start AHCI host, request IRQs and register it
2502 * @host: target ATA host
2503 * @sht: scsi_host_template to use when registering the host
2506 * Inherited from calling layer (may sleep).
2509 * 0 on success, -errno otherwise.
2511 int ahci_host_activate(struct ata_host
*host
, struct scsi_host_template
*sht
)
2513 struct ahci_host_priv
*hpriv
= host
->private_data
;
2514 int irq
= hpriv
->irq
;
2517 if (hpriv
->flags
& (AHCI_HFLAG_MULTI_MSI
| AHCI_HFLAG_MULTI_MSIX
))
2518 rc
= ahci_host_activate_multi_irqs(host
, sht
);
2519 else if (hpriv
->flags
& AHCI_HFLAG_EDGE_IRQ
)
2520 rc
= ata_host_activate(host
, irq
, ahci_single_edge_irq_intr
,
2523 rc
= ata_host_activate(host
, irq
, ahci_single_level_irq_intr
,
2527 EXPORT_SYMBOL_GPL(ahci_host_activate
);
2529 MODULE_AUTHOR("Jeff Garzik");
2530 MODULE_DESCRIPTION("Common AHCI SATA low-level routines");
2531 MODULE_LICENSE("GPL");