?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ftrace: do_each_pid_task() needs rcu lock
[linux-2.6/cjktty.git] / drivers / ata / sata_nv.c
blobc49ad0e61b6f2550a8b3f0b84122aa6f0bc0b40b
1 /*
2 * sata_nv.c - NVIDIA nForce SATA
3 *
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
6 *
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
21 *
22 *
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/DocBook/libata.*
25 *
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
30 * hotplug info, etc.
31 *
32 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
36 *
37 */
38
39 #include <linux/kernel.h>
40 #include <linux/module.h>
41 #include <linux/pci.h>
42 #include <linux/init.h>
43 #include <linux/blkdev.h>
44 #include <linux/delay.h>
45 #include <linux/interrupt.h>
46 #include <linux/device.h>
47 #include <scsi/scsi_host.h>
48 #include <scsi/scsi_device.h>
49 #include <linux/libata.h>
50
51 #define DRV_NAME "sata_nv"
52 #define DRV_VERSION "3.5"
53
54 #define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
55
56 enum {
57 NV_MMIO_BAR = 5,
58
59 NV_PORTS = 2,
60 NV_PIO_MASK = 0x1f,
61 NV_MWDMA_MASK = 0x07,
62 NV_UDMA_MASK = 0x7f,
63 NV_PORT0_SCR_REG_OFFSET = 0x00,
64 NV_PORT1_SCR_REG_OFFSET = 0x40,
65
66 /* INT_STATUS/ENABLE */
67 NV_INT_STATUS = 0x10,
68 NV_INT_ENABLE = 0x11,
69 NV_INT_STATUS_CK804 = 0x440,
70 NV_INT_ENABLE_CK804 = 0x441,
71
72 /* INT_STATUS/ENABLE bits */
73 NV_INT_DEV = 0x01,
74 NV_INT_PM = 0x02,
75 NV_INT_ADDED = 0x04,
76 NV_INT_REMOVED = 0x08,
77
78 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
79
80 NV_INT_ALL = 0x0f,
81 NV_INT_MASK = NV_INT_DEV |
82 NV_INT_ADDED | NV_INT_REMOVED,
83
84 /* INT_CONFIG */
85 NV_INT_CONFIG = 0x12,
86 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
87
88 // For PCI config register 20
89 NV_MCP_SATA_CFG_20 = 0x50,
90 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
91 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
92 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
93 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
94 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
95
96 NV_ADMA_MAX_CPBS = 32,
97 NV_ADMA_CPB_SZ = 128,
98 NV_ADMA_APRD_SZ = 16,
99 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
100 NV_ADMA_APRD_SZ,
101 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
102 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
103 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
104 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
105
106 /* BAR5 offset to ADMA general registers */
107 NV_ADMA_GEN = 0x400,
108 NV_ADMA_GEN_CTL = 0x00,
109 NV_ADMA_NOTIFIER_CLEAR = 0x30,
110
111 /* BAR5 offset to ADMA ports */
112 NV_ADMA_PORT = 0x480,
113
114 /* size of ADMA port register space */
115 NV_ADMA_PORT_SIZE = 0x100,
116
117 /* ADMA port registers */
118 NV_ADMA_CTL = 0x40,
119 NV_ADMA_CPB_COUNT = 0x42,
120 NV_ADMA_NEXT_CPB_IDX = 0x43,
121 NV_ADMA_STAT = 0x44,
122 NV_ADMA_CPB_BASE_LOW = 0x48,
123 NV_ADMA_CPB_BASE_HIGH = 0x4C,
124 NV_ADMA_APPEND = 0x50,
125 NV_ADMA_NOTIFIER = 0x68,
126 NV_ADMA_NOTIFIER_ERROR = 0x6C,
127
128 /* NV_ADMA_CTL register bits */
129 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
130 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
131 NV_ADMA_CTL_GO = (1 << 7),
132 NV_ADMA_CTL_AIEN = (1 << 8),
133 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
134 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
135
136 /* CPB response flag bits */
137 NV_CPB_RESP_DONE = (1 << 0),
138 NV_CPB_RESP_ATA_ERR = (1 << 3),
139 NV_CPB_RESP_CMD_ERR = (1 << 4),
140 NV_CPB_RESP_CPB_ERR = (1 << 7),
141
142 /* CPB control flag bits */
143 NV_CPB_CTL_CPB_VALID = (1 << 0),
144 NV_CPB_CTL_QUEUE = (1 << 1),
145 NV_CPB_CTL_APRD_VALID = (1 << 2),
146 NV_CPB_CTL_IEN = (1 << 3),
147 NV_CPB_CTL_FPDMA = (1 << 4),
148
149 /* APRD flags */
150 NV_APRD_WRITE = (1 << 1),
151 NV_APRD_END = (1 << 2),
152 NV_APRD_CONT = (1 << 3),
153
154 /* NV_ADMA_STAT flags */
155 NV_ADMA_STAT_TIMEOUT = (1 << 0),
156 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
157 NV_ADMA_STAT_HOTPLUG = (1 << 2),
158 NV_ADMA_STAT_CPBERR = (1 << 4),
159 NV_ADMA_STAT_SERROR = (1 << 5),
160 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
161 NV_ADMA_STAT_IDLE = (1 << 8),
162 NV_ADMA_STAT_LEGACY = (1 << 9),
163 NV_ADMA_STAT_STOPPED = (1 << 10),
164 NV_ADMA_STAT_DONE = (1 << 12),
165 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
166 NV_ADMA_STAT_TIMEOUT,
167
168 /* port flags */
169 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
170 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
171
172 /* MCP55 reg offset */
173 NV_CTL_MCP55 = 0x400,
174 NV_INT_STATUS_MCP55 = 0x440,
175 NV_INT_ENABLE_MCP55 = 0x444,
176 NV_NCQ_REG_MCP55 = 0x448,
177
178 /* MCP55 */
179 NV_INT_ALL_MCP55 = 0xffff,
180 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
181 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
182
183 /* SWNCQ ENABLE BITS*/
184 NV_CTL_PRI_SWNCQ = 0x02,
185 NV_CTL_SEC_SWNCQ = 0x04,
186
187 /* SW NCQ status bits*/
188 NV_SWNCQ_IRQ_DEV = (1 << 0),
189 NV_SWNCQ_IRQ_PM = (1 << 1),
190 NV_SWNCQ_IRQ_ADDED = (1 << 2),
191 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
192
193 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
194 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
195 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
196 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
197
198 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
199 NV_SWNCQ_IRQ_REMOVED,
200
201 };
202
203 /* ADMA Physical Region Descriptor - one SG segment */
204 struct nv_adma_prd {
205 __le64 addr;
206 __le32 len;
207 u8 flags;
208 u8 packet_len;
209 __le16 reserved;
210 };
211
212 enum nv_adma_regbits {
213 CMDEND = (1 << 15), /* end of command list */
214 WNB = (1 << 14), /* wait-not-BSY */
215 IGN = (1 << 13), /* ignore this entry */
216 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
217 DA2 = (1 << (2 + 8)),
218 DA1 = (1 << (1 + 8)),
219 DA0 = (1 << (0 + 8)),
220 };
221
222 /* ADMA Command Parameter Block
223 The first 5 SG segments are stored inside the Command Parameter Block itself.
224 If there are more than 5 segments the remainder are stored in a separate
225 memory area indicated by next_aprd. */
226 struct nv_adma_cpb {
227 u8 resp_flags; /* 0 */
228 u8 reserved1; /* 1 */
229 u8 ctl_flags; /* 2 */
230 /* len is length of taskfile in 64 bit words */
231 u8 len; /* 3 */
232 u8 tag; /* 4 */
233 u8 next_cpb_idx; /* 5 */
234 __le16 reserved2; /* 6-7 */
235 __le16 tf[12]; /* 8-31 */
236 struct nv_adma_prd aprd[5]; /* 32-111 */
237 __le64 next_aprd; /* 112-119 */
238 __le64 reserved3; /* 120-127 */
239 };
240
241
242 struct nv_adma_port_priv {
243 struct nv_adma_cpb *cpb;
244 dma_addr_t cpb_dma;
245 struct nv_adma_prd *aprd;
246 dma_addr_t aprd_dma;
247 void __iomem *ctl_block;
248 void __iomem *gen_block;
249 void __iomem *notifier_clear_block;
250 u64 adma_dma_mask;
251 u8 flags;
252 int last_issue_ncq;
253 };
254
255 struct nv_host_priv {
256 unsigned long type;
257 };
258
259 struct defer_queue {
260 u32 defer_bits;
261 unsigned int head;
262 unsigned int tail;
263 unsigned int tag[ATA_MAX_QUEUE];
264 };
265
266 enum ncq_saw_flag_list {
267 ncq_saw_d2h = (1U << 0),
268 ncq_saw_dmas = (1U << 1),
269 ncq_saw_sdb = (1U << 2),
270 ncq_saw_backout = (1U << 3),
271 };
272
273 struct nv_swncq_port_priv {
274 struct ata_prd *prd; /* our SG list */
275 dma_addr_t prd_dma; /* and its DMA mapping */
276 void __iomem *sactive_block;
277 void __iomem *irq_block;
278 void __iomem *tag_block;
279 u32 qc_active;
280
281 unsigned int last_issue_tag;
282
283 /* fifo circular queue to store deferral command */
284 struct defer_queue defer_queue;
285
286 /* for NCQ interrupt analysis */
287 u32 dhfis_bits;
288 u32 dmafis_bits;
289 u32 sdbfis_bits;
290
291 unsigned int ncq_flags;
292 };
293
294
295 #define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
296
297 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
298 #ifdef CONFIG_PM
299 static int nv_pci_device_resume(struct pci_dev *pdev);
300 #endif
301 static void nv_ck804_host_stop(struct ata_host *host);
302 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
303 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
304 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
305 static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
306 static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
307
308 static int nv_noclassify_hardreset(struct ata_link *link, unsigned int *class,
309 unsigned long deadline);
310 static void nv_nf2_freeze(struct ata_port *ap);
311 static void nv_nf2_thaw(struct ata_port *ap);
312 static void nv_ck804_freeze(struct ata_port *ap);
313 static void nv_ck804_thaw(struct ata_port *ap);
314 static int nv_adma_slave_config(struct scsi_device *sdev);
315 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
316 static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
317 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
318 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
319 static void nv_adma_irq_clear(struct ata_port *ap);
320 static int nv_adma_port_start(struct ata_port *ap);
321 static void nv_adma_port_stop(struct ata_port *ap);
322 #ifdef CONFIG_PM
323 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
324 static int nv_adma_port_resume(struct ata_port *ap);
325 #endif
326 static void nv_adma_freeze(struct ata_port *ap);
327 static void nv_adma_thaw(struct ata_port *ap);
328 static void nv_adma_error_handler(struct ata_port *ap);
329 static void nv_adma_host_stop(struct ata_host *host);
330 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
331 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
332
333 static void nv_mcp55_thaw(struct ata_port *ap);
334 static void nv_mcp55_freeze(struct ata_port *ap);
335 static void nv_swncq_error_handler(struct ata_port *ap);
336 static int nv_swncq_slave_config(struct scsi_device *sdev);
337 static int nv_swncq_port_start(struct ata_port *ap);
338 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
339 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
340 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
341 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
342 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
343 #ifdef CONFIG_PM
344 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
345 static int nv_swncq_port_resume(struct ata_port *ap);
346 #endif
347
348 enum nv_host_type
349 {
350 GENERIC,
351 NFORCE2,
352 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
353 CK804,
354 ADMA,
355 MCP5x,
356 SWNCQ,
357 };
358
359 static const struct pci_device_id nv_pci_tbl[] = {
360 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
361 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
362 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
363 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
364 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
365 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
366 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
367 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), MCP5x },
368 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), MCP5x },
369 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), MCP5x },
370 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), MCP5x },
371 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
372 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
373 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
374
375 { } /* terminate list */
376 };
377
378 static struct pci_driver nv_pci_driver = {
379 .name = DRV_NAME,
380 .id_table = nv_pci_tbl,
381 .probe = nv_init_one,
382 #ifdef CONFIG_PM
383 .suspend = ata_pci_device_suspend,
384 .resume = nv_pci_device_resume,
385 #endif
386 .remove = ata_pci_remove_one,
387 };
388
389 static struct scsi_host_template nv_sht = {
390 ATA_BMDMA_SHT(DRV_NAME),
391 };
392
393 static struct scsi_host_template nv_adma_sht = {
394 ATA_NCQ_SHT(DRV_NAME),
395 .can_queue = NV_ADMA_MAX_CPBS,
396 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
397 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
398 .slave_configure = nv_adma_slave_config,
399 };
400
401 static struct scsi_host_template nv_swncq_sht = {
402 ATA_NCQ_SHT(DRV_NAME),
403 .can_queue = ATA_MAX_QUEUE,
404 .sg_tablesize = LIBATA_MAX_PRD,
405 .dma_boundary = ATA_DMA_BOUNDARY,
406 .slave_configure = nv_swncq_slave_config,
407 };
408
409 static struct ata_port_operations nv_common_ops = {
410 .inherits = &ata_bmdma_port_ops,
411 .scr_read = nv_scr_read,
412 .scr_write = nv_scr_write,
413 };
414
415 /* OSDL bz11195 reports that link doesn't come online after hardreset
416 * on generic nv's and there have been several other similar reports
417 * on linux-ide. Disable hardreset for generic nv's.
418 */
419 static struct ata_port_operations nv_generic_ops = {
420 .inherits = &nv_common_ops,
421 .hardreset = ATA_OP_NULL,
422 };
423
424 /* OSDL bz3352 reports that nf2/3 controllers can't determine device
425 * signature reliably. Also, the following thread reports detection
426 * failure on cold boot with the standard debouncing timing.
427 *
428 * http://thread.gmane.org/gmane.linux.ide/34098
429 *
430 * Debounce with hotplug timing and request follow-up SRST.
431 */
432 static struct ata_port_operations nv_nf2_ops = {
433 .inherits = &nv_common_ops,
434 .freeze = nv_nf2_freeze,
435 .thaw = nv_nf2_thaw,
436 .hardreset = nv_noclassify_hardreset,
437 };
438
439 /* CK804 finally gets hardreset right */
440 static struct ata_port_operations nv_ck804_ops = {
441 .inherits = &nv_common_ops,
442 .freeze = nv_ck804_freeze,
443 .thaw = nv_ck804_thaw,
444 .host_stop = nv_ck804_host_stop,
445 };
446
447 static struct ata_port_operations nv_adma_ops = {
448 .inherits = &nv_ck804_ops,
449
450 .check_atapi_dma = nv_adma_check_atapi_dma,
451 .sff_tf_read = nv_adma_tf_read,
452 .qc_defer = ata_std_qc_defer,
453 .qc_prep = nv_adma_qc_prep,
454 .qc_issue = nv_adma_qc_issue,
455 .sff_irq_clear = nv_adma_irq_clear,
456
457 .freeze = nv_adma_freeze,
458 .thaw = nv_adma_thaw,
459 .error_handler = nv_adma_error_handler,
460 .post_internal_cmd = nv_adma_post_internal_cmd,
461
462 .port_start = nv_adma_port_start,
463 .port_stop = nv_adma_port_stop,
464 #ifdef CONFIG_PM
465 .port_suspend = nv_adma_port_suspend,
466 .port_resume = nv_adma_port_resume,
467 #endif
468 .host_stop = nv_adma_host_stop,
469 };
470
471 /* Kernel bz#12351 reports that when SWNCQ is enabled, for hotplug to
472 * work, hardreset should be used and hardreset can't report proper
473 * signature, which suggests that mcp5x is closer to nf2 as long as
474 * reset quirkiness is concerned. Define separate ops for mcp5x with
475 * nv_noclassify_hardreset().
476 */
477 static struct ata_port_operations nv_mcp5x_ops = {
478 .inherits = &nv_common_ops,
479 .hardreset = nv_noclassify_hardreset,
480 };
481
482 static struct ata_port_operations nv_swncq_ops = {
483 .inherits = &nv_mcp5x_ops,
484
485 .qc_defer = ata_std_qc_defer,
486 .qc_prep = nv_swncq_qc_prep,
487 .qc_issue = nv_swncq_qc_issue,
488
489 .freeze = nv_mcp55_freeze,
490 .thaw = nv_mcp55_thaw,
491 .error_handler = nv_swncq_error_handler,
492
493 #ifdef CONFIG_PM
494 .port_suspend = nv_swncq_port_suspend,
495 .port_resume = nv_swncq_port_resume,
496 #endif
497 .port_start = nv_swncq_port_start,
498 };
499
500 struct nv_pi_priv {
501 irq_handler_t irq_handler;
502 struct scsi_host_template *sht;
503 };
504
505 #define NV_PI_PRIV(_irq_handler, _sht) \
506 &(struct nv_pi_priv){ .irq_handler = _irq_handler, .sht = _sht }
507
508 static const struct ata_port_info nv_port_info[] = {
509 /* generic */
510 {
511 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
512 .pio_mask = NV_PIO_MASK,
513 .mwdma_mask = NV_MWDMA_MASK,
514 .udma_mask = NV_UDMA_MASK,
515 .port_ops = &nv_generic_ops,
516 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
517 },
518 /* nforce2/3 */
519 {
520 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
521 .pio_mask = NV_PIO_MASK,
522 .mwdma_mask = NV_MWDMA_MASK,
523 .udma_mask = NV_UDMA_MASK,
524 .port_ops = &nv_nf2_ops,
525 .private_data = NV_PI_PRIV(nv_nf2_interrupt, &nv_sht),
526 },
527 /* ck804 */
528 {
529 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
530 .pio_mask = NV_PIO_MASK,
531 .mwdma_mask = NV_MWDMA_MASK,
532 .udma_mask = NV_UDMA_MASK,
533 .port_ops = &nv_ck804_ops,
534 .private_data = NV_PI_PRIV(nv_ck804_interrupt, &nv_sht),
535 },
536 /* ADMA */
537 {
538 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
539 ATA_FLAG_MMIO | ATA_FLAG_NCQ,
540 .pio_mask = NV_PIO_MASK,
541 .mwdma_mask = NV_MWDMA_MASK,
542 .udma_mask = NV_UDMA_MASK,
543 .port_ops = &nv_adma_ops,
544 .private_data = NV_PI_PRIV(nv_adma_interrupt, &nv_adma_sht),
545 },
546 /* MCP5x */
547 {
548 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
549 .pio_mask = NV_PIO_MASK,
550 .mwdma_mask = NV_MWDMA_MASK,
551 .udma_mask = NV_UDMA_MASK,
552 .port_ops = &nv_mcp5x_ops,
553 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
554 },
555 /* SWNCQ */
556 {
557 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
558 ATA_FLAG_NCQ,
559 .pio_mask = NV_PIO_MASK,
560 .mwdma_mask = NV_MWDMA_MASK,
561 .udma_mask = NV_UDMA_MASK,
562 .port_ops = &nv_swncq_ops,
563 .private_data = NV_PI_PRIV(nv_swncq_interrupt, &nv_swncq_sht),
564 },
565 };
566
567 MODULE_AUTHOR("NVIDIA");
568 MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
569 MODULE_LICENSE("GPL");
570 MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
571 MODULE_VERSION(DRV_VERSION);
572
573 static int adma_enabled;
574 static int swncq_enabled = 1;
575
576 static void nv_adma_register_mode(struct ata_port *ap)
577 {
578 struct nv_adma_port_priv *pp = ap->private_data;
579 void __iomem *mmio = pp->ctl_block;
580 u16 tmp, status;
581 int count = 0;
582
583 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
584 return;
585
586 status = readw(mmio + NV_ADMA_STAT);
587 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
588 ndelay(50);
589 status = readw(mmio + NV_ADMA_STAT);
590 count++;
591 }
592 if (count == 20)
593 ata_port_printk(ap, KERN_WARNING,
594 "timeout waiting for ADMA IDLE, stat=0x%hx\n",
595 status);
596
597 tmp = readw(mmio + NV_ADMA_CTL);
598 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
599
600 count = 0;
601 status = readw(mmio + NV_ADMA_STAT);
602 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
603 ndelay(50);
604 status = readw(mmio + NV_ADMA_STAT);
605 count++;
606 }
607 if (count == 20)
608 ata_port_printk(ap, KERN_WARNING,
609 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
610 status);
611
612 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
613 }
614
615 static void nv_adma_mode(struct ata_port *ap)
616 {
617 struct nv_adma_port_priv *pp = ap->private_data;
618 void __iomem *mmio = pp->ctl_block;
619 u16 tmp, status;
620 int count = 0;
621
622 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
623 return;
624
625 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
626
627 tmp = readw(mmio + NV_ADMA_CTL);
628 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
629
630 status = readw(mmio + NV_ADMA_STAT);
631 while (((status & NV_ADMA_STAT_LEGACY) ||
632 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
633 ndelay(50);
634 status = readw(mmio + NV_ADMA_STAT);
635 count++;
636 }
637 if (count == 20)
638 ata_port_printk(ap, KERN_WARNING,
639 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
640 status);
641
642 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
643 }
644
645 static int nv_adma_slave_config(struct scsi_device *sdev)
646 {
647 struct ata_port *ap = ata_shost_to_port(sdev->host);
648 struct nv_adma_port_priv *pp = ap->private_data;
649 struct nv_adma_port_priv *port0, *port1;
650 struct scsi_device *sdev0, *sdev1;
651 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
652 unsigned long segment_boundary, flags;
653 unsigned short sg_tablesize;
654 int rc;
655 int adma_enable;
656 u32 current_reg, new_reg, config_mask;
657
658 rc = ata_scsi_slave_config(sdev);
659
660 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
661 /* Not a proper libata device, ignore */
662 return rc;
663
664 spin_lock_irqsave(ap->lock, flags);
665
666 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
667 /*
668 * NVIDIA reports that ADMA mode does not support ATAPI commands.
669 * Therefore ATAPI commands are sent through the legacy interface.
670 * However, the legacy interface only supports 32-bit DMA.
671 * Restrict DMA parameters as required by the legacy interface
672 * when an ATAPI device is connected.
673 */
674 segment_boundary = ATA_DMA_BOUNDARY;
675 /* Subtract 1 since an extra entry may be needed for padding, see
676 libata-scsi.c */
677 sg_tablesize = LIBATA_MAX_PRD - 1;
678
679 /* Since the legacy DMA engine is in use, we need to disable ADMA
680 on the port. */
681 adma_enable = 0;
682 nv_adma_register_mode(ap);
683 } else {
684 segment_boundary = NV_ADMA_DMA_BOUNDARY;
685 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
686 adma_enable = 1;
687 }
688
689 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
690
691 if (ap->port_no == 1)
692 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
693 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
694 else
695 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
696 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
697
698 if (adma_enable) {
699 new_reg = current_reg | config_mask;
700 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
701 } else {
702 new_reg = current_reg & ~config_mask;
703 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
704 }
705
706 if (current_reg != new_reg)
707 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
708
709 port0 = ap->host->ports[0]->private_data;
710 port1 = ap->host->ports[1]->private_data;
711 sdev0 = ap->host->ports[0]->link.device[0].sdev;
712 sdev1 = ap->host->ports[1]->link.device[0].sdev;
713 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
714 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
715 /** We have to set the DMA mask to 32-bit if either port is in
716 ATAPI mode, since they are on the same PCI device which is
717 used for DMA mapping. If we set the mask we also need to set
718 the bounce limit on both ports to ensure that the block
719 layer doesn't feed addresses that cause DMA mapping to
720 choke. If either SCSI device is not allocated yet, it's OK
721 since that port will discover its correct setting when it
722 does get allocated.
723 Note: Setting 32-bit mask should not fail. */
724 if (sdev0)
725 blk_queue_bounce_limit(sdev0->request_queue,
726 ATA_DMA_MASK);
727 if (sdev1)
728 blk_queue_bounce_limit(sdev1->request_queue,
729 ATA_DMA_MASK);
730
731 pci_set_dma_mask(pdev, ATA_DMA_MASK);
732 } else {
733 /** This shouldn't fail as it was set to this value before */
734 pci_set_dma_mask(pdev, pp->adma_dma_mask);
735 if (sdev0)
736 blk_queue_bounce_limit(sdev0->request_queue,
737 pp->adma_dma_mask);
738 if (sdev1)
739 blk_queue_bounce_limit(sdev1->request_queue,
740 pp->adma_dma_mask);
741 }
742
743 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
744 blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize);
745 ata_port_printk(ap, KERN_INFO,
746 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
747 (unsigned long long)*ap->host->dev->dma_mask,
748 segment_boundary, sg_tablesize);
749
750 spin_unlock_irqrestore(ap->lock, flags);
751
752 return rc;
753 }
754
755 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
756 {
757 struct nv_adma_port_priv *pp = qc->ap->private_data;
758 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
759 }
760
761 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
762 {
763 /* Other than when internal or pass-through commands are executed,
764 the only time this function will be called in ADMA mode will be
765 if a command fails. In the failure case we don't care about going
766 into register mode with ADMA commands pending, as the commands will
767 all shortly be aborted anyway. We assume that NCQ commands are not
768 issued via passthrough, which is the only way that switching into
769 ADMA mode could abort outstanding commands. */
770 nv_adma_register_mode(ap);
771
772 ata_sff_tf_read(ap, tf);
773 }
774
775 static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
776 {
777 unsigned int idx = 0;
778
779 if (tf->flags & ATA_TFLAG_ISADDR) {
780 if (tf->flags & ATA_TFLAG_LBA48) {
781 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
782 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
783 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
784 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
785 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
786 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
787 } else
788 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
789
790 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
791 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
792 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
793 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
794 }
795
796 if (tf->flags & ATA_TFLAG_DEVICE)
797 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
798
799 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
800
801 while (idx < 12)
802 cpb[idx++] = cpu_to_le16(IGN);
803
804 return idx;
805 }
806
807 static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
808 {
809 struct nv_adma_port_priv *pp = ap->private_data;
810 u8 flags = pp->cpb[cpb_num].resp_flags;
811
812 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
813
814 if (unlikely((force_err ||
815 flags & (NV_CPB_RESP_ATA_ERR |
816 NV_CPB_RESP_CMD_ERR |
817 NV_CPB_RESP_CPB_ERR)))) {
818 struct ata_eh_info *ehi = &ap->link.eh_info;
819 int freeze = 0;
820
821 ata_ehi_clear_desc(ehi);
822 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
823 if (flags & NV_CPB_RESP_ATA_ERR) {
824 ata_ehi_push_desc(ehi, "ATA error");
825 ehi->err_mask |= AC_ERR_DEV;
826 } else if (flags & NV_CPB_RESP_CMD_ERR) {
827 ata_ehi_push_desc(ehi, "CMD error");
828 ehi->err_mask |= AC_ERR_DEV;
829 } else if (flags & NV_CPB_RESP_CPB_ERR) {
830 ata_ehi_push_desc(ehi, "CPB error");
831 ehi->err_mask |= AC_ERR_SYSTEM;
832 freeze = 1;
833 } else {
834 /* notifier error, but no error in CPB flags? */
835 ata_ehi_push_desc(ehi, "unknown");
836 ehi->err_mask |= AC_ERR_OTHER;
837 freeze = 1;
838 }
839 /* Kill all commands. EH will determine what actually failed. */
840 if (freeze)
841 ata_port_freeze(ap);
842 else
843 ata_port_abort(ap);
844 return 1;
845 }
846
847 if (likely(flags & NV_CPB_RESP_DONE)) {
848 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
849 VPRINTK("CPB flags done, flags=0x%x\n", flags);
850 if (likely(qc)) {
851 DPRINTK("Completing qc from tag %d\n", cpb_num);
852 ata_qc_complete(qc);
853 } else {
854 struct ata_eh_info *ehi = &ap->link.eh_info;
855 /* Notifier bits set without a command may indicate the drive
856 is misbehaving. Raise host state machine violation on this
857 condition. */
858 ata_port_printk(ap, KERN_ERR,
859 "notifier for tag %d with no cmd?\n",
860 cpb_num);
861 ehi->err_mask |= AC_ERR_HSM;
862 ehi->action |= ATA_EH_RESET;
863 ata_port_freeze(ap);
864 return 1;
865 }
866 }
867 return 0;
868 }
869
870 static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
871 {
872 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
873
874 /* freeze if hotplugged */
875 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
876 ata_port_freeze(ap);
877 return 1;
878 }
879
880 /* bail out if not our interrupt */
881 if (!(irq_stat & NV_INT_DEV))
882 return 0;
883
884 /* DEV interrupt w/ no active qc? */
885 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
886 ata_sff_check_status(ap);
887 return 1;
888 }
889
890 /* handle interrupt */
891 return ata_sff_host_intr(ap, qc);
892 }
893
894 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
895 {
896 struct ata_host *host = dev_instance;
897 int i, handled = 0;
898 u32 notifier_clears[2];
899
900 spin_lock(&host->lock);
901
902 for (i = 0; i < host->n_ports; i++) {
903 struct ata_port *ap = host->ports[i];
904 notifier_clears[i] = 0;
905
906 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
907 struct nv_adma_port_priv *pp = ap->private_data;
908 void __iomem *mmio = pp->ctl_block;
909 u16 status;
910 u32 gen_ctl;
911 u32 notifier, notifier_error;
912
913 /* if ADMA is disabled, use standard ata interrupt handler */
914 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
915 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
916 >> (NV_INT_PORT_SHIFT * i);
917 handled += nv_host_intr(ap, irq_stat);
918 continue;
919 }
920
921 /* if in ATA register mode, check for standard interrupts */
922 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
923 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
924 >> (NV_INT_PORT_SHIFT * i);
925 if (ata_tag_valid(ap->link.active_tag))
926 /** NV_INT_DEV indication seems unreliable at times
927 at least in ADMA mode. Force it on always when a
928 command is active, to prevent losing interrupts. */
929 irq_stat |= NV_INT_DEV;
930 handled += nv_host_intr(ap, irq_stat);
931 }
932
933 notifier = readl(mmio + NV_ADMA_NOTIFIER);
934 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
935 notifier_clears[i] = notifier | notifier_error;
936
937 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
938
939 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
940 !notifier_error)
941 /* Nothing to do */
942 continue;
943
944 status = readw(mmio + NV_ADMA_STAT);
945
946 /* Clear status. Ensure the controller sees the clearing before we start
947 looking at any of the CPB statuses, so that any CPB completions after
948 this point in the handler will raise another interrupt. */
949 writew(status, mmio + NV_ADMA_STAT);
950 readw(mmio + NV_ADMA_STAT); /* flush posted write */
951 rmb();
952
953 handled++; /* irq handled if we got here */
954
955 /* freeze if hotplugged or controller error */
956 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
957 NV_ADMA_STAT_HOTUNPLUG |
958 NV_ADMA_STAT_TIMEOUT |
959 NV_ADMA_STAT_SERROR))) {
960 struct ata_eh_info *ehi = &ap->link.eh_info;
961
962 ata_ehi_clear_desc(ehi);
963 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
964 if (status & NV_ADMA_STAT_TIMEOUT) {
965 ehi->err_mask |= AC_ERR_SYSTEM;
966 ata_ehi_push_desc(ehi, "timeout");
967 } else if (status & NV_ADMA_STAT_HOTPLUG) {
968 ata_ehi_hotplugged(ehi);
969 ata_ehi_push_desc(ehi, "hotplug");
970 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
971 ata_ehi_hotplugged(ehi);
972 ata_ehi_push_desc(ehi, "hot unplug");
973 } else if (status & NV_ADMA_STAT_SERROR) {
974 /* let libata analyze SError and figure out the cause */
975 ata_ehi_push_desc(ehi, "SError");
976 } else
977 ata_ehi_push_desc(ehi, "unknown");
978 ata_port_freeze(ap);
979 continue;
980 }
981
982 if (status & (NV_ADMA_STAT_DONE |
983 NV_ADMA_STAT_CPBERR |
984 NV_ADMA_STAT_CMD_COMPLETE)) {
985 u32 check_commands = notifier_clears[i];
986 int pos, error = 0;
987
988 if (status & NV_ADMA_STAT_CPBERR) {
989 /* Check all active commands */
990 if (ata_tag_valid(ap->link.active_tag))
991 check_commands = 1 <<
992 ap->link.active_tag;
993 else
994 check_commands = ap->
995 link.sactive;
996 }
997
998 /** Check CPBs for completed commands */
999 while ((pos = ffs(check_commands)) && !error) {
1000 pos--;
1001 error = nv_adma_check_cpb(ap, pos,
1002 notifier_error & (1 << pos));
1003 check_commands &= ~(1 << pos);
1004 }
1005 }
1006 }
1007 }
1008
1009 if (notifier_clears[0] || notifier_clears[1]) {
1010 /* Note: Both notifier clear registers must be written
1011 if either is set, even if one is zero, according to NVIDIA. */
1012 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1013 writel(notifier_clears[0], pp->notifier_clear_block);
1014 pp = host->ports[1]->private_data;
1015 writel(notifier_clears[1], pp->notifier_clear_block);
1016 }
1017
1018 spin_unlock(&host->lock);
1019
1020 return IRQ_RETVAL(handled);
1021 }
1022
1023 static void nv_adma_freeze(struct ata_port *ap)
1024 {
1025 struct nv_adma_port_priv *pp = ap->private_data;
1026 void __iomem *mmio = pp->ctl_block;
1027 u16 tmp;
1028
1029 nv_ck804_freeze(ap);
1030
1031 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1032 return;
1033
1034 /* clear any outstanding CK804 notifications */
1035 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1036 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1037
1038 /* Disable interrupt */
1039 tmp = readw(mmio + NV_ADMA_CTL);
1040 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1041 mmio + NV_ADMA_CTL);
1042 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1043 }
1044
1045 static void nv_adma_thaw(struct ata_port *ap)
1046 {
1047 struct nv_adma_port_priv *pp = ap->private_data;
1048 void __iomem *mmio = pp->ctl_block;
1049 u16 tmp;
1050
1051 nv_ck804_thaw(ap);
1052
1053 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1054 return;
1055
1056 /* Enable interrupt */
1057 tmp = readw(mmio + NV_ADMA_CTL);
1058 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1059 mmio + NV_ADMA_CTL);
1060 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1061 }
1062
1063 static void nv_adma_irq_clear(struct ata_port *ap)
1064 {
1065 struct nv_adma_port_priv *pp = ap->private_data;
1066 void __iomem *mmio = pp->ctl_block;
1067 u32 notifier_clears[2];
1068
1069 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
1070 ata_sff_irq_clear(ap);
1071 return;
1072 }
1073
1074 /* clear any outstanding CK804 notifications */
1075 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1076 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1077
1078 /* clear ADMA status */
1079 writew(0xffff, mmio + NV_ADMA_STAT);
1080
1081 /* clear notifiers - note both ports need to be written with
1082 something even though we are only clearing on one */
1083 if (ap->port_no == 0) {
1084 notifier_clears[0] = 0xFFFFFFFF;
1085 notifier_clears[1] = 0;
1086 } else {
1087 notifier_clears[0] = 0;
1088 notifier_clears[1] = 0xFFFFFFFF;
1089 }
1090 pp = ap->host->ports[0]->private_data;
1091 writel(notifier_clears[0], pp->notifier_clear_block);
1092 pp = ap->host->ports[1]->private_data;
1093 writel(notifier_clears[1], pp->notifier_clear_block);
1094 }
1095
1096 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
1097 {
1098 struct nv_adma_port_priv *pp = qc->ap->private_data;
1099
1100 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
1101 ata_sff_post_internal_cmd(qc);
1102 }
1103
1104 static int nv_adma_port_start(struct ata_port *ap)
1105 {
1106 struct device *dev = ap->host->dev;
1107 struct nv_adma_port_priv *pp;
1108 int rc;
1109 void *mem;
1110 dma_addr_t mem_dma;
1111 void __iomem *mmio;
1112 struct pci_dev *pdev = to_pci_dev(dev);
1113 u16 tmp;
1114
1115 VPRINTK("ENTER\n");
1116
1117 /* Ensure DMA mask is set to 32-bit before allocating legacy PRD and
1118 pad buffers */
1119 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1120 if (rc)
1121 return rc;
1122 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1123 if (rc)
1124 return rc;
1125
1126 rc = ata_port_start(ap);
1127 if (rc)
1128 return rc;
1129
1130 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1131 if (!pp)
1132 return -ENOMEM;
1133
1134 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
1135 ap->port_no * NV_ADMA_PORT_SIZE;
1136 pp->ctl_block = mmio;
1137 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
1138 pp->notifier_clear_block = pp->gen_block +
1139 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1140
1141 /* Now that the legacy PRD and padding buffer are allocated we can
1142 safely raise the DMA mask to allocate the CPB/APRD table.
1143 These are allowed to fail since we store the value that ends up
1144 being used to set as the bounce limit in slave_config later if
1145 needed. */
1146 pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1147 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1148 pp->adma_dma_mask = *dev->dma_mask;
1149
1150 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1151 &mem_dma, GFP_KERNEL);
1152 if (!mem)
1153 return -ENOMEM;
1154 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1155
1156 /*
1157 * First item in chunk of DMA memory:
1158 * 128-byte command parameter block (CPB)
1159 * one for each command tag
1160 */
1161 pp->cpb = mem;
1162 pp->cpb_dma = mem_dma;
1163
1164 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1165 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1166
1167 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1168 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1169
1170 /*
1171 * Second item: block of ADMA_SGTBL_LEN s/g entries
1172 */
1173 pp->aprd = mem;
1174 pp->aprd_dma = mem_dma;
1175
1176 ap->private_data = pp;
1177
1178 /* clear any outstanding interrupt conditions */
1179 writew(0xffff, mmio + NV_ADMA_STAT);
1180
1181 /* initialize port variables */
1182 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1183
1184 /* clear CPB fetch count */
1185 writew(0, mmio + NV_ADMA_CPB_COUNT);
1186
1187 /* clear GO for register mode, enable interrupt */
1188 tmp = readw(mmio + NV_ADMA_CTL);
1189 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1190 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1191
1192 tmp = readw(mmio + NV_ADMA_CTL);
1193 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1194 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1195 udelay(1);
1196 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1197 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1198
1199 return 0;
1200 }
1201
1202 static void nv_adma_port_stop(struct ata_port *ap)
1203 {
1204 struct nv_adma_port_priv *pp = ap->private_data;
1205 void __iomem *mmio = pp->ctl_block;
1206
1207 VPRINTK("ENTER\n");
1208 writew(0, mmio + NV_ADMA_CTL);
1209 }
1210
1211 #ifdef CONFIG_PM
1212 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1213 {
1214 struct nv_adma_port_priv *pp = ap->private_data;
1215 void __iomem *mmio = pp->ctl_block;
1216
1217 /* Go to register mode - clears GO */
1218 nv_adma_register_mode(ap);
1219
1220 /* clear CPB fetch count */
1221 writew(0, mmio + NV_ADMA_CPB_COUNT);
1222
1223 /* disable interrupt, shut down port */
1224 writew(0, mmio + NV_ADMA_CTL);
1225
1226 return 0;
1227 }
1228
1229 static int nv_adma_port_resume(struct ata_port *ap)
1230 {
1231 struct nv_adma_port_priv *pp = ap->private_data;
1232 void __iomem *mmio = pp->ctl_block;
1233 u16 tmp;
1234
1235 /* set CPB block location */
1236 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1237 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1238
1239 /* clear any outstanding interrupt conditions */
1240 writew(0xffff, mmio + NV_ADMA_STAT);
1241
1242 /* initialize port variables */
1243 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1244
1245 /* clear CPB fetch count */
1246 writew(0, mmio + NV_ADMA_CPB_COUNT);
1247
1248 /* clear GO for register mode, enable interrupt */
1249 tmp = readw(mmio + NV_ADMA_CTL);
1250 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1251 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1252
1253 tmp = readw(mmio + NV_ADMA_CTL);
1254 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1255 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1256 udelay(1);
1257 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1258 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1259
1260 return 0;
1261 }
1262 #endif
1263
1264 static void nv_adma_setup_port(struct ata_port *ap)
1265 {
1266 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1267 struct ata_ioports *ioport = &ap->ioaddr;
1268
1269 VPRINTK("ENTER\n");
1270
1271 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
1272
1273 ioport->cmd_addr = mmio;
1274 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
1275 ioport->error_addr =
1276 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1277 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1278 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1279 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1280 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1281 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
1282 ioport->status_addr =
1283 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
1284 ioport->altstatus_addr =
1285 ioport->ctl_addr = mmio + 0x20;
1286 }
1287
1288 static int nv_adma_host_init(struct ata_host *host)
1289 {
1290 struct pci_dev *pdev = to_pci_dev(host->dev);
1291 unsigned int i;
1292 u32 tmp32;
1293
1294 VPRINTK("ENTER\n");
1295
1296 /* enable ADMA on the ports */
1297 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1298 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1299 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1300 NV_MCP_SATA_CFG_20_PORT1_EN |
1301 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1302
1303 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1304
1305 for (i = 0; i < host->n_ports; i++)
1306 nv_adma_setup_port(host->ports[i]);
1307
1308 return 0;
1309 }
1310
1311 static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1312 struct scatterlist *sg,
1313 int idx,
1314 struct nv_adma_prd *aprd)
1315 {
1316 u8 flags = 0;
1317 if (qc->tf.flags & ATA_TFLAG_WRITE)
1318 flags |= NV_APRD_WRITE;
1319 if (idx == qc->n_elem - 1)
1320 flags |= NV_APRD_END;
1321 else if (idx != 4)
1322 flags |= NV_APRD_CONT;
1323
1324 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1325 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
1326 aprd->flags = flags;
1327 aprd->packet_len = 0;
1328 }
1329
1330 static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1331 {
1332 struct nv_adma_port_priv *pp = qc->ap->private_data;
1333 struct nv_adma_prd *aprd;
1334 struct scatterlist *sg;
1335 unsigned int si;
1336
1337 VPRINTK("ENTER\n");
1338
1339 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1340 aprd = (si < 5) ? &cpb->aprd[si] :
1341 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (si-5)];
1342 nv_adma_fill_aprd(qc, sg, si, aprd);
1343 }
1344 if (si > 5)
1345 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
1346 else
1347 cpb->next_aprd = cpu_to_le64(0);
1348 }
1349
1350 static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1351 {
1352 struct nv_adma_port_priv *pp = qc->ap->private_data;
1353
1354 /* ADMA engine can only be used for non-ATAPI DMA commands,
1355 or interrupt-driven no-data commands. */
1356 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
1357 (qc->tf.flags & ATA_TFLAG_POLLING))
1358 return 1;
1359
1360 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
1361 (qc->tf.protocol == ATA_PROT_NODATA))
1362 return 0;
1363
1364 return 1;
1365 }
1366
1367 static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1368 {
1369 struct nv_adma_port_priv *pp = qc->ap->private_data;
1370 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1371 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
1372 NV_CPB_CTL_IEN;
1373
1374 if (nv_adma_use_reg_mode(qc)) {
1375 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1376 (qc->flags & ATA_QCFLAG_DMAMAP));
1377 nv_adma_register_mode(qc->ap);
1378 ata_sff_qc_prep(qc);
1379 return;
1380 }
1381
1382 cpb->resp_flags = NV_CPB_RESP_DONE;
1383 wmb();
1384 cpb->ctl_flags = 0;
1385 wmb();
1386
1387 cpb->len = 3;
1388 cpb->tag = qc->tag;
1389 cpb->next_cpb_idx = 0;
1390
1391 /* turn on NCQ flags for NCQ commands */
1392 if (qc->tf.protocol == ATA_PROT_NCQ)
1393 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1394
1395 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1396
1397 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1398
1399 if (qc->flags & ATA_QCFLAG_DMAMAP) {
1400 nv_adma_fill_sg(qc, cpb);
1401 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1402 } else
1403 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
1404
1405 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1406 until we are finished filling in all of the contents */
1407 wmb();
1408 cpb->ctl_flags = ctl_flags;
1409 wmb();
1410 cpb->resp_flags = 0;
1411 }
1412
1413 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1414 {
1415 struct nv_adma_port_priv *pp = qc->ap->private_data;
1416 void __iomem *mmio = pp->ctl_block;
1417 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
1418
1419 VPRINTK("ENTER\n");
1420
1421 /* We can't handle result taskfile with NCQ commands, since
1422 retrieving the taskfile switches us out of ADMA mode and would abort
1423 existing commands. */
1424 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1425 (qc->flags & ATA_QCFLAG_RESULT_TF))) {
1426 ata_dev_printk(qc->dev, KERN_ERR,
1427 "NCQ w/ RESULT_TF not allowed\n");
1428 return AC_ERR_SYSTEM;
1429 }
1430
1431 if (nv_adma_use_reg_mode(qc)) {
1432 /* use ATA register mode */
1433 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
1434 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1435 (qc->flags & ATA_QCFLAG_DMAMAP));
1436 nv_adma_register_mode(qc->ap);
1437 return ata_sff_qc_issue(qc);
1438 } else
1439 nv_adma_mode(qc->ap);
1440
1441 /* write append register, command tag in lower 8 bits
1442 and (number of cpbs to append -1) in top 8 bits */
1443 wmb();
1444
1445 if (curr_ncq != pp->last_issue_ncq) {
1446 /* Seems to need some delay before switching between NCQ and
1447 non-NCQ commands, else we get command timeouts and such. */
1448 udelay(20);
1449 pp->last_issue_ncq = curr_ncq;
1450 }
1451
1452 writew(qc->tag, mmio + NV_ADMA_APPEND);
1453
1454 DPRINTK("Issued tag %u\n", qc->tag);
1455
1456 return 0;
1457 }
1458
1459 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1460 {
1461 struct ata_host *host = dev_instance;
1462 unsigned int i;
1463 unsigned int handled = 0;
1464 unsigned long flags;
1465
1466 spin_lock_irqsave(&host->lock, flags);
1467
1468 for (i = 0; i < host->n_ports; i++) {
1469 struct ata_port *ap;
1470
1471 ap = host->ports[i];
1472 if (ap &&
1473 !(ap->flags & ATA_FLAG_DISABLED)) {
1474 struct ata_queued_cmd *qc;
1475
1476 qc = ata_qc_from_tag(ap, ap->link.active_tag);
1477 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING)))
1478 handled += ata_sff_host_intr(ap, qc);
1479 else
1480 // No request pending? Clear interrupt status
1481 // anyway, in case there's one pending.
1482 ap->ops->sff_check_status(ap);
1483 }
1484
1485 }
1486
1487 spin_unlock_irqrestore(&host->lock, flags);
1488
1489 return IRQ_RETVAL(handled);
1490 }
1491
1492 static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
1493 {
1494 int i, handled = 0;
1495
1496 for (i = 0; i < host->n_ports; i++) {
1497 struct ata_port *ap = host->ports[i];
1498
1499 if (ap && !(ap->flags & ATA_FLAG_DISABLED))
1500 handled += nv_host_intr(ap, irq_stat);
1501
1502 irq_stat >>= NV_INT_PORT_SHIFT;
1503 }
1504
1505 return IRQ_RETVAL(handled);
1506 }
1507
1508 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
1509 {
1510 struct ata_host *host = dev_instance;
1511 u8 irq_stat;
1512 irqreturn_t ret;
1513
1514 spin_lock(&host->lock);
1515 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
1516 ret = nv_do_interrupt(host, irq_stat);
1517 spin_unlock(&host->lock);
1518
1519 return ret;
1520 }
1521
1522 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
1523 {
1524 struct ata_host *host = dev_instance;
1525 u8 irq_stat;
1526 irqreturn_t ret;
1527
1528 spin_lock(&host->lock);
1529 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1530 ret = nv_do_interrupt(host, irq_stat);
1531 spin_unlock(&host->lock);
1532
1533 return ret;
1534 }
1535
1536 static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
1537 {
1538 if (sc_reg > SCR_CONTROL)
1539 return -EINVAL;
1540
1541 *val = ioread32(link->ap->ioaddr.scr_addr + (sc_reg * 4));
1542 return 0;
1543 }
1544
1545 static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
1546 {
1547 if (sc_reg > SCR_CONTROL)
1548 return -EINVAL;
1549
1550 iowrite32(val, link->ap->ioaddr.scr_addr + (sc_reg * 4));
1551 return 0;
1552 }
1553
1554 static int nv_noclassify_hardreset(struct ata_link *link, unsigned int *class,
1555 unsigned long deadline)
1556 {
1557 bool online;
1558 int rc;
1559
1560 rc = sata_link_hardreset(link, sata_deb_timing_hotplug, deadline,
1561 &online, NULL);
1562 return online ? -EAGAIN : rc;
1563 }
1564
1565 static void nv_nf2_freeze(struct ata_port *ap)
1566 {
1567 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1568 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1569 u8 mask;
1570
1571 mask = ioread8(scr_addr + NV_INT_ENABLE);
1572 mask &= ~(NV_INT_ALL << shift);
1573 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1574 }
1575
1576 static void nv_nf2_thaw(struct ata_port *ap)
1577 {
1578 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1579 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1580 u8 mask;
1581
1582 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
1583
1584 mask = ioread8(scr_addr + NV_INT_ENABLE);
1585 mask |= (NV_INT_MASK << shift);
1586 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1587 }
1588
1589 static void nv_ck804_freeze(struct ata_port *ap)
1590 {
1591 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1592 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1593 u8 mask;
1594
1595 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1596 mask &= ~(NV_INT_ALL << shift);
1597 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1598 }
1599
1600 static void nv_ck804_thaw(struct ata_port *ap)
1601 {
1602 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1603 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1604 u8 mask;
1605
1606 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1607
1608 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1609 mask |= (NV_INT_MASK << shift);
1610 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1611 }
1612
1613 static void nv_mcp55_freeze(struct ata_port *ap)
1614 {
1615 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1616 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1617 u32 mask;
1618
1619 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1620
1621 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1622 mask &= ~(NV_INT_ALL_MCP55 << shift);
1623 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1624 ata_sff_freeze(ap);
1625 }
1626
1627 static void nv_mcp55_thaw(struct ata_port *ap)
1628 {
1629 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1630 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1631 u32 mask;
1632
1633 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1634
1635 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1636 mask |= (NV_INT_MASK_MCP55 << shift);
1637 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1638 ata_sff_thaw(ap);
1639 }
1640
1641 static void nv_adma_error_handler(struct ata_port *ap)
1642 {
1643 struct nv_adma_port_priv *pp = ap->private_data;
1644 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
1645 void __iomem *mmio = pp->ctl_block;
1646 int i;
1647 u16 tmp;
1648
1649 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
1650 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1651 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1652 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1653 u32 status = readw(mmio + NV_ADMA_STAT);
1654 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1655 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
1656
1657 ata_port_printk(ap, KERN_ERR,
1658 "EH in ADMA mode, notifier 0x%X "
1659 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1660 "next cpb count 0x%X next cpb idx 0x%x\n",
1661 notifier, notifier_error, gen_ctl, status,
1662 cpb_count, next_cpb_idx);
1663
1664 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
1665 struct nv_adma_cpb *cpb = &pp->cpb[i];
1666 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
1667 ap->link.sactive & (1 << i))
1668 ata_port_printk(ap, KERN_ERR,
1669 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1670 i, cpb->ctl_flags, cpb->resp_flags);
1671 }
1672 }
1673
1674 /* Push us back into port register mode for error handling. */
1675 nv_adma_register_mode(ap);
1676
1677 /* Mark all of the CPBs as invalid to prevent them from
1678 being executed */
1679 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
1680 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1681
1682 /* clear CPB fetch count */
1683 writew(0, mmio + NV_ADMA_CPB_COUNT);
1684
1685 /* Reset channel */
1686 tmp = readw(mmio + NV_ADMA_CTL);
1687 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1688 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1689 udelay(1);
1690 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1691 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1692 }
1693
1694 ata_sff_error_handler(ap);
1695 }
1696
1697 static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1698 {
1699 struct nv_swncq_port_priv *pp = ap->private_data;
1700 struct defer_queue *dq = &pp->defer_queue;
1701
1702 /* queue is full */
1703 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1704 dq->defer_bits |= (1 << qc->tag);
1705 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag;
1706 }
1707
1708 static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1709 {
1710 struct nv_swncq_port_priv *pp = ap->private_data;
1711 struct defer_queue *dq = &pp->defer_queue;
1712 unsigned int tag;
1713
1714 if (dq->head == dq->tail) /* null queue */
1715 return NULL;
1716
1717 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1718 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1719 WARN_ON(!(dq->defer_bits & (1 << tag)));
1720 dq->defer_bits &= ~(1 << tag);
1721
1722 return ata_qc_from_tag(ap, tag);
1723 }
1724
1725 static void nv_swncq_fis_reinit(struct ata_port *ap)
1726 {
1727 struct nv_swncq_port_priv *pp = ap->private_data;
1728
1729 pp->dhfis_bits = 0;
1730 pp->dmafis_bits = 0;
1731 pp->sdbfis_bits = 0;
1732 pp->ncq_flags = 0;
1733 }
1734
1735 static void nv_swncq_pp_reinit(struct ata_port *ap)
1736 {
1737 struct nv_swncq_port_priv *pp = ap->private_data;
1738 struct defer_queue *dq = &pp->defer_queue;
1739
1740 dq->head = 0;
1741 dq->tail = 0;
1742 dq->defer_bits = 0;
1743 pp->qc_active = 0;
1744 pp->last_issue_tag = ATA_TAG_POISON;
1745 nv_swncq_fis_reinit(ap);
1746 }
1747
1748 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1749 {
1750 struct nv_swncq_port_priv *pp = ap->private_data;
1751
1752 writew(fis, pp->irq_block);
1753 }
1754
1755 static void __ata_bmdma_stop(struct ata_port *ap)
1756 {
1757 struct ata_queued_cmd qc;
1758
1759 qc.ap = ap;
1760 ata_bmdma_stop(&qc);
1761 }
1762
1763 static void nv_swncq_ncq_stop(struct ata_port *ap)
1764 {
1765 struct nv_swncq_port_priv *pp = ap->private_data;
1766 unsigned int i;
1767 u32 sactive;
1768 u32 done_mask;
1769
1770 ata_port_printk(ap, KERN_ERR,
1771 "EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n",
1772 ap->qc_active, ap->link.sactive);
1773 ata_port_printk(ap, KERN_ERR,
1774 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1775 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1776 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1777 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1778
1779 ata_port_printk(ap, KERN_ERR, "ATA_REG 0x%X ERR_REG 0x%X\n",
1780 ap->ops->sff_check_status(ap),
1781 ioread8(ap->ioaddr.error_addr));
1782
1783 sactive = readl(pp->sactive_block);
1784 done_mask = pp->qc_active ^ sactive;
1785
1786 ata_port_printk(ap, KERN_ERR, "tag : dhfis dmafis sdbfis sacitve\n");
1787 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1788 u8 err = 0;
1789 if (pp->qc_active & (1 << i))
1790 err = 0;
1791 else if (done_mask & (1 << i))
1792 err = 1;
1793 else
1794 continue;
1795
1796 ata_port_printk(ap, KERN_ERR,
1797 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1798 (pp->dhfis_bits >> i) & 0x1,
1799 (pp->dmafis_bits >> i) & 0x1,
1800 (pp->sdbfis_bits >> i) & 0x1,
1801 (sactive >> i) & 0x1,
1802 (err ? "error! tag doesn't exit" : " "));
1803 }
1804
1805 nv_swncq_pp_reinit(ap);
1806 ap->ops->sff_irq_clear(ap);
1807 __ata_bmdma_stop(ap);
1808 nv_swncq_irq_clear(ap, 0xffff);
1809 }
1810
1811 static void nv_swncq_error_handler(struct ata_port *ap)
1812 {
1813 struct ata_eh_context *ehc = &ap->link.eh_context;
1814
1815 if (ap->link.sactive) {
1816 nv_swncq_ncq_stop(ap);
1817 ehc->i.action |= ATA_EH_RESET;
1818 }
1819
1820 ata_sff_error_handler(ap);
1821 }
1822
1823 #ifdef CONFIG_PM
1824 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1825 {
1826 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1827 u32 tmp;
1828
1829 /* clear irq */
1830 writel(~0, mmio + NV_INT_STATUS_MCP55);
1831
1832 /* disable irq */
1833 writel(0, mmio + NV_INT_ENABLE_MCP55);
1834
1835 /* disable swncq */
1836 tmp = readl(mmio + NV_CTL_MCP55);
1837 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1838 writel(tmp, mmio + NV_CTL_MCP55);
1839
1840 return 0;
1841 }
1842
1843 static int nv_swncq_port_resume(struct ata_port *ap)
1844 {
1845 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1846 u32 tmp;
1847
1848 /* clear irq */
1849 writel(~0, mmio + NV_INT_STATUS_MCP55);
1850
1851 /* enable irq */
1852 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1853
1854 /* enable swncq */
1855 tmp = readl(mmio + NV_CTL_MCP55);
1856 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1857
1858 return 0;
1859 }
1860 #endif
1861
1862 static void nv_swncq_host_init(struct ata_host *host)
1863 {
1864 u32 tmp;
1865 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1866 struct pci_dev *pdev = to_pci_dev(host->dev);
1867 u8 regval;
1868
1869 /* disable ECO 398 */
1870 pci_read_config_byte(pdev, 0x7f, &regval);
1871 regval &= ~(1 << 7);
1872 pci_write_config_byte(pdev, 0x7f, regval);
1873
1874 /* enable swncq */
1875 tmp = readl(mmio + NV_CTL_MCP55);
1876 VPRINTK("HOST_CTL:0x%X\n", tmp);
1877 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1878
1879 /* enable irq intr */
1880 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1881 VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1882 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1883
1884 /* clear port irq */
1885 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1886 }
1887
1888 static int nv_swncq_slave_config(struct scsi_device *sdev)
1889 {
1890 struct ata_port *ap = ata_shost_to_port(sdev->host);
1891 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1892 struct ata_device *dev;
1893 int rc;
1894 u8 rev;
1895 u8 check_maxtor = 0;
1896 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1897
1898 rc = ata_scsi_slave_config(sdev);
1899 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1900 /* Not a proper libata device, ignore */
1901 return rc;
1902
1903 dev = &ap->link.device[sdev->id];
1904 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1905 return rc;
1906
1907 /* if MCP51 and Maxtor, then disable ncq */
1908 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1909 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1910 check_maxtor = 1;
1911
1912 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1913 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1914 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1915 pci_read_config_byte(pdev, 0x8, &rev);
1916 if (rev <= 0xa2)
1917 check_maxtor = 1;
1918 }
1919
1920 if (!check_maxtor)
1921 return rc;
1922
1923 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1924
1925 if (strncmp(model_num, "Maxtor", 6) == 0) {
1926 ata_scsi_change_queue_depth(sdev, 1);
1927 ata_dev_printk(dev, KERN_NOTICE,
1928 "Disabling SWNCQ mode (depth %x)\n", sdev->queue_depth);
1929 }
1930
1931 return rc;
1932 }
1933
1934 static int nv_swncq_port_start(struct ata_port *ap)
1935 {
1936 struct device *dev = ap->host->dev;
1937 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1938 struct nv_swncq_port_priv *pp;
1939 int rc;
1940
1941 rc = ata_port_start(ap);
1942 if (rc)
1943 return rc;
1944
1945 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1946 if (!pp)
1947 return -ENOMEM;
1948
1949 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1950 &pp->prd_dma, GFP_KERNEL);
1951 if (!pp->prd)
1952 return -ENOMEM;
1953 memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
1954
1955 ap->private_data = pp;
1956 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1957 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1958 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1959
1960 return 0;
1961 }
1962
1963 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
1964 {
1965 if (qc->tf.protocol != ATA_PROT_NCQ) {
1966 ata_sff_qc_prep(qc);
1967 return;
1968 }
1969
1970 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
1971 return;
1972
1973 nv_swncq_fill_sg(qc);
1974 }
1975
1976 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
1977 {
1978 struct ata_port *ap = qc->ap;
1979 struct scatterlist *sg;
1980 struct nv_swncq_port_priv *pp = ap->private_data;
1981 struct ata_prd *prd;
1982 unsigned int si, idx;
1983
1984 prd = pp->prd + ATA_MAX_PRD * qc->tag;
1985
1986 idx = 0;
1987 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1988 u32 addr, offset;
1989 u32 sg_len, len;
1990
1991 addr = (u32)sg_dma_address(sg);
1992 sg_len = sg_dma_len(sg);
1993
1994 while (sg_len) {
1995 offset = addr & 0xffff;
1996 len = sg_len;
1997 if ((offset + sg_len) > 0x10000)
1998 len = 0x10000 - offset;
1999
2000 prd[idx].addr = cpu_to_le32(addr);
2001 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
2002
2003 idx++;
2004 sg_len -= len;
2005 addr += len;
2006 }
2007 }
2008
2009 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
2010 }
2011
2012 static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2013 struct ata_queued_cmd *qc)
2014 {
2015 struct nv_swncq_port_priv *pp = ap->private_data;
2016
2017 if (qc == NULL)
2018 return 0;
2019
2020 DPRINTK("Enter\n");
2021
2022 writel((1 << qc->tag), pp->sactive_block);
2023 pp->last_issue_tag = qc->tag;
2024 pp->dhfis_bits &= ~(1 << qc->tag);
2025 pp->dmafis_bits &= ~(1 << qc->tag);
2026 pp->qc_active |= (0x1 << qc->tag);
2027
2028 ap->ops->sff_tf_load(ap, &qc->tf); /* load tf registers */
2029 ap->ops->sff_exec_command(ap, &qc->tf);
2030
2031 DPRINTK("Issued tag %u\n", qc->tag);
2032
2033 return 0;
2034 }
2035
2036 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2037 {
2038 struct ata_port *ap = qc->ap;
2039 struct nv_swncq_port_priv *pp = ap->private_data;
2040
2041 if (qc->tf.protocol != ATA_PROT_NCQ)
2042 return ata_sff_qc_issue(qc);
2043
2044 DPRINTK("Enter\n");
2045
2046 if (!pp->qc_active)
2047 nv_swncq_issue_atacmd(ap, qc);
2048 else
2049 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2050
2051 return 0;
2052 }
2053
2054 static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2055 {
2056 u32 serror;
2057 struct ata_eh_info *ehi = &ap->link.eh_info;
2058
2059 ata_ehi_clear_desc(ehi);
2060
2061 /* AHCI needs SError cleared; otherwise, it might lock up */
2062 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2063 sata_scr_write(&ap->link, SCR_ERROR, serror);
2064
2065 /* analyze @irq_stat */
2066 if (fis & NV_SWNCQ_IRQ_ADDED)
2067 ata_ehi_push_desc(ehi, "hot plug");
2068 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2069 ata_ehi_push_desc(ehi, "hot unplug");
2070
2071 ata_ehi_hotplugged(ehi);
2072
2073 /* okay, let's hand over to EH */
2074 ehi->serror |= serror;
2075
2076 ata_port_freeze(ap);
2077 }
2078
2079 static int nv_swncq_sdbfis(struct ata_port *ap)
2080 {
2081 struct ata_queued_cmd *qc;
2082 struct nv_swncq_port_priv *pp = ap->private_data;
2083 struct ata_eh_info *ehi = &ap->link.eh_info;
2084 u32 sactive;
2085 int nr_done = 0;
2086 u32 done_mask;
2087 int i;
2088 u8 host_stat;
2089 u8 lack_dhfis = 0;
2090
2091 host_stat = ap->ops->bmdma_status(ap);
2092 if (unlikely(host_stat & ATA_DMA_ERR)) {
2093 /* error when transfering data to/from memory */
2094 ata_ehi_clear_desc(ehi);
2095 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2096 ehi->err_mask |= AC_ERR_HOST_BUS;
2097 ehi->action |= ATA_EH_RESET;
2098 return -EINVAL;
2099 }
2100
2101 ap->ops->sff_irq_clear(ap);
2102 __ata_bmdma_stop(ap);
2103
2104 sactive = readl(pp->sactive_block);
2105 done_mask = pp->qc_active ^ sactive;
2106
2107 if (unlikely(done_mask & sactive)) {
2108 ata_ehi_clear_desc(ehi);
2109 ata_ehi_push_desc(ehi, "illegal SWNCQ:qc_active transition"
2110 "(%08x->%08x)", pp->qc_active, sactive);
2111 ehi->err_mask |= AC_ERR_HSM;
2112 ehi->action |= ATA_EH_RESET;
2113 return -EINVAL;
2114 }
2115 for (i = 0; i < ATA_MAX_QUEUE; i++) {
2116 if (!(done_mask & (1 << i)))
2117 continue;
2118
2119 qc = ata_qc_from_tag(ap, i);
2120 if (qc) {
2121 ata_qc_complete(qc);
2122 pp->qc_active &= ~(1 << i);
2123 pp->dhfis_bits &= ~(1 << i);
2124 pp->dmafis_bits &= ~(1 << i);
2125 pp->sdbfis_bits |= (1 << i);
2126 nr_done++;
2127 }
2128 }
2129
2130 if (!ap->qc_active) {
2131 DPRINTK("over\n");
2132 nv_swncq_pp_reinit(ap);
2133 return nr_done;
2134 }
2135
2136 if (pp->qc_active & pp->dhfis_bits)
2137 return nr_done;
2138
2139 if ((pp->ncq_flags & ncq_saw_backout) ||
2140 (pp->qc_active ^ pp->dhfis_bits))
2141 /* if the controller cann't get a device to host register FIS,
2142 * The driver needs to reissue the new command.
2143 */
2144 lack_dhfis = 1;
2145
2146 DPRINTK("id 0x%x QC: qc_active 0x%x,"
2147 "SWNCQ:qc_active 0x%X defer_bits %X "
2148 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2149 ap->print_id, ap->qc_active, pp->qc_active,
2150 pp->defer_queue.defer_bits, pp->dhfis_bits,
2151 pp->dmafis_bits, pp->last_issue_tag);
2152
2153 nv_swncq_fis_reinit(ap);
2154
2155 if (lack_dhfis) {
2156 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2157 nv_swncq_issue_atacmd(ap, qc);
2158 return nr_done;
2159 }
2160
2161 if (pp->defer_queue.defer_bits) {
2162 /* send deferral queue command */
2163 qc = nv_swncq_qc_from_dq(ap);
2164 WARN_ON(qc == NULL);
2165 nv_swncq_issue_atacmd(ap, qc);
2166 }
2167
2168 return nr_done;
2169 }
2170
2171 static inline u32 nv_swncq_tag(struct ata_port *ap)
2172 {
2173 struct nv_swncq_port_priv *pp = ap->private_data;
2174 u32 tag;
2175
2176 tag = readb(pp->tag_block) >> 2;
2177 return (tag & 0x1f);
2178 }
2179
2180 static int nv_swncq_dmafis(struct ata_port *ap)
2181 {
2182 struct ata_queued_cmd *qc;
2183 unsigned int rw;
2184 u8 dmactl;
2185 u32 tag;
2186 struct nv_swncq_port_priv *pp = ap->private_data;
2187
2188 __ata_bmdma_stop(ap);
2189 tag = nv_swncq_tag(ap);
2190
2191 DPRINTK("dma setup tag 0x%x\n", tag);
2192 qc = ata_qc_from_tag(ap, tag);
2193
2194 if (unlikely(!qc))
2195 return 0;
2196
2197 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2198
2199 /* load PRD table addr. */
2200 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,
2201 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2202
2203 /* specify data direction, triple-check start bit is clear */
2204 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2205 dmactl &= ~ATA_DMA_WR;
2206 if (!rw)
2207 dmactl |= ATA_DMA_WR;
2208
2209 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2210
2211 return 1;
2212 }
2213
2214 static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2215 {
2216 struct nv_swncq_port_priv *pp = ap->private_data;
2217 struct ata_queued_cmd *qc;
2218 struct ata_eh_info *ehi = &ap->link.eh_info;
2219 u32 serror;
2220 u8 ata_stat;
2221 int rc = 0;
2222
2223 ata_stat = ap->ops->sff_check_status(ap);
2224 nv_swncq_irq_clear(ap, fis);
2225 if (!fis)
2226 return;
2227
2228 if (ap->pflags & ATA_PFLAG_FROZEN)
2229 return;
2230
2231 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2232 nv_swncq_hotplug(ap, fis);
2233 return;
2234 }
2235
2236 if (!pp->qc_active)
2237 return;
2238
2239 if (ap->ops->scr_read(&ap->link, SCR_ERROR, &serror))
2240 return;
2241 ap->ops->scr_write(&ap->link, SCR_ERROR, serror);
2242
2243 if (ata_stat & ATA_ERR) {
2244 ata_ehi_clear_desc(ehi);
2245 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2246 ehi->err_mask |= AC_ERR_DEV;
2247 ehi->serror |= serror;
2248 ehi->action |= ATA_EH_RESET;
2249 ata_port_freeze(ap);
2250 return;
2251 }
2252
2253 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2254 /* If the IRQ is backout, driver must issue
2255 * the new command again some time later.
2256 */
2257 pp->ncq_flags |= ncq_saw_backout;
2258 }
2259
2260 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2261 pp->ncq_flags |= ncq_saw_sdb;
2262 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2263 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2264 ap->print_id, pp->qc_active, pp->dhfis_bits,
2265 pp->dmafis_bits, readl(pp->sactive_block));
2266 rc = nv_swncq_sdbfis(ap);
2267 if (rc < 0)
2268 goto irq_error;
2269 }
2270
2271 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2272 /* The interrupt indicates the new command
2273 * was transmitted correctly to the drive.
2274 */
2275 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2276 pp->ncq_flags |= ncq_saw_d2h;
2277 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2278 ata_ehi_push_desc(ehi, "illegal fis transaction");
2279 ehi->err_mask |= AC_ERR_HSM;
2280 ehi->action |= ATA_EH_RESET;
2281 goto irq_error;
2282 }
2283
2284 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2285 !(pp->ncq_flags & ncq_saw_dmas)) {
2286 ata_stat = ap->ops->sff_check_status(ap);
2287 if (ata_stat & ATA_BUSY)
2288 goto irq_exit;
2289
2290 if (pp->defer_queue.defer_bits) {
2291 DPRINTK("send next command\n");
2292 qc = nv_swncq_qc_from_dq(ap);
2293 nv_swncq_issue_atacmd(ap, qc);
2294 }
2295 }
2296 }
2297
2298 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2299 /* program the dma controller with appropriate PRD buffers
2300 * and start the DMA transfer for requested command.
2301 */
2302 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2303 pp->ncq_flags |= ncq_saw_dmas;
2304 rc = nv_swncq_dmafis(ap);
2305 }
2306
2307 irq_exit:
2308 return;
2309 irq_error:
2310 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2311 ata_port_freeze(ap);
2312 return;
2313 }
2314
2315 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2316 {
2317 struct ata_host *host = dev_instance;
2318 unsigned int i;
2319 unsigned int handled = 0;
2320 unsigned long flags;
2321 u32 irq_stat;
2322
2323 spin_lock_irqsave(&host->lock, flags);
2324
2325 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2326
2327 for (i = 0; i < host->n_ports; i++) {
2328 struct ata_port *ap = host->ports[i];
2329
2330 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
2331 if (ap->link.sactive) {
2332 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2333 handled = 1;
2334 } else {
2335 if (irq_stat) /* reserve Hotplug */
2336 nv_swncq_irq_clear(ap, 0xfff0);
2337
2338 handled += nv_host_intr(ap, (u8)irq_stat);
2339 }
2340 }
2341 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2342 }
2343
2344 spin_unlock_irqrestore(&host->lock, flags);
2345
2346 return IRQ_RETVAL(handled);
2347 }
2348
2349 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
2350 {
2351 static int printed_version;
2352 const struct ata_port_info *ppi[] = { NULL, NULL };
2353 struct nv_pi_priv *ipriv;
2354 struct ata_host *host;
2355 struct nv_host_priv *hpriv;
2356 int rc;
2357 u32 bar;
2358 void __iomem *base;
2359 unsigned long type = ent->driver_data;
2360
2361 // Make sure this is a SATA controller by counting the number of bars
2362 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2363 // it's an IDE controller and we ignore it.
2364 for (bar = 0; bar < 6; bar++)
2365 if (pci_resource_start(pdev, bar) == 0)
2366 return -ENODEV;
2367
2368 if (!printed_version++)
2369 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
2370
2371 rc = pcim_enable_device(pdev);
2372 if (rc)
2373 return rc;
2374
2375 /* determine type and allocate host */
2376 if (type == CK804 && adma_enabled) {
2377 dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
2378 type = ADMA;
2379 } else if (type == MCP5x && swncq_enabled) {
2380 dev_printk(KERN_NOTICE, &pdev->dev, "Using SWNCQ mode\n");
2381 type = SWNCQ;
2382 }
2383
2384 ppi[0] = &nv_port_info[type];
2385 ipriv = ppi[0]->private_data;
2386 rc = ata_pci_sff_prepare_host(pdev, ppi, &host);
2387 if (rc)
2388 return rc;
2389
2390 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
2391 if (!hpriv)
2392 return -ENOMEM;
2393 hpriv->type = type;
2394 host->private_data = hpriv;
2395
2396 /* request and iomap NV_MMIO_BAR */
2397 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2398 if (rc)
2399 return rc;
2400
2401 /* configure SCR access */
2402 base = host->iomap[NV_MMIO_BAR];
2403 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2404 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
2405
2406 /* enable SATA space for CK804 */
2407 if (type >= CK804) {
2408 u8 regval;
2409
2410 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2411 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2412 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2413 }
2414
2415 /* init ADMA */
2416 if (type == ADMA) {
2417 rc = nv_adma_host_init(host);
2418 if (rc)
2419 return rc;
2420 } else if (type == SWNCQ)
2421 nv_swncq_host_init(host);
2422
2423 pci_set_master(pdev);
2424 return ata_host_activate(host, pdev->irq, ipriv->irq_handler,
2425 IRQF_SHARED, ipriv->sht);
2426 }
2427
2428 #ifdef CONFIG_PM
2429 static int nv_pci_device_resume(struct pci_dev *pdev)
2430 {
2431 struct ata_host *host = dev_get_drvdata(&pdev->dev);
2432 struct nv_host_priv *hpriv = host->private_data;
2433 int rc;
2434
2435 rc = ata_pci_device_do_resume(pdev);
2436 if (rc)
2437 return rc;
2438
2439 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
2440 if (hpriv->type >= CK804) {
2441 u8 regval;
2442
2443 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2444 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2445 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2446 }
2447 if (hpriv->type == ADMA) {
2448 u32 tmp32;
2449 struct nv_adma_port_priv *pp;
2450 /* enable/disable ADMA on the ports appropriately */
2451 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2452
2453 pp = host->ports[0]->private_data;
2454 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2455 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2456 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2457 else
2458 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
2459 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2460 pp = host->ports[1]->private_data;
2461 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2462 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
2463 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2464 else
2465 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
2466 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2467
2468 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2469 }
2470 }
2471
2472 ata_host_resume(host);
2473
2474 return 0;
2475 }
2476 #endif
2477
2478 static void nv_ck804_host_stop(struct ata_host *host)
2479 {
2480 struct pci_dev *pdev = to_pci_dev(host->dev);
2481 u8 regval;
2482
2483 /* disable SATA space for CK804 */
2484 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2485 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2486 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2487 }
2488
2489 static void nv_adma_host_stop(struct ata_host *host)
2490 {
2491 struct pci_dev *pdev = to_pci_dev(host->dev);
2492 u32 tmp32;
2493
2494 /* disable ADMA on the ports */
2495 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2496 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2497 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2498 NV_MCP_SATA_CFG_20_PORT1_EN |
2499 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2500
2501 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2502
2503 nv_ck804_host_stop(host);
2504 }
2505
2506 static int __init nv_init(void)
2507 {
2508 return pci_register_driver(&nv_pci_driver);
2509 }
2510
2511 static void __exit nv_exit(void)
2512 {
2513 pci_unregister_driver(&nv_pci_driver);
2514 }
2515
2516 module_init(nv_init);
2517 module_exit(nv_exit);
2518 module_param_named(adma, adma_enabled, bool, 0444);
2519 MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: true)");
2520 module_param_named(swncq, swncq_enabled, bool, 0444);
2521 MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: true)");
2522
CJK support for tty framebuffer vt