2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
7 * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms and conditions of the GNU General Public License,
11 * version 2, as published by the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
22 * The full GNU General Public License is included in this distribution in
23 * the file called "COPYING".
27 * Copyright(c) 2004-2009 Intel Corporation. All rights reserved.
29 * Redistribution and use in source and binary forms, with or without
30 * modification, are permitted provided that the following conditions are met:
32 * * Redistributions of source code must retain the above copyright
33 * notice, this list of conditions and the following disclaimer.
34 * * Redistributions in binary form must reproduce the above copyright
35 * notice, this list of conditions and the following disclaimer in
36 * the documentation and/or other materials provided with the
38 * * Neither the name of Intel Corporation nor the names of its
39 * contributors may be used to endorse or promote products derived
40 * from this software without specific prior written permission.
42 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
43 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
44 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
45 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
46 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
47 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
48 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
49 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
50 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
51 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
52 * POSSIBILITY OF SUCH DAMAGE.
56 * Support routines for v3+ hardware
59 #include <linux/pci.h>
60 #include <linux/dmaengine.h>
61 #include <linux/dma-mapping.h>
62 #include "registers.h"
67 static void ioat3_dma_unmap(struct ioat2_dma_chan
*ioat
,
68 struct ioat_ring_ent
*desc
)
70 struct ioat_chan_common
*chan
= &ioat
->base
;
71 struct pci_dev
*pdev
= chan
->device
->pdev
;
72 size_t len
= desc
->len
;
73 size_t offset
= len
- desc
->hw
->size
;
74 struct dma_async_tx_descriptor
*tx
= &desc
->txd
;
75 enum dma_ctrl_flags flags
= tx
->flags
;
77 switch (desc
->hw
->ctl_f
.op
) {
79 ioat_dma_unmap(chan
, flags
, len
, desc
->hw
);
82 struct ioat_fill_descriptor
*hw
= desc
->fill
;
84 if (!(flags
& DMA_COMPL_SKIP_DEST_UNMAP
))
85 ioat_unmap(pdev
, hw
->dst_addr
- offset
, len
,
86 PCI_DMA_FROMDEVICE
, flags
, 1);
90 dev_err(&pdev
->dev
, "%s: unknown op type: %#x\n",
91 __func__
, desc
->hw
->ctl_f
.op
);
96 static void __cleanup(struct ioat2_dma_chan
*ioat
, unsigned long phys_complete
)
98 struct ioat_chan_common
*chan
= &ioat
->base
;
99 struct ioat_ring_ent
*desc
;
100 bool seen_current
= false;
104 dev_dbg(to_dev(chan
), "%s: head: %#x tail: %#x issued: %#x\n",
105 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
);
107 active
= ioat2_ring_active(ioat
);
108 for (i
= 0; i
< active
&& !seen_current
; i
++) {
109 struct dma_async_tx_descriptor
*tx
;
111 prefetch(ioat2_get_ring_ent(ioat
, ioat
->tail
+ i
+ 1));
112 desc
= ioat2_get_ring_ent(ioat
, ioat
->tail
+ i
);
113 dump_desc_dbg(ioat
, desc
);
116 chan
->completed_cookie
= tx
->cookie
;
117 ioat3_dma_unmap(ioat
, desc
);
120 tx
->callback(tx
->callback_param
);
125 if (tx
->phys
== phys_complete
)
129 BUG_ON(!seen_current
); /* no active descs have written a completion? */
130 chan
->last_completion
= phys_complete
;
131 if (ioat
->head
== ioat
->tail
) {
132 dev_dbg(to_dev(chan
), "%s: cancel completion timeout\n",
134 clear_bit(IOAT_COMPLETION_PENDING
, &chan
->state
);
135 mod_timer(&chan
->timer
, jiffies
+ IDLE_TIMEOUT
);
139 static void ioat3_cleanup(struct ioat2_dma_chan
*ioat
)
141 struct ioat_chan_common
*chan
= &ioat
->base
;
142 unsigned long phys_complete
;
144 prefetch(chan
->completion
);
146 if (!spin_trylock_bh(&chan
->cleanup_lock
))
149 if (!ioat_cleanup_preamble(chan
, &phys_complete
)) {
150 spin_unlock_bh(&chan
->cleanup_lock
);
154 if (!spin_trylock_bh(&ioat
->ring_lock
)) {
155 spin_unlock_bh(&chan
->cleanup_lock
);
159 __cleanup(ioat
, phys_complete
);
161 spin_unlock_bh(&ioat
->ring_lock
);
162 spin_unlock_bh(&chan
->cleanup_lock
);
165 static void ioat3_cleanup_tasklet(unsigned long data
)
167 struct ioat2_dma_chan
*ioat
= (void *) data
;
170 writew(IOAT_CHANCTRL_RUN
, ioat
->base
.reg_base
+ IOAT_CHANCTRL_OFFSET
);
173 static void ioat3_restart_channel(struct ioat2_dma_chan
*ioat
)
175 struct ioat_chan_common
*chan
= &ioat
->base
;
176 unsigned long phys_complete
;
179 status
= ioat_chansts(chan
);
180 if (is_ioat_active(status
) || is_ioat_idle(status
))
182 while (is_ioat_active(status
) || is_ioat_idle(status
)) {
183 status
= ioat_chansts(chan
);
187 if (ioat_cleanup_preamble(chan
, &phys_complete
))
188 __cleanup(ioat
, phys_complete
);
190 __ioat2_restart_chan(ioat
);
193 static void ioat3_timer_event(unsigned long data
)
195 struct ioat2_dma_chan
*ioat
= (void *) data
;
196 struct ioat_chan_common
*chan
= &ioat
->base
;
198 spin_lock_bh(&chan
->cleanup_lock
);
199 if (test_bit(IOAT_COMPLETION_PENDING
, &chan
->state
)) {
200 unsigned long phys_complete
;
203 spin_lock_bh(&ioat
->ring_lock
);
204 status
= ioat_chansts(chan
);
206 /* when halted due to errors check for channel
207 * programming errors before advancing the completion state
209 if (is_ioat_halted(status
)) {
212 chanerr
= readl(chan
->reg_base
+ IOAT_CHANERR_OFFSET
);
213 BUG_ON(is_ioat_bug(chanerr
));
216 /* if we haven't made progress and we have already
217 * acknowledged a pending completion once, then be more
218 * forceful with a restart
220 if (ioat_cleanup_preamble(chan
, &phys_complete
))
221 __cleanup(ioat
, phys_complete
);
222 else if (test_bit(IOAT_COMPLETION_ACK
, &chan
->state
))
223 ioat3_restart_channel(ioat
);
225 set_bit(IOAT_COMPLETION_ACK
, &chan
->state
);
226 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
228 spin_unlock_bh(&ioat
->ring_lock
);
232 /* if the ring is idle, empty, and oversized try to step
235 spin_lock_bh(&ioat
->ring_lock
);
236 active
= ioat2_ring_active(ioat
);
237 if (active
== 0 && ioat
->alloc_order
> ioat_get_alloc_order())
238 reshape_ring(ioat
, ioat
->alloc_order
-1);
239 spin_unlock_bh(&ioat
->ring_lock
);
241 /* keep shrinking until we get back to our minimum
244 if (ioat
->alloc_order
> ioat_get_alloc_order())
245 mod_timer(&chan
->timer
, jiffies
+ IDLE_TIMEOUT
);
247 spin_unlock_bh(&chan
->cleanup_lock
);
250 static enum dma_status
251 ioat3_is_complete(struct dma_chan
*c
, dma_cookie_t cookie
,
252 dma_cookie_t
*done
, dma_cookie_t
*used
)
254 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
256 if (ioat_is_complete(c
, cookie
, done
, used
) == DMA_SUCCESS
)
261 return ioat_is_complete(c
, cookie
, done
, used
);
264 static struct dma_async_tx_descriptor
*
265 ioat3_prep_memset_lock(struct dma_chan
*c
, dma_addr_t dest
, int value
,
266 size_t len
, unsigned long flags
)
268 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
269 struct ioat_ring_ent
*desc
;
270 size_t total_len
= len
;
271 struct ioat_fill_descriptor
*fill
;
273 u64 src_data
= (0x0101010101010101ULL
) * (value
& 0xff);
277 num_descs
= ioat2_xferlen_to_descs(ioat
, len
);
278 if (likely(num_descs
) &&
279 ioat2_alloc_and_lock(&idx
, ioat
, num_descs
) == 0)
283 for (i
= 0; i
< num_descs
; i
++) {
284 size_t xfer_size
= min_t(size_t, len
, 1 << ioat
->xfercap_log
);
286 desc
= ioat2_get_ring_ent(ioat
, idx
+ i
);
289 fill
->size
= xfer_size
;
290 fill
->src_data
= src_data
;
291 fill
->dst_addr
= dest
;
293 fill
->ctl_f
.op
= IOAT_OP_FILL
;
297 dump_desc_dbg(ioat
, desc
);
300 desc
->txd
.flags
= flags
;
301 desc
->len
= total_len
;
302 fill
->ctl_f
.int_en
= !!(flags
& DMA_PREP_INTERRUPT
);
303 fill
->ctl_f
.fence
= !!(flags
& DMA_PREP_FENCE
);
304 fill
->ctl_f
.compl_write
= 1;
305 dump_desc_dbg(ioat
, desc
);
307 /* we leave the channel locked to ensure in order submission */
311 int __devinit
ioat3_dma_probe(struct ioatdma_device
*device
, int dca
)
313 struct pci_dev
*pdev
= device
->pdev
;
314 struct dma_device
*dma
;
316 struct ioat_chan_common
*chan
;
321 device
->enumerate_channels
= ioat2_enumerate_channels
;
322 device
->cleanup_tasklet
= ioat3_cleanup_tasklet
;
323 device
->timer_fn
= ioat3_timer_event
;
324 dma
= &device
->common
;
325 dma
->device_prep_dma_memcpy
= ioat2_dma_prep_memcpy_lock
;
326 dma
->device_issue_pending
= ioat2_issue_pending
;
327 dma
->device_alloc_chan_resources
= ioat2_alloc_chan_resources
;
328 dma
->device_free_chan_resources
= ioat2_free_chan_resources
;
329 dma
->device_is_tx_complete
= ioat3_is_complete
;
330 cap
= readl(device
->reg_base
+ IOAT_DMA_CAP_OFFSET
);
331 if (cap
& IOAT_CAP_FILL_BLOCK
) {
332 dma_cap_set(DMA_MEMSET
, dma
->cap_mask
);
333 dma
->device_prep_dma_memset
= ioat3_prep_memset_lock
;
336 /* -= IOAT ver.3 workarounds =- */
337 /* Write CHANERRMSK_INT with 3E07h to mask out the errors
338 * that can cause stability issues for IOAT ver.3
340 pci_write_config_dword(pdev
, IOAT_PCI_CHANERRMASK_INT_OFFSET
, 0x3e07);
342 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
343 * (workaround for spurious config parity error after restart)
345 pci_read_config_word(pdev
, IOAT_PCI_DEVICE_ID_OFFSET
, &dev_id
);
346 if (dev_id
== PCI_DEVICE_ID_INTEL_IOAT_TBG0
)
347 pci_write_config_dword(pdev
, IOAT_PCI_DMAUNCERRSTS_OFFSET
, 0x10);
349 err
= ioat_probe(device
);
352 ioat_set_tcp_copy_break(262144);
354 list_for_each_entry(c
, &dma
->channels
, device_node
) {
355 chan
= to_chan_common(c
);
356 writel(IOAT_DMA_DCA_ANY_CPU
,
357 chan
->reg_base
+ IOAT_DCACTRL_OFFSET
);
360 err
= ioat_register(device
);
364 device
->dca
= ioat3_dca_init(pdev
, device
->reg_base
);