2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2009 Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
24 * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
25 * does asynchronous data movement and checksumming operations.
28 #include <linux/init.h>
29 #include <linux/module.h>
30 #include <linux/slab.h>
31 #include <linux/pci.h>
32 #include <linux/interrupt.h>
33 #include <linux/dmaengine.h>
34 #include <linux/delay.h>
35 #include <linux/dma-mapping.h>
36 #include <linux/workqueue.h>
37 #include <linux/i7300_idle.h>
40 #include "registers.h"
43 int ioat_ring_alloc_order
= 8;
44 module_param(ioat_ring_alloc_order
, int, 0644);
45 MODULE_PARM_DESC(ioat_ring_alloc_order
,
46 "ioat2+: allocate 2^n descriptors per channel"
47 " (default: 8 max: 16)");
48 static int ioat_ring_max_alloc_order
= IOAT_MAX_ORDER
;
49 module_param(ioat_ring_max_alloc_order
, int, 0644);
50 MODULE_PARM_DESC(ioat_ring_max_alloc_order
,
51 "ioat2+: upper limit for ring size (default: 16)");
53 void __ioat2_issue_pending(struct ioat2_dma_chan
*ioat
)
55 struct ioat_chan_common
*chan
= &ioat
->base
;
57 ioat
->dmacount
+= ioat2_ring_pending(ioat
);
58 ioat
->issued
= ioat
->head
;
59 writew(ioat
->dmacount
, chan
->reg_base
+ IOAT_CHAN_DMACOUNT_OFFSET
);
61 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
62 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
, ioat
->dmacount
);
65 void ioat2_issue_pending(struct dma_chan
*c
)
67 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
69 if (ioat2_ring_pending(ioat
)) {
70 spin_lock_bh(&ioat
->prep_lock
);
71 __ioat2_issue_pending(ioat
);
72 spin_unlock_bh(&ioat
->prep_lock
);
77 * ioat2_update_pending - log pending descriptors
78 * @ioat: ioat2+ channel
80 * Check if the number of unsubmitted descriptors has exceeded the
81 * watermark. Called with prep_lock held
83 static void ioat2_update_pending(struct ioat2_dma_chan
*ioat
)
85 if (ioat2_ring_pending(ioat
) > ioat_pending_level
)
86 __ioat2_issue_pending(ioat
);
89 static void __ioat2_start_null_desc(struct ioat2_dma_chan
*ioat
)
91 struct ioat_ring_ent
*desc
;
92 struct ioat_dma_descriptor
*hw
;
94 if (ioat2_ring_space(ioat
) < 1) {
95 dev_err(to_dev(&ioat
->base
),
96 "Unable to start null desc - ring full\n");
100 dev_dbg(to_dev(&ioat
->base
), "%s: head: %#x tail: %#x issued: %#x\n",
101 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
);
102 desc
= ioat2_get_ring_ent(ioat
, ioat
->head
);
107 hw
->ctl_f
.int_en
= 1;
108 hw
->ctl_f
.compl_write
= 1;
109 /* set size to non-zero value (channel returns error when size is 0) */
110 hw
->size
= NULL_DESC_BUFFER_SIZE
;
113 async_tx_ack(&desc
->txd
);
114 ioat2_set_chainaddr(ioat
, desc
->txd
.phys
);
115 dump_desc_dbg(ioat
, desc
);
118 __ioat2_issue_pending(ioat
);
121 static void ioat2_start_null_desc(struct ioat2_dma_chan
*ioat
)
123 spin_lock_bh(&ioat
->prep_lock
);
124 __ioat2_start_null_desc(ioat
);
125 spin_unlock_bh(&ioat
->prep_lock
);
128 static void __cleanup(struct ioat2_dma_chan
*ioat
, unsigned long phys_complete
)
130 struct ioat_chan_common
*chan
= &ioat
->base
;
131 struct dma_async_tx_descriptor
*tx
;
132 struct ioat_ring_ent
*desc
;
133 bool seen_current
= false;
135 int idx
= ioat
->tail
, i
;
137 dev_dbg(to_dev(chan
), "%s: head: %#x tail: %#x issued: %#x\n",
138 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
);
140 active
= ioat2_ring_active(ioat
);
141 for (i
= 0; i
< active
&& !seen_current
; i
++) {
142 smp_read_barrier_depends();
143 prefetch(ioat2_get_ring_ent(ioat
, idx
+ i
+ 1));
144 desc
= ioat2_get_ring_ent(ioat
, idx
+ i
);
146 dump_desc_dbg(ioat
, desc
);
148 ioat_dma_unmap(chan
, tx
->flags
, desc
->len
, desc
->hw
);
149 chan
->completed_cookie
= tx
->cookie
;
152 tx
->callback(tx
->callback_param
);
157 if (tx
->phys
== phys_complete
)
160 smp_mb(); /* finish all descriptor reads before incrementing tail */
161 ioat
->tail
= idx
+ i
;
162 BUG_ON(active
&& !seen_current
); /* no active descs have written a completion? */
164 chan
->last_completion
= phys_complete
;
165 if (active
- i
== 0) {
166 dev_dbg(to_dev(chan
), "%s: cancel completion timeout\n",
168 clear_bit(IOAT_COMPLETION_PENDING
, &chan
->state
);
169 mod_timer(&chan
->timer
, jiffies
+ IDLE_TIMEOUT
);
174 * ioat2_cleanup - clean finished descriptors (advance tail pointer)
175 * @chan: ioat channel to be cleaned up
177 static void ioat2_cleanup(struct ioat2_dma_chan
*ioat
)
179 struct ioat_chan_common
*chan
= &ioat
->base
;
180 unsigned long phys_complete
;
182 spin_lock_bh(&chan
->cleanup_lock
);
183 if (ioat_cleanup_preamble(chan
, &phys_complete
))
184 __cleanup(ioat
, phys_complete
);
185 spin_unlock_bh(&chan
->cleanup_lock
);
188 void ioat2_cleanup_event(unsigned long data
)
190 struct ioat2_dma_chan
*ioat
= to_ioat2_chan((void *) data
);
193 writew(IOAT_CHANCTRL_RUN
, ioat
->base
.reg_base
+ IOAT_CHANCTRL_OFFSET
);
196 void __ioat2_restart_chan(struct ioat2_dma_chan
*ioat
)
198 struct ioat_chan_common
*chan
= &ioat
->base
;
200 /* set the tail to be re-issued */
201 ioat
->issued
= ioat
->tail
;
203 set_bit(IOAT_COMPLETION_PENDING
, &chan
->state
);
204 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
206 dev_dbg(to_dev(chan
),
207 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
208 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
, ioat
->dmacount
);
210 if (ioat2_ring_pending(ioat
)) {
211 struct ioat_ring_ent
*desc
;
213 desc
= ioat2_get_ring_ent(ioat
, ioat
->tail
);
214 ioat2_set_chainaddr(ioat
, desc
->txd
.phys
);
215 __ioat2_issue_pending(ioat
);
217 __ioat2_start_null_desc(ioat
);
220 int ioat2_quiesce(struct ioat_chan_common
*chan
, unsigned long tmo
)
222 unsigned long end
= jiffies
+ tmo
;
226 status
= ioat_chansts(chan
);
227 if (is_ioat_active(status
) || is_ioat_idle(status
))
229 while (is_ioat_active(status
) || is_ioat_idle(status
)) {
230 if (tmo
&& time_after(jiffies
, end
)) {
234 status
= ioat_chansts(chan
);
241 int ioat2_reset_sync(struct ioat_chan_common
*chan
, unsigned long tmo
)
243 unsigned long end
= jiffies
+ tmo
;
247 while (ioat_reset_pending(chan
)) {
248 if (end
&& time_after(jiffies
, end
)) {
258 static void ioat2_restart_channel(struct ioat2_dma_chan
*ioat
)
260 struct ioat_chan_common
*chan
= &ioat
->base
;
261 unsigned long phys_complete
;
263 ioat2_quiesce(chan
, 0);
264 if (ioat_cleanup_preamble(chan
, &phys_complete
))
265 __cleanup(ioat
, phys_complete
);
267 __ioat2_restart_chan(ioat
);
270 void ioat2_timer_event(unsigned long data
)
272 struct ioat2_dma_chan
*ioat
= to_ioat2_chan((void *) data
);
273 struct ioat_chan_common
*chan
= &ioat
->base
;
275 if (test_bit(IOAT_COMPLETION_PENDING
, &chan
->state
)) {
276 unsigned long phys_complete
;
279 status
= ioat_chansts(chan
);
281 /* when halted due to errors check for channel
282 * programming errors before advancing the completion state
284 if (is_ioat_halted(status
)) {
287 chanerr
= readl(chan
->reg_base
+ IOAT_CHANERR_OFFSET
);
288 dev_err(to_dev(chan
), "%s: Channel halted (%x)\n",
290 if (test_bit(IOAT_RUN
, &chan
->state
))
291 BUG_ON(is_ioat_bug(chanerr
));
292 else /* we never got off the ground */
296 /* if we haven't made progress and we have already
297 * acknowledged a pending completion once, then be more
298 * forceful with a restart
300 spin_lock_bh(&chan
->cleanup_lock
);
301 if (ioat_cleanup_preamble(chan
, &phys_complete
)) {
302 __cleanup(ioat
, phys_complete
);
303 } else if (test_bit(IOAT_COMPLETION_ACK
, &chan
->state
)) {
304 spin_lock_bh(&ioat
->prep_lock
);
305 ioat2_restart_channel(ioat
);
306 spin_unlock_bh(&ioat
->prep_lock
);
308 set_bit(IOAT_COMPLETION_ACK
, &chan
->state
);
309 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
311 spin_unlock_bh(&chan
->cleanup_lock
);
315 /* if the ring is idle, empty, and oversized try to step
318 spin_lock_bh(&chan
->cleanup_lock
);
319 spin_lock_bh(&ioat
->prep_lock
);
320 active
= ioat2_ring_active(ioat
);
321 if (active
== 0 && ioat
->alloc_order
> ioat_get_alloc_order())
322 reshape_ring(ioat
, ioat
->alloc_order
-1);
323 spin_unlock_bh(&ioat
->prep_lock
);
324 spin_unlock_bh(&chan
->cleanup_lock
);
326 /* keep shrinking until we get back to our minimum
329 if (ioat
->alloc_order
> ioat_get_alloc_order())
330 mod_timer(&chan
->timer
, jiffies
+ IDLE_TIMEOUT
);
334 static int ioat2_reset_hw(struct ioat_chan_common
*chan
)
336 /* throw away whatever the channel was doing and get it initialized */
339 ioat2_quiesce(chan
, msecs_to_jiffies(100));
341 chanerr
= readl(chan
->reg_base
+ IOAT_CHANERR_OFFSET
);
342 writel(chanerr
, chan
->reg_base
+ IOAT_CHANERR_OFFSET
);
344 return ioat2_reset_sync(chan
, msecs_to_jiffies(200));
348 * ioat2_enumerate_channels - find and initialize the device's channels
349 * @device: the device to be enumerated
351 int ioat2_enumerate_channels(struct ioatdma_device
*device
)
353 struct ioat2_dma_chan
*ioat
;
354 struct device
*dev
= &device
->pdev
->dev
;
355 struct dma_device
*dma
= &device
->common
;
359 INIT_LIST_HEAD(&dma
->channels
);
360 dma
->chancnt
= readb(device
->reg_base
+ IOAT_CHANCNT_OFFSET
);
361 dma
->chancnt
&= 0x1f; /* bits [4:0] valid */
362 if (dma
->chancnt
> ARRAY_SIZE(device
->idx
)) {
363 dev_warn(dev
, "(%d) exceeds max supported channels (%zu)\n",
364 dma
->chancnt
, ARRAY_SIZE(device
->idx
));
365 dma
->chancnt
= ARRAY_SIZE(device
->idx
);
367 xfercap_log
= readb(device
->reg_base
+ IOAT_XFERCAP_OFFSET
);
368 xfercap_log
&= 0x1f; /* bits [4:0] valid */
369 if (xfercap_log
== 0)
371 dev_dbg(dev
, "%s: xfercap = %d\n", __func__
, 1 << xfercap_log
);
373 #ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
374 if (i7300_idle_platform_probe(NULL
, NULL
, 1) == 0)
377 for (i
= 0; i
< dma
->chancnt
; i
++) {
378 ioat
= devm_kzalloc(dev
, sizeof(*ioat
), GFP_KERNEL
);
382 ioat_init_channel(device
, &ioat
->base
, i
);
383 ioat
->xfercap_log
= xfercap_log
;
384 spin_lock_init(&ioat
->prep_lock
);
385 if (device
->reset_hw(&ioat
->base
)) {
394 static dma_cookie_t
ioat2_tx_submit_unlock(struct dma_async_tx_descriptor
*tx
)
396 struct dma_chan
*c
= tx
->chan
;
397 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
398 struct ioat_chan_common
*chan
= &ioat
->base
;
399 dma_cookie_t cookie
= c
->cookie
;
406 dev_dbg(to_dev(&ioat
->base
), "%s: cookie: %d\n", __func__
, cookie
);
408 if (!test_and_set_bit(IOAT_COMPLETION_PENDING
, &chan
->state
))
409 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
411 /* make descriptor updates visible before advancing ioat->head,
412 * this is purposefully not smp_wmb() since we are also
413 * publishing the descriptor updates to a dma device
417 ioat
->head
+= ioat
->produce
;
419 ioat2_update_pending(ioat
);
420 spin_unlock_bh(&ioat
->prep_lock
);
425 static struct ioat_ring_ent
*ioat2_alloc_ring_ent(struct dma_chan
*chan
, gfp_t flags
)
427 struct ioat_dma_descriptor
*hw
;
428 struct ioat_ring_ent
*desc
;
429 struct ioatdma_device
*dma
;
432 dma
= to_ioatdma_device(chan
->device
);
433 hw
= pci_pool_alloc(dma
->dma_pool
, flags
, &phys
);
436 memset(hw
, 0, sizeof(*hw
));
438 desc
= kmem_cache_alloc(ioat2_cache
, flags
);
440 pci_pool_free(dma
->dma_pool
, hw
, phys
);
443 memset(desc
, 0, sizeof(*desc
));
445 dma_async_tx_descriptor_init(&desc
->txd
, chan
);
446 desc
->txd
.tx_submit
= ioat2_tx_submit_unlock
;
448 desc
->txd
.phys
= phys
;
452 static void ioat2_free_ring_ent(struct ioat_ring_ent
*desc
, struct dma_chan
*chan
)
454 struct ioatdma_device
*dma
;
456 dma
= to_ioatdma_device(chan
->device
);
457 pci_pool_free(dma
->dma_pool
, desc
->hw
, desc
->txd
.phys
);
458 kmem_cache_free(ioat2_cache
, desc
);
461 static struct ioat_ring_ent
**ioat2_alloc_ring(struct dma_chan
*c
, int order
, gfp_t flags
)
463 struct ioat_ring_ent
**ring
;
464 int descs
= 1 << order
;
467 if (order
> ioat_get_max_alloc_order())
470 /* allocate the array to hold the software ring */
471 ring
= kcalloc(descs
, sizeof(*ring
), flags
);
474 for (i
= 0; i
< descs
; i
++) {
475 ring
[i
] = ioat2_alloc_ring_ent(c
, flags
);
478 ioat2_free_ring_ent(ring
[i
], c
);
482 set_desc_id(ring
[i
], i
);
486 for (i
= 0; i
< descs
-1; i
++) {
487 struct ioat_ring_ent
*next
= ring
[i
+1];
488 struct ioat_dma_descriptor
*hw
= ring
[i
]->hw
;
490 hw
->next
= next
->txd
.phys
;
492 ring
[i
]->hw
->next
= ring
[0]->txd
.phys
;
497 void ioat2_free_chan_resources(struct dma_chan
*c
);
499 /* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
500 * @chan: channel to be initialized
502 int ioat2_alloc_chan_resources(struct dma_chan
*c
)
504 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
505 struct ioat_chan_common
*chan
= &ioat
->base
;
506 struct ioat_ring_ent
**ring
;
510 /* have we already been set up? */
512 return 1 << ioat
->alloc_order
;
514 /* Setup register to interrupt and write completion status on error */
515 writew(IOAT_CHANCTRL_RUN
, chan
->reg_base
+ IOAT_CHANCTRL_OFFSET
);
517 /* allocate a completion writeback area */
518 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
519 chan
->completion
= pci_pool_alloc(chan
->device
->completion_pool
,
520 GFP_KERNEL
, &chan
->completion_dma
);
521 if (!chan
->completion
)
524 memset(chan
->completion
, 0, sizeof(*chan
->completion
));
525 writel(((u64
) chan
->completion_dma
) & 0x00000000FFFFFFFF,
526 chan
->reg_base
+ IOAT_CHANCMP_OFFSET_LOW
);
527 writel(((u64
) chan
->completion_dma
) >> 32,
528 chan
->reg_base
+ IOAT_CHANCMP_OFFSET_HIGH
);
530 order
= ioat_get_alloc_order();
531 ring
= ioat2_alloc_ring(c
, order
, GFP_KERNEL
);
535 spin_lock_bh(&chan
->cleanup_lock
);
536 spin_lock_bh(&ioat
->prep_lock
);
541 ioat
->alloc_order
= order
;
542 spin_unlock_bh(&ioat
->prep_lock
);
543 spin_unlock_bh(&chan
->cleanup_lock
);
545 tasklet_enable(&chan
->cleanup_task
);
546 ioat2_start_null_desc(ioat
);
548 /* check that we got off the ground */
550 status
= ioat_chansts(chan
);
551 if (is_ioat_active(status
) || is_ioat_idle(status
)) {
552 set_bit(IOAT_RUN
, &chan
->state
);
553 return 1 << ioat
->alloc_order
;
555 u32 chanerr
= readl(chan
->reg_base
+ IOAT_CHANERR_OFFSET
);
557 dev_WARN(to_dev(chan
),
558 "failed to start channel chanerr: %#x\n", chanerr
);
559 ioat2_free_chan_resources(c
);
564 bool reshape_ring(struct ioat2_dma_chan
*ioat
, int order
)
566 /* reshape differs from normal ring allocation in that we want
567 * to allocate a new software ring while only
568 * extending/truncating the hardware ring
570 struct ioat_chan_common
*chan
= &ioat
->base
;
571 struct dma_chan
*c
= &chan
->common
;
572 const u16 curr_size
= ioat2_ring_size(ioat
);
573 const u16 active
= ioat2_ring_active(ioat
);
574 const u16 new_size
= 1 << order
;
575 struct ioat_ring_ent
**ring
;
578 if (order
> ioat_get_max_alloc_order())
581 /* double check that we have at least 1 free descriptor */
582 if (active
== curr_size
)
585 /* when shrinking, verify that we can hold the current active
586 * set in the new ring
588 if (active
>= new_size
)
591 /* allocate the array to hold the software ring */
592 ring
= kcalloc(new_size
, sizeof(*ring
), GFP_NOWAIT
);
596 /* allocate/trim descriptors as needed */
597 if (new_size
> curr_size
) {
598 /* copy current descriptors to the new ring */
599 for (i
= 0; i
< curr_size
; i
++) {
600 u16 curr_idx
= (ioat
->tail
+i
) & (curr_size
-1);
601 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
603 ring
[new_idx
] = ioat
->ring
[curr_idx
];
604 set_desc_id(ring
[new_idx
], new_idx
);
607 /* add new descriptors to the ring */
608 for (i
= curr_size
; i
< new_size
; i
++) {
609 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
611 ring
[new_idx
] = ioat2_alloc_ring_ent(c
, GFP_NOWAIT
);
612 if (!ring
[new_idx
]) {
614 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
616 ioat2_free_ring_ent(ring
[new_idx
], c
);
621 set_desc_id(ring
[new_idx
], new_idx
);
624 /* hw link new descriptors */
625 for (i
= curr_size
-1; i
< new_size
; i
++) {
626 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
627 struct ioat_ring_ent
*next
= ring
[(new_idx
+1) & (new_size
-1)];
628 struct ioat_dma_descriptor
*hw
= ring
[new_idx
]->hw
;
630 hw
->next
= next
->txd
.phys
;
633 struct ioat_dma_descriptor
*hw
;
634 struct ioat_ring_ent
*next
;
636 /* copy current descriptors to the new ring, dropping the
637 * removed descriptors
639 for (i
= 0; i
< new_size
; i
++) {
640 u16 curr_idx
= (ioat
->tail
+i
) & (curr_size
-1);
641 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
643 ring
[new_idx
] = ioat
->ring
[curr_idx
];
644 set_desc_id(ring
[new_idx
], new_idx
);
647 /* free deleted descriptors */
648 for (i
= new_size
; i
< curr_size
; i
++) {
649 struct ioat_ring_ent
*ent
;
651 ent
= ioat2_get_ring_ent(ioat
, ioat
->tail
+i
);
652 ioat2_free_ring_ent(ent
, c
);
655 /* fix up hardware ring */
656 hw
= ring
[(ioat
->tail
+new_size
-1) & (new_size
-1)]->hw
;
657 next
= ring
[(ioat
->tail
+new_size
) & (new_size
-1)];
658 hw
->next
= next
->txd
.phys
;
661 dev_dbg(to_dev(chan
), "%s: allocated %d descriptors\n",
666 ioat
->alloc_order
= order
;
672 * ioat2_check_space_lock - verify space and grab ring producer lock
673 * @ioat: ioat2,3 channel (ring) to operate on
674 * @num_descs: allocation length
676 int ioat2_check_space_lock(struct ioat2_dma_chan
*ioat
, int num_descs
)
678 struct ioat_chan_common
*chan
= &ioat
->base
;
682 spin_lock_bh(&ioat
->prep_lock
);
683 /* never allow the last descriptor to be consumed, we need at
684 * least one free at all times to allow for on-the-fly ring
687 if (likely(ioat2_ring_space(ioat
) > num_descs
)) {
688 dev_dbg(to_dev(chan
), "%s: num_descs: %d (%x:%x:%x)\n",
689 __func__
, num_descs
, ioat
->head
, ioat
->tail
, ioat
->issued
);
690 ioat
->produce
= num_descs
;
691 return 0; /* with ioat->prep_lock held */
693 retry
= test_and_set_bit(IOAT_RESHAPE_PENDING
, &chan
->state
);
694 spin_unlock_bh(&ioat
->prep_lock
);
696 /* is another cpu already trying to expand the ring? */
700 spin_lock_bh(&chan
->cleanup_lock
);
701 spin_lock_bh(&ioat
->prep_lock
);
702 retry
= reshape_ring(ioat
, ioat
->alloc_order
+ 1);
703 clear_bit(IOAT_RESHAPE_PENDING
, &chan
->state
);
704 spin_unlock_bh(&ioat
->prep_lock
);
705 spin_unlock_bh(&chan
->cleanup_lock
);
707 /* if we were able to expand the ring retry the allocation */
711 if (printk_ratelimit())
712 dev_dbg(to_dev(chan
), "%s: ring full! num_descs: %d (%x:%x:%x)\n",
713 __func__
, num_descs
, ioat
->head
, ioat
->tail
, ioat
->issued
);
715 /* progress reclaim in the allocation failure case we may be
716 * called under bh_disabled so we need to trigger the timer
719 if (jiffies
> chan
->timer
.expires
&& timer_pending(&chan
->timer
)) {
720 struct ioatdma_device
*device
= chan
->device
;
722 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
723 device
->timer_fn((unsigned long) &chan
->common
);
729 struct dma_async_tx_descriptor
*
730 ioat2_dma_prep_memcpy_lock(struct dma_chan
*c
, dma_addr_t dma_dest
,
731 dma_addr_t dma_src
, size_t len
, unsigned long flags
)
733 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
734 struct ioat_dma_descriptor
*hw
;
735 struct ioat_ring_ent
*desc
;
736 dma_addr_t dst
= dma_dest
;
737 dma_addr_t src
= dma_src
;
738 size_t total_len
= len
;
739 int num_descs
, idx
, i
;
741 num_descs
= ioat2_xferlen_to_descs(ioat
, len
);
742 if (likely(num_descs
) && ioat2_check_space_lock(ioat
, num_descs
) == 0)
748 size_t copy
= min_t(size_t, len
, 1 << ioat
->xfercap_log
);
750 desc
= ioat2_get_ring_ent(ioat
, idx
+ i
);
761 dump_desc_dbg(ioat
, desc
);
762 } while (++i
< num_descs
);
764 desc
->txd
.flags
= flags
;
765 desc
->len
= total_len
;
766 hw
->ctl_f
.int_en
= !!(flags
& DMA_PREP_INTERRUPT
);
767 hw
->ctl_f
.fence
= !!(flags
& DMA_PREP_FENCE
);
768 hw
->ctl_f
.compl_write
= 1;
769 dump_desc_dbg(ioat
, desc
);
770 /* we leave the channel locked to ensure in order submission */
776 * ioat2_free_chan_resources - release all the descriptors
777 * @chan: the channel to be cleaned
779 void ioat2_free_chan_resources(struct dma_chan
*c
)
781 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
782 struct ioat_chan_common
*chan
= &ioat
->base
;
783 struct ioatdma_device
*device
= chan
->device
;
784 struct ioat_ring_ent
*desc
;
785 const u16 total_descs
= 1 << ioat
->alloc_order
;
789 /* Before freeing channel resources first check
790 * if they have been previously allocated for this channel.
795 tasklet_disable(&chan
->cleanup_task
);
796 del_timer_sync(&chan
->timer
);
797 device
->cleanup_fn((unsigned long) c
);
798 device
->reset_hw(chan
);
799 clear_bit(IOAT_RUN
, &chan
->state
);
801 spin_lock_bh(&chan
->cleanup_lock
);
802 spin_lock_bh(&ioat
->prep_lock
);
803 descs
= ioat2_ring_space(ioat
);
804 dev_dbg(to_dev(chan
), "freeing %d idle descriptors\n", descs
);
805 for (i
= 0; i
< descs
; i
++) {
806 desc
= ioat2_get_ring_ent(ioat
, ioat
->head
+ i
);
807 ioat2_free_ring_ent(desc
, c
);
810 if (descs
< total_descs
)
811 dev_err(to_dev(chan
), "Freeing %d in use descriptors!\n",
812 total_descs
- descs
);
814 for (i
= 0; i
< total_descs
- descs
; i
++) {
815 desc
= ioat2_get_ring_ent(ioat
, ioat
->tail
+ i
);
816 dump_desc_dbg(ioat
, desc
);
817 ioat2_free_ring_ent(desc
, c
);
822 ioat
->alloc_order
= 0;
823 pci_pool_free(device
->completion_pool
, chan
->completion
,
824 chan
->completion_dma
);
825 spin_unlock_bh(&ioat
->prep_lock
);
826 spin_unlock_bh(&chan
->cleanup_lock
);
828 chan
->last_completion
= 0;
829 chan
->completion_dma
= 0;
833 static ssize_t
ring_size_show(struct dma_chan
*c
, char *page
)
835 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
837 return sprintf(page
, "%d\n", (1 << ioat
->alloc_order
) & ~1);
839 static struct ioat_sysfs_entry ring_size_attr
= __ATTR_RO(ring_size
);
841 static ssize_t
ring_active_show(struct dma_chan
*c
, char *page
)
843 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
845 /* ...taken outside the lock, no need to be precise */
846 return sprintf(page
, "%d\n", ioat2_ring_active(ioat
));
848 static struct ioat_sysfs_entry ring_active_attr
= __ATTR_RO(ring_active
);
850 static struct attribute
*ioat2_attrs
[] = {
851 &ring_size_attr
.attr
,
852 &ring_active_attr
.attr
,
854 &ioat_version_attr
.attr
,
858 struct kobj_type ioat2_ktype
= {
859 .sysfs_ops
= &ioat_sysfs_ops
,
860 .default_attrs
= ioat2_attrs
,
863 int __devinit
ioat2_dma_probe(struct ioatdma_device
*device
, int dca
)
865 struct pci_dev
*pdev
= device
->pdev
;
866 struct dma_device
*dma
;
868 struct ioat_chan_common
*chan
;
871 device
->enumerate_channels
= ioat2_enumerate_channels
;
872 device
->reset_hw
= ioat2_reset_hw
;
873 device
->cleanup_fn
= ioat2_cleanup_event
;
874 device
->timer_fn
= ioat2_timer_event
;
875 device
->self_test
= ioat_dma_self_test
;
876 dma
= &device
->common
;
877 dma
->device_prep_dma_memcpy
= ioat2_dma_prep_memcpy_lock
;
878 dma
->device_issue_pending
= ioat2_issue_pending
;
879 dma
->device_alloc_chan_resources
= ioat2_alloc_chan_resources
;
880 dma
->device_free_chan_resources
= ioat2_free_chan_resources
;
881 dma
->device_tx_status
= ioat_dma_tx_status
;
883 err
= ioat_probe(device
);
886 ioat_set_tcp_copy_break(2048);
888 list_for_each_entry(c
, &dma
->channels
, device_node
) {
889 chan
= to_chan_common(c
);
890 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE
| IOAT_DMA_DCA_ANY_CPU
,
891 chan
->reg_base
+ IOAT_DCACTRL_OFFSET
);
894 err
= ioat_register(device
);
898 ioat_kobject_add(device
, &ioat2_ktype
);
901 device
->dca
= ioat2_dca_init(pdev
, device
->reg_base
);