2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2009 Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
24 * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
25 * does asynchronous data movement and checksumming operations.
28 #include <linux/init.h>
29 #include <linux/module.h>
30 #include <linux/slab.h>
31 #include <linux/pci.h>
32 #include <linux/interrupt.h>
33 #include <linux/dmaengine.h>
34 #include <linux/delay.h>
35 #include <linux/dma-mapping.h>
36 #include <linux/workqueue.h>
37 #include <linux/i7300_idle.h>
40 #include "registers.h"
43 int ioat_ring_alloc_order
= 8;
44 module_param(ioat_ring_alloc_order
, int, 0644);
45 MODULE_PARM_DESC(ioat_ring_alloc_order
,
46 "ioat2+: allocate 2^n descriptors per channel"
47 " (default: 8 max: 16)");
48 static int ioat_ring_max_alloc_order
= IOAT_MAX_ORDER
;
49 module_param(ioat_ring_max_alloc_order
, int, 0644);
50 MODULE_PARM_DESC(ioat_ring_max_alloc_order
,
51 "ioat2+: upper limit for ring size (default: 16)");
53 void __ioat2_issue_pending(struct ioat2_dma_chan
*ioat
)
55 struct ioat_chan_common
*chan
= &ioat
->base
;
57 ioat
->dmacount
+= ioat2_ring_pending(ioat
);
58 ioat
->issued
= ioat
->head
;
59 writew(ioat
->dmacount
, chan
->reg_base
+ IOAT_CHAN_DMACOUNT_OFFSET
);
61 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
62 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
, ioat
->dmacount
);
65 void ioat2_issue_pending(struct dma_chan
*c
)
67 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
69 if (ioat2_ring_pending(ioat
)) {
70 spin_lock_bh(&ioat
->prep_lock
);
71 __ioat2_issue_pending(ioat
);
72 spin_unlock_bh(&ioat
->prep_lock
);
77 * ioat2_update_pending - log pending descriptors
78 * @ioat: ioat2+ channel
80 * Check if the number of unsubmitted descriptors has exceeded the
81 * watermark. Called with prep_lock held
83 static void ioat2_update_pending(struct ioat2_dma_chan
*ioat
)
85 if (ioat2_ring_pending(ioat
) > ioat_pending_level
)
86 __ioat2_issue_pending(ioat
);
89 static void __ioat2_start_null_desc(struct ioat2_dma_chan
*ioat
)
91 struct ioat_ring_ent
*desc
;
92 struct ioat_dma_descriptor
*hw
;
94 if (ioat2_ring_space(ioat
) < 1) {
95 dev_err(to_dev(&ioat
->base
),
96 "Unable to start null desc - ring full\n");
100 dev_dbg(to_dev(&ioat
->base
), "%s: head: %#x tail: %#x issued: %#x\n",
101 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
);
102 desc
= ioat2_get_ring_ent(ioat
, ioat
->head
);
107 hw
->ctl_f
.int_en
= 1;
108 hw
->ctl_f
.compl_write
= 1;
109 /* set size to non-zero value (channel returns error when size is 0) */
110 hw
->size
= NULL_DESC_BUFFER_SIZE
;
113 async_tx_ack(&desc
->txd
);
114 ioat2_set_chainaddr(ioat
, desc
->txd
.phys
);
115 dump_desc_dbg(ioat
, desc
);
118 __ioat2_issue_pending(ioat
);
121 static void ioat2_start_null_desc(struct ioat2_dma_chan
*ioat
)
123 spin_lock_bh(&ioat
->prep_lock
);
124 __ioat2_start_null_desc(ioat
);
125 spin_unlock_bh(&ioat
->prep_lock
);
128 static void __cleanup(struct ioat2_dma_chan
*ioat
, unsigned long phys_complete
)
130 struct ioat_chan_common
*chan
= &ioat
->base
;
131 struct dma_async_tx_descriptor
*tx
;
132 struct ioat_ring_ent
*desc
;
133 bool seen_current
= false;
135 int idx
= ioat
->tail
, i
;
137 dev_dbg(to_dev(chan
), "%s: head: %#x tail: %#x issued: %#x\n",
138 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
);
140 active
= ioat2_ring_active(ioat
);
141 for (i
= 0; i
< active
&& !seen_current
; i
++) {
142 smp_read_barrier_depends();
143 prefetch(ioat2_get_ring_ent(ioat
, idx
+ i
+ 1));
144 desc
= ioat2_get_ring_ent(ioat
, idx
+ i
);
146 dump_desc_dbg(ioat
, desc
);
148 ioat_dma_unmap(chan
, tx
->flags
, desc
->len
, desc
->hw
);
149 chan
->completed_cookie
= tx
->cookie
;
152 tx
->callback(tx
->callback_param
);
157 if (tx
->phys
== phys_complete
)
160 smp_mb(); /* finish all descriptor reads before incrementing tail */
161 ioat
->tail
= idx
+ i
;
162 BUG_ON(active
&& !seen_current
); /* no active descs have written a completion? */
164 chan
->last_completion
= phys_complete
;
165 if (active
- i
== 0) {
166 dev_dbg(to_dev(chan
), "%s: cancel completion timeout\n",
168 clear_bit(IOAT_COMPLETION_PENDING
, &chan
->state
);
169 mod_timer(&chan
->timer
, jiffies
+ IDLE_TIMEOUT
);
174 * ioat2_cleanup - clean finished descriptors (advance tail pointer)
175 * @chan: ioat channel to be cleaned up
177 static void ioat2_cleanup(struct ioat2_dma_chan
*ioat
)
179 struct ioat_chan_common
*chan
= &ioat
->base
;
180 unsigned long phys_complete
;
182 spin_lock_bh(&chan
->cleanup_lock
);
183 if (ioat_cleanup_preamble(chan
, &phys_complete
))
184 __cleanup(ioat
, phys_complete
);
185 spin_unlock_bh(&chan
->cleanup_lock
);
188 void ioat2_cleanup_event(unsigned long data
)
190 struct ioat2_dma_chan
*ioat
= to_ioat2_chan((void *) data
);
193 writew(IOAT_CHANCTRL_RUN
, ioat
->base
.reg_base
+ IOAT_CHANCTRL_OFFSET
);
196 void __ioat2_restart_chan(struct ioat2_dma_chan
*ioat
)
198 struct ioat_chan_common
*chan
= &ioat
->base
;
200 /* set the tail to be re-issued */
201 ioat
->issued
= ioat
->tail
;
203 set_bit(IOAT_COMPLETION_PENDING
, &chan
->state
);
204 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
206 dev_dbg(to_dev(chan
),
207 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
208 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
, ioat
->dmacount
);
210 if (ioat2_ring_pending(ioat
)) {
211 struct ioat_ring_ent
*desc
;
213 desc
= ioat2_get_ring_ent(ioat
, ioat
->tail
);
214 ioat2_set_chainaddr(ioat
, desc
->txd
.phys
);
215 __ioat2_issue_pending(ioat
);
217 __ioat2_start_null_desc(ioat
);
220 int ioat2_quiesce(struct ioat_chan_common
*chan
, unsigned long tmo
)
222 unsigned long end
= jiffies
+ tmo
;
226 status
= ioat_chansts(chan
);
227 if (is_ioat_active(status
) || is_ioat_idle(status
))
229 while (is_ioat_active(status
) || is_ioat_idle(status
)) {
230 if (tmo
&& time_after(jiffies
, end
)) {
234 status
= ioat_chansts(chan
);
241 int ioat2_reset_sync(struct ioat_chan_common
*chan
, unsigned long tmo
)
243 unsigned long end
= jiffies
+ tmo
;
247 while (ioat_reset_pending(chan
)) {
248 if (end
&& time_after(jiffies
, end
)) {
258 static void ioat2_restart_channel(struct ioat2_dma_chan
*ioat
)
260 struct ioat_chan_common
*chan
= &ioat
->base
;
261 unsigned long phys_complete
;
263 ioat2_quiesce(chan
, 0);
264 if (ioat_cleanup_preamble(chan
, &phys_complete
))
265 __cleanup(ioat
, phys_complete
);
267 __ioat2_restart_chan(ioat
);
270 void ioat2_timer_event(unsigned long data
)
272 struct ioat2_dma_chan
*ioat
= to_ioat2_chan((void *) data
);
273 struct ioat_chan_common
*chan
= &ioat
->base
;
275 if (test_bit(IOAT_COMPLETION_PENDING
, &chan
->state
)) {
276 unsigned long phys_complete
;
279 status
= ioat_chansts(chan
);
281 /* when halted due to errors check for channel
282 * programming errors before advancing the completion state
284 if (is_ioat_halted(status
)) {
287 chanerr
= readl(chan
->reg_base
+ IOAT_CHANERR_OFFSET
);
288 dev_err(to_dev(chan
), "%s: Channel halted (%x)\n",
290 BUG_ON(is_ioat_bug(chanerr
));
293 /* if we haven't made progress and we have already
294 * acknowledged a pending completion once, then be more
295 * forceful with a restart
297 spin_lock_bh(&chan
->cleanup_lock
);
298 if (ioat_cleanup_preamble(chan
, &phys_complete
)) {
299 __cleanup(ioat
, phys_complete
);
300 } else if (test_bit(IOAT_COMPLETION_ACK
, &chan
->state
)) {
301 spin_lock_bh(&ioat
->prep_lock
);
302 ioat2_restart_channel(ioat
);
303 spin_unlock_bh(&ioat
->prep_lock
);
305 set_bit(IOAT_COMPLETION_ACK
, &chan
->state
);
306 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
308 spin_unlock_bh(&chan
->cleanup_lock
);
312 /* if the ring is idle, empty, and oversized try to step
315 spin_lock_bh(&chan
->cleanup_lock
);
316 spin_lock_bh(&ioat
->prep_lock
);
317 active
= ioat2_ring_active(ioat
);
318 if (active
== 0 && ioat
->alloc_order
> ioat_get_alloc_order())
319 reshape_ring(ioat
, ioat
->alloc_order
-1);
320 spin_unlock_bh(&ioat
->prep_lock
);
321 spin_unlock_bh(&chan
->cleanup_lock
);
323 /* keep shrinking until we get back to our minimum
326 if (ioat
->alloc_order
> ioat_get_alloc_order())
327 mod_timer(&chan
->timer
, jiffies
+ IDLE_TIMEOUT
);
331 static int ioat2_reset_hw(struct ioat_chan_common
*chan
)
333 /* throw away whatever the channel was doing and get it initialized */
336 ioat2_quiesce(chan
, msecs_to_jiffies(100));
338 chanerr
= readl(chan
->reg_base
+ IOAT_CHANERR_OFFSET
);
339 writel(chanerr
, chan
->reg_base
+ IOAT_CHANERR_OFFSET
);
341 return ioat2_reset_sync(chan
, msecs_to_jiffies(200));
345 * ioat2_enumerate_channels - find and initialize the device's channels
346 * @device: the device to be enumerated
348 int ioat2_enumerate_channels(struct ioatdma_device
*device
)
350 struct ioat2_dma_chan
*ioat
;
351 struct device
*dev
= &device
->pdev
->dev
;
352 struct dma_device
*dma
= &device
->common
;
356 INIT_LIST_HEAD(&dma
->channels
);
357 dma
->chancnt
= readb(device
->reg_base
+ IOAT_CHANCNT_OFFSET
);
358 dma
->chancnt
&= 0x1f; /* bits [4:0] valid */
359 if (dma
->chancnt
> ARRAY_SIZE(device
->idx
)) {
360 dev_warn(dev
, "(%d) exceeds max supported channels (%zu)\n",
361 dma
->chancnt
, ARRAY_SIZE(device
->idx
));
362 dma
->chancnt
= ARRAY_SIZE(device
->idx
);
364 xfercap_log
= readb(device
->reg_base
+ IOAT_XFERCAP_OFFSET
);
365 xfercap_log
&= 0x1f; /* bits [4:0] valid */
366 if (xfercap_log
== 0)
368 dev_dbg(dev
, "%s: xfercap = %d\n", __func__
, 1 << xfercap_log
);
370 /* FIXME which i/oat version is i7300? */
371 #ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
372 if (i7300_idle_platform_probe(NULL
, NULL
, 1) == 0)
375 for (i
= 0; i
< dma
->chancnt
; i
++) {
376 ioat
= devm_kzalloc(dev
, sizeof(*ioat
), GFP_KERNEL
);
380 ioat_init_channel(device
, &ioat
->base
, i
);
381 ioat
->xfercap_log
= xfercap_log
;
382 spin_lock_init(&ioat
->prep_lock
);
383 if (device
->reset_hw(&ioat
->base
)) {
392 static dma_cookie_t
ioat2_tx_submit_unlock(struct dma_async_tx_descriptor
*tx
)
394 struct dma_chan
*c
= tx
->chan
;
395 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
396 struct ioat_chan_common
*chan
= &ioat
->base
;
397 dma_cookie_t cookie
= c
->cookie
;
404 dev_dbg(to_dev(&ioat
->base
), "%s: cookie: %d\n", __func__
, cookie
);
406 if (!test_and_set_bit(IOAT_COMPLETION_PENDING
, &chan
->state
))
407 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
409 /* make descriptor updates visible before advancing ioat->head,
410 * this is purposefully not smp_wmb() since we are also
411 * publishing the descriptor updates to a dma device
415 ioat
->head
+= ioat
->produce
;
417 ioat2_update_pending(ioat
);
418 spin_unlock_bh(&ioat
->prep_lock
);
423 static struct ioat_ring_ent
*ioat2_alloc_ring_ent(struct dma_chan
*chan
, gfp_t flags
)
425 struct ioat_dma_descriptor
*hw
;
426 struct ioat_ring_ent
*desc
;
427 struct ioatdma_device
*dma
;
430 dma
= to_ioatdma_device(chan
->device
);
431 hw
= pci_pool_alloc(dma
->dma_pool
, flags
, &phys
);
434 memset(hw
, 0, sizeof(*hw
));
436 desc
= kmem_cache_alloc(ioat2_cache
, flags
);
438 pci_pool_free(dma
->dma_pool
, hw
, phys
);
441 memset(desc
, 0, sizeof(*desc
));
443 dma_async_tx_descriptor_init(&desc
->txd
, chan
);
444 desc
->txd
.tx_submit
= ioat2_tx_submit_unlock
;
446 desc
->txd
.phys
= phys
;
450 static void ioat2_free_ring_ent(struct ioat_ring_ent
*desc
, struct dma_chan
*chan
)
452 struct ioatdma_device
*dma
;
454 dma
= to_ioatdma_device(chan
->device
);
455 pci_pool_free(dma
->dma_pool
, desc
->hw
, desc
->txd
.phys
);
456 kmem_cache_free(ioat2_cache
, desc
);
459 static struct ioat_ring_ent
**ioat2_alloc_ring(struct dma_chan
*c
, int order
, gfp_t flags
)
461 struct ioat_ring_ent
**ring
;
462 int descs
= 1 << order
;
465 if (order
> ioat_get_max_alloc_order())
468 /* allocate the array to hold the software ring */
469 ring
= kcalloc(descs
, sizeof(*ring
), flags
);
472 for (i
= 0; i
< descs
; i
++) {
473 ring
[i
] = ioat2_alloc_ring_ent(c
, flags
);
476 ioat2_free_ring_ent(ring
[i
], c
);
480 set_desc_id(ring
[i
], i
);
484 for (i
= 0; i
< descs
-1; i
++) {
485 struct ioat_ring_ent
*next
= ring
[i
+1];
486 struct ioat_dma_descriptor
*hw
= ring
[i
]->hw
;
488 hw
->next
= next
->txd
.phys
;
490 ring
[i
]->hw
->next
= ring
[0]->txd
.phys
;
495 /* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
496 * @chan: channel to be initialized
498 int ioat2_alloc_chan_resources(struct dma_chan
*c
)
500 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
501 struct ioat_chan_common
*chan
= &ioat
->base
;
502 struct ioat_ring_ent
**ring
;
505 /* have we already been set up? */
507 return 1 << ioat
->alloc_order
;
509 /* Setup register to interrupt and write completion status on error */
510 writew(IOAT_CHANCTRL_RUN
, chan
->reg_base
+ IOAT_CHANCTRL_OFFSET
);
512 /* allocate a completion writeback area */
513 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
514 chan
->completion
= pci_pool_alloc(chan
->device
->completion_pool
,
515 GFP_KERNEL
, &chan
->completion_dma
);
516 if (!chan
->completion
)
519 memset(chan
->completion
, 0, sizeof(*chan
->completion
));
520 writel(((u64
) chan
->completion_dma
) & 0x00000000FFFFFFFF,
521 chan
->reg_base
+ IOAT_CHANCMP_OFFSET_LOW
);
522 writel(((u64
) chan
->completion_dma
) >> 32,
523 chan
->reg_base
+ IOAT_CHANCMP_OFFSET_HIGH
);
525 order
= ioat_get_alloc_order();
526 ring
= ioat2_alloc_ring(c
, order
, GFP_KERNEL
);
530 spin_lock_bh(&chan
->cleanup_lock
);
531 spin_lock_bh(&ioat
->prep_lock
);
536 ioat
->alloc_order
= order
;
537 spin_unlock_bh(&ioat
->prep_lock
);
538 spin_unlock_bh(&chan
->cleanup_lock
);
540 tasklet_enable(&chan
->cleanup_task
);
541 ioat2_start_null_desc(ioat
);
543 return 1 << ioat
->alloc_order
;
546 bool reshape_ring(struct ioat2_dma_chan
*ioat
, int order
)
548 /* reshape differs from normal ring allocation in that we want
549 * to allocate a new software ring while only
550 * extending/truncating the hardware ring
552 struct ioat_chan_common
*chan
= &ioat
->base
;
553 struct dma_chan
*c
= &chan
->common
;
554 const u16 curr_size
= ioat2_ring_size(ioat
);
555 const u16 active
= ioat2_ring_active(ioat
);
556 const u16 new_size
= 1 << order
;
557 struct ioat_ring_ent
**ring
;
560 if (order
> ioat_get_max_alloc_order())
563 /* double check that we have at least 1 free descriptor */
564 if (active
== curr_size
)
567 /* when shrinking, verify that we can hold the current active
568 * set in the new ring
570 if (active
>= new_size
)
573 /* allocate the array to hold the software ring */
574 ring
= kcalloc(new_size
, sizeof(*ring
), GFP_NOWAIT
);
578 /* allocate/trim descriptors as needed */
579 if (new_size
> curr_size
) {
580 /* copy current descriptors to the new ring */
581 for (i
= 0; i
< curr_size
; i
++) {
582 u16 curr_idx
= (ioat
->tail
+i
) & (curr_size
-1);
583 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
585 ring
[new_idx
] = ioat
->ring
[curr_idx
];
586 set_desc_id(ring
[new_idx
], new_idx
);
589 /* add new descriptors to the ring */
590 for (i
= curr_size
; i
< new_size
; i
++) {
591 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
593 ring
[new_idx
] = ioat2_alloc_ring_ent(c
, GFP_NOWAIT
);
594 if (!ring
[new_idx
]) {
596 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
598 ioat2_free_ring_ent(ring
[new_idx
], c
);
603 set_desc_id(ring
[new_idx
], new_idx
);
606 /* hw link new descriptors */
607 for (i
= curr_size
-1; i
< new_size
; i
++) {
608 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
609 struct ioat_ring_ent
*next
= ring
[(new_idx
+1) & (new_size
-1)];
610 struct ioat_dma_descriptor
*hw
= ring
[new_idx
]->hw
;
612 hw
->next
= next
->txd
.phys
;
615 struct ioat_dma_descriptor
*hw
;
616 struct ioat_ring_ent
*next
;
618 /* copy current descriptors to the new ring, dropping the
619 * removed descriptors
621 for (i
= 0; i
< new_size
; i
++) {
622 u16 curr_idx
= (ioat
->tail
+i
) & (curr_size
-1);
623 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
625 ring
[new_idx
] = ioat
->ring
[curr_idx
];
626 set_desc_id(ring
[new_idx
], new_idx
);
629 /* free deleted descriptors */
630 for (i
= new_size
; i
< curr_size
; i
++) {
631 struct ioat_ring_ent
*ent
;
633 ent
= ioat2_get_ring_ent(ioat
, ioat
->tail
+i
);
634 ioat2_free_ring_ent(ent
, c
);
637 /* fix up hardware ring */
638 hw
= ring
[(ioat
->tail
+new_size
-1) & (new_size
-1)]->hw
;
639 next
= ring
[(ioat
->tail
+new_size
) & (new_size
-1)];
640 hw
->next
= next
->txd
.phys
;
643 dev_dbg(to_dev(chan
), "%s: allocated %d descriptors\n",
648 ioat
->alloc_order
= order
;
654 * ioat2_check_space_lock - verify space and grab ring producer lock
655 * @ioat: ioat2,3 channel (ring) to operate on
656 * @num_descs: allocation length
658 int ioat2_check_space_lock(struct ioat2_dma_chan
*ioat
, int num_descs
)
660 struct ioat_chan_common
*chan
= &ioat
->base
;
664 spin_lock_bh(&ioat
->prep_lock
);
665 /* never allow the last descriptor to be consumed, we need at
666 * least one free at all times to allow for on-the-fly ring
669 if (likely(ioat2_ring_space(ioat
) > num_descs
)) {
670 dev_dbg(to_dev(chan
), "%s: num_descs: %d (%x:%x:%x)\n",
671 __func__
, num_descs
, ioat
->head
, ioat
->tail
, ioat
->issued
);
672 ioat
->produce
= num_descs
;
673 return 0; /* with ioat->prep_lock held */
675 retry
= test_and_set_bit(IOAT_RESHAPE_PENDING
, &chan
->state
);
676 spin_unlock_bh(&ioat
->prep_lock
);
678 /* is another cpu already trying to expand the ring? */
682 spin_lock_bh(&chan
->cleanup_lock
);
683 spin_lock_bh(&ioat
->prep_lock
);
684 retry
= reshape_ring(ioat
, ioat
->alloc_order
+ 1);
685 clear_bit(IOAT_RESHAPE_PENDING
, &chan
->state
);
686 spin_unlock_bh(&ioat
->prep_lock
);
687 spin_unlock_bh(&chan
->cleanup_lock
);
689 /* if we were able to expand the ring retry the allocation */
693 if (printk_ratelimit())
694 dev_dbg(to_dev(chan
), "%s: ring full! num_descs: %d (%x:%x:%x)\n",
695 __func__
, num_descs
, ioat
->head
, ioat
->tail
, ioat
->issued
);
697 /* progress reclaim in the allocation failure case we may be
698 * called under bh_disabled so we need to trigger the timer
701 if (jiffies
> chan
->timer
.expires
&& timer_pending(&chan
->timer
)) {
702 struct ioatdma_device
*device
= chan
->device
;
704 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
705 device
->timer_fn((unsigned long) &chan
->common
);
711 struct dma_async_tx_descriptor
*
712 ioat2_dma_prep_memcpy_lock(struct dma_chan
*c
, dma_addr_t dma_dest
,
713 dma_addr_t dma_src
, size_t len
, unsigned long flags
)
715 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
716 struct ioat_dma_descriptor
*hw
;
717 struct ioat_ring_ent
*desc
;
718 dma_addr_t dst
= dma_dest
;
719 dma_addr_t src
= dma_src
;
720 size_t total_len
= len
;
721 int num_descs
, idx
, i
;
723 num_descs
= ioat2_xferlen_to_descs(ioat
, len
);
724 if (likely(num_descs
) && ioat2_check_space_lock(ioat
, num_descs
) == 0)
730 size_t copy
= min_t(size_t, len
, 1 << ioat
->xfercap_log
);
732 desc
= ioat2_get_ring_ent(ioat
, idx
+ i
);
743 dump_desc_dbg(ioat
, desc
);
744 } while (++i
< num_descs
);
746 desc
->txd
.flags
= flags
;
747 desc
->len
= total_len
;
748 hw
->ctl_f
.int_en
= !!(flags
& DMA_PREP_INTERRUPT
);
749 hw
->ctl_f
.fence
= !!(flags
& DMA_PREP_FENCE
);
750 hw
->ctl_f
.compl_write
= 1;
751 dump_desc_dbg(ioat
, desc
);
752 /* we leave the channel locked to ensure in order submission */
758 * ioat2_free_chan_resources - release all the descriptors
759 * @chan: the channel to be cleaned
761 void ioat2_free_chan_resources(struct dma_chan
*c
)
763 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
764 struct ioat_chan_common
*chan
= &ioat
->base
;
765 struct ioatdma_device
*device
= chan
->device
;
766 struct ioat_ring_ent
*desc
;
767 const u16 total_descs
= 1 << ioat
->alloc_order
;
771 /* Before freeing channel resources first check
772 * if they have been previously allocated for this channel.
777 tasklet_disable(&chan
->cleanup_task
);
778 del_timer_sync(&chan
->timer
);
779 device
->cleanup_fn((unsigned long) c
);
780 device
->reset_hw(chan
);
782 spin_lock_bh(&chan
->cleanup_lock
);
783 spin_lock_bh(&ioat
->prep_lock
);
784 descs
= ioat2_ring_space(ioat
);
785 dev_dbg(to_dev(chan
), "freeing %d idle descriptors\n", descs
);
786 for (i
= 0; i
< descs
; i
++) {
787 desc
= ioat2_get_ring_ent(ioat
, ioat
->head
+ i
);
788 ioat2_free_ring_ent(desc
, c
);
791 if (descs
< total_descs
)
792 dev_err(to_dev(chan
), "Freeing %d in use descriptors!\n",
793 total_descs
- descs
);
795 for (i
= 0; i
< total_descs
- descs
; i
++) {
796 desc
= ioat2_get_ring_ent(ioat
, ioat
->tail
+ i
);
797 dump_desc_dbg(ioat
, desc
);
798 ioat2_free_ring_ent(desc
, c
);
803 ioat
->alloc_order
= 0;
804 pci_pool_free(device
->completion_pool
, chan
->completion
,
805 chan
->completion_dma
);
806 spin_unlock_bh(&ioat
->prep_lock
);
807 spin_unlock_bh(&chan
->cleanup_lock
);
809 chan
->last_completion
= 0;
810 chan
->completion_dma
= 0;
814 static ssize_t
ring_size_show(struct dma_chan
*c
, char *page
)
816 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
818 return sprintf(page
, "%d\n", (1 << ioat
->alloc_order
) & ~1);
820 static struct ioat_sysfs_entry ring_size_attr
= __ATTR_RO(ring_size
);
822 static ssize_t
ring_active_show(struct dma_chan
*c
, char *page
)
824 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
826 /* ...taken outside the lock, no need to be precise */
827 return sprintf(page
, "%d\n", ioat2_ring_active(ioat
));
829 static struct ioat_sysfs_entry ring_active_attr
= __ATTR_RO(ring_active
);
831 static struct attribute
*ioat2_attrs
[] = {
832 &ring_size_attr
.attr
,
833 &ring_active_attr
.attr
,
835 &ioat_version_attr
.attr
,
839 struct kobj_type ioat2_ktype
= {
840 .sysfs_ops
= &ioat_sysfs_ops
,
841 .default_attrs
= ioat2_attrs
,
844 int __devinit
ioat2_dma_probe(struct ioatdma_device
*device
, int dca
)
846 struct pci_dev
*pdev
= device
->pdev
;
847 struct dma_device
*dma
;
849 struct ioat_chan_common
*chan
;
852 device
->enumerate_channels
= ioat2_enumerate_channels
;
853 device
->reset_hw
= ioat2_reset_hw
;
854 device
->cleanup_fn
= ioat2_cleanup_event
;
855 device
->timer_fn
= ioat2_timer_event
;
856 device
->self_test
= ioat_dma_self_test
;
857 dma
= &device
->common
;
858 dma
->device_prep_dma_memcpy
= ioat2_dma_prep_memcpy_lock
;
859 dma
->device_issue_pending
= ioat2_issue_pending
;
860 dma
->device_alloc_chan_resources
= ioat2_alloc_chan_resources
;
861 dma
->device_free_chan_resources
= ioat2_free_chan_resources
;
862 dma
->device_tx_status
= ioat_tx_status
;
864 err
= ioat_probe(device
);
867 ioat_set_tcp_copy_break(2048);
869 list_for_each_entry(c
, &dma
->channels
, device_node
) {
870 chan
= to_chan_common(c
);
871 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE
| IOAT_DMA_DCA_ANY_CPU
,
872 chan
->reg_base
+ IOAT_DCACTRL_OFFSET
);
875 err
= ioat_register(device
);
879 ioat_kobject_add(device
, &ioat2_ktype
);
882 device
->dca
= ioat2_dca_init(pdev
, device
->reg_base
);