2 * scsi_dma.c Copyright (C) 2000 Eric Youngdale
4 * mid-level SCSI DMA bounce buffer allocator
9 #include <linux/config.h>
10 #include <linux/module.h>
11 #include <linux/blk.h>
16 #include "constants.h"
19 #include <linux/kmod.h>
23 * PAGE_SIZE must be a multiple of the sector size (512). True
24 * for all reasonably recent architectures (even the VAX...).
26 #define SECTOR_SIZE 512
27 #define SECTORS_PER_PAGE (PAGE_SIZE/SECTOR_SIZE)
29 #if SECTORS_PER_PAGE <= 8
30 typedef unsigned char FreeSectorBitmap
;
31 #elif SECTORS_PER_PAGE <= 32
32 typedef unsigned int FreeSectorBitmap
;
38 * Used for access to internal allocator used for DMA safe buffers.
40 static spinlock_t allocator_request_lock
= SPIN_LOCK_UNLOCKED
;
42 static FreeSectorBitmap
*dma_malloc_freelist
= NULL
;
43 static int need_isa_bounce_buffers
;
44 static unsigned int dma_sectors
= 0;
45 unsigned int scsi_dma_free_sectors
= 0;
46 unsigned int scsi_need_isa_buffer
= 0;
47 static unsigned char **dma_malloc_pages
= NULL
;
50 * Function: scsi_malloc
52 * Purpose: Allocate memory from the DMA-safe pool.
54 * Arguments: len - amount of memory we need.
56 * Lock status: No locks assumed to be held. This function is SMP-safe.
58 * Returns: Pointer to memory block.
60 * Notes: Prior to the new queue code, this function was not SMP-safe.
61 * This function can only allocate in units of sectors
64 * We cannot use the normal system allocator becuase we need
65 * to be able to guarantee that we can process a complete disk
66 * I/O request without touching the system allocator. Think
67 * about it - if the system were heavily swapping, and tried to
68 * write out a block of memory to disk, and the SCSI code needed
69 * to allocate more memory in order to be able to write the
70 * data to disk, you would wedge the system.
72 void *scsi_malloc(unsigned int len
)
74 unsigned int nbits
, mask
;
78 if (len
% SECTOR_SIZE
!= 0 || len
> PAGE_SIZE
)
82 mask
= (1 << nbits
) - 1;
84 spin_lock_irqsave(&allocator_request_lock
, flags
);
86 for (i
= 0; i
< dma_sectors
/ SECTORS_PER_PAGE
; i
++)
87 for (j
= 0; j
<= SECTORS_PER_PAGE
- nbits
; j
++) {
88 if ((dma_malloc_freelist
[i
] & (mask
<< j
)) == 0) {
89 dma_malloc_freelist
[i
] |= (mask
<< j
);
90 scsi_dma_free_sectors
-= nbits
;
92 SCSI_LOG_MLQUEUE(3, printk("SMalloc: %d %p [From:%p]\n", len
, dma_malloc_pages
[i
] + (j
<< 9)));
93 printk("SMalloc: %d %p [From:%p]\n", len
, dma_malloc_pages
[i
] + (j
<< 9));
95 spin_unlock_irqrestore(&allocator_request_lock
, flags
);
96 return (void *) ((unsigned long) dma_malloc_pages
[i
] + (j
<< 9));
99 spin_unlock_irqrestore(&allocator_request_lock
, flags
);
100 return NULL
; /* Nope. No more */
104 * Function: scsi_free
106 * Purpose: Free memory into the DMA-safe pool.
108 * Arguments: ptr - data block we are freeing.
109 * len - size of block we are freeing.
111 * Lock status: No locks assumed to be held. This function is SMP-safe.
115 * Notes: This function *must* only be used to free memory
116 * allocated from scsi_malloc().
118 * Prior to the new queue code, this function was not SMP-safe.
119 * This function can only allocate in units of sectors
122 int scsi_free(void *obj
, unsigned int len
)
124 unsigned int page
, sector
, nbits
, mask
;
128 unsigned long ret
= 0;
131 __asm__
__volatile__("move\t%0,$31":"=r"(ret
));
133 ret
= __builtin_return_address(0);
135 printk("scsi_free %p %d\n", obj
, len
);
136 SCSI_LOG_MLQUEUE(3, printk("SFree: %p %d\n", obj
, len
));
139 spin_lock_irqsave(&allocator_request_lock
, flags
);
141 for (page
= 0; page
< dma_sectors
/ SECTORS_PER_PAGE
; page
++) {
142 unsigned long page_addr
= (unsigned long) dma_malloc_pages
[page
];
143 if ((unsigned long) obj
>= page_addr
&&
144 (unsigned long) obj
< page_addr
+ PAGE_SIZE
) {
145 sector
= (((unsigned long) obj
) - page_addr
) >> 9;
148 mask
= (1 << nbits
) - 1;
150 if (sector
+ nbits
> SECTORS_PER_PAGE
)
151 panic("scsi_free:Bad memory alignment");
153 if ((dma_malloc_freelist
[page
] &
154 (mask
<< sector
)) != (mask
<< sector
)) {
156 printk("scsi_free(obj=%p, len=%d) called from %08lx\n",
159 panic("scsi_free:Trying to free unused memory");
161 scsi_dma_free_sectors
+= nbits
;
162 dma_malloc_freelist
[page
] &= ~(mask
<< sector
);
163 spin_unlock_irqrestore(&allocator_request_lock
, flags
);
167 panic("scsi_free:Bad offset");
172 * Function: scsi_resize_dma_pool
174 * Purpose: Ensure that the DMA pool is sufficiently large to be
175 * able to guarantee that we can always process I/O requests
176 * without calling the system allocator.
180 * Lock status: No locks assumed to be held. This function is SMP-safe.
184 * Notes: Prior to the new queue code, this function was not SMP-safe.
185 * Go through the device list and recompute the most appropriate
186 * size for the dma pool. Then grab more memory (as required).
188 void scsi_resize_dma_pool(void)
193 struct Scsi_Host
*shpnt
;
194 struct Scsi_Host
*host
= NULL
;
196 FreeSectorBitmap
*new_dma_malloc_freelist
= NULL
;
197 unsigned int new_dma_sectors
= 0;
198 unsigned int new_need_isa_buffer
= 0;
199 unsigned char **new_dma_malloc_pages
= NULL
;
200 int out_of_space
= 0;
202 spin_lock_irqsave(&allocator_request_lock
, flags
);
204 if (!scsi_hostlist
) {
206 * Free up the DMA pool.
208 if (scsi_dma_free_sectors
!= dma_sectors
)
209 panic("SCSI DMA pool memory leak %d %d\n", scsi_dma_free_sectors
, dma_sectors
);
211 for (i
= 0; i
< dma_sectors
/ SECTORS_PER_PAGE
; i
++)
212 free_pages((unsigned long) dma_malloc_pages
[i
], 0);
213 if (dma_malloc_pages
)
214 kfree((char *) dma_malloc_pages
);
215 dma_malloc_pages
= NULL
;
216 if (dma_malloc_freelist
)
217 kfree((char *) dma_malloc_freelist
);
218 dma_malloc_freelist
= NULL
;
220 scsi_dma_free_sectors
= 0;
221 spin_unlock_irqrestore(&allocator_request_lock
, flags
);
224 /* Next, check to see if we need to extend the DMA buffer pool */
226 new_dma_sectors
= 2 * SECTORS_PER_PAGE
; /* Base value we use */
228 if (__pa(high_memory
) - 1 > ISA_DMA_THRESHOLD
)
229 need_isa_bounce_buffers
= 1;
231 need_isa_bounce_buffers
= 0;
234 for (shpnt
= scsi_hostlist
; shpnt
; shpnt
= shpnt
->next
)
235 new_dma_sectors
+= SECTORS_PER_PAGE
; /* Increment for each host */
237 for (host
= scsi_hostlist
; host
; host
= host
->next
) {
238 for (SDpnt
= host
->host_queue
; SDpnt
; SDpnt
= SDpnt
->next
) {
240 * sd and sr drivers allocate scatterlists.
241 * sr drivers may allocate for each command 1x2048 or 2x1024 extra
242 * buffers for 2k sector size and 1k fs.
243 * sg driver allocates buffers < 4k.
244 * st driver does not need buffers from the dma pool.
245 * estimate 4k buffer/command for devices of unknown type (should panic).
247 if (SDpnt
->type
== TYPE_WORM
|| SDpnt
->type
== TYPE_ROM
||
248 SDpnt
->type
== TYPE_DISK
|| SDpnt
->type
== TYPE_MOD
) {
249 int nents
= host
->sg_tablesize
;
250 #ifdef DMA_CHUNK_SIZE
251 /* If the architecture does DMA sg merging, make sure
252 we count with at least 64 entries even for HBAs
253 which handle very few sg entries. */
254 if (nents
< 64) nents
= 64;
256 new_dma_sectors
+= ((nents
*
257 sizeof(struct scatterlist
) + 511) >> 9) *
259 if (SDpnt
->type
== TYPE_WORM
|| SDpnt
->type
== TYPE_ROM
)
260 new_dma_sectors
+= (2048 >> 9) * SDpnt
->queue_depth
;
261 } else if (SDpnt
->type
== TYPE_SCANNER
||
262 SDpnt
->type
== TYPE_PROCESSOR
||
263 SDpnt
->type
== TYPE_COMM
||
264 SDpnt
->type
== TYPE_MEDIUM_CHANGER
||
265 SDpnt
->type
== TYPE_ENCLOSURE
) {
266 new_dma_sectors
+= (4096 >> 9) * SDpnt
->queue_depth
;
268 if (SDpnt
->type
!= TYPE_TAPE
) {
269 printk("resize_dma_pool: unknown device type %d\n", SDpnt
->type
);
270 new_dma_sectors
+= (4096 >> 9) * SDpnt
->queue_depth
;
274 if (host
->unchecked_isa_dma
&&
275 need_isa_bounce_buffers
&&
276 SDpnt
->type
!= TYPE_TAPE
) {
277 new_dma_sectors
+= (PAGE_SIZE
>> 9) * host
->sg_tablesize
*
279 new_need_isa_buffer
++;
285 printk("resize_dma_pool: needed dma sectors = %d\n", new_dma_sectors
);
288 /* limit DMA memory to 32MB: */
289 new_dma_sectors
= (new_dma_sectors
+ 15) & 0xfff0;
292 * We never shrink the buffers - this leads to
293 * race conditions that I would rather not even think
296 #if 0 /* Why do this? No gain and risks out_of_space */
297 if (new_dma_sectors
< dma_sectors
)
298 new_dma_sectors
= dma_sectors
;
300 if (new_dma_sectors
<= dma_sectors
) {
301 spin_unlock_irqrestore(&allocator_request_lock
, flags
);
302 return; /* best to quit while we are in front */
305 for (k
= 0; k
< 20; ++k
) { /* just in case */
307 size
= (new_dma_sectors
/ SECTORS_PER_PAGE
) *
308 sizeof(FreeSectorBitmap
);
309 new_dma_malloc_freelist
= (FreeSectorBitmap
*)
310 kmalloc(size
, GFP_ATOMIC
);
311 if (new_dma_malloc_freelist
) {
312 memset(new_dma_malloc_freelist
, 0, size
);
313 size
= (new_dma_sectors
/ SECTORS_PER_PAGE
) *
314 sizeof(*new_dma_malloc_pages
);
315 new_dma_malloc_pages
= (unsigned char **)
316 kmalloc(size
, GFP_ATOMIC
);
317 if (!new_dma_malloc_pages
) {
318 size
= (new_dma_sectors
/ SECTORS_PER_PAGE
) *
319 sizeof(FreeSectorBitmap
);
320 kfree((char *) new_dma_malloc_freelist
);
323 memset(new_dma_malloc_pages
, 0, size
);
328 if ((!out_of_space
) && (new_dma_sectors
> dma_sectors
)) {
329 for (i
= dma_sectors
/ SECTORS_PER_PAGE
;
330 i
< new_dma_sectors
/ SECTORS_PER_PAGE
; i
++) {
331 new_dma_malloc_pages
[i
] = (unsigned char *)
332 __get_free_pages(GFP_ATOMIC
| GFP_DMA
, 0);
333 if (!new_dma_malloc_pages
[i
])
336 if (i
!= new_dma_sectors
/ SECTORS_PER_PAGE
) { /* clean up */
340 for (i
= 0; i
< k
; ++i
)
341 free_pages((unsigned long) new_dma_malloc_pages
[i
], 0);
344 if (out_of_space
) { /* try scaling down new_dma_sectors request */
345 printk("scsi::resize_dma_pool: WARNING, dma_sectors=%u, "
346 "wanted=%u, scaling\n", dma_sectors
, new_dma_sectors
);
347 if (new_dma_sectors
< (8 * SECTORS_PER_PAGE
))
348 break; /* pretty well hopeless ... */
349 new_dma_sectors
= (new_dma_sectors
* 3) / 4;
350 new_dma_sectors
= (new_dma_sectors
+ 15) & 0xfff0;
351 if (new_dma_sectors
<= dma_sectors
)
352 break; /* stick with what we have got */
354 break; /* found space ... */
355 } /* end of for loop */
357 spin_unlock_irqrestore(&allocator_request_lock
, flags
);
358 scsi_need_isa_buffer
= new_need_isa_buffer
; /* some useful info */
359 printk(" WARNING, not enough memory, pool not expanded\n");
362 /* When we dick with the actual DMA list, we need to
365 if (dma_malloc_freelist
) {
366 size
= (dma_sectors
/ SECTORS_PER_PAGE
) * sizeof(FreeSectorBitmap
);
367 memcpy(new_dma_malloc_freelist
, dma_malloc_freelist
, size
);
368 kfree((char *) dma_malloc_freelist
);
370 dma_malloc_freelist
= new_dma_malloc_freelist
;
372 if (dma_malloc_pages
) {
373 size
= (dma_sectors
/ SECTORS_PER_PAGE
) * sizeof(*dma_malloc_pages
);
374 memcpy(new_dma_malloc_pages
, dma_malloc_pages
, size
);
375 kfree((char *) dma_malloc_pages
);
377 scsi_dma_free_sectors
+= new_dma_sectors
- dma_sectors
;
378 dma_malloc_pages
= new_dma_malloc_pages
;
379 dma_sectors
= new_dma_sectors
;
380 scsi_need_isa_buffer
= new_need_isa_buffer
;
382 spin_unlock_irqrestore(&allocator_request_lock
, flags
);
385 printk("resize_dma_pool: dma free sectors = %d\n", scsi_dma_free_sectors
);
386 printk("resize_dma_pool: dma sectors = %d\n", dma_sectors
);
387 printk("resize_dma_pool: need isa buffers = %d\n", scsi_need_isa_buffer
);
392 * Function: scsi_init_minimal_dma_pool
394 * Purpose: Allocate a minimal (1-page) DMA pool.
398 * Lock status: No locks assumed to be held. This function is SMP-safe.
404 int scsi_init_minimal_dma_pool(void)
410 spin_lock_irqsave(&allocator_request_lock
, flags
);
412 dma_sectors
= PAGE_SIZE
/ SECTOR_SIZE
;
413 scsi_dma_free_sectors
= dma_sectors
;
415 * Set up a minimal DMA buffer list - this will be used during scan_scsis
419 /* One bit per sector to indicate free/busy */
420 size
= (dma_sectors
/ SECTORS_PER_PAGE
) * sizeof(FreeSectorBitmap
);
421 dma_malloc_freelist
= (FreeSectorBitmap
*)
422 kmalloc(size
, GFP_ATOMIC
);
423 if (dma_malloc_freelist
) {
424 memset(dma_malloc_freelist
, 0, size
);
425 /* One pointer per page for the page list */
426 dma_malloc_pages
= (unsigned char **) kmalloc(
427 (dma_sectors
/ SECTORS_PER_PAGE
) * sizeof(*dma_malloc_pages
),
429 if (dma_malloc_pages
) {
430 memset(dma_malloc_pages
, 0, size
);
431 dma_malloc_pages
[0] = (unsigned char *)
432 __get_free_pages(GFP_ATOMIC
| GFP_DMA
, 0);
433 if (dma_malloc_pages
[0])
438 if (dma_malloc_freelist
) {
439 kfree((char *) dma_malloc_freelist
);
440 if (dma_malloc_pages
)
441 kfree((char *) dma_malloc_pages
);
443 spin_unlock_irqrestore(&allocator_request_lock
, flags
);
444 printk("scsi::init_module: failed, out of memory\n");
448 spin_unlock_irqrestore(&allocator_request_lock
, flags
);