?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ddraw: Move the wined3d_texture_update_desc() call into ddraw_surface_create_wined3d_...
[wine.git] / dlls / ntdll / heap.c
blob6688fab96906a125f924c24f5ebafb6bb8a859e5
1 /*
2 * Win32 heap functions
3 *
4 * Copyright 1996 Alexandre Julliard
5 * Copyright 1998 Ulrich Weigand
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
20 */
21
22 #include <assert.h>
23 #include <stdlib.h>
24 #include <stdarg.h>
25 #include <stdio.h>
26 #include <string.h>
27
28 #define RUNNING_ON_VALGRIND 0 /* FIXME */
29
30 #include "ntstatus.h"
31 #define WIN32_NO_STATUS
32 #define NONAMELESSUNION
33 #include "windef.h"
34 #include "winnt.h"
35 #include "winternl.h"
36 #include "ntdll_misc.h"
37 #include "wine/list.h"
38 #include "wine/debug.h"
39
40 WINE_DEFAULT_DEBUG_CHANNEL(heap);
41
42 /* HeapCompatibilityInformation values */
43
44 #define HEAP_STD 0
45 #define HEAP_LAL 1
46 #define HEAP_LFH 2
47
48
49 /* undocumented RtlWalkHeap structure */
50
51 struct rtl_heap_entry
52 {
53 LPVOID lpData;
54 SIZE_T cbData; /* differs from PROCESS_HEAP_ENTRY */
55 BYTE cbOverhead;
56 BYTE iRegionIndex;
57 WORD wFlags; /* value differs from PROCESS_HEAP_ENTRY */
58 union {
59 struct {
60 HANDLE hMem;
61 DWORD dwReserved[3];
62 } Block;
63 struct {
64 DWORD dwCommittedSize;
65 DWORD dwUnCommittedSize;
66 LPVOID lpFirstBlock;
67 LPVOID lpLastBlock;
68 } Region;
69 };
70 };
71
72 /* rtl_heap_entry flags, names made up */
73
74 #define RTL_HEAP_ENTRY_BUSY 0x0001
75 #define RTL_HEAP_ENTRY_REGION 0x0002
76 #define RTL_HEAP_ENTRY_BLOCK 0x0010
77 #define RTL_HEAP_ENTRY_UNCOMMITTED 0x1000
78 #define RTL_HEAP_ENTRY_COMMITTED 0x4000
79 #define RTL_HEAP_ENTRY_LFH 0x8000
80
81
82 /* header for heap blocks */
83
84 #define REGION_ALIGN 0x10000
85 #define BLOCK_ALIGN (2 * sizeof(void *))
86
87 struct DECLSPEC_ALIGN(8) block
88 {
89 /* block size in multiple of BLOCK_ALIGN */
90 WORD block_size;
91 /* unused size (used block) / high size bits (free block) */
92 WORD tail_size;
93 /* offset to region base / first group block (LFH block) */
94 WORD base_offset;
95 BYTE block_type;
96 BYTE block_flags;
97 };
98
99 C_ASSERT( sizeof(struct block) == 8 );
100
101 /* block specific flags */
102
103 #define BLOCK_FLAG_FREE 0x01
104 #define BLOCK_FLAG_PREV_FREE 0x02
105 #define BLOCK_FLAG_FREE_LINK 0x03
106 #define BLOCK_FLAG_LARGE 0x04
107 #define BLOCK_FLAG_LFH 0x08 /* block is handled by the LFH frontend */
108 #define BLOCK_FLAG_USER_INFO 0x10 /* user flags up to 0xf0 */
109 #define BLOCK_FLAG_USER_MASK 0xf0
110
111 #define BLOCK_USER_FLAGS( heap_flags ) (((heap_flags) >> 4) & BLOCK_FLAG_USER_MASK)
112 #define HEAP_USER_FLAGS( block_flags ) (((block_flags) & BLOCK_FLAG_USER_MASK) << 4)
113
114 /* entry to link free blocks in free lists */
115
116 struct DECLSPEC_ALIGN(BLOCK_ALIGN) entry
117 {
118 struct block block;
119 struct list entry;
120 };
121
122 C_ASSERT( sizeof(struct entry) == 2 * BLOCK_ALIGN );
123
124 typedef struct
125 {
126 SIZE_T __pad[sizeof(SIZE_T) / sizeof(DWORD)];
127 struct list entry; /* entry in heap large blocks list */
128 SIZE_T data_size; /* size of user data */
129 SIZE_T block_size; /* total size of virtual memory block */
130 void *user_value;
131 struct block block;
132 } ARENA_LARGE;
133
134 /* block must be last and aligned */
135 C_ASSERT( sizeof(ARENA_LARGE) == offsetof(ARENA_LARGE, block) + sizeof(struct block) );
136 C_ASSERT( sizeof(ARENA_LARGE) == 4 * BLOCK_ALIGN );
137
138 #define BLOCK_TYPE_USED 'u'
139 #define BLOCK_TYPE_DEAD 'D'
140 #define BLOCK_TYPE_FREE 'F'
141 #define BLOCK_TYPE_LARGE 'L'
142
143 #define BLOCK_FILL_USED 0x55
144 #define BLOCK_FILL_TAIL 0xab
145 #define BLOCK_FILL_FREE 0xfeeefeee
146
147 #define ROUND_ADDR(addr, mask) ((void *)((UINT_PTR)(addr) & ~(UINT_PTR)(mask)))
148 #define ROUND_SIZE(size, mask) ((((SIZE_T)(size) + (mask)) & ~(SIZE_T)(mask)))
149 #define FIELD_MAX(type, field) (((SIZE_T)1 << (sizeof(((type *)0)->field) * 8)) - 1)
150
151 #define HEAP_MIN_BLOCK_SIZE ROUND_SIZE(sizeof(struct entry) + BLOCK_ALIGN, BLOCK_ALIGN - 1)
152
153 C_ASSERT( sizeof(struct block) <= HEAP_MIN_BLOCK_SIZE );
154 C_ASSERT( sizeof(struct entry) <= HEAP_MIN_BLOCK_SIZE );
155
156 /* used block size is coded into block_size */
157 #define HEAP_MAX_USED_BLOCK_SIZE (FIELD_MAX( struct block, block_size ) * BLOCK_ALIGN)
158 /* free block size is coded into block_size + tail_size */
159 #define HEAP_MAX_FREE_BLOCK_SIZE (HEAP_MAX_USED_BLOCK_SIZE + (FIELD_MAX( struct block, tail_size ) << 16) * BLOCK_ALIGN)
160 /* subheap distance is coded into base_offset */
161 #define HEAP_MAX_BLOCK_REGION_SIZE (FIELD_MAX( struct block, base_offset ) * REGION_ALIGN)
162
163 C_ASSERT( HEAP_MAX_USED_BLOCK_SIZE == 512 * 1024 * (sizeof(void *) / 4) - BLOCK_ALIGN );
164 C_ASSERT( HEAP_MAX_FREE_BLOCK_SIZE >= 128 * 1024 * 1024 * (sizeof(void *) / 4) - BLOCK_ALIGN );
165 C_ASSERT( HEAP_MAX_BLOCK_REGION_SIZE >= 128 * 1024 * 1024 * (sizeof(void *) / 4) - BLOCK_ALIGN );
166 C_ASSERT( HEAP_MAX_FREE_BLOCK_SIZE >= HEAP_MAX_BLOCK_REGION_SIZE );
167
168 /* minimum size to start allocating large blocks */
169 #define HEAP_MIN_LARGE_BLOCK_SIZE (HEAP_MAX_USED_BLOCK_SIZE - 0x1000)
170
171 /* There will be a free list bucket for every arena size up to and including this value */
172 #define HEAP_MAX_SMALL_FREE_LIST 0x100
173 C_ASSERT( HEAP_MAX_SMALL_FREE_LIST % BLOCK_ALIGN == 0 );
174 #define HEAP_NB_SMALL_FREE_LISTS (((HEAP_MAX_SMALL_FREE_LIST - HEAP_MIN_BLOCK_SIZE) / BLOCK_ALIGN) + 1)
175
176 /* Max size of the blocks on the free lists above HEAP_MAX_SMALL_FREE_LIST */
177 static const SIZE_T free_list_sizes[] =
178 {
179 0x200, 0x400, 0x1000, ~(SIZE_T)0
180 };
181 #define HEAP_NB_FREE_LISTS (ARRAY_SIZE(free_list_sizes) + HEAP_NB_SMALL_FREE_LISTS)
182
183 typedef struct DECLSPEC_ALIGN(BLOCK_ALIGN) tagSUBHEAP
184 {
185 SIZE_T __pad[sizeof(SIZE_T) / sizeof(DWORD)];
186 SIZE_T block_size;
187 SIZE_T data_size;
188 struct list entry;
189 void *user_value;
190 struct block block;
191 } SUBHEAP;
192
193 /* block must be last and aligned */
194 C_ASSERT( sizeof(SUBHEAP) == offsetof(SUBHEAP, block) + sizeof(struct block) );
195 C_ASSERT( sizeof(SUBHEAP) == 4 * BLOCK_ALIGN );
196
197
198 /* LFH block size bins */
199
200 #define BIN_SIZE_MIN_0 0
201 #define BIN_SIZE_MIN_1 0x100
202 #define BIN_SIZE_MIN_2 0x200
203 #define BIN_SIZE_MIN_3 0x400
204 #define BIN_SIZE_MIN_4 0x800
205 #define BIN_SIZE_MIN_5 0x1000
206 #define BIN_SIZE_MIN_6 0x2000
207 #define BIN_SIZE_MIN_7 0x4000
208 #define BIN_SIZE_MAX 0x8000
209
210 #define BIN_SIZE_STEP_0 (16)
211 #define BIN_SIZE_STEP_1 (BIN_SIZE_MIN_1 >> 4)
212 #define BIN_SIZE_STEP_2 (BIN_SIZE_MIN_2 >> 4)
213 #define BIN_SIZE_STEP_3 (BIN_SIZE_MIN_3 >> 4)
214 #define BIN_SIZE_STEP_4 (BIN_SIZE_MIN_4 >> 4)
215 #define BIN_SIZE_STEP_5 (BIN_SIZE_MIN_5 >> 4)
216 #define BIN_SIZE_STEP_6 (BIN_SIZE_MIN_6 >> 4)
217 #define BIN_SIZE_STEP_7 (BIN_SIZE_MIN_7 >> 4)
218
219 #define BLOCK_BIN_SIZE_N( n, bin ) (BIN_SIZE_MIN_##n + (bin + 1) * BIN_SIZE_STEP_##n)
220 #define BLOCK_SIZE_BIN_N( n, size ) (((size) - 1 - BIN_SIZE_MIN_##n) / BIN_SIZE_STEP_##n)
221
222 #define BLOCK_BIN_SIZE( bin ) ((bin) >= 0x80 ? ~(SIZE_T)0 : \
223 (bin) >= 0x70 ? BLOCK_BIN_SIZE_N( 7, bin - 0x70 ) : \
224 (bin) >= 0x60 ? BLOCK_BIN_SIZE_N( 6, bin - 0x60 ) : \
225 (bin) >= 0x50 ? BLOCK_BIN_SIZE_N( 5, bin - 0x50 ) : \
226 (bin) >= 0x40 ? BLOCK_BIN_SIZE_N( 4, bin - 0x40 ) : \
227 (bin) >= 0x30 ? BLOCK_BIN_SIZE_N( 3, bin - 0x30 ) : \
228 (bin) >= 0x20 ? BLOCK_BIN_SIZE_N( 2, bin - 0x20 ) : \
229 (bin) >= 0x10 ? BLOCK_BIN_SIZE_N( 1, bin - 0x10 ) : \
230 BLOCK_BIN_SIZE_N( 0, bin ))
231
232 #define BLOCK_SIZE_BIN( size ) ((size) > BIN_SIZE_MAX ? 0x80 : \
233 (size) > BIN_SIZE_MIN_7 ? 0x70 + BLOCK_SIZE_BIN_N( 7, size ) : \
234 (size) > BIN_SIZE_MIN_6 ? 0x60 + BLOCK_SIZE_BIN_N( 6, size ) : \
235 (size) > BIN_SIZE_MIN_5 ? 0x50 + BLOCK_SIZE_BIN_N( 5, size ) : \
236 (size) > BIN_SIZE_MIN_4 ? 0x40 + BLOCK_SIZE_BIN_N( 4, size ) : \
237 (size) > BIN_SIZE_MIN_3 ? 0x30 + BLOCK_SIZE_BIN_N( 3, size ) : \
238 (size) > BIN_SIZE_MIN_2 ? 0x20 + BLOCK_SIZE_BIN_N( 2, size ) : \
239 (size) > BIN_SIZE_MIN_1 ? 0x10 + BLOCK_SIZE_BIN_N( 1, size ) : \
240 (size) <= BIN_SIZE_MIN_0 ? 0 : BLOCK_SIZE_BIN_N( 0, size ))
241
242 #define BLOCK_SIZE_BIN_COUNT (BLOCK_SIZE_BIN( BIN_SIZE_MAX + 1 ) + 1)
243
244 /* macros sanity checks */
245 C_ASSERT( BLOCK_SIZE_BIN( 0 ) == 0 );
246 C_ASSERT( BLOCK_SIZE_BIN( 0x10 ) == 0 );
247 C_ASSERT( BLOCK_BIN_SIZE( 0 ) == BIN_SIZE_MIN_0 + 1 * BIN_SIZE_STEP_0 );
248 C_ASSERT( BLOCK_SIZE_BIN( 0x11 ) == 1 );
249 C_ASSERT( BLOCK_BIN_SIZE( 1 ) == BIN_SIZE_MIN_0 + 2 * BIN_SIZE_STEP_0 );
250 C_ASSERT( BLOCK_SIZE_BIN( BIN_SIZE_MAX ) == 0x7f );
251 C_ASSERT( BLOCK_BIN_SIZE( 0x7f ) == BIN_SIZE_MAX );
252 C_ASSERT( BLOCK_SIZE_BIN( BIN_SIZE_MAX + 1 ) == 0x80 );
253 C_ASSERT( BLOCK_BIN_SIZE( 0x80 ) == ~(SIZE_T)0 );
254
255 /* difference between block classes and all possible validation overhead must fit into block tail_size */
256 C_ASSERT( BIN_SIZE_STEP_7 + 3 * BLOCK_ALIGN <= FIELD_MAX( struct block, tail_size ) );
257
258 static BYTE affinity_mapping[] = {20,6,31,15,14,29,27,4,18,24,26,13,0,9,2,30,17,7,23,25,10,19,12,3,22,21,5,16,1,28,11,8};
259 static LONG next_thread_affinity;
260
261 /* a bin, tracking heap blocks of a certain size */
262 struct bin
263 {
264 /* counters for LFH activation */
265 LONG count_alloc;
266 LONG count_freed;
267 LONG enabled;
268
269 /* list of groups with free blocks */
270 SLIST_HEADER groups;
271
272 /* array of affinity reserved groups, interleaved with other bins to keep
273 * all pointers of the same affinity and different bin grouped together,
274 * and pointers of the same bin and different affinity away from each other,
275 * hopefully in separate cache lines.
276 */
277 struct group **affinity_group_base;
278 };
279
280 static inline struct group **bin_get_affinity_group( struct bin *bin, BYTE affinity )
281 {
282 return bin->affinity_group_base + affinity * BLOCK_SIZE_BIN_COUNT;
283 }
284
285 struct heap
286 { /* win32/win64 */
287 DWORD_PTR unknown1[2]; /* 0000/0000 */
288 DWORD ffeeffee; /* 0008/0010 */
289 DWORD auto_flags; /* 000c/0014 */
290 DWORD_PTR unknown2[7]; /* 0010/0018 */
291 DWORD unknown3[2]; /* 002c/0050 */
292 DWORD_PTR unknown4[3]; /* 0034/0058 */
293 DWORD flags; /* 0040/0070 */
294 DWORD force_flags; /* 0044/0074 */
295 /* end of the Windows 10 compatible struct layout */
296
297 LONG compat_info; /* HeapCompatibilityInformation / heap frontend type */
298 struct list entry; /* Entry in process heap list */
299 struct list subheap_list; /* Sub-heap list */
300 struct list large_list; /* Large blocks list */
301 SIZE_T grow_size; /* Size of next subheap for growing heap */
302 SIZE_T min_size; /* Minimum committed size */
303 DWORD magic; /* Magic number */
304 DWORD pending_pos; /* Position in pending free requests ring */
305 struct block **pending_free; /* Ring buffer for pending free requests */
306 RTL_CRITICAL_SECTION cs;
307 struct entry free_lists[HEAP_NB_FREE_LISTS];
308 struct bin *bins;
309 SUBHEAP subheap;
310 };
311
312 /* subheap must be last and aligned */
313 C_ASSERT( sizeof(struct heap) == offsetof(struct heap, subheap) + sizeof(SUBHEAP) );
314 C_ASSERT( sizeof(struct heap) % BLOCK_ALIGN == 0 );
315 C_ASSERT( offsetof(struct heap, subheap) <= REGION_ALIGN - 1 );
316
317 #define HEAP_MAGIC ((DWORD)('H' | ('E'<<8) | ('A'<<16) | ('P'<<24)))
318
319 #define HEAP_DEF_SIZE (0x40000 * BLOCK_ALIGN)
320 #define MAX_FREE_PENDING 1024 /* max number of free requests to delay */
321
322 /* some undocumented flags (names are made up) */
323 #define HEAP_PRIVATE 0x00001000
324 #define HEAP_ADD_USER_INFO 0x00000100
325 #define HEAP_USER_FLAGS_MASK 0x00000f00
326 #define HEAP_PAGE_ALLOCS 0x01000000
327 #define HEAP_VALIDATE 0x10000000
328 #define HEAP_VALIDATE_ALL 0x20000000
329 #define HEAP_VALIDATE_PARAMS 0x40000000
330 #define HEAP_CHECKING_ENABLED 0x80000000
331
332 static struct heap *process_heap; /* main process heap */
333
334 /* check if memory range a contains memory range b */
335 static inline BOOL contains( const void *a, SIZE_T a_size, const void *b, SIZE_T b_size )
336 {
337 const void *a_end = (char *)a + a_size, *b_end = (char *)b + b_size;
338 return a <= b && b <= b_end && b_end <= a_end;
339 }
340
341 static inline UINT block_get_flags( const struct block *block )
342 {
343 return block->block_flags;
344 }
345
346 static inline UINT block_get_type( const struct block *block )
347 {
348 return block->block_type;
349 }
350
351 static inline void block_set_base( struct block *block, const void *base )
352 {
353 const char *offset = ROUND_ADDR( block, REGION_ALIGN - 1 );
354 block->base_offset = (offset - (char *)base) / REGION_ALIGN;
355 }
356
357 static inline void block_set_type( struct block *block, UINT type )
358 {
359 block->block_type = type;
360 }
361
362 static inline SUBHEAP *block_get_subheap( const struct heap *heap, const struct block *block )
363 {
364 char *offset = ROUND_ADDR( block, REGION_ALIGN - 1 );
365 void *base = offset - block->base_offset * REGION_ALIGN;
366 if (base != (void *)heap) return base;
367 else return (SUBHEAP *)&heap->subheap;
368 }
369
370 static inline UINT block_get_overhead( const struct block *block )
371 {
372 if (block_get_flags( block ) & BLOCK_FLAG_FREE) return sizeof(*block) + sizeof(struct list);
373 return sizeof(*block) + block->tail_size;
374 }
375
376 /* return the size of a block, including its header */
377 static inline UINT block_get_size( const struct block *block )
378 {
379 UINT block_size = block->block_size;
380 if (block_get_flags( block ) & BLOCK_FLAG_FREE) block_size += (UINT)block->tail_size << 16;
381 return block_size * BLOCK_ALIGN;
382 }
383
384 static inline void block_set_size( struct block *block, UINT block_size )
385 {
386 block_size /= BLOCK_ALIGN;
387 if (block_get_flags( block ) & BLOCK_FLAG_FREE) block->tail_size = block_size >> 16;
388 block->block_size = block_size;
389 }
390
391 static inline void block_set_flags( struct block *block, BYTE clear, BYTE set )
392 {
393 UINT block_size = block_get_size( block );
394 block->block_flags &= ~clear;
395 block->block_flags |= set;
396 block_set_size( block, block_size );
397 }
398
399 static inline void *subheap_base( const SUBHEAP *subheap )
400 {
401 return ROUND_ADDR( subheap, REGION_ALIGN - 1 );
402 }
403
404 static inline SIZE_T subheap_overhead( const SUBHEAP *subheap )
405 {
406 return (char *)&subheap->block - (char *)subheap_base( subheap );
407 }
408
409 static inline SIZE_T subheap_size( const SUBHEAP *subheap )
410 {
411 return subheap->block_size + subheap_overhead( subheap );
412 }
413
414 static inline const void *subheap_commit_end( const SUBHEAP *subheap )
415 {
416 return (char *)(subheap + 1) + subheap->data_size;
417 }
418
419 static void subheap_set_bounds( SUBHEAP *subheap, char *commit_end, char *end )
420 {
421 subheap->block_size = end - (char *)&subheap->block;
422 subheap->data_size = commit_end - (char *)(subheap + 1);
423 }
424
425 static inline void *first_block( const SUBHEAP *subheap )
426 {
427 return (void *)&subheap->block;
428 }
429
430 static inline const void *last_block( const SUBHEAP *subheap )
431 {
432 return (char *)subheap_commit_end( subheap ) - sizeof(struct block);
433 }
434
435 static inline struct block *next_block( const SUBHEAP *subheap, const struct block *block )
436 {
437 const char *data = (char *)(block + 1), *next, *last = last_block( subheap );
438 next = (char *)block + block_get_size( block );
439 if (!contains( data, last - (char *)data, next, sizeof(*block) )) return NULL;
440 return (struct block *)next;
441 }
442
443 static inline BOOL check_subheap( const SUBHEAP *subheap, const struct heap *heap )
444 {
445 if (subheap->user_value != heap) return FALSE;
446
447 return contains( &subheap->block, subheap->block_size, subheap + 1, subheap->data_size );
448 }
449
450 static BOOL heap_validate( const struct heap *heap );
451
452 /* mark a block of memory as innacessible for debugging purposes */
453 static inline void valgrind_make_noaccess( void const *ptr, SIZE_T size )
454 {
455 #if defined(VALGRIND_MAKE_MEM_NOACCESS)
456 VALGRIND_DISCARD( VALGRIND_MAKE_MEM_NOACCESS( ptr, size ) );
457 #elif defined(VALGRIND_MAKE_NOACCESS)
458 VALGRIND_DISCARD( VALGRIND_MAKE_NOACCESS( ptr, size ) );
459 #endif
460 }
461
462 /* mark a block of memory as initialized for debugging purposes */
463 static inline void valgrind_make_readable( void const *ptr, SIZE_T size )
464 {
465 #if defined(VALGRIND_MAKE_MEM_DEFINED)
466 VALGRIND_DISCARD( VALGRIND_MAKE_MEM_DEFINED( ptr, size ) );
467 #elif defined(VALGRIND_MAKE_READABLE)
468 VALGRIND_DISCARD( VALGRIND_MAKE_READABLE( ptr, size ) );
469 #endif
470 }
471
472 /* mark a block of memory as uninitialized for debugging purposes */
473 static inline void valgrind_make_writable( void const *ptr, SIZE_T size )
474 {
475 #if defined(VALGRIND_MAKE_MEM_UNDEFINED)
476 VALGRIND_DISCARD( VALGRIND_MAKE_MEM_UNDEFINED( ptr, size ) );
477 #elif defined(VALGRIND_MAKE_WRITABLE)
478 VALGRIND_DISCARD( VALGRIND_MAKE_WRITABLE( ptr, size ) );
479 #endif
480 }
481
482 /* mark a block of memory as free for debugging purposes */
483 static inline void mark_block_free( void *ptr, SIZE_T size, DWORD flags )
484 {
485 if (flags & HEAP_FREE_CHECKING_ENABLED)
486 {
487 SIZE_T i;
488 for (i = 0; i < size / sizeof(DWORD); i++) ((DWORD *)ptr)[i] = BLOCK_FILL_FREE;
489 }
490 valgrind_make_noaccess( ptr, size );
491 }
492
493 /* mark a block of memory as a tail block */
494 static inline void mark_block_tail( struct block *block, DWORD flags )
495 {
496 char *tail = (char *)block + block_get_size( block ) - block->tail_size;
497 if (flags & HEAP_TAIL_CHECKING_ENABLED)
498 {
499 valgrind_make_writable( tail, BLOCK_ALIGN );
500 memset( tail, BLOCK_FILL_TAIL, BLOCK_ALIGN );
501 }
502 valgrind_make_noaccess( tail, BLOCK_ALIGN );
503 if (flags & HEAP_ADD_USER_INFO)
504 {
505 if (flags & HEAP_TAIL_CHECKING_ENABLED || RUNNING_ON_VALGRIND) tail += BLOCK_ALIGN;
506 valgrind_make_writable( tail + sizeof(void *), sizeof(void *) );
507 memset( tail + sizeof(void *), 0, sizeof(void *) );
508 }
509 }
510
511 /* initialize contents of a newly created block of memory */
512 static inline void initialize_block( struct block *block, SIZE_T old_size, SIZE_T size, DWORD flags )
513 {
514 char *data = (char *)(block + 1);
515
516 if (size <= old_size) return;
517
518 if (flags & HEAP_ZERO_MEMORY)
519 {
520 valgrind_make_writable( data + old_size, size - old_size );
521 memset( data + old_size, 0, size - old_size );
522 }
523 else if (flags & HEAP_FREE_CHECKING_ENABLED)
524 {
525 valgrind_make_writable( data + old_size, size - old_size );
526 memset( data + old_size, BLOCK_FILL_USED, size - old_size );
527 }
528 }
529
530 /* notify that a new block of memory has been allocated for debugging purposes */
531 static inline void valgrind_notify_alloc( void const *ptr, SIZE_T size, BOOL init )
532 {
533 #ifdef VALGRIND_MALLOCLIKE_BLOCK
534 VALGRIND_MALLOCLIKE_BLOCK( ptr, size, 0, init );
535 #endif
536 }
537
538 /* notify that a block of memory has been freed for debugging purposes */
539 static inline void valgrind_notify_free( void const *ptr )
540 {
541 #ifdef VALGRIND_FREELIKE_BLOCK
542 VALGRIND_FREELIKE_BLOCK( ptr, 0 );
543 #endif
544 }
545
546 static inline void valgrind_notify_resize( void const *ptr, SIZE_T size_old, SIZE_T size_new )
547 {
548 #ifdef VALGRIND_RESIZEINPLACE_BLOCK
549 /* zero is not a valid size */
550 VALGRIND_RESIZEINPLACE_BLOCK( ptr, size_old ? size_old : 1, size_new ? size_new : 1, 0 );
551 #endif
552 }
553
554 static void valgrind_notify_free_all( SUBHEAP *subheap, const struct heap *heap )
555 {
556 #ifdef VALGRIND_FREELIKE_BLOCK
557 struct block *block;
558
559 if (!RUNNING_ON_VALGRIND) return;
560 if (!check_subheap( subheap, heap )) return;
561
562 for (block = first_block( subheap ); block; block = next_block( subheap, block ))
563 {
564 if (block_get_flags( block ) & BLOCK_FLAG_FREE) continue;
565 if (block_get_type( block ) == BLOCK_TYPE_USED) valgrind_notify_free( block + 1 );
566 }
567 #endif
568 }
569
570 /* locate a free list entry of the appropriate size */
571 /* size is the size of the whole block including the arena header */
572 static inline struct entry *find_free_list( struct heap *heap, SIZE_T block_size, BOOL last )
573 {
574 struct entry *list, *end = heap->free_lists + ARRAY_SIZE(heap->free_lists);
575 unsigned int i;
576
577 if (block_size <= HEAP_MAX_SMALL_FREE_LIST)
578 i = (block_size - HEAP_MIN_BLOCK_SIZE) / BLOCK_ALIGN;
579 else for (i = HEAP_NB_SMALL_FREE_LISTS; i < HEAP_NB_FREE_LISTS - 1; i++)
580 if (block_size <= free_list_sizes[i - HEAP_NB_SMALL_FREE_LISTS]) break;
581
582 list = heap->free_lists + i;
583 if (last && ++list == end) list = heap->free_lists;
584 return list;
585 }
586
587 /* get the memory protection type to use for a given heap */
588 static inline ULONG get_protection_type( DWORD flags )
589 {
590 return (flags & HEAP_CREATE_ENABLE_EXECUTE) ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE;
591 }
592
593 static RTL_CRITICAL_SECTION_DEBUG process_heap_cs_debug =
594 {
595 0, 0, NULL, /* will be set later */
596 { &process_heap_cs_debug.ProcessLocksList, &process_heap_cs_debug.ProcessLocksList },
597 0, 0, { (DWORD_PTR)(__FILE__ ": main process heap section") }
598 };
599
600 static inline ULONG heap_get_flags( const struct heap *heap, ULONG flags )
601 {
602 if (flags & (HEAP_TAIL_CHECKING_ENABLED | HEAP_FREE_CHECKING_ENABLED)) flags |= HEAP_CHECKING_ENABLED;
603 flags &= HEAP_GENERATE_EXCEPTIONS | HEAP_NO_SERIALIZE | HEAP_ZERO_MEMORY | HEAP_REALLOC_IN_PLACE_ONLY | HEAP_CHECKING_ENABLED | HEAP_USER_FLAGS_MASK;
604 return heap->flags | flags;
605 }
606
607 static inline void heap_lock( struct heap *heap, ULONG flags )
608 {
609 if (flags & HEAP_NO_SERIALIZE) return;
610 RtlEnterCriticalSection( &heap->cs );
611 }
612
613 static inline void heap_unlock( struct heap *heap, ULONG flags )
614 {
615 if (flags & HEAP_NO_SERIALIZE) return;
616 RtlLeaveCriticalSection( &heap->cs );
617 }
618
619 static void heap_set_status( const struct heap *heap, ULONG flags, NTSTATUS status )
620 {
621 if (status == STATUS_NO_MEMORY && (flags & HEAP_GENERATE_EXCEPTIONS)) RtlRaiseStatus( status );
622 if (status) RtlSetLastWin32ErrorAndNtStatusFromNtStatus( status );
623 }
624
625 static size_t get_free_list_block_size( unsigned int index )
626 {
627 if (index < HEAP_NB_SMALL_FREE_LISTS) return HEAP_MIN_BLOCK_SIZE + index * BLOCK_ALIGN;
628 return free_list_sizes[index - HEAP_NB_SMALL_FREE_LISTS];
629 }
630
631 static void heap_dump( const struct heap *heap )
632 {
633 const struct block *block;
634 const ARENA_LARGE *large;
635 const SUBHEAP *subheap;
636 unsigned int i;
637
638 TRACE( "heap: %p\n", heap );
639 TRACE( " next %p\n", LIST_ENTRY( heap->entry.next, struct heap, entry ) );
640
641 TRACE( " bins:\n" );
642 for (i = 0; heap->bins && i < BLOCK_SIZE_BIN_COUNT; i++)
643 {
644 const struct bin *bin = heap->bins + i;
645 ULONG alloc = ReadNoFence( &bin->count_alloc ), freed = ReadNoFence( &bin->count_freed );
646 if (!alloc && !freed) continue;
647 TRACE( " %3u: size %#4Ix, alloc %ld, freed %ld, enabled %lu\n", i, BLOCK_BIN_SIZE( i ),
648 alloc, freed, ReadNoFence( &bin->enabled ) );
649 }
650
651 TRACE( " free_lists: %p\n", heap->free_lists );
652 for (i = 0; i < HEAP_NB_FREE_LISTS; i++)
653 TRACE( " %p: size %#8Ix, prev %p, next %p\n", heap->free_lists + i, get_free_list_block_size( i ),
654 LIST_ENTRY( heap->free_lists[i].entry.prev, struct entry, entry ),
655 LIST_ENTRY( heap->free_lists[i].entry.next, struct entry, entry ) );
656
657 TRACE( " subheaps: %p\n", &heap->subheap_list );
658 LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry )
659 {
660 SIZE_T free_size = 0, used_size = 0, overhead = 0;
661 const char *base = subheap_base( subheap );
662
663 TRACE( " %p: base %p first %p last %p end %p\n", subheap, base, first_block( subheap ),
664 last_block( subheap ), base + subheap_size( subheap ) );
665
666 if (!check_subheap( subheap, heap )) return;
667
668 overhead += subheap_overhead( subheap );
669 for (block = first_block( subheap ); block; block = next_block( subheap, block ))
670 {
671 if (block_get_flags( block ) & BLOCK_FLAG_FREE)
672 {
673 TRACE( " %p: (free) type %#10x, size %#8x, flags %#4x, prev %p, next %p\n", block,
674 block_get_type( block ), block_get_size( block ), block_get_flags( block ),
675 LIST_ENTRY( ((struct entry *)block)->entry.prev, struct entry, entry ),
676 LIST_ENTRY( ((struct entry *)block)->entry.next, struct entry, entry ) );
677
678 overhead += block_get_overhead( block );
679 free_size += block_get_size( block ) - block_get_overhead( block );
680 }
681 else
682 {
683 TRACE( " %p: (used) type %#10x, size %#8x, flags %#4x, unused %#4x", block,
684 block_get_type( block ), block_get_size( block ), block_get_flags( block ),
685 block->tail_size );
686 if (!(block_get_flags( block ) & BLOCK_FLAG_PREV_FREE)) TRACE( "\n" );
687 else TRACE( ", back %p\n", *((struct block **)block - 1) );
688
689 overhead += block_get_overhead( block );
690 used_size += block_get_size( block ) - block_get_overhead( block );
691 }
692 }
693
694 TRACE( " total %#Ix, used %#Ix, free %#Ix, overhead %#Ix (%Iu%%)\n", used_size + free_size + overhead,
695 used_size, free_size, overhead, (overhead * 100) / subheap_size( subheap ) );
696 }
697
698 TRACE( " large blocks: %p\n", &heap->large_list );
699 LIST_FOR_EACH_ENTRY( large, &heap->large_list, ARENA_LARGE, entry )
700 {
701 TRACE( " %p: (large) type %#10x, size %#8x, flags %#4x, total_size %#10Ix, alloc_size %#10Ix, prev %p, next %p\n",
702 large, block_get_type( &large->block ), block_get_size( &large->block ), block_get_flags( &large->block ), large->block_size,
703 large->data_size, LIST_ENTRY( large->entry.prev, ARENA_LARGE, entry ), LIST_ENTRY( large->entry.next, ARENA_LARGE, entry ) );
704 }
705
706 if (heap->pending_free)
707 {
708 TRACE( " pending blocks: %p\n", heap->pending_free );
709 for (i = 0; i < MAX_FREE_PENDING; ++i)
710 {
711 if (!(block = heap->pending_free[i])) break;
712
713 TRACE( " %c%p: (pend) type %#10x, size %#8x, flags %#4x, unused %#4x", i == heap->pending_pos ? '*' : ' ',
714 block, block_get_type( block ), block_get_size( block ), block_get_flags( block ), block->tail_size );
715 if (!(block_get_flags( block ) & BLOCK_FLAG_PREV_FREE)) TRACE( "\n" );
716 else TRACE( ", back %p\n", *((struct block **)block - 1) );
717 }
718 }
719 }
720
721 static const char *debugstr_heap_entry( struct rtl_heap_entry *entry )
722 {
723 const char *str = wine_dbg_sprintf( "data %p, size %#Ix, overhead %#x, region %#x, flags %#x", entry->lpData,
724 entry->cbData, entry->cbOverhead, entry->iRegionIndex, entry->wFlags );
725 if (!(entry->wFlags & RTL_HEAP_ENTRY_REGION)) return str;
726 return wine_dbg_sprintf( "%s, commit %#lx, uncommit %#lx, first %p, last %p", str, entry->Region.dwCommittedSize,
727 entry->Region.dwUnCommittedSize, entry->Region.lpFirstBlock, entry->Region.lpLastBlock );
728 }
729
730 static struct heap *unsafe_heap_from_handle( HANDLE handle, ULONG flags, ULONG *heap_flags )
731 {
732 struct heap *heap = handle;
733 BOOL valid = TRUE;
734
735 if (!heap || (heap->magic != HEAP_MAGIC))
736 {
737 ERR( "Invalid handle %p!\n", handle );
738 return NULL;
739 }
740 if (heap->flags & HEAP_VALIDATE_ALL)
741 {
742 heap_lock( heap, 0 );
743 valid = heap_validate( heap );
744 heap_unlock( heap, 0 );
745
746 if (!valid && TRACE_ON(heap))
747 {
748 heap_dump( heap );
749 assert( FALSE );
750 }
751 }
752
753 *heap_flags = heap_get_flags( heap, flags );
754 return valid ? heap : NULL;
755 }
756
757
758 static SUBHEAP *find_subheap( const struct heap *heap, const struct block *block, BOOL heap_walk )
759 {
760 SUBHEAP *subheap;
761
762 LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry )
763 {
764 SIZE_T blocks_size = (char *)last_block( subheap ) - (char *)first_block( subheap );
765 if (!check_subheap( subheap, heap )) return NULL;
766 if (contains( first_block( subheap ), blocks_size, block, sizeof(*block) )) return subheap;
767 /* outside of blocks region, possible corruption or heap_walk */
768 if (contains( subheap_base( subheap ), subheap_size( subheap ), block, 1 )) return heap_walk ? subheap : NULL;
769 }
770
771 return NULL;
772 }
773
774
775 static inline BOOL subheap_commit( const struct heap *heap, SUBHEAP *subheap, const struct block *block, SIZE_T block_size )
776 {
777 const char *end = (char *)subheap_base( subheap ) + subheap_size( subheap ), *commit_end;
778 SIZE_T size;
779 void *addr;
780
781 commit_end = (char *)block + block_size + sizeof(struct entry);
782 commit_end = ROUND_ADDR( (char *)commit_end + REGION_ALIGN - 1, REGION_ALIGN - 1 );
783
784 if (commit_end > end) commit_end = end;
785 if (commit_end <= (char *)subheap_commit_end( subheap )) return TRUE;
786
787 addr = (void *)subheap_commit_end( subheap );
788 size = commit_end - (char *)addr;
789
790 if (NtAllocateVirtualMemory( NtCurrentProcess(), &addr, 0, &size, MEM_COMMIT,
791 get_protection_type( heap->flags ) ))
792 {
793 WARN( "Could not commit %#Ix bytes at %p for heap %p\n", size, addr, heap );
794 return FALSE;
795 }
796
797 subheap->data_size = (char *)commit_end - (char *)(subheap + 1);
798 return TRUE;
799 }
800
801 static inline BOOL subheap_decommit( const struct heap *heap, SUBHEAP *subheap, const void *commit_end )
802 {
803 char *base = subheap_base( subheap );
804 SIZE_T size;
805 void *addr;
806
807 commit_end = ROUND_ADDR( (char *)commit_end + REGION_ALIGN - 1, REGION_ALIGN - 1 );
808 if (subheap == &heap->subheap) commit_end = max( (char *)commit_end, (char *)base + heap->min_size );
809 if (commit_end >= subheap_commit_end( subheap )) return TRUE;
810
811 size = (char *)subheap_commit_end( subheap ) - (char *)commit_end;
812 addr = (void *)commit_end;
813
814 if (NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_DECOMMIT ))
815 {
816 WARN( "Could not decommit %#Ix bytes at %p for heap %p\n", size, addr, heap );
817 return FALSE;
818 }
819
820 subheap->data_size = (char *)commit_end - (char *)(subheap + 1);
821 return TRUE;
822 }
823
824 static void block_init_free( struct block *block, ULONG flags, SUBHEAP *subheap, SIZE_T block_size )
825 {
826 const char *end = (char *)block + block_size, *commit_end = subheap_commit_end( subheap );
827 struct entry *entry = (struct entry *)block;
828
829 valgrind_make_writable( block, sizeof(*entry) );
830 block_set_type( block, BLOCK_TYPE_FREE );
831 block_set_base( block, subheap_base( subheap ) );
832 block_set_flags( block, ~0, BLOCK_FLAG_FREE );
833 block_set_size( block, block_size );
834
835 /* If debugging, erase the freed block content */
836
837 if (end > commit_end) end = commit_end;
838 if (end > (char *)(entry + 1)) mark_block_free( entry + 1, end - (char *)(entry + 1), flags );
839 }
840
841 static void insert_free_block( struct heap *heap, ULONG flags, SUBHEAP *subheap, struct block *block )
842 {
843 struct entry *entry = (struct entry *)block, *list;
844 struct block *next;
845
846 if ((next = next_block( subheap, block )))
847 {
848 /* set the next block PREV_FREE flag and back pointer */
849 block_set_flags( next, 0, BLOCK_FLAG_PREV_FREE );
850 valgrind_make_writable( (struct block **)next - 1, sizeof(struct block *) );
851 *((struct block **)next - 1) = block;
852 }
853
854 list = find_free_list( heap, block_get_size( block ), !next );
855 if (!next) list_add_before( &list->entry, &entry->entry );
856 else list_add_after( &list->entry, &entry->entry );
857 }
858
859
860 static struct block *heap_delay_free( struct heap *heap, ULONG flags, struct block *block )
861 {
862 struct block *tmp;
863
864 if (!heap->pending_free) return block;
865
866 block_set_type( block, BLOCK_TYPE_DEAD );
867 mark_block_free( block + 1, block_get_size( block ) - sizeof(*block), flags );
868
869 heap_lock( heap, flags );
870
871 tmp = heap->pending_free[heap->pending_pos];
872 heap->pending_free[heap->pending_pos] = block;
873 heap->pending_pos = (heap->pending_pos + 1) % MAX_FREE_PENDING;
874
875 heap_unlock( heap, flags );
876
877 return tmp;
878 }
879
880
881 static NTSTATUS heap_free_block( struct heap *heap, ULONG flags, struct block *block )
882 {
883 SUBHEAP *subheap = block_get_subheap( heap, block );
884 SIZE_T block_size = block_get_size( block );
885 struct entry *entry;
886 struct block *next;
887
888 if ((next = next_block( subheap, block )) && (block_get_flags( next ) & BLOCK_FLAG_FREE))
889 {
890 /* merge with next block if it is free */
891 entry = (struct entry *)next;
892 block_size += block_get_size( &entry->block );
893 list_remove( &entry->entry );
894 next = next_block( subheap, next );
895 }
896
897 if (block_get_flags( block ) & BLOCK_FLAG_PREV_FREE)
898 {
899 /* merge with previous block if it is free */
900 entry = *((struct entry **)block - 1);
901 block_size += block_get_size( &entry->block );
902 list_remove( &entry->entry );
903 block = &entry->block;
904 }
905
906 if (block == first_block( subheap ) && !next && subheap != &heap->subheap)
907 {
908 /* free the subheap if it's empty and not the main one */
909 void *addr = subheap_base( subheap );
910 SIZE_T size = 0;
911
912 list_remove( &subheap->entry );
913 NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE );
914 return STATUS_SUCCESS;
915 }
916
917 block_init_free( block, flags, subheap, block_size );
918 insert_free_block( heap, flags, subheap, block );
919
920 /* keep room for a full committed block as hysteresis */
921 if (!next) subheap_decommit( heap, subheap, (char *)((struct entry *)block + 1) + REGION_ALIGN );
922
923 return STATUS_SUCCESS;
924 }
925
926
927 static struct block *split_block( struct heap *heap, ULONG flags, struct block *block,
928 SIZE_T old_block_size, SIZE_T block_size )
929 {
930 SUBHEAP *subheap = block_get_subheap( heap, block );
931
932 if (old_block_size >= block_size + HEAP_MIN_BLOCK_SIZE)
933 {
934 block_set_size( block, block_size );
935 return next_block( subheap, block );
936 }
937
938 block_set_size( block, old_block_size );
939 return NULL;
940 }
941
942
943 static void *allocate_region( struct heap *heap, ULONG flags, SIZE_T *region_size, SIZE_T *commit_size )
944 {
945 void *addr = NULL;
946 NTSTATUS status;
947
948 if (heap && !(flags & HEAP_GROWABLE))
949 {
950 WARN( "Heap %p isn't growable, cannot allocate %#Ix bytes\n", heap, *region_size );
951 return NULL;
952 }
953
954 /* allocate the memory block */
955 if ((status = NtAllocateVirtualMemory( NtCurrentProcess(), &addr, 0, region_size, MEM_RESERVE,
956 get_protection_type( flags ) )))
957 {
958 WARN( "Could not allocate %#Ix bytes, status %#lx\n", *region_size, status );
959 return NULL;
960 }
961 if ((status = NtAllocateVirtualMemory( NtCurrentProcess(), &addr, 0, commit_size, MEM_COMMIT,
962 get_protection_type( flags ) )))
963 {
964 WARN( "Could not commit %#Ix bytes, status %#lx\n", *commit_size, status );
965 return NULL;
966 }
967
968 return addr;
969 }
970
971
972 static NTSTATUS heap_allocate_large( struct heap *heap, ULONG flags, SIZE_T block_size,
973 SIZE_T size, void **ret )
974 {
975 ARENA_LARGE *arena;
976 SIZE_T total_size = ROUND_SIZE( sizeof(*arena) + size, REGION_ALIGN - 1 );
977 struct block *block;
978
979 if (total_size < size) return STATUS_NO_MEMORY; /* overflow */
980 if (!(arena = allocate_region( heap, flags, &total_size, &total_size ))) return STATUS_NO_MEMORY;
981
982 block = &arena->block;
983 arena->data_size = size;
984 arena->block_size = (char *)arena + total_size - (char *)block;
985
986 block_set_type( block, BLOCK_TYPE_LARGE );
987 block_set_base( block, arena );
988 block_set_flags( block, ~0, BLOCK_FLAG_LARGE | BLOCK_USER_FLAGS( flags ) );
989 block_set_size( block, 0 );
990
991 heap_lock( heap, flags );
992 list_add_tail( &heap->large_list, &arena->entry );
993 heap_unlock( heap, flags );
994
995 valgrind_make_noaccess( (char *)block + sizeof(*block) + arena->data_size,
996 arena->block_size - sizeof(*block) - arena->data_size );
997
998 *ret = block + 1;
999 return STATUS_SUCCESS;
1000 }
1001
1002
1003 static NTSTATUS heap_free_large( struct heap *heap, ULONG flags, struct block *block )
1004 {
1005 ARENA_LARGE *arena = CONTAINING_RECORD( block, ARENA_LARGE, block );
1006 LPVOID address = arena;
1007 SIZE_T size = 0;
1008
1009 heap_lock( heap, flags );
1010 list_remove( &arena->entry );
1011 heap_unlock( heap, flags );
1012
1013 return NtFreeVirtualMemory( NtCurrentProcess(), &address, &size, MEM_RELEASE );
1014 }
1015
1016
1017 /***********************************************************************
1018 * find_large_block
1019 */
1020 static BOOL find_large_block( const struct heap *heap, const struct block *block )
1021 {
1022 ARENA_LARGE *arena;
1023
1024 LIST_FOR_EACH_ENTRY( arena, &heap->large_list, ARENA_LARGE, entry )
1025 if (block == &arena->block) return TRUE;
1026
1027 return FALSE;
1028 }
1029
1030 static BOOL validate_large_block( const struct heap *heap, const struct block *block )
1031 {
1032 const ARENA_LARGE *arena = CONTAINING_RECORD( block, ARENA_LARGE, block );
1033 const char *err = NULL;
1034
1035 if (ROUND_ADDR( block, REGION_ALIGN - 1 ) != arena)
1036 err = "invalid block BLOCK_ALIGN";
1037 else if (block_get_size( block ))
1038 err = "invalid block size";
1039 else if (!(block_get_flags( block ) & BLOCK_FLAG_LARGE))
1040 err = "invalid block flags";
1041 else if (block_get_type( block ) != BLOCK_TYPE_LARGE)
1042 err = "invalid block type";
1043 else if (!contains( block, arena->block_size, block + 1, arena->data_size ))
1044 err = "invalid block size";
1045
1046 if (err)
1047 {
1048 ERR( "heap %p, block %p: %s\n", heap, block, err );
1049 if (TRACE_ON(heap)) heap_dump( heap );
1050 }
1051
1052 return !err;
1053 }
1054
1055
1056 static SUBHEAP *create_subheap( struct heap *heap, DWORD flags, SIZE_T total_size, SIZE_T commit_size )
1057 {
1058 SIZE_T block_size;
1059 SUBHEAP *subheap;
1060
1061 commit_size = ROUND_SIZE( max( commit_size, REGION_ALIGN ), REGION_ALIGN - 1 );
1062 total_size = min( max( commit_size, total_size ), 0xffff0000 ); /* don't allow a heap larger than 4GB */
1063
1064 if (!(subheap = allocate_region( heap, flags, &total_size, &commit_size ))) return NULL;
1065
1066 subheap->user_value = heap;
1067 subheap_set_bounds( subheap, (char *)subheap + commit_size, (char *)subheap + total_size );
1068 block_size = (SIZE_T)ROUND_ADDR( subheap_size( subheap ) - subheap_overhead( subheap ), BLOCK_ALIGN - 1 );
1069 block_init_free( first_block( subheap ), flags, subheap, block_size );
1070
1071 list_add_head( &heap->subheap_list, &subheap->entry );
1072
1073 return subheap;
1074 }
1075
1076
1077 static struct block *find_free_block( struct heap *heap, ULONG flags, SIZE_T block_size )
1078 {
1079 struct list *ptr = &find_free_list( heap, block_size, FALSE )->entry;
1080 struct entry *entry;
1081 struct block *block;
1082 SIZE_T total_size;
1083 SUBHEAP *subheap;
1084
1085 /* Find a suitable free list, and in it find a block large enough */
1086
1087 while ((ptr = list_next( &heap->free_lists[0].entry, ptr )))
1088 {
1089 entry = LIST_ENTRY( ptr, struct entry, entry );
1090 block = &entry->block;
1091 if (block_get_flags( block ) == BLOCK_FLAG_FREE_LINK) continue;
1092 if (block_get_size( block ) >= block_size)
1093 {
1094 if (!subheap_commit( heap, block_get_subheap( heap, block ), block, block_size )) return NULL;
1095 list_remove( &entry->entry );
1096 return block;
1097 }
1098 }
1099
1100 /* make sure we can fit the block and a free entry at the end */
1101 total_size = sizeof(SUBHEAP) + block_size + sizeof(struct entry);
1102 if (total_size < block_size) return NULL; /* overflow */
1103
1104 if ((subheap = create_subheap( heap, flags, max( heap->grow_size, total_size ), total_size )))
1105 {
1106 if (heap->grow_size <= HEAP_MAX_FREE_BLOCK_SIZE / 2) heap->grow_size *= 2;
1107 }
1108 else while (!subheap) /* shrink the grow size again if we are running out of space */
1109 {
1110 if (heap->grow_size <= total_size || heap->grow_size <= 4 * 1024 * 1024) return NULL;
1111 heap->grow_size /= 2;
1112 subheap = create_subheap( heap, flags, max( heap->grow_size, total_size ), total_size );
1113 }
1114
1115 TRACE( "created new sub-heap %p of %#Ix bytes for heap %p\n", subheap, subheap_size( subheap ), heap );
1116
1117 return first_block( subheap );
1118 }
1119
1120
1121 static BOOL is_valid_free_block( const struct heap *heap, const struct block *block )
1122 {
1123 const SUBHEAP *subheap;
1124 unsigned int i;
1125
1126 if ((subheap = find_subheap( heap, block, FALSE ))) return TRUE;
1127 for (i = 0; i < HEAP_NB_FREE_LISTS; i++) if (block == &heap->free_lists[i].block) return TRUE;
1128 return FALSE;
1129 }
1130
1131 static BOOL validate_free_block( const struct heap *heap, const SUBHEAP *subheap, const struct block *block )
1132 {
1133 const char *err = NULL, *base = subheap_base( subheap ), *commit_end = subheap_commit_end( subheap );
1134 const struct entry *entry = (struct entry *)block;
1135 const struct block *prev, *next;
1136 DWORD flags = heap->flags;
1137
1138 if ((ULONG_PTR)(block + 1) % BLOCK_ALIGN)
1139 err = "invalid block BLOCK_ALIGN";
1140 else if (block_get_type( block ) != BLOCK_TYPE_FREE)
1141 err = "invalid block header";
1142 else if (!(block_get_flags( block ) & BLOCK_FLAG_FREE) || (block_get_flags( block ) & BLOCK_FLAG_PREV_FREE))
1143 err = "invalid block flags";
1144 else if (!contains( base, subheap_size( subheap ), block, block_get_size( block ) ))
1145 err = "invalid block size";
1146 else if (!is_valid_free_block( heap, (next = &LIST_ENTRY( entry->entry.next, struct entry, entry )->block) ))
1147 err = "invalid next free block pointer";
1148 else if (!(block_get_flags( next ) & BLOCK_FLAG_FREE) || block_get_type( next ) != BLOCK_TYPE_FREE)
1149 err = "invalid next free block header";
1150 else if (!is_valid_free_block( heap, (prev = &LIST_ENTRY( entry->entry.prev, struct entry, entry )->block) ))
1151 err = "invalid previous free block pointer";
1152 else if (!(block_get_flags( prev ) & BLOCK_FLAG_FREE) || block_get_type( prev ) != BLOCK_TYPE_FREE)
1153 err = "invalid previous free block header";
1154 else if ((next = next_block( subheap, block )))
1155 {
1156 if (!(block_get_flags( next ) & BLOCK_FLAG_PREV_FREE))
1157 err = "invalid next block flags";
1158 if (*((struct block **)next - 1) != block)
1159 err = "invalid next block back pointer";
1160 }
1161
1162 if (!err && (flags & HEAP_FREE_CHECKING_ENABLED))
1163 {
1164 const char *ptr = (char *)(entry + 1), *end = (char *)block + block_get_size( block );
1165 if (next) end -= sizeof(struct block *);
1166 if (end > commit_end) end = commit_end;
1167 while (!err && ptr < end)
1168 {
1169 if (*(DWORD *)ptr != BLOCK_FILL_FREE) err = "free block overwritten";
1170 ptr += sizeof(DWORD);
1171 }
1172 }
1173
1174 if (err)
1175 {
1176 ERR( "heap %p, block %p: %s\n", heap, block, err );
1177 if (TRACE_ON(heap)) heap_dump( heap );
1178 }
1179
1180 return !err;
1181 }
1182
1183
1184 static BOOL validate_used_block( const struct heap *heap, const SUBHEAP *subheap, const struct block *block,
1185 unsigned int expected_block_type )
1186 {
1187 const char *err = NULL, *base = subheap_base( subheap ), *commit_end = subheap_commit_end( subheap );
1188 DWORD flags = heap->flags;
1189 const struct block *next;
1190 int i;
1191
1192 if ((ULONG_PTR)(block + 1) % BLOCK_ALIGN)
1193 err = "invalid block BLOCK_ALIGN";
1194 else if (block_get_type( block ) != BLOCK_TYPE_USED && block_get_type( block ) != BLOCK_TYPE_DEAD)
1195 err = "invalid block header";
1196 else if (expected_block_type && block_get_type( block ) != expected_block_type)
1197 err = "invalid block type";
1198 else if (block_get_flags( block ) & BLOCK_FLAG_FREE)
1199 err = "invalid block flags";
1200 else if (!contains( base, commit_end - base, block, block_get_size( block ) ))
1201 err = "invalid block size";
1202 else if (block->tail_size > block_get_size( block ) - sizeof(*block))
1203 err = "invalid block unused size";
1204 else if ((next = next_block( subheap, block )) && (block_get_flags( next ) & BLOCK_FLAG_PREV_FREE) &&
1205 /* LFH blocks do not use BLOCK_FLAG_PREV_FREE or back pointer */
1206 !(block_get_flags( block ) & BLOCK_FLAG_LFH))
1207 err = "invalid next block flags";
1208 else if (block_get_flags( block ) & BLOCK_FLAG_PREV_FREE)
1209 {
1210 const struct block *prev = *((struct block **)block - 1);
1211 if (!is_valid_free_block( heap, prev ))
1212 err = "invalid previous block pointer";
1213 else if (!(block_get_flags( prev ) & BLOCK_FLAG_FREE) || block_get_type( prev ) != BLOCK_TYPE_FREE)
1214 err = "invalid previous block flags";
1215 if ((char *)prev + block_get_size( prev ) != (char *)block)
1216 err = "invalid previous block size";
1217 }
1218
1219 if (!err && block_get_type( block ) == BLOCK_TYPE_DEAD)
1220 {
1221 const char *ptr = (char *)(block + 1), *end = (char *)block + block_get_size( block );
1222 while (!err && ptr < end)
1223 {
1224 if (*(DWORD *)ptr != BLOCK_FILL_FREE) err = "free block overwritten";
1225 ptr += sizeof(DWORD);
1226 }
1227 }
1228 else if (!err && (flags & HEAP_TAIL_CHECKING_ENABLED))
1229 {
1230 const unsigned char *tail = (unsigned char *)block + block_get_size( block ) - block->tail_size;
1231 for (i = 0; !err && i < BLOCK_ALIGN; i++) if (tail[i] != BLOCK_FILL_TAIL) err = "invalid block tail";
1232 }
1233
1234 if (err)
1235 {
1236 ERR( "heap %p, block %p: %s\n", heap, block, err );
1237 if (TRACE_ON(heap)) heap_dump( heap );
1238 }
1239
1240 return !err;
1241 }
1242
1243
1244 static BOOL heap_validate_ptr( const struct heap *heap, const void *ptr )
1245 {
1246 const struct block *block = (struct block *)ptr - 1;
1247 const SUBHEAP *subheap;
1248
1249 if (!(subheap = find_subheap( heap, block, FALSE )))
1250 {
1251 if (!find_large_block( heap, block ))
1252 {
1253 WARN("heap %p, ptr %p: block region not found\n", heap, ptr );
1254 return FALSE;
1255 }
1256
1257 return validate_large_block( heap, block );
1258 }
1259
1260 return validate_used_block( heap, subheap, block, BLOCK_TYPE_USED );
1261 }
1262
1263 static BOOL heap_validate( const struct heap *heap )
1264 {
1265 const ARENA_LARGE *large_arena;
1266 const struct block *block;
1267 const SUBHEAP *subheap;
1268
1269 LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry )
1270 {
1271 if (!check_subheap( subheap, heap ))
1272 {
1273 ERR( "heap %p, subheap %p corrupted sizes or user_value\n", heap, subheap );
1274 if (TRACE_ON(heap)) heap_dump( heap );
1275 return FALSE;
1276 }
1277
1278 for (block = first_block( subheap ); block; block = next_block( subheap, block ))
1279 {
1280 if (block_get_flags( block ) & BLOCK_FLAG_FREE)
1281 {
1282 if (!validate_free_block( heap, subheap, block )) return FALSE;
1283 }
1284 else
1285 {
1286 if (!validate_used_block( heap, subheap, block, 0 )) return FALSE;
1287 }
1288 }
1289 }
1290
1291 if (heap->pending_free)
1292 {
1293 unsigned int i;
1294
1295 for (i = 0; i < MAX_FREE_PENDING; i++)
1296 {
1297 if (!(block = heap->pending_free[i])) break;
1298
1299 subheap = find_subheap( heap, block, FALSE );
1300 if (!subheap)
1301 {
1302 ERR( "heap %p: cannot find valid subheap for delayed freed block %p\n", heap, block );
1303 if (TRACE_ON(heap)) heap_dump( heap );
1304 return FALSE;
1305 }
1306
1307 if (!validate_used_block( heap, subheap, block, BLOCK_TYPE_DEAD )) return FALSE;
1308 }
1309
1310 for (; i < MAX_FREE_PENDING; i++)
1311 {
1312 if ((block = heap->pending_free[i]))
1313 {
1314 ERR( "heap %p: unexpected delayed freed block %p at slot %u\n", heap, block, i );
1315 if (TRACE_ON(heap)) heap_dump( heap );
1316 return FALSE;
1317 }
1318 }
1319 }
1320
1321 LIST_FOR_EACH_ENTRY( large_arena, &heap->large_list, ARENA_LARGE, entry )
1322 if (!validate_large_block( heap, &large_arena->block )) return FALSE;
1323
1324 return TRUE;
1325 }
1326
1327 static inline struct block *unsafe_block_from_ptr( struct heap *heap, ULONG flags, const void *ptr )
1328 {
1329 struct block *block = (struct block *)ptr - 1;
1330 const char *err = NULL;
1331 SUBHEAP *subheap;
1332
1333 if (flags & HEAP_VALIDATE)
1334 {
1335 heap_lock( heap, flags );
1336 if (!heap_validate_ptr( heap, ptr )) block = NULL;
1337 heap_unlock( heap, flags );
1338 return block;
1339 }
1340
1341 if ((ULONG_PTR)ptr % BLOCK_ALIGN)
1342 err = "invalid ptr alignment";
1343 else if (block_get_type( block ) == BLOCK_TYPE_DEAD)
1344 err = "delayed freed block";
1345 else if (block_get_type( block ) == BLOCK_TYPE_FREE)
1346 err = "already freed block";
1347 else if (block_get_flags( block ) & BLOCK_FLAG_LFH)
1348 {
1349 /* LFH block base_offset points to the group, not the subheap */
1350 if (block_get_type( block ) != BLOCK_TYPE_USED)
1351 err = "invalid block type";
1352 }
1353 else if ((subheap = block_get_subheap( heap, block )) >= (SUBHEAP *)block)
1354 err = "invalid base offset";
1355 else if (block_get_type( block ) == BLOCK_TYPE_USED)
1356 {
1357 const char *base = subheap_base( subheap ), *commit_end = subheap_commit_end( subheap );
1358 if (subheap->user_value != heap) err = "mismatching heap";
1359 else if (!contains( base, commit_end - base, block, block_get_size( block ) )) err = "invalid block size";
1360 }
1361 else if (block_get_type( block ) == BLOCK_TYPE_LARGE)
1362 {
1363 ARENA_LARGE *large = subheap_base( subheap );
1364 if (block != &large->block) err = "invalid large block";
1365 }
1366 else
1367 {
1368 err = "invalid block type";
1369 }
1370
1371 if (err) WARN( "heap %p, block %p: %s\n", heap, block, err );
1372 return err ? NULL : block;
1373 }
1374
1375 static DWORD heap_flags_from_global_flag( DWORD flag )
1376 {
1377 DWORD ret = 0;
1378
1379 if (flag & FLG_HEAP_ENABLE_TAIL_CHECK)
1380 ret |= HEAP_TAIL_CHECKING_ENABLED;
1381 if (flag & FLG_HEAP_ENABLE_FREE_CHECK)
1382 ret |= HEAP_FREE_CHECKING_ENABLED;
1383 if (flag & FLG_HEAP_VALIDATE_PARAMETERS)
1384 ret |= HEAP_VALIDATE_PARAMS | HEAP_TAIL_CHECKING_ENABLED | HEAP_FREE_CHECKING_ENABLED;
1385 if (flag & FLG_HEAP_VALIDATE_ALL)
1386 ret |= HEAP_VALIDATE_ALL | HEAP_TAIL_CHECKING_ENABLED | HEAP_FREE_CHECKING_ENABLED;
1387 if (flag & FLG_HEAP_DISABLE_COALESCING)
1388 ret |= HEAP_DISABLE_COALESCE_ON_FREE;
1389 if (flag & FLG_HEAP_PAGE_ALLOCS)
1390 ret |= HEAP_PAGE_ALLOCS;
1391
1392 return ret;
1393 }
1394
1395 /***********************************************************************
1396 * heap_set_debug_flags
1397 */
1398 static void heap_set_debug_flags( HANDLE handle )
1399 {
1400 ULONG global_flags = RtlGetNtGlobalFlags();
1401 DWORD dummy, flags, force_flags;
1402 struct heap *heap;
1403
1404 if (TRACE_ON(heap)) global_flags |= FLG_HEAP_VALIDATE_ALL;
1405 if (WARN_ON(heap)) global_flags |= FLG_HEAP_VALIDATE_PARAMETERS;
1406
1407 heap = unsafe_heap_from_handle( handle, 0, &dummy );
1408 flags = heap_flags_from_global_flag( global_flags );
1409 force_flags = (heap->flags | flags) & ~(HEAP_SHARED|HEAP_DISABLE_COALESCE_ON_FREE);
1410
1411 if (global_flags & FLG_HEAP_ENABLE_TAGGING) flags |= HEAP_SHARED;
1412 if (!(global_flags & FLG_HEAP_PAGE_ALLOCS)) force_flags &= ~(HEAP_GROWABLE|HEAP_PRIVATE);
1413
1414 if (RUNNING_ON_VALGRIND) flags = 0; /* no sense in validating since Valgrind catches accesses */
1415
1416 heap->flags |= flags;
1417 heap->force_flags |= force_flags;
1418
1419 if (flags & (HEAP_FREE_CHECKING_ENABLED | HEAP_TAIL_CHECKING_ENABLED)) /* fix existing blocks */
1420 {
1421 struct block *block;
1422 SUBHEAP *subheap;
1423
1424 LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry )
1425 {
1426 const char *commit_end = subheap_commit_end( subheap );
1427
1428 if (!check_subheap( subheap, heap )) break;
1429
1430 for (block = first_block( subheap ); block; block = next_block( subheap, block ))
1431 {
1432 if (block_get_flags( block ) & BLOCK_FLAG_FREE)
1433 {
1434 char *data = (char *)block + block_get_overhead( block ), *end = (char *)block + block_get_size( block );
1435 if (next_block( subheap, block )) end -= sizeof(struct block *);
1436 if (end > commit_end) mark_block_free( data, commit_end - data, flags );
1437 else mark_block_free( data, end - data, flags );
1438 }
1439 else
1440 {
1441 if (block_get_type( block ) == BLOCK_TYPE_DEAD) mark_block_free( block + 1, block_get_size( block ) - sizeof(*block), flags );
1442 else mark_block_tail( block, flags );
1443 }
1444 }
1445 }
1446 }
1447
1448 if ((heap->flags & HEAP_GROWABLE) && !heap->pending_free &&
1449 ((flags & HEAP_FREE_CHECKING_ENABLED) || RUNNING_ON_VALGRIND))
1450 {
1451 heap->pending_free = RtlAllocateHeap( handle, HEAP_ZERO_MEMORY,
1452 MAX_FREE_PENDING * sizeof(*heap->pending_free) );
1453 heap->pending_pos = 0;
1454 }
1455 }
1456
1457
1458 /***********************************************************************
1459 * RtlCreateHeap (NTDLL.@)
1460 *
1461 * Create a new Heap.
1462 *
1463 * PARAMS
1464 * flags [I] HEAP_ flags from "winnt.h"
1465 * addr [I] Desired base address
1466 * totalSize [I] Total size of the heap, or 0 for a growable heap
1467 * commitSize [I] Amount of heap space to commit
1468 * unknown [I] Not yet understood
1469 * definition [I] Heap definition
1470 *
1471 * RETURNS
1472 * Success: A HANDLE to the newly created heap.
1473 * Failure: a NULL HANDLE.
1474 */
1475 HANDLE WINAPI RtlCreateHeap( ULONG flags, void *addr, SIZE_T total_size, SIZE_T commit_size,
1476 void *unknown, RTL_HEAP_DEFINITION *definition )
1477 {
1478 struct entry *entry;
1479 struct heap *heap;
1480 SIZE_T block_size;
1481 SUBHEAP *subheap;
1482 unsigned int i;
1483
1484 TRACE( "flags %#lx, addr %p, total_size %#Ix, commit_size %#Ix, unknown %p, definition %p\n",
1485 flags, addr, total_size, commit_size, unknown, definition );
1486
1487 flags &= ~(HEAP_TAIL_CHECKING_ENABLED|HEAP_FREE_CHECKING_ENABLED);
1488 if (process_heap) flags |= HEAP_PRIVATE;
1489 if (!process_heap || !total_size || (flags & HEAP_SHARED)) flags |= HEAP_GROWABLE;
1490 if (!total_size) total_size = HEAP_DEF_SIZE;
1491
1492 if (!(heap = addr))
1493 {
1494 if (!commit_size) commit_size = REGION_ALIGN;
1495 total_size = min( max( total_size, commit_size ), 0xffff0000 ); /* don't allow a heap larger than 4GB */
1496 commit_size = min( total_size, ROUND_SIZE( commit_size, REGION_ALIGN - 1 ) );
1497 if (!(heap = allocate_region( NULL, flags, &total_size, &commit_size ))) return 0;
1498 }
1499
1500 heap->ffeeffee = 0xffeeffee;
1501 heap->auto_flags = (flags & HEAP_GROWABLE);
1502 heap->flags = (flags & ~HEAP_SHARED);
1503 heap->compat_info = HEAP_STD;
1504 heap->magic = HEAP_MAGIC;
1505 heap->grow_size = max( HEAP_DEF_SIZE, total_size );
1506 heap->min_size = commit_size;
1507 list_init( &heap->subheap_list );
1508 list_init( &heap->large_list );
1509
1510 list_init( &heap->free_lists[0].entry );
1511 for (i = 0, entry = heap->free_lists; i < HEAP_NB_FREE_LISTS; i++, entry++)
1512 {
1513 block_set_flags( &entry->block, ~0, BLOCK_FLAG_FREE_LINK );
1514 block_set_size( &entry->block, 0 );
1515 block_set_type( &entry->block, BLOCK_TYPE_FREE );
1516 block_set_base( &entry->block, heap );
1517 if (i) list_add_after( &entry[-1].entry, &entry->entry );
1518 }
1519
1520 if (!process_heap) /* do it by hand to avoid memory allocations */
1521 {
1522 heap->cs.DebugInfo = &process_heap_cs_debug;
1523 heap->cs.LockCount = -1;
1524 heap->cs.RecursionCount = 0;
1525 heap->cs.OwningThread = 0;
1526 heap->cs.LockSemaphore = 0;
1527 heap->cs.SpinCount = 0;
1528 process_heap_cs_debug.CriticalSection = &heap->cs;
1529 }
1530 else
1531 {
1532 RtlInitializeCriticalSection( &heap->cs );
1533 heap->cs.DebugInfo->Spare[0] = (DWORD_PTR)(__FILE__ ": heap.cs");
1534 }
1535
1536 subheap = &heap->subheap;
1537 subheap->user_value = heap;
1538 subheap_set_bounds( subheap, (char *)heap + commit_size, (char *)heap + total_size );
1539 block_size = (SIZE_T)ROUND_ADDR( subheap_size( subheap ) - subheap_overhead( subheap ), BLOCK_ALIGN - 1 );
1540 block_init_free( first_block( subheap ), flags, subheap, block_size );
1541
1542 insert_free_block( heap, flags, subheap, first_block( subheap ) );
1543 list_add_head( &heap->subheap_list, &subheap->entry );
1544
1545 heap_set_debug_flags( heap );
1546
1547 if (heap->flags & HEAP_GROWABLE)
1548 {
1549 SIZE_T size = (sizeof(struct bin) + sizeof(struct group *) * ARRAY_SIZE(affinity_mapping)) * BLOCK_SIZE_BIN_COUNT;
1550 NtAllocateVirtualMemory( NtCurrentProcess(), (void *)&heap->bins,
1551 0, &size, MEM_COMMIT, PAGE_READWRITE );
1552
1553 for (i = 0; heap->bins && i < BLOCK_SIZE_BIN_COUNT; ++i)
1554 {
1555 RtlInitializeSListHead( &heap->bins[i].groups );
1556 /* offset affinity_group_base to interleave the bin affinity group pointers */
1557 heap->bins[i].affinity_group_base = (struct group **)(heap->bins + BLOCK_SIZE_BIN_COUNT) + i;
1558 }
1559 }
1560
1561 /* link it into the per-process heap list */
1562 if (process_heap)
1563 {
1564 RtlEnterCriticalSection( &process_heap->cs );
1565 list_add_head( &process_heap->entry, &heap->entry );
1566 RtlLeaveCriticalSection( &process_heap->cs );
1567 }
1568 else if (!addr)
1569 {
1570 process_heap = heap; /* assume the first heap we create is the process main heap */
1571 list_init( &process_heap->entry );
1572 }
1573
1574 return heap;
1575 }
1576
1577
1578 /***********************************************************************
1579 * RtlDestroyHeap (NTDLL.@)
1580 *
1581 * Destroy a Heap created with RtlCreateHeap().
1582 *
1583 * PARAMS
1584 * heap [I] Heap to destroy.
1585 *
1586 * RETURNS
1587 * Success: A NULL HANDLE, if heap is NULL or it was destroyed
1588 * Failure: The Heap handle, if heap is the process heap.
1589 */
1590 HANDLE WINAPI RtlDestroyHeap( HANDLE handle )
1591 {
1592 SUBHEAP *subheap, *next;
1593 ARENA_LARGE *arena, *arena_next;
1594 struct block **pending, **tmp;
1595 struct heap *heap;
1596 ULONG heap_flags;
1597 SIZE_T size;
1598 void *addr;
1599
1600 TRACE( "handle %p\n", handle );
1601
1602 if (!(heap = unsafe_heap_from_handle( handle, 0, &heap_flags )) && handle &&
1603 (((struct heap *)handle)->flags & HEAP_VALIDATE_PARAMS) &&
1604 NtCurrentTeb()->Peb->BeingDebugged)
1605 {
1606 DbgPrint( "Attempt to destroy an invalid heap\n" );
1607 DbgBreakPoint();
1608 }
1609 if (!heap) return handle;
1610
1611 if ((pending = heap->pending_free))
1612 {
1613 heap->pending_free = NULL;
1614 for (tmp = pending; *tmp && tmp != pending + MAX_FREE_PENDING; ++tmp)
1615 heap_free_block( heap, heap->flags, *tmp );
1616 RtlFreeHeap( handle, 0, pending );
1617 }
1618
1619 if (heap == process_heap) return handle; /* cannot delete the main process heap */
1620
1621 /* remove it from the per-process list */
1622 RtlEnterCriticalSection( &process_heap->cs );
1623 list_remove( &heap->entry );
1624 RtlLeaveCriticalSection( &process_heap->cs );
1625
1626 heap->cs.DebugInfo->Spare[0] = 0;
1627 RtlDeleteCriticalSection( &heap->cs );
1628
1629 LIST_FOR_EACH_ENTRY_SAFE( arena, arena_next, &heap->large_list, ARENA_LARGE, entry )
1630 {
1631 list_remove( &arena->entry );
1632 size = 0;
1633 addr = arena;
1634 NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE );
1635 }
1636 LIST_FOR_EACH_ENTRY_SAFE( subheap, next, &heap->subheap_list, SUBHEAP, entry )
1637 {
1638 if (subheap == &heap->subheap) continue; /* do this one last */
1639 valgrind_notify_free_all( subheap, heap );
1640 list_remove( &subheap->entry );
1641 size = 0;
1642 addr = ROUND_ADDR( subheap, REGION_ALIGN - 1 );
1643 NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE );
1644 }
1645 valgrind_notify_free_all( &heap->subheap, heap );
1646 if ((addr = heap->bins))
1647 {
1648 size = 0;
1649 NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE );
1650 }
1651 size = 0;
1652 addr = heap;
1653 NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE );
1654 return 0;
1655 }
1656
1657 static SIZE_T heap_get_block_size( const struct heap *heap, ULONG flags, SIZE_T size )
1658 {
1659 static const ULONG padd_flags = HEAP_VALIDATE | HEAP_VALIDATE_ALL | HEAP_VALIDATE_PARAMS | HEAP_ADD_USER_INFO;
1660 static const ULONG check_flags = HEAP_TAIL_CHECKING_ENABLED | HEAP_FREE_CHECKING_ENABLED | HEAP_CHECKING_ENABLED;
1661 SIZE_T overhead, block_size;
1662
1663 if ((flags & check_flags)) overhead = BLOCK_ALIGN;
1664 else overhead = sizeof(struct block);
1665
1666 if ((flags & HEAP_TAIL_CHECKING_ENABLED) || RUNNING_ON_VALGRIND) overhead += BLOCK_ALIGN;
1667 if (flags & padd_flags) overhead += BLOCK_ALIGN;
1668
1669 if (size < BLOCK_ALIGN) size = BLOCK_ALIGN;
1670 block_size = ROUND_SIZE( size + overhead, BLOCK_ALIGN - 1 );
1671
1672 if (block_size < size) return ~0U; /* overflow */
1673 if (block_size < HEAP_MIN_BLOCK_SIZE) block_size = HEAP_MIN_BLOCK_SIZE;
1674 return block_size;
1675 }
1676
1677 static NTSTATUS heap_allocate_block( struct heap *heap, ULONG flags, SIZE_T block_size, SIZE_T size, void **ret )
1678 {
1679 struct block *block, *next;
1680 SIZE_T old_block_size;
1681 SUBHEAP *subheap;
1682
1683 /* Locate a suitable free block */
1684
1685 if (!(block = find_free_block( heap, flags, block_size ))) return STATUS_NO_MEMORY;
1686 /* read the free block size, changing block type or flags may alter it */
1687 old_block_size = block_get_size( block );
1688 subheap = block_get_subheap( heap, block );
1689
1690 if ((next = split_block( heap, flags, block, old_block_size, block_size )))
1691 {
1692 block_init_free( next, flags, subheap, old_block_size - block_size );
1693 insert_free_block( heap, flags, subheap, next );
1694 }
1695
1696 block_set_type( block, BLOCK_TYPE_USED );
1697 block_set_flags( block, ~0, BLOCK_USER_FLAGS( flags ) );
1698 block->tail_size = block_get_size( block ) - sizeof(*block) - size;
1699 initialize_block( block, 0, size, flags );
1700 mark_block_tail( block, flags );
1701
1702 if ((next = next_block( subheap, block ))) block_set_flags( next, BLOCK_FLAG_PREV_FREE, 0 );
1703
1704 *ret = block + 1;
1705 return STATUS_SUCCESS;
1706 }
1707
1708 /* Low Fragmentation Heap frontend */
1709
1710 /* header for every LFH block group */
1711 struct DECLSPEC_ALIGN(BLOCK_ALIGN) group
1712 {
1713 SLIST_ENTRY entry;
1714 /* one bit for each free block and the highest bit for GROUP_FLAG_FREE */
1715 LONG free_bits;
1716 /* affinity of the thread which last allocated from this group */
1717 LONG affinity;
1718 /* first block of a group, required for alignment */
1719 struct block first_block;
1720 };
1721
1722 #define GROUP_BLOCK_COUNT (sizeof(((struct group *)0)->free_bits) * 8 - 1)
1723 #define GROUP_FLAG_FREE (1u << GROUP_BLOCK_COUNT)
1724
1725 static inline UINT block_get_group_index( const struct block *block )
1726 {
1727 return block->base_offset;
1728 }
1729
1730 static inline struct group *block_get_group( const struct block *block )
1731 {
1732 SIZE_T block_size = block_get_size( block );
1733 void *first_block = (char *)block - block_get_group_index( block ) * block_size;
1734 return CONTAINING_RECORD( first_block, struct group, first_block );
1735 }
1736
1737 static inline void block_set_group( struct block *block, SIZE_T block_size, const struct group *group )
1738 {
1739 SIZE_T offset = (char *)block - (char *)&group->first_block;
1740 block->base_offset = offset / block_size;
1741 }
1742
1743 static inline struct block *group_get_block( struct group *group, SIZE_T block_size, UINT index )
1744 {
1745 char *first_block = (char *)&group->first_block;
1746 return (struct block *)(first_block + index * block_size);
1747 }
1748
1749 /* lookup a free block using the group free_bits, the current thread must own the group */
1750 static inline struct block *group_find_free_block( struct group *group, SIZE_T block_size )
1751 {
1752 ULONG i, free_bits = ReadNoFence( &group->free_bits );
1753 /* free_bits will never be 0 as the group is unlinked when it's fully used */
1754 BitScanForward( &i, free_bits );
1755 InterlockedAnd( &group->free_bits, ~(1 << i) );
1756 return group_get_block( group, block_size, i );
1757 }
1758
1759 /* allocate a new group block using non-LFH allocation, returns a group owned by current thread */
1760 static struct group *group_allocate( struct heap *heap, ULONG flags, SIZE_T block_size )
1761 {
1762 SIZE_T i, group_size, group_block_size;
1763 struct group *group;
1764 NTSTATUS status;
1765
1766 group_size = offsetof( struct group, first_block ) + GROUP_BLOCK_COUNT * block_size;
1767 group_block_size = heap_get_block_size( heap, flags, group_size );
1768
1769 heap_lock( heap, flags );
1770
1771 if (group_block_size >= HEAP_MIN_LARGE_BLOCK_SIZE)
1772 status = heap_allocate_large( heap, flags & ~HEAP_ZERO_MEMORY, group_block_size, group_size, (void **)&group );
1773 else
1774 status = heap_allocate_block( heap, flags & ~HEAP_ZERO_MEMORY, group_block_size, group_size, (void **)&group );
1775
1776 heap_unlock( heap, flags );
1777
1778 if (status) return NULL;
1779
1780 block_set_flags( (struct block *)group - 1, 0, BLOCK_FLAG_LFH );
1781 group->free_bits = ~GROUP_FLAG_FREE;
1782
1783 for (i = 0; i < GROUP_BLOCK_COUNT; ++i)
1784 {
1785 struct block *block = group_get_block( group, block_size, i );
1786 valgrind_make_writable( block, sizeof(*block) );
1787 block_set_type( block, BLOCK_TYPE_FREE );
1788 block_set_flags( block, ~0, BLOCK_FLAG_FREE | BLOCK_FLAG_LFH );
1789 block_set_group( block, block_size, group );
1790 block_set_size( block, block_size );
1791 mark_block_free( block + 1, (char *)block + block_size - (char *)(block + 1), flags );
1792 }
1793
1794 return group;
1795 }
1796
1797 /* release a fully freed group to the non-LFH subheap, group must be owned by current thread */
1798 static NTSTATUS group_release( struct heap *heap, ULONG flags, struct bin *bin, struct group *group )
1799 {
1800 struct block *block = (struct block *)group - 1;
1801 NTSTATUS status;
1802
1803 heap_lock( heap, flags );
1804
1805 block_set_flags( block, BLOCK_FLAG_LFH, 0 );
1806
1807 if (block_get_flags( block ) & BLOCK_FLAG_LARGE)
1808 status = heap_free_large( heap, flags, block );
1809 else
1810 status = heap_free_block( heap, flags, block );
1811
1812 heap_unlock( heap, flags );
1813
1814 return status;
1815 }
1816
1817 static inline ULONG heap_current_thread_affinity(void)
1818 {
1819 ULONG affinity;
1820
1821 if (!(affinity = NtCurrentTeb()->HeapVirtualAffinity))
1822 {
1823 affinity = InterlockedIncrement( &next_thread_affinity );
1824 affinity = affinity_mapping[affinity % ARRAY_SIZE(affinity_mapping)];
1825 NtCurrentTeb()->HeapVirtualAffinity = affinity;
1826 }
1827
1828 return affinity;
1829 }
1830
1831 /* acquire a group from the bin, thread takes ownership of a shared group or allocates a new one */
1832 static struct group *heap_acquire_bin_group( struct heap *heap, ULONG flags, SIZE_T block_size, struct bin *bin )
1833 {
1834 ULONG affinity = NtCurrentTeb()->HeapVirtualAffinity;
1835 struct group *group;
1836 SLIST_ENTRY *entry;
1837
1838 if ((group = InterlockedExchangePointer( (void *)bin_get_affinity_group( bin, affinity ), NULL )))
1839 return group;
1840
1841 if ((entry = RtlInterlockedPopEntrySList( &bin->groups )))
1842 return CONTAINING_RECORD( entry, struct group, entry );
1843
1844 return group_allocate( heap, flags, block_size );
1845 }
1846
1847 /* release a thread owned and fully freed group to the bin shared group, or free its memory */
1848 static NTSTATUS heap_release_bin_group( struct heap *heap, ULONG flags, struct bin *bin, struct group *group )
1849 {
1850 ULONG affinity = group->affinity;
1851
1852 /* using InterlockedExchangePointer here would possibly return a group that has used blocks,
1853 * we prefer keeping our fully freed group instead for reduced memory consumption.
1854 */
1855 if (!InterlockedCompareExchangePointer( (void *)bin_get_affinity_group( bin, affinity ), group, NULL ))
1856 return STATUS_SUCCESS;
1857
1858 /* try re-using the block group instead of releasing it */
1859 if (RtlQueryDepthSList( &bin->groups ) <= ARRAY_SIZE(affinity_mapping))
1860 {
1861 RtlInterlockedPushEntrySList( &bin->groups, &group->entry );
1862 return STATUS_SUCCESS;
1863 }
1864
1865 return group_release( heap, flags, bin, group );
1866 }
1867
1868 static struct block *find_free_bin_block( struct heap *heap, ULONG flags, SIZE_T block_size, struct bin *bin )
1869 {
1870 ULONG affinity = heap_current_thread_affinity();
1871 struct block *block;
1872 struct group *group;
1873
1874 /* acquire a group, the thread will own it and no other thread can clear free bits.
1875 * some other thread might still set the free bits if they are freeing blocks.
1876 */
1877 if (!(group = heap_acquire_bin_group( heap, flags, block_size, bin ))) return NULL;
1878 group->affinity = affinity;
1879
1880 block = group_find_free_block( group, block_size );
1881
1882 /* serialize with heap_free_block_lfh: atomically set GROUP_FLAG_FREE when the free bits are all 0. */
1883 if (ReadNoFence( &group->free_bits ) || InterlockedCompareExchange( &group->free_bits, GROUP_FLAG_FREE, 0 ))
1884 {
1885 /* if GROUP_FLAG_FREE isn't set, thread is responsible for putting it back into group list. */
1886 if ((group = InterlockedExchangePointer( (void *)bin_get_affinity_group( bin, affinity ), group )))
1887 RtlInterlockedPushEntrySList( &bin->groups, &group->entry );
1888 }
1889
1890 return block;
1891 }
1892
1893 static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T block_size,
1894 SIZE_T size, void **ret )
1895 {
1896 struct bin *bin, *last = heap->bins + BLOCK_SIZE_BIN_COUNT - 1;
1897 struct block *block;
1898
1899 bin = heap->bins + BLOCK_SIZE_BIN( block_size );
1900 if (bin == last) return STATUS_UNSUCCESSFUL;
1901
1902 /* paired with WriteRelease in bin_try_enable. */
1903 if (!ReadAcquire( &bin->enabled )) return STATUS_UNSUCCESSFUL;
1904
1905 block_size = BLOCK_BIN_SIZE( BLOCK_SIZE_BIN( block_size ) );
1906
1907 if ((block = find_free_bin_block( heap, flags, block_size, bin )))
1908 {
1909 block_set_type( block, BLOCK_TYPE_USED );
1910 block_set_flags( block, ~BLOCK_FLAG_LFH, BLOCK_USER_FLAGS( flags ) );
1911 block->tail_size = block_size - sizeof(*block) - size;
1912 initialize_block( block, 0, size, flags );
1913 mark_block_tail( block, flags );
1914 *ret = block + 1;
1915 }
1916
1917 return block ? STATUS_SUCCESS : STATUS_NO_MEMORY;
1918 }
1919
1920 static NTSTATUS heap_free_block_lfh( struct heap *heap, ULONG flags, struct block *block )
1921 {
1922 struct bin *bin, *last = heap->bins + BLOCK_SIZE_BIN_COUNT - 1;
1923 SIZE_T i, block_size = block_get_size( block );
1924 struct group *group = block_get_group( block );
1925 NTSTATUS status = STATUS_SUCCESS;
1926
1927 if (!(block_get_flags( block ) & BLOCK_FLAG_LFH)) return STATUS_UNSUCCESSFUL;
1928
1929 bin = heap->bins + BLOCK_SIZE_BIN( block_size );
1930 if (bin == last) return STATUS_UNSUCCESSFUL;
1931
1932 i = block_get_group_index( block );
1933 valgrind_make_writable( block, sizeof(*block) );
1934 block_set_type( block, BLOCK_TYPE_FREE );
1935 block_set_flags( block, ~BLOCK_FLAG_LFH, BLOCK_FLAG_FREE );
1936 mark_block_free( block + 1, (char *)block + block_size - (char *)(block + 1), flags );
1937
1938 /* if this was the last used block in a group and GROUP_FLAG_FREE was set */
1939 if (InterlockedOr( &group->free_bits, 1 << i ) == ~(1 << i))
1940 {
1941 /* thread now owns the group, and can release it to its bin */
1942 group->free_bits = ~GROUP_FLAG_FREE;
1943 status = heap_release_bin_group( heap, flags, bin, group );
1944 }
1945
1946 return status;
1947 }
1948
1949 static void bin_try_enable( struct heap *heap, struct bin *bin )
1950 {
1951 ULONG alloc = ReadNoFence( &bin->count_alloc ), freed = ReadNoFence( &bin->count_freed );
1952 SIZE_T block_size = BLOCK_BIN_SIZE( bin - heap->bins );
1953 BOOL enable = FALSE;
1954
1955 if (bin == heap->bins && alloc > 0x10) enable = TRUE;
1956 else if (bin - heap->bins < 0x30 && alloc > 0x800) enable = TRUE;
1957 else if (bin - heap->bins < 0x30 && alloc - freed > 0x10) enable = TRUE;
1958 else if (alloc - freed > 0x400000 / block_size) enable = TRUE;
1959 if (!enable) return;
1960
1961 if (ReadNoFence( &heap->compat_info ) != HEAP_LFH)
1962 {
1963 ULONG info = HEAP_LFH;
1964 RtlSetHeapInformation( heap, HeapCompatibilityInformation, &info, sizeof(info) );
1965 }
1966
1967 /* paired with ReadAcquire in heap_allocate_block_lfh.
1968 *
1969 * The acq/rel barrier on the enabled flag is protecting compat_info
1970 * (i.e. compat_info := LFH happens-before enabled := TRUE), so that
1971 * a caller that observes LFH block allocation (alloc request
1972 * succeeds without heap lock) will never observe HEAP_STD when it
1973 * queries the heap.
1974 */
1975 WriteRelease( &bin->enabled, TRUE );
1976 }
1977
1978 static void heap_thread_detach_bin_groups( struct heap *heap )
1979 {
1980 ULONG i, affinity = NtCurrentTeb()->HeapVirtualAffinity;
1981
1982 if (!heap->bins) return;
1983
1984 for (i = 0; i < BLOCK_SIZE_BIN_COUNT; ++i)
1985 {
1986 struct bin *bin = heap->bins + i;
1987 struct group *group;
1988 if (!(group = InterlockedExchangePointer( (void *)bin_get_affinity_group( bin, affinity ), NULL ))) continue;
1989 RtlInterlockedPushEntrySList( &bin->groups, &group->entry );
1990 }
1991 }
1992
1993 void heap_thread_detach(void)
1994 {
1995 struct heap *heap;
1996
1997 RtlEnterCriticalSection( &process_heap->cs );
1998
1999 LIST_FOR_EACH_ENTRY( heap, &process_heap->entry, struct heap, entry )
2000 heap_thread_detach_bin_groups( heap );
2001
2002 heap_thread_detach_bin_groups( process_heap );
2003
2004 RtlLeaveCriticalSection( &process_heap->cs );
2005 }
2006
2007 /***********************************************************************
2008 * RtlAllocateHeap (NTDLL.@)
2009 */
2010 void *WINAPI DECLSPEC_HOTPATCH RtlAllocateHeap( HANDLE handle, ULONG flags, SIZE_T size )
2011 {
2012 struct heap *heap;
2013 SIZE_T block_size;
2014 void *ptr = NULL;
2015 ULONG heap_flags;
2016 NTSTATUS status;
2017
2018 if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
2019 status = STATUS_INVALID_HANDLE;
2020 else if ((block_size = heap_get_block_size( heap, heap_flags, size )) == ~0U)
2021 status = STATUS_NO_MEMORY;
2022 else if (block_size >= HEAP_MIN_LARGE_BLOCK_SIZE)
2023 status = heap_allocate_large( heap, heap_flags, block_size, size, &ptr );
2024 else if (heap->bins && !heap_allocate_block_lfh( heap, heap_flags, block_size, size, &ptr ))
2025 status = STATUS_SUCCESS;
2026 else
2027 {
2028 heap_lock( heap, heap_flags );
2029 status = heap_allocate_block( heap, heap_flags, block_size, size, &ptr );
2030 heap_unlock( heap, heap_flags );
2031
2032 if (!status && heap->bins)
2033 {
2034 SIZE_T bin = BLOCK_SIZE_BIN( block_get_size( (struct block *)ptr - 1 ) );
2035 InterlockedIncrement( &heap->bins[bin].count_alloc );
2036 if (!ReadNoFence( &heap->bins[bin].enabled )) bin_try_enable( heap, &heap->bins[bin] );
2037 }
2038 }
2039
2040 if (!status) valgrind_notify_alloc( ptr, size, flags & HEAP_ZERO_MEMORY );
2041
2042 TRACE( "handle %p, flags %#lx, size %#Ix, return %p, status %#lx.\n", handle, flags, size, ptr, status );
2043 heap_set_status( heap, flags, status );
2044 return ptr;
2045 }
2046
2047
2048 /***********************************************************************
2049 * RtlFreeHeap (NTDLL.@)
2050 */
2051 BOOLEAN WINAPI DECLSPEC_HOTPATCH RtlFreeHeap( HANDLE handle, ULONG flags, void *ptr )
2052 {
2053 struct block *block;
2054 struct heap *heap;
2055 ULONG heap_flags;
2056 NTSTATUS status;
2057
2058 if (!ptr) return TRUE;
2059
2060 valgrind_notify_free( ptr );
2061
2062 if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
2063 status = STATUS_INVALID_PARAMETER;
2064 else if (!(block = unsafe_block_from_ptr( heap, heap_flags, ptr )))
2065 status = STATUS_INVALID_PARAMETER;
2066 else if (block_get_flags( block ) & BLOCK_FLAG_LARGE)
2067 status = heap_free_large( heap, heap_flags, block );
2068 else if (!(block = heap_delay_free( heap, heap_flags, block )))
2069 status = STATUS_SUCCESS;
2070 else if (!heap_free_block_lfh( heap, heap_flags, block ))
2071 status = STATUS_SUCCESS;
2072 else
2073 {
2074 SIZE_T block_size = block_get_size( block ), bin = BLOCK_SIZE_BIN( block_size );
2075
2076 heap_lock( heap, heap_flags );
2077 status = heap_free_block( heap, heap_flags, block );
2078 heap_unlock( heap, heap_flags );
2079
2080 if (!status && heap->bins) InterlockedIncrement( &heap->bins[bin].count_freed );
2081 }
2082
2083 TRACE( "handle %p, flags %#lx, ptr %p, return %u, status %#lx.\n", handle, flags, ptr, !status, status );
2084 heap_set_status( heap, flags, status );
2085 return !status;
2086 }
2087
2088 static NTSTATUS heap_resize_large( struct heap *heap, ULONG flags, struct block *block, SIZE_T block_size,
2089 SIZE_T size, SIZE_T *old_size, void **ret )
2090 {
2091 ARENA_LARGE *large = CONTAINING_RECORD( block, ARENA_LARGE, block );
2092 SIZE_T old_block_size = large->block_size;
2093 *old_size = large->data_size;
2094
2095 if (old_block_size < block_size) return STATUS_NO_MEMORY;
2096
2097 /* FIXME: we could remap zero-pages instead */
2098 valgrind_notify_resize( block + 1, *old_size, size );
2099 initialize_block( block, *old_size, size, flags );
2100
2101 large->data_size = size;
2102 valgrind_make_noaccess( (char *)block + sizeof(*block) + large->data_size,
2103 old_block_size - sizeof(*block) - large->data_size );
2104
2105 *ret = block + 1;
2106 return STATUS_SUCCESS;
2107 }
2108
2109 static NTSTATUS heap_resize_block( struct heap *heap, ULONG flags, struct block *block, SIZE_T block_size,
2110 SIZE_T size, SIZE_T old_block_size, SIZE_T *old_size, void **ret )
2111 {
2112 SUBHEAP *subheap = block_get_subheap( heap, block );
2113 struct block *next;
2114
2115 if (block_size > old_block_size)
2116 {
2117 /* need to grow block, make sure it's followed by large enough free block */
2118 if (!(next = next_block( subheap, block ))) return STATUS_NO_MEMORY;
2119 if (!(block_get_flags( next ) & BLOCK_FLAG_FREE)) return STATUS_NO_MEMORY;
2120 if (block_size > old_block_size + block_get_size( next )) return STATUS_NO_MEMORY;
2121 if (!subheap_commit( heap, subheap, block, block_size )) return STATUS_NO_MEMORY;
2122 }
2123
2124 if ((next = next_block( subheap, block )) && (block_get_flags( next ) & BLOCK_FLAG_FREE))
2125 {
2126 /* merge with next block if it is free */
2127 struct entry *entry = (struct entry *)next;
2128 list_remove( &entry->entry );
2129 old_block_size += block_get_size( next );
2130 }
2131
2132 if ((next = split_block( heap, flags, block, old_block_size, block_size )))
2133 {
2134 block_init_free( next, flags, subheap, old_block_size - block_size );
2135 insert_free_block( heap, flags, subheap, next );
2136 }
2137
2138 valgrind_notify_resize( block + 1, *old_size, size );
2139 block_set_flags( block, BLOCK_FLAG_USER_MASK & ~BLOCK_FLAG_USER_INFO, BLOCK_USER_FLAGS( flags ) );
2140 block->tail_size = block_get_size( block ) - sizeof(*block) - size;
2141 initialize_block( block, *old_size, size, flags );
2142 mark_block_tail( block, flags );
2143
2144 if ((next = next_block( subheap, block ))) block_set_flags( next, BLOCK_FLAG_PREV_FREE, 0 );
2145
2146 *ret = block + 1;
2147 return STATUS_SUCCESS;
2148 }
2149
2150 static NTSTATUS heap_resize_block_lfh( struct block *block, ULONG flags, SIZE_T block_size, SIZE_T size, SIZE_T *old_size, void **ret )
2151 {
2152 /* as native LFH does it with different block size: refuse to resize even though we could */
2153 if (ROUND_SIZE( *old_size, BLOCK_ALIGN - 1) != ROUND_SIZE( size, BLOCK_ALIGN - 1)) return STATUS_NO_MEMORY;
2154 if (size >= *old_size) return STATUS_NO_MEMORY;
2155
2156 block_set_flags( block, BLOCK_FLAG_USER_MASK & ~BLOCK_FLAG_USER_INFO, BLOCK_USER_FLAGS( flags ) );
2157 block->tail_size = block_size - sizeof(*block) - size;
2158 initialize_block( block, *old_size, size, flags );
2159 mark_block_tail( block, flags );
2160
2161 *ret = block + 1;
2162 return STATUS_SUCCESS;
2163 }
2164
2165 static NTSTATUS heap_resize_in_place( struct heap *heap, ULONG flags, struct block *block, SIZE_T block_size,
2166 SIZE_T size, SIZE_T *old_size, void **ret )
2167 {
2168 SIZE_T old_bin, old_block_size;
2169 NTSTATUS status;
2170
2171 if (block_get_flags( block ) & BLOCK_FLAG_LARGE)
2172 return heap_resize_large( heap, flags, block, block_size, size, old_size, ret );
2173
2174 old_block_size = block_get_size( block );
2175 *old_size = old_block_size - block_get_overhead( block );
2176 old_bin = BLOCK_SIZE_BIN( old_block_size );
2177
2178 if (block_size >= HEAP_MIN_LARGE_BLOCK_SIZE) return STATUS_NO_MEMORY; /* growing small block to large block */
2179
2180 if (block_get_flags( block ) & BLOCK_FLAG_LFH)
2181 return heap_resize_block_lfh( block, flags, block_size, size, old_size, ret );
2182
2183 heap_lock( heap, flags );
2184 status = heap_resize_block( heap, flags, block, block_size, size, old_block_size, old_size, ret );
2185 heap_unlock( heap, flags );
2186
2187 if (!status && heap->bins)
2188 {
2189 SIZE_T new_bin = BLOCK_SIZE_BIN( block_size );
2190 InterlockedIncrement( &heap->bins[old_bin].count_freed );
2191 InterlockedIncrement( &heap->bins[new_bin].count_alloc );
2192 if (!ReadNoFence( &heap->bins[new_bin].enabled )) bin_try_enable( heap, &heap->bins[new_bin] );
2193 }
2194
2195 return status;
2196 }
2197
2198 /***********************************************************************
2199 * RtlReAllocateHeap (NTDLL.@)
2200 */
2201 void *WINAPI RtlReAllocateHeap( HANDLE handle, ULONG flags, void *ptr, SIZE_T size )
2202 {
2203 SIZE_T block_size, old_size;
2204 struct block *block;
2205 struct heap *heap;
2206 ULONG heap_flags;
2207 void *ret = NULL;
2208 NTSTATUS status;
2209
2210 if (!ptr) return NULL;
2211
2212 if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
2213 status = STATUS_INVALID_HANDLE;
2214 else if ((block_size = heap_get_block_size( heap, heap_flags, size )) == ~0U)
2215 status = STATUS_NO_MEMORY;
2216 else if (!(block = unsafe_block_from_ptr( heap, heap_flags, ptr )))
2217 status = STATUS_INVALID_PARAMETER;
2218 else if ((status = heap_resize_in_place( heap, heap_flags, block, block_size, size,
2219 &old_size, &ret )))
2220 {
2221 if (flags & HEAP_REALLOC_IN_PLACE_ONLY)
2222 status = STATUS_NO_MEMORY;
2223 else if (!(ret = RtlAllocateHeap( heap, flags, size )))
2224 status = STATUS_NO_MEMORY;
2225 else
2226 {
2227 memcpy( ret, ptr, min( size, old_size ) );
2228 RtlFreeHeap( heap, flags, ptr );
2229 status = STATUS_SUCCESS;
2230 }
2231 }
2232
2233 TRACE( "handle %p, flags %#lx, ptr %p, size %#Ix, return %p, status %#lx.\n", handle, flags, ptr, size, ret, status );
2234 heap_set_status( heap, flags, status );
2235 return ret;
2236 }
2237
2238
2239 /***********************************************************************
2240 * RtlCompactHeap (NTDLL.@)
2241 *
2242 * Compact the free space in a Heap.
2243 *
2244 * PARAMS
2245 * heap [I] Heap that block was allocated from
2246 * flags [I] HEAP_ flags from "winnt.h"
2247 *
2248 * RETURNS
2249 * The number of bytes compacted.
2250 *
2251 * NOTES
2252 * This function is a harmless stub.
2253 */
2254 ULONG WINAPI RtlCompactHeap( HANDLE handle, ULONG flags )
2255 {
2256 static BOOL reported;
2257 if (!reported++) FIXME( "handle %p, flags %#lx stub!\n", handle, flags );
2258 return 0;
2259 }
2260
2261
2262 /***********************************************************************
2263 * RtlLockHeap (NTDLL.@)
2264 *
2265 * Lock a Heap.
2266 *
2267 * PARAMS
2268 * heap [I] Heap to lock
2269 *
2270 * RETURNS
2271 * Success: TRUE. The Heap is locked.
2272 * Failure: FALSE, if heap is invalid.
2273 */
2274 BOOLEAN WINAPI RtlLockHeap( HANDLE handle )
2275 {
2276 struct heap *heap;
2277 ULONG heap_flags;
2278 if (!(heap = unsafe_heap_from_handle( handle, 0, &heap_flags ))) return FALSE;
2279 heap_lock( heap, heap_flags );
2280 return TRUE;
2281 }
2282
2283
2284 /***********************************************************************
2285 * RtlUnlockHeap (NTDLL.@)
2286 *
2287 * Unlock a Heap.
2288 *
2289 * PARAMS
2290 * heap [I] Heap to unlock
2291 *
2292 * RETURNS
2293 * Success: TRUE. The Heap is unlocked.
2294 * Failure: FALSE, if heap is invalid.
2295 */
2296 BOOLEAN WINAPI RtlUnlockHeap( HANDLE handle )
2297 {
2298 struct heap *heap;
2299 ULONG heap_flags;
2300 if (!(heap = unsafe_heap_from_handle( handle, 0, &heap_flags ))) return FALSE;
2301 heap_unlock( heap, heap_flags );
2302 return TRUE;
2303 }
2304
2305
2306 static NTSTATUS heap_size( const struct heap *heap, struct block *block, SIZE_T *size )
2307 {
2308 if (block_get_flags( block ) & BLOCK_FLAG_LARGE)
2309 {
2310 const ARENA_LARGE *large_arena = CONTAINING_RECORD( block, ARENA_LARGE, block );
2311 *size = large_arena->data_size;
2312 }
2313 else *size = block_get_size( block ) - block_get_overhead( block );
2314
2315 return STATUS_SUCCESS;
2316 }
2317
2318 /***********************************************************************
2319 * RtlSizeHeap (NTDLL.@)
2320 */
2321 SIZE_T WINAPI RtlSizeHeap( HANDLE handle, ULONG flags, const void *ptr )
2322 {
2323 SIZE_T size = ~(SIZE_T)0;
2324 struct block *block;
2325 struct heap *heap;
2326 ULONG heap_flags;
2327 NTSTATUS status;
2328
2329 if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
2330 status = STATUS_INVALID_PARAMETER;
2331 else if (!(block = unsafe_block_from_ptr( heap, heap_flags, ptr )))
2332 status = STATUS_INVALID_PARAMETER;
2333 else
2334 {
2335 heap_lock( heap, heap_flags );
2336 status = heap_size( heap, block, &size );
2337 heap_unlock( heap, heap_flags );
2338 }
2339
2340 TRACE( "handle %p, flags %#lx, ptr %p, return %#Ix, status %#lx.\n", handle, flags, ptr, size, status );
2341 heap_set_status( heap, flags, status );
2342 return size;
2343 }
2344
2345
2346 /***********************************************************************
2347 * RtlValidateHeap (NTDLL.@)
2348 */
2349 BOOLEAN WINAPI RtlValidateHeap( HANDLE handle, ULONG flags, const void *ptr )
2350 {
2351 struct heap *heap;
2352 ULONG heap_flags;
2353 BOOLEAN ret;
2354
2355 if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
2356 ret = FALSE;
2357 else
2358 {
2359 heap_lock( heap, heap_flags );
2360 if (ptr) ret = heap_validate_ptr( heap, ptr );
2361 else ret = heap_validate( heap );
2362 heap_unlock( heap, heap_flags );
2363 }
2364
2365 TRACE( "handle %p, flags %#lx, ptr %p, return %u.\n", handle, flags, ptr, !!ret );
2366 return ret;
2367 }
2368
2369
2370 static NTSTATUS heap_walk_blocks( const struct heap *heap, const SUBHEAP *subheap,
2371 const struct block *block, struct rtl_heap_entry *entry )
2372 {
2373 const char *base = subheap_base( subheap ), *commit_end = subheap_commit_end( subheap ), *end = base + subheap_size( subheap );
2374 const struct block *blocks = first_block( subheap );
2375
2376 if (entry->lpData == commit_end) return STATUS_NO_MORE_ENTRIES;
2377 if (entry->lpData == base) block = blocks;
2378 else if (!(block = next_block( subheap, block )))
2379 {
2380 entry->lpData = (void *)commit_end;
2381 entry->cbData = end - commit_end;
2382 entry->cbOverhead = 0;
2383 entry->iRegionIndex = 0;
2384 entry->wFlags = RTL_HEAP_ENTRY_UNCOMMITTED;
2385 return STATUS_SUCCESS;
2386 }
2387
2388 if (block_get_flags( block ) & BLOCK_FLAG_FREE)
2389 {
2390 entry->lpData = (char *)block + block_get_overhead( block );
2391 entry->cbData = block_get_size( block ) - block_get_overhead( block );
2392 /* FIXME: last free block should not include uncommitted range, which also has its own overhead */
2393 if (!contains( blocks, commit_end - (char *)blocks, block, block_get_size( block ) ))
2394 entry->cbData = commit_end - (char *)entry->lpData - 4 * BLOCK_ALIGN;
2395 entry->cbOverhead = 2 * BLOCK_ALIGN;
2396 entry->iRegionIndex = 0;
2397 entry->wFlags = 0;
2398 }
2399 else
2400 {
2401 entry->lpData = (void *)(block + 1);
2402 entry->cbData = block_get_size( block ) - block_get_overhead( block );
2403 entry->cbOverhead = block_get_overhead( block );
2404 entry->iRegionIndex = 0;
2405 entry->wFlags = RTL_HEAP_ENTRY_COMMITTED|RTL_HEAP_ENTRY_BLOCK|RTL_HEAP_ENTRY_BUSY;
2406 }
2407
2408 return STATUS_SUCCESS;
2409 }
2410
2411 static NTSTATUS heap_walk( const struct heap *heap, struct rtl_heap_entry *entry )
2412 {
2413 const char *data = entry->lpData;
2414 const ARENA_LARGE *large = NULL;
2415 const struct block *block;
2416 const struct list *next;
2417 const SUBHEAP *subheap;
2418 NTSTATUS status;
2419 char *base;
2420
2421 if (!data || entry->wFlags & RTL_HEAP_ENTRY_REGION) block = (struct block *)data;
2422 else if (entry->wFlags & RTL_HEAP_ENTRY_BUSY) block = (struct block *)data - 1;
2423 else block = (struct block *)(data - sizeof(struct list)) - 1;
2424
2425 if (find_large_block( heap, block ))
2426 {
2427 large = CONTAINING_RECORD( block, ARENA_LARGE, block );
2428 next = &large->entry;
2429 }
2430 else if ((subheap = find_subheap( heap, block, TRUE )))
2431 {
2432 if (!(status = heap_walk_blocks( heap, subheap, block, entry ))) return STATUS_SUCCESS;
2433 else if (status != STATUS_NO_MORE_ENTRIES) return status;
2434 next = &subheap->entry;
2435 }
2436 else
2437 {
2438 if (entry->lpData) return STATUS_INVALID_PARAMETER;
2439 next = &heap->subheap_list;
2440 }
2441
2442 if (!large && (next = list_next( &heap->subheap_list, next )))
2443 {
2444 subheap = LIST_ENTRY( next, SUBHEAP, entry );
2445 base = subheap_base( subheap );
2446 entry->lpData = base;
2447 entry->cbData = subheap_overhead( subheap );
2448 entry->cbOverhead = 0;
2449 entry->iRegionIndex = 0;
2450 entry->wFlags = RTL_HEAP_ENTRY_REGION;
2451 entry->Region.dwCommittedSize = (char *)subheap_commit_end( subheap ) - base;
2452 entry->Region.dwUnCommittedSize = subheap_size( subheap ) - entry->Region.dwCommittedSize;
2453 entry->Region.lpFirstBlock = base + entry->cbData;
2454 entry->Region.lpLastBlock = base + subheap_size( subheap );
2455 return STATUS_SUCCESS;
2456 }
2457
2458 if (!next) next = &heap->large_list;
2459 if ((next = list_next( &heap->large_list, next )))
2460 {
2461 large = LIST_ENTRY( next, ARENA_LARGE, entry );
2462 entry->lpData = (void *)(large + 1);
2463 entry->cbData = large->data_size;
2464 entry->cbOverhead = 0;
2465 entry->iRegionIndex = 64;
2466 entry->wFlags = RTL_HEAP_ENTRY_COMMITTED|RTL_HEAP_ENTRY_BLOCK|RTL_HEAP_ENTRY_BUSY;
2467 return STATUS_SUCCESS;
2468 }
2469
2470 return STATUS_NO_MORE_ENTRIES;
2471 }
2472
2473 /***********************************************************************
2474 * RtlWalkHeap (NTDLL.@)
2475 */
2476 NTSTATUS WINAPI RtlWalkHeap( HANDLE handle, void *entry_ptr )
2477 {
2478 struct rtl_heap_entry *entry = entry_ptr;
2479 struct heap *heap;
2480 ULONG heap_flags;
2481 NTSTATUS status;
2482
2483 if (!entry) return STATUS_INVALID_PARAMETER;
2484
2485 if (!(heap = unsafe_heap_from_handle( handle, 0, &heap_flags )))
2486 status = STATUS_INVALID_HANDLE;
2487 else
2488 {
2489 heap_lock( heap, heap_flags );
2490 status = heap_walk( heap, entry );
2491 heap_unlock( heap, heap_flags );
2492 }
2493
2494 TRACE( "handle %p, entry %p %s, return %#lx\n", handle, entry,
2495 status ? "<empty>" : debugstr_heap_entry(entry), status );
2496 return status;
2497 }
2498
2499
2500 /***********************************************************************
2501 * RtlGetProcessHeaps (NTDLL.@)
2502 *
2503 * Get the Heaps belonging to the current process.
2504 *
2505 * PARAMS
2506 * count [I] size of heaps
2507 * heaps [O] Destination array for heap HANDLE's
2508 *
2509 * RETURNS
2510 * Success: The number of Heaps allocated by the process.
2511 * Failure: 0.
2512 */
2513 ULONG WINAPI RtlGetProcessHeaps( ULONG count, HANDLE *heaps )
2514 {
2515 ULONG total = 1; /* main heap */
2516 struct list *ptr;
2517
2518 RtlEnterCriticalSection( &process_heap->cs );
2519 LIST_FOR_EACH( ptr, &process_heap->entry ) total++;
2520 if (total <= count)
2521 {
2522 *heaps++ = process_heap;
2523 LIST_FOR_EACH( ptr, &process_heap->entry )
2524 *heaps++ = LIST_ENTRY( ptr, struct heap, entry );
2525 }
2526 RtlLeaveCriticalSection( &process_heap->cs );
2527 return total;
2528 }
2529
2530 /***********************************************************************
2531 * RtlQueryHeapInformation (NTDLL.@)
2532 */
2533 NTSTATUS WINAPI RtlQueryHeapInformation( HANDLE handle, HEAP_INFORMATION_CLASS info_class,
2534 void *info, SIZE_T size_in, SIZE_T *size_out )
2535 {
2536 struct heap *heap;
2537 ULONG flags;
2538
2539 TRACE( "handle %p, info_class %u, info %p, size_in %Iu, size_out %p.\n", handle, info_class, info, size_in, size_out );
2540
2541 switch (info_class)
2542 {
2543 case HeapCompatibilityInformation:
2544 if (!(heap = unsafe_heap_from_handle( handle, 0, &flags ))) return STATUS_ACCESS_VIOLATION;
2545 if (size_out) *size_out = sizeof(ULONG);
2546 if (size_in < sizeof(ULONG)) return STATUS_BUFFER_TOO_SMALL;
2547 *(ULONG *)info = ReadNoFence( &heap->compat_info );
2548 return STATUS_SUCCESS;
2549
2550 default:
2551 FIXME( "HEAP_INFORMATION_CLASS %u not implemented!\n", info_class );
2552 return STATUS_INVALID_INFO_CLASS;
2553 }
2554 }
2555
2556 /***********************************************************************
2557 * RtlSetHeapInformation (NTDLL.@)
2558 */
2559 NTSTATUS WINAPI RtlSetHeapInformation( HANDLE handle, HEAP_INFORMATION_CLASS info_class, void *info, SIZE_T size )
2560 {
2561 struct heap *heap;
2562 ULONG flags;
2563
2564 TRACE( "handle %p, info_class %u, info %p, size %Iu.\n", handle, info_class, info, size );
2565
2566 switch (info_class)
2567 {
2568 case HeapCompatibilityInformation:
2569 {
2570 ULONG compat_info;
2571
2572 if (size < sizeof(ULONG)) return STATUS_BUFFER_TOO_SMALL;
2573 if (!(heap = unsafe_heap_from_handle( handle, 0, &flags ))) return STATUS_INVALID_HANDLE;
2574 if (heap->flags & HEAP_NO_SERIALIZE) return STATUS_INVALID_PARAMETER;
2575
2576 compat_info = *(ULONG *)info;
2577 if (compat_info != HEAP_STD && compat_info != HEAP_LFH)
2578 {
2579 FIXME( "HeapCompatibilityInformation %lu not implemented!\n", compat_info );
2580 return STATUS_UNSUCCESSFUL;
2581 }
2582 if (InterlockedCompareExchange( &heap->compat_info, compat_info, HEAP_STD ) != HEAP_STD)
2583 return STATUS_UNSUCCESSFUL;
2584 return STATUS_SUCCESS;
2585 }
2586
2587 default:
2588 FIXME( "HEAP_INFORMATION_CLASS %u not implemented!\n", info_class );
2589 return STATUS_SUCCESS;
2590 }
2591 }
2592
2593 /***********************************************************************
2594 * RtlGetUserInfoHeap (NTDLL.@)
2595 */
2596 BOOLEAN WINAPI RtlGetUserInfoHeap( HANDLE handle, ULONG flags, void *ptr, void **user_value, ULONG *user_flags )
2597 {
2598 NTSTATUS status = STATUS_SUCCESS;
2599 struct block *block;
2600 struct heap *heap;
2601 ULONG heap_flags;
2602 char *tmp;
2603
2604 TRACE( "handle %p, flags %#lx, ptr %p, user_value %p, user_flags %p semi-stub!\n",
2605 handle, flags, ptr, user_value, user_flags );
2606
2607 *user_flags = 0;
2608
2609 if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
2610 status = STATUS_SUCCESS;
2611 else if (!(block = unsafe_block_from_ptr( heap, heap_flags, ptr )))
2612 {
2613 status = STATUS_INVALID_PARAMETER;
2614 *user_value = 0;
2615 }
2616 else if (!(*user_flags = HEAP_USER_FLAGS(block_get_flags( block ))))
2617 WARN( "Block %p wasn't allocated with user info\n", ptr );
2618 else if (block_get_flags( block ) & BLOCK_FLAG_LARGE)
2619 {
2620 const ARENA_LARGE *large = CONTAINING_RECORD( block, ARENA_LARGE, block );
2621 *user_flags = *user_flags & ~HEAP_ADD_USER_INFO;
2622 *user_value = large->user_value;
2623 }
2624 else
2625 {
2626 heap_lock( heap, heap_flags );
2627
2628 tmp = (char *)block + block_get_size( block ) - block->tail_size + sizeof(void *);
2629 if ((heap_flags & HEAP_TAIL_CHECKING_ENABLED) || RUNNING_ON_VALGRIND) tmp += BLOCK_ALIGN;
2630 *user_flags = *user_flags & ~HEAP_ADD_USER_INFO;
2631 *user_value = *(void **)tmp;
2632
2633 heap_unlock( heap, heap_flags );
2634 }
2635
2636 heap_set_status( heap, flags, status );
2637 return !status;
2638 }
2639
2640 /***********************************************************************
2641 * RtlSetUserValueHeap (NTDLL.@)
2642 */
2643 BOOLEAN WINAPI RtlSetUserValueHeap( HANDLE handle, ULONG flags, void *ptr, void *user_value )
2644 {
2645 struct block *block;
2646 struct heap *heap;
2647 ULONG heap_flags;
2648 BOOLEAN ret;
2649 char *tmp;
2650
2651 TRACE( "handle %p, flags %#lx, ptr %p, user_value %p.\n", handle, flags, ptr, user_value );
2652
2653 if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
2654 ret = TRUE;
2655 else if (!(block = unsafe_block_from_ptr( heap, heap_flags, ptr )))
2656 ret = FALSE;
2657 else if (!(block_get_flags( block ) & BLOCK_FLAG_USER_INFO))
2658 ret = FALSE;
2659 else if (block_get_flags( block ) & BLOCK_FLAG_LARGE)
2660 {
2661 ARENA_LARGE *large = CONTAINING_RECORD( block, ARENA_LARGE, block );
2662 large->user_value = user_value;
2663 ret = TRUE;
2664 }
2665 else
2666 {
2667 heap_lock( heap, heap_flags );
2668
2669 tmp = (char *)block + block_get_size( block ) - block->tail_size + sizeof(void *);
2670 if ((heap_flags & HEAP_TAIL_CHECKING_ENABLED) || RUNNING_ON_VALGRIND) tmp += BLOCK_ALIGN;
2671 *(void **)tmp = user_value;
2672 ret = TRUE;
2673
2674 heap_unlock( heap, heap_flags );
2675 }
2676
2677 return ret;
2678 }
2679
2680 /***********************************************************************
2681 * RtlSetUserFlagsHeap (NTDLL.@)
2682 */
2683 BOOLEAN WINAPI RtlSetUserFlagsHeap( HANDLE handle, ULONG flags, void *ptr, ULONG clear, ULONG set )
2684 {
2685 struct block *block;
2686 struct heap *heap;
2687 ULONG heap_flags;
2688 BOOLEAN ret;
2689
2690 TRACE( "handle %p, flags %#lx, ptr %p, clear %#lx, set %#lx.\n", handle, flags, ptr, clear, set );
2691
2692 if ((clear | set) & ~(0xe00))
2693 {
2694 SetLastError( ERROR_INVALID_PARAMETER );
2695 return FALSE;
2696 }
2697
2698 if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
2699 ret = TRUE;
2700 else if (!(block = unsafe_block_from_ptr( heap, heap_flags, ptr )))
2701 ret = FALSE;
2702 else if (!(block_get_flags( block ) & BLOCK_FLAG_USER_INFO))
2703 ret = FALSE;
2704 else
2705 {
2706 block_set_flags( block, BLOCK_USER_FLAGS( clear ), BLOCK_USER_FLAGS( set ) );
2707 ret = TRUE;
2708 }
2709
2710 return ret;
2711 }
Windows API implementation