1 /* Copyright (C) 2012-2013
2 Free Software Foundation
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3, or (at your option)
11 GCC is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 Under Section 7 of GPL version 3, you are granted additional
17 permissions described in the GCC Runtime Library Exception, version
18 3.1, as published by the Free Software Foundation.
20 You should have received a copy of the GNU General Public License and
21 a copy of the GCC Runtime Library Exception along with this program;
22 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 <http://www.gnu.org/licenses/>. */
25 /* This file is part of the vtable security feature implementation.
26 The vtable security feature is designed to detect when a virtual
27 call is about to be made through an invalid vtable pointer
28 (possibly due to data corruption or malicious attacks). The
29 compiler finds every virtual call, and inserts a verification call
30 before the virtual call. The verification call takes the actual
31 vtable pointer value in the object through which the virtual call
32 is being made, and compares the vtable pointer against a set of all
33 valid vtable pointers that the object could contain (this set is
34 based on the declared type of the object). If the pointer is in
35 the valid set, execution is allowed to continue; otherwise the
38 There are several pieces needed in order to make this work: 1. For
39 every virtual class in the program (i.e. a class that contains
40 virtual methods), we need to build the set of all possible valid
41 vtables that an object of that class could point to. This includes
42 vtables for any class(es) that inherit from the class under
43 consideration. 2. For every such data set we build up, we need a
44 way to find and reference the data set. This is complicated by the
45 fact that the real vtable addresses are not known until runtime,
46 when the program is loaded into memory, but we need to reference the
47 sets at compile time when we are inserting verification calls into
48 the program. 3. We need to find every virtual call in the program,
49 and insert the verification call (with the appropriate arguments)
50 before the virtual call. 4. We need some runtime library pieces:
51 the code to build up the data sets at runtime; the code to actually
52 perform the verification using the data sets; and some code to set
53 protections on the data sets, so they themselves do not become
56 To find and reference the set of valid vtable pointers for any given
57 virtual class, we create a special global varible for each virtual
58 class. We refer to this as the "vtable map variable" for that
59 class. The vtable map variable has the type "void *", and is
60 initialized by the compiler to NULL. At runtime when the set of
61 valid vtable pointers for a virtual class, e.g. class Foo, is built,
62 the vtable map variable for class Foo is made to point to the set.
63 During compile time, when the compiler is inserting verification
64 calls into the program, it passes the vtable map variable for the
65 appropriate class to the verification call, so that at runtime the
66 verification call can find the appropriate data set.
68 The actual set of valid vtable pointers for a polymorphic class,
69 e.g. class Foo, cannot be built until runtime, when the vtables get
70 loaded into memory and their addresses are known. But the knowledge
71 about which vtables belong in which class' hierarchy is only known
72 at compile time. Therefore at compile time we collect class
73 hierarchy and vtable information about every virtual class, and we
74 generate calls to build up the data sets at runtime. To build the
75 data sets, we call one of the functions we add to the runtime
76 library, __VLTRegisterPair. __VLTRegisterPair takes two arguments,
77 a vtable map variable and the address of a vtable. If the vtable
78 map variable is currently NULL, it creates a new data set (hash
79 table), makes the vtable map variable point to the new data set, and
80 inserts the vtable address into the data set. If the vtable map
81 variable is not NULL, it just inserts the vtable address into the
82 data set. In order to make sure that our data sets are built before
83 any verification calls happen, we create a special constructor
84 initialization function for each compilation unit, give it a very
85 high initialization priority, and insert all of our calls to
86 __VLTRegisterPair into our special constructor initialization
89 /* This file contains the main externally visible runtime library
90 functions for vtable verification: __VLTChangePermission,
91 __VLTRegisterPair, and __VLTVerifyVtablePointer. It also contains
92 debug versions __VLTRegisterPairDebug and
93 __VLTVerifyVtablePointerDebug, which have extra parameters in order
94 to make it easier to debug verification failures.
96 The final piece of functionality implemented in this file is symbol
97 resolution for multiple instances of the same vtable map variable.
98 If the same virtual class is used in two different compilation
99 units, then each compilation unit will create a vtable map variable
100 for the class. We need all instances of the same vtable map
101 variable to point to the same (single) set of valid vtable
102 pointers for the class, so we wrote our own hashtable-based symbol
103 resolution for vtable map variables (with a tiny optimization in
104 the case where there is only one instance of the variable).
106 There are two other important pieces to the runtime for vtable
107 verification besides the main pieces that go into libstdc++.so: two
108 special tiny shared libraries, libvtv_init.so and libvtv_stubs.so.
109 libvtv_init.so is built from vtv_init.cc. It is designed to help
110 minimize the calls made to mprotect (see the comments in
111 vtv_init.cc for more details). Anything compiled with
112 "-fvtable-verify=std" must be linked with libvtv_init.so (the gcc
113 driver has been modified to do this). vtv_stubs.so is built from
114 vtv_stubs.cc. It replaces the main runtime functions
115 (__VLTChangePermissino, __VLTRegisterPair and
116 __VLTVerifyVtablePointer) with stub functions that do nothing. If
117 a programmer has a library that was built with verification, but
118 wishes to not have verification turned on, the programmer can link
119 in the vtv_stubs.so library. */
124 #if defined (__CYGWIN__) || defined (__MINGW32__)
126 #include <winternl.h>
129 #include <execinfo.h>
133 #if !defined (__CYGWIN__) && !defined (__MINGW32__)
134 #include <sys/mman.h>
141 /* For gthreads suppport */
142 #include <bits/c++config.h>
143 #include <ext/concurrence.h>
145 #include "vtv_utils.h"
146 #include "vtv_malloc.h"
150 #include "vtv_fail.h"
152 #include "vtv-change-permission.h"
154 #if defined (__CYGWIN__) || defined (__MINGW32__)
155 // porting: fix link error to libc
156 void __fortify_fail (const char * msg
){
157 OutputDebugString(msg
);
163 /* __fortify_fail is a function in glibc that calls __libc_message,
164 causing it to print out a program termination error message
165 (including the name of the binary being terminated), a stack
166 trace where the error occurred, and a memory map dump. Ideally
167 we would have called __libc_message directly, but that function
168 does not appear to be accessible to functions outside glibc,
169 whereas __fortify_fail is. We call __fortify_fail from
170 __vtv_really_fail. We looked at calling __libc_fatal, which is
171 externally accessible, but it does not do the back trace and
174 extern void __fortify_fail (const char *) __attribute__((noreturn
));
179 /* The following variables are used only for debugging and performance
180 tuning purposes. Therefore they do not need to be "protected".
181 They cannot be used to attack the vtable verification system and if
182 they become corrupted it will not affect the correctness or
183 security of any of the rest of the vtable verification feature. */
185 unsigned int num_calls_to_regset
= 0;
186 unsigned int num_calls_to_regpair
= 0;
187 unsigned int num_calls_to_verify_vtable
= 0;
188 unsigned long long regset_cycles
= 0;
189 unsigned long long regpair_cycles
= 0;
190 unsigned long long verify_vtable_cycles
= 0;
192 /* Be careful about initialization of statics in this file. Some of
193 the routines below are called before any runtime initialization for
194 statics in this file will be done. For example, dont try to
195 initialize any of these statics with a runtime call (for ex:
196 sysconf). The initialization will happen after calls to the routines
197 to protect/unprotec the vtabla_map variables */
199 /* No need to mark the following variables with VTV_PROTECTED_VAR.
200 These are either const or are only used for debugging/tracing.
201 debugging/tracing will not be ON on production environments */
203 static const bool debug_hash
= HASHTABLE_STATS
;
206 static const int debug_functions
= 1;
207 static const int debug_init
= 1;
208 static const int debug_verify_vtable
= 1;
210 static const int debug_functions
= 0;
211 static const int debug_init
= 0;
212 static const int debug_verify_vtable
= 0;
215 /* Global file descriptor variables for logging, tracing and debugging. */
217 static int init_log_fd
= -1;
218 static int verify_vtable_log_fd
= -1;
220 /* This holds a formatted error logging message, to be written to the
221 vtable verify failures log. */
222 static char debug_log_message
[1024];
225 #ifdef __GTHREAD_MUTEX_INIT
226 static __gthread_mutex_t change_permissions_lock
= __GTHREAD_MUTEX_INIT
;
228 static __gthread_mutex_t change_permissions_lock
;
238 static inline unsigned long long
239 get_cycle_count (void)
245 accumulate_cycle_count (unsigned long long *sum
, unsigned long long start
)
247 unsigned long long end
= rdtsc();
248 *sum
= *sum
+ (end
- start
);
252 increment_num_calls (unsigned int *num_calls
)
254 *num_calls
= *num_calls
+ 1;
259 static inline unsigned long long
260 get_cycle_count (void)
262 return (unsigned long long) 0;
266 accumulate_cycle_count (unsigned long long *sum
__attribute__((__unused__
)),
267 unsigned long long start
__attribute__((__unused__
)))
273 increment_num_calls (unsigned int *num_calls
__attribute__((__unused__
)))
280 /* Types needed by insert_only_hash_sets. */
281 typedef uintptr_t int_vptr
;
283 /* The set of valid vtable pointers for each virtual class is stored
284 in a hash table. This is the hashing function used for the hash
285 table. For more information on the implementation of the hash
286 table, see the class insert_only_hash_sets in vtv_set.h. */
290 /* Hash function, used to convert vtable pointer, V, (a memory
291 address) into an index into the hash table. */
293 operator() (int_vptr v
) const
295 const uint32_t x
= 0x7a35e4d9;
296 const int shift
= (sizeof (v
) == 8) ? 23 : 21;
298 return v
^ (v
>> shift
);
302 /* This is the memory allocator used to create the hash table data
303 sets of valid vtable pointers. We use VTV_malloc in order to keep
304 track of which pages have been allocated, so we can update the
305 protections on those pages appropriately. See the class
306 insert_only_hash_sets in vtv_set.h for more information. */
308 struct vptr_set_alloc
310 /* Memory allocator operator. N is the number of bytes to be
313 operator() (size_t n
) const
315 return __vtv_malloc (n
);
319 /* Instantiate the template classes (in vtv_set.h) for our particular
321 typedef insert_only_hash_sets
<int_vptr
, vptr_hash
, vptr_set_alloc
> vtv_sets
;
322 typedef vtv_sets::insert_only_hash_set vtv_set
;
323 typedef vtv_set
* vtv_set_handle
;
324 typedef vtv_set_handle
* vtv_set_handle_handle
;
326 /* Records for caching the section header information that we have
327 read out of the file(s) on disk (in dl_iterate_phdr_callback), to
328 avoid having to re-open and re-read the same file multiple
333 #if defined (__CYGWIN__) || defined (__MINGW32__)
334 uintptr_t dlpi_addr
; /* The header address in the INFO record,
335 passed in from dl_iterate_phdr. */
336 uintptr_t mp_low
; /* Start address of the .vtable_map_vars
337 section in memory. */
339 ElfW (Addr
) dlpi_addr
; /* The header address in the INFO record,
340 passed in from dl_iterate_phdr. */
341 ElfW (Addr
) mp_low
; /* Start address of the .vtable_map_vars
342 section in memory. */
344 size_t mp_size
; /* Size of the .vtable_map_vars section in
348 /* Array for caching the section header information, read from file,
349 to avoid re-opening and re-reading the same file over-and-over
352 #define MAX_ENTRIES 250
353 static struct sect_hdr_data vtv_sect_info_cache
[MAX_ENTRIES
] VTV_PROTECTED_VAR
;
355 unsigned int num_cache_entries VTV_PROTECTED_VAR
= 0;
357 /* This function takes the LOAD_ADDR for an object opened by the
358 dynamic loader, and checks the array of cached file data to see if
359 there is an entry with the same addres. If it finds such an entry,
360 it returns the record for that entry; otherwise it returns
363 #if defined (__CYGWIN__) || defined (__MINGW32__)
364 struct sect_hdr_data
*
365 search_cached_file_data (uintptr_t load_addr
)
367 struct sect_hdr_data
*
368 search_cached_file_data (ElfW (Addr
) load_addr
)
372 for (i
= 0; i
< num_cache_entries
; ++i
)
374 if (vtv_sect_info_cache
[i
].dlpi_addr
== load_addr
)
375 return &(vtv_sect_info_cache
[i
]);
381 /* This function tries to read COUNT bytes out of the file referred to
382 by FD into the buffer BUF. It returns the actual number of bytes
383 it succeeded in reading. */
386 ReadPersistent (int fd
, void *buf
, size_t count
)
388 char *buf0
= (char *) buf
;
389 size_t num_bytes
= 0;
390 while (num_bytes
< count
)
393 len
= read (fd
, buf0
+ num_bytes
, count
- num_bytes
);
404 /* This function tries to read COUNT bytes, starting at OFFSET from
405 the file referred to by FD, and put them into BUF. It calls
406 ReadPersistent to help it do so. It returns the actual number of
407 bytes read, or -1 if it fails altogether. */
410 ReadFromOffset (int fd
, void *buf
, const size_t count
, const off_t offset
)
412 off_t off
= lseek (fd
, offset
, SEEK_SET
);
413 if (off
!= (off_t
) -1)
414 return ReadPersistent (fd
, buf
, count
);
418 /* The function takes a MESSAGE and attempts to write it to the vtable
419 memory protection log (for debugging purposes). If the file is not
420 open, it attempts to open the file first. */
423 log_memory_protection_data (char *message
)
425 static int log_fd
= -1;
428 log_fd
= __vtv_open_log ("vtv_memory_protection_data.log");
430 __vtv_add_to_log (log_fd
, "%s", message
);
433 #if defined (__CYGWIN__) || defined (__MINGW32__)
435 read_section_offset_and_length (char *name
,
437 const char *sect_name
,
443 struct sect_hdr_data
*cached_data
= NULL
;
445 /* Check to see if we already have the data for this file. */
446 cached_data
= search_cached_file_data (addr
);
450 *sect_offset
= cached_data
->mp_low
;
451 *sect_len
= cached_data
->mp_size
;
455 // check for DOS Header magic bytes
456 if (*(WORD
*)addr
== 0x5A4D)
458 int name_len
= strlen (sect_name
);
461 /* Attempt to open the binary file on disk. */
462 if (strlen (name
) == 0)
467 fd
= open (name
, O_RDONLY
| O_BINARY
);
471 /* Find the section header information in memory. */
472 PIMAGE_DOS_HEADER pDosHeader
= (PIMAGE_DOS_HEADER
)addr
;
473 PIMAGE_NT_HEADERS pNtHeaders
= (PIMAGE_NT_HEADERS
)((char *)addr
474 + pDosHeader
->e_lfanew
);
475 PIMAGE_FILE_HEADER pFileHeader
= &pNtHeaders
->FileHeader
;
477 DWORD PointerToStringTable
= pFileHeader
->PointerToSymbolTable
478 + (pFileHeader
->NumberOfSymbols
*0x12);
480 PIMAGE_SECTION_HEADER sect_hdr
=
481 (PIMAGE_SECTION_HEADER
)((char *)&pNtHeaders
->OptionalHeader
482 + pFileHeader
->SizeOfOptionalHeader
);
484 /* Loop through all the section headers, looking for one whose
485 name is ".vtable_map_vars". */
487 for (int i
= 0; i
< pFileHeader
->NumberOfSections
&& !found
; ++i
)
489 char header_name
[64];
491 /* Check if we have to get the section name from the COFF string
493 if (sect_hdr
[i
].Name
[0] == '/')
495 if (atoi((const char*)sect_hdr
[i
].Name
+1) == 0)
500 off_t name_offset
= PointerToStringTable
501 + atoi((const char*)sect_hdr
[i
].Name
+1);
503 size_t bytes_read
= ReadFromOffset (fd
, &header_name
, 64,
506 VTV_ASSERT (bytes_read
> 0);
510 memcpy (&header_name
, sect_hdr
[i
].Name
,
511 sizeof (sect_hdr
[i
].Name
));
514 if (memcmp (header_name
, sect_name
, name_len
) == 0)
516 /* We found the section; get its load offset and
518 *sect_offset
= sect_hdr
[i
].VirtualAddress
;
519 if (sect_hdr
[i
].Misc
.VirtualSize
% VTV_PAGE_SIZE
!= 0)
520 *sect_len
= sect_hdr
[i
].Misc
.VirtualSize
+ VTV_PAGE_SIZE
521 - (sect_hdr
[i
].Misc
.VirtualSize
% VTV_PAGE_SIZE
);
523 *sect_len
= sect_hdr
[i
].Misc
.VirtualSize
;
531 if (*sect_offset
!= 0 && *sect_len
!= 0)
533 /* Calculate the page location in memory, making sure the
534 address is page-aligned. */
535 uintptr_t start_addr
= addr
+ *sect_offset
;
536 *sect_offset
= start_addr
& ~(VTV_PAGE_SIZE
- 1);
537 *sect_len
= *sect_len
- 1;
539 /* Since we got this far, we must not have found these pages in
540 the cache, so add them to it. NOTE: We could get here either
541 while making everything read-only or while making everything
542 read-write. We will only update the cache if we get here on
543 a read-write (to make absolutely sure the cache is writable
544 -- also the read-write pass should come before the read-only
546 if ((mprotect_flags
& PROT_WRITE
)
547 && num_cache_entries
< MAX_ENTRIES
)
549 vtv_sect_info_cache
[num_cache_entries
].dlpi_addr
= addr
;
550 vtv_sect_info_cache
[num_cache_entries
].mp_low
= *sect_offset
;
551 vtv_sect_info_cache
[num_cache_entries
].mp_size
= *sect_len
;
558 read_section_offset_and_length (struct dl_phdr_info
*info
,
559 const char *sect_name
,
562 ElfW (Word
) *sect_len
)
564 char program_name
[PATH_MAX
];
567 struct sect_hdr_data
*cached_data
= NULL
;
568 const ElfW (Phdr
) *phdr_info
= info
->dlpi_phdr
;
569 const ElfW (Ehdr
) *ehdr_info
=
570 (const ElfW (Ehdr
) *) (info
->dlpi_addr
+ info
->dlpi_phdr
[0].p_vaddr
571 - info
->dlpi_phdr
[0].p_offset
);
574 /* Get the name of the main executable. This may or may not include
575 arguments passed to the program. Find the first space, assume it
576 is the start of the argument list, and change it to a '\0'. */
577 snprintf (program_name
, sizeof (program_name
), program_invocation_name
);
579 /* Check to see if we already have the data for this file. */
580 cached_data
= search_cached_file_data (info
->dlpi_addr
);
584 *sect_offset
= cached_data
->mp_low
;
585 *sect_len
= cached_data
->mp_size
;
589 /* Find the first non-escaped space in the program name and make it
590 the end of the string. */
591 cptr
= strchr (program_name
, ' ');
592 if (cptr
!= NULL
&& cptr
[-1] != '\\')
595 if ((phdr_info
->p_type
== PT_PHDR
|| phdr_info
->p_type
== PT_LOAD
)
596 && (ehdr_info
->e_shoff
&& ehdr_info
->e_shnum
))
598 int name_len
= strlen (sect_name
);
601 /* Attempt to open the binary file on disk. */
602 if (strlen (info
->dlpi_name
) == 0)
604 /* If the constructor initialization function was put into
605 the preinit array, then this function will get called
606 while handling preinit array stuff, in which case
607 program_invocation_name has not been initialized. In
608 that case we can get the filename of the executable from
610 if (strlen (program_name
) > 0)
612 if (phdr_info
->p_type
== PT_PHDR
)
613 fd
= open (program_name
, O_RDONLY
);
616 fd
= open ("/proc/self/exe", O_RDONLY
);
619 fd
= open (info
->dlpi_name
, O_RDONLY
);
623 /* Find the section header information in the file. */
624 ElfW (Half
) strtab_idx
= ehdr_info
->e_shstrndx
;
625 ElfW (Shdr
) shstrtab
;
626 off_t shstrtab_offset
= ehdr_info
->e_shoff
+
627 (ehdr_info
->e_shentsize
* strtab_idx
);
628 size_t bytes_read
= ReadFromOffset (fd
, &shstrtab
, sizeof (shstrtab
),
630 VTV_ASSERT (bytes_read
== sizeof (shstrtab
));
632 ElfW (Shdr
) sect_hdr
;
634 /* This code will be needed once we have crated libvtv.so. */
635 bool is_libvtv
= false;
638 if (strstr (info->dlpi_name, "libvtv.so"))
642 /* Loop through all the section headers, looking for one whose
643 name is ".vtable_map_vars". */
645 for (int i
= 0; i
< ehdr_info
->e_shnum
&& !found
; ++i
)
647 off_t offset
= ehdr_info
->e_shoff
+ (ehdr_info
->e_shentsize
* i
);
649 bytes_read
= ReadFromOffset (fd
, §_hdr
, sizeof (sect_hdr
),
652 VTV_ASSERT (bytes_read
== sizeof (sect_hdr
));
654 char header_name
[64];
655 off_t name_offset
= shstrtab
.sh_offset
+ sect_hdr
.sh_name
;
657 bytes_read
= ReadFromOffset (fd
, &header_name
, 64, name_offset
);
659 VTV_ASSERT (bytes_read
> 0);
661 if (memcmp (header_name
, sect_name
, name_len
) == 0)
663 /* We found the section; get its load offset and
665 *sect_offset
= sect_hdr
.sh_addr
;
667 *sect_len
= sect_hdr
.sh_size
- VTV_PAGE_SIZE
;
669 *sect_len
= sect_hdr
.sh_size
;
677 if (*sect_offset
!= 0 && *sect_len
!= 0)
679 /* Calculate the page location in memory, making sure the
680 address is page-aligned. */
681 ElfW (Addr
) start_addr
= (const ElfW (Addr
)) info
->dlpi_addr
683 *sect_offset
= start_addr
& ~(VTV_PAGE_SIZE
- 1);
684 *sect_len
= *sect_len
- 1;
686 /* Since we got this far, we must not have found these pages in
687 the cache, so add them to it. NOTE: We could get here either
688 while making everything read-only or while making everything
689 read-write. We will only update the cache if we get here on
690 a read-write (to make absolutely sure the cache is writable
691 -- also the read-write pass should come before the read-only
693 if ((mprotect_flags
& PROT_WRITE
)
694 && num_cache_entries
< MAX_ENTRIES
)
696 vtv_sect_info_cache
[num_cache_entries
].dlpi_addr
= info
->dlpi_addr
;
697 vtv_sect_info_cache
[num_cache_entries
].mp_low
= *sect_offset
;
698 vtv_sect_info_cache
[num_cache_entries
].mp_size
= *sect_len
;
705 #if defined (__CYGWIN__) || defined (__MINGW32__)
706 /* This function is used to iterate over all loaded modules and searches
707 for a section called ".vtable_map_vars". The only interaction with
708 the binary file on disk of the module is to read section names in the
709 COFF string table. If the module contains a ".vtable_map_vars" section,
710 read section offset and size from the section header of the loaded module.
711 Call 'mprotect' on those pages, setting the protection either to
712 read-only or read-write, depending on what's in data.
713 The calls to change the protection occur in vtv_unprotect_vtable_vars
714 and vtv_protect_vtable_vars. */
717 iterate_modules (void *data
)
719 int * mprotect_flags
= (int *) data
;
720 off_t map_sect_offset
= 0;
721 WORD map_sect_len
= 0;
723 const char *map_sect_name
= VTV_PROTECTED_VARS_SECTION
;
728 hProcess
= GetCurrentProcess ();
730 if (NULL
== hProcess
)
733 if (EnumProcessModules (hProcess
, hMods
, sizeof (hMods
), &cbNeeded
))
735 /* Iterate over all loaded modules. */
736 for (unsigned int i
= 0; i
< (cbNeeded
/ sizeof (HMODULE
)); i
++)
738 char szModName
[MAX_PATH
];
740 if (GetModuleFileNameExA (hProcess
, hMods
[i
], szModName
,
745 read_section_offset_and_length (szModName
,
746 (uintptr_t) hMods
[i
],
754 snprintf (buffer
, sizeof(buffer
),
755 " Looking at load module %s to change permissions to %s\n",
757 (*mprotect_flags
& PROT_WRITE
) ? "READ/WRITE" : "READ-ONLY");
758 log_memory_protection_data (buffer
);
761 /* See if we actually found the section. */
762 if (map_sect_offset
&& map_sect_len
)
764 unsigned long long start
;
769 snprintf (buffer
, sizeof (buffer
),
770 " (%s): Protecting %p to %p\n",
772 (void *) map_sect_offset
,
773 (void *) (map_sect_offset
+ map_sect_len
));
774 log_memory_protection_data (buffer
);
777 /* Change the protections on the pages for the section. */
779 start
= get_cycle_count ();
780 result
= mprotect ((void *) map_sect_offset
, map_sect_len
,
782 accumulate_cycle_count (&mprotect_cycles
, start
);
787 snprintf (buffer
, sizeof (buffer
),
788 "Failed called to mprotect for %s error: ",
789 (*mprotect_flags
& PROT_WRITE
) ?
790 "READ/WRITE" : "READ-ONLY");
791 log_memory_protection_data (buffer
);
800 snprintf (buffer
, sizeof (buffer
),
801 "mprotect'ed range [%p, %p]\n",
802 (void *) map_sect_offset
,
803 (char *) map_sect_offset
+ map_sect_len
);
804 log_memory_protection_data (buffer
);
807 increment_num_calls (&num_calls_to_mprotect
);
808 /* num_pages_protected += (map_sect_len + VTV_PAGE_SIZE - 1)
810 num_pages_protected
+= (map_sect_len
+ 4096 - 1) / 4096;
817 CloseHandle(hProcess
);
822 /* This is the callback function used by dl_iterate_phdr (which is
823 called from vtv_unprotect_vtable_vars and vtv_protect_vtable_vars).
824 It attempts to find the binary file on disk for the INFO record
825 that dl_iterate_phdr passes in; open the binary file, and read its
826 section header information. If the file contains a
827 ".vtable_map_vars" section, read the section offset and size. Use
828 the section offset and size, in conjunction with the data in INFO
829 to locate the pages in memory where the section is. Call
830 'mprotect' on those pages, setting the protection either to
831 read-only or read-write, depending on what's in DATA. */
834 dl_iterate_phdr_callback (struct dl_phdr_info
*info
, size_t, void *data
)
836 int * mprotect_flags
= (int *) data
;
837 off_t map_sect_offset
= 0;
838 ElfW (Word
) map_sect_len
= 0;
840 char program_name
[1024];
841 const char *map_sect_name
= VTV_PROTECTED_VARS_SECTION
;
843 /* Check to see if this is the record for the Linux Virtual Dynamic
844 Shared Object (linux-vdso.so.1), which exists only in memory (and
845 therefore cannot be read from disk). */
847 if (strcmp (info
->dlpi_name
, "linux-vdso.so.1") == 0)
850 if (strlen (info
->dlpi_name
) == 0
851 && info
->dlpi_addr
!= 0)
854 /* Get the name of the main executable. This may or may not include
855 arguments passed to the program. Find the first space, assume it
856 is the start of the argument list, and change it to a '\0'. */
857 snprintf (program_name
, sizeof (program_name
), program_invocation_name
);
859 read_section_offset_and_length (info
, map_sect_name
, *mprotect_flags
,
860 &map_sect_offset
, &map_sect_len
);
864 snprintf (buffer
, sizeof(buffer
),
865 " Looking at load module %s to change permissions to %s\n",
866 ((strlen (info
->dlpi_name
) == 0) ? program_name
868 (*mprotect_flags
& PROT_WRITE
) ? "READ/WRITE" : "READ-ONLY");
869 log_memory_protection_data (buffer
);
872 /* See if we actually found the section. */
873 if (map_sect_offset
&& map_sect_len
)
875 unsigned long long start
;
880 snprintf (buffer
, sizeof (buffer
),
881 " (%s): Protecting %p to %p\n",
882 ((strlen (info
->dlpi_name
) == 0) ? program_name
884 (void *) map_sect_offset
,
885 (void *) (map_sect_offset
+ map_sect_len
));
886 log_memory_protection_data (buffer
);
889 /* Change the protections on the pages for the section. */
891 start
= get_cycle_count ();
892 result
= mprotect ((void *) map_sect_offset
, map_sect_len
,
894 accumulate_cycle_count (&mprotect_cycles
, start
);
899 snprintf (buffer
, sizeof (buffer
),
900 "Failed called to mprotect for %s error: ",
901 (*mprotect_flags
& PROT_WRITE
) ?
902 "READ/WRITE" : "READ-ONLY");
903 log_memory_protection_data (buffer
);
912 snprintf (buffer
, sizeof (buffer
),
913 "mprotect'ed range [%p, %p]\n",
914 (void *) map_sect_offset
,
915 (char *) map_sect_offset
+ map_sect_len
);
916 log_memory_protection_data (buffer
);
919 increment_num_calls (&num_calls_to_mprotect
);
920 /* num_pages_protected += (map_sect_len + VTV_PAGE_SIZE - 1) / VTV_PAGE_SIZE; */
921 num_pages_protected
+= (map_sect_len
+ 4096 - 1) / 4096;
928 /* This function explicitly changes the protection (read-only or read-write)
929 on the vtv_sect_info_cache, which is used for speeding up look ups in the
930 function dl_iterate_phdr_callback. This data structure needs to be
931 explicitly made read-write before any calls to dl_iterate_phdr_callback,
932 because otherwise it may still be read-only when dl_iterate_phdr_callback
933 attempts to write to it.
935 More detailed explanation: dl_iterate_phdr_callback finds all the
936 .vtable_map_vars sections in all loaded objects (including the main program)
937 and (depending on where it was called from) either makes all the pages in the
938 sections read-write or read-only. The vtv_sect_info_cache should be in the
939 .vtable_map_vars section for libstdc++.so, which means that normally it would
940 be read-only until libstdc++.so is processed by dl_iterate_phdr_callback
941 (on the read-write pass), after which it will be writable. But if any loaded
942 object gets processed before libstdc++.so, it will attempt to update the
943 data cache, which will still be read-only, and cause a seg fault. Hence
944 we need a special function, called before dl_iterate_phdr_callback, that
945 will make the data cache writable. */
948 change_protections_on_phdr_cache (int protection_flag
)
950 char * low_address
= (char *) &(vtv_sect_info_cache
);
951 size_t cache_size
= MAX_ENTRIES
* sizeof (struct sect_hdr_data
);
953 low_address
= (char *) ((uintptr_t) low_address
& ~(VTV_PAGE_SIZE
- 1));
955 if (mprotect ((void *) low_address
, cache_size
, protection_flag
) == -1)
959 /* Unprotect all the vtable map vars and other side data that is used
960 to keep the core hash_map data. All of these data have been put
961 into relro sections */
964 vtv_unprotect_vtable_vars (void)
968 mprotect_flags
= PROT_READ
| PROT_WRITE
;
969 change_protections_on_phdr_cache (mprotect_flags
);
970 #if defined (__CYGWIN__) || defined (__MINGW32__)
971 iterate_modules ((void *) &mprotect_flags
);
973 dl_iterate_phdr (dl_iterate_phdr_callback
, (void *) &mprotect_flags
);
977 /* Protect all the vtable map vars and other side data that is used
978 to keep the core hash_map data. All of these data have been put
979 into relro sections */
982 vtv_protect_vtable_vars (void)
986 mprotect_flags
= PROT_READ
;
987 #if defined (__CYGWIN__) || defined (__MINGW32__)
988 iterate_modules ((void *) &mprotect_flags
);
990 dl_iterate_phdr (dl_iterate_phdr_callback
, (void *) &mprotect_flags
);
992 change_protections_on_phdr_cache (mprotect_flags
);
995 #ifndef __GTHREAD_MUTEX_INIT
997 initialize_change_permissions_mutexes ()
999 __GTHREAD_MUTEX_INIT_FUNCTION (&change_permissions_lock
);
1003 /* Variables needed for getting the statistics about the hashtable set. */
1005 _AtomicStatCounter stat_contains
= 0;
1006 _AtomicStatCounter stat_insert
= 0;
1007 _AtomicStatCounter stat_resize
= 0;
1008 _AtomicStatCounter stat_create
= 0;
1009 _AtomicStatCounter stat_probes_in_non_trivial_set
= 0;
1010 _AtomicStatCounter stat_contains_size0
= 0;
1011 _AtomicStatCounter stat_contains_size1
= 0;
1012 _AtomicStatCounter stat_contains_size2
= 0;
1013 _AtomicStatCounter stat_contains_size3
= 0;
1014 _AtomicStatCounter stat_contains_size4
= 0;
1015 _AtomicStatCounter stat_contains_size5
= 0;
1016 _AtomicStatCounter stat_contains_size6
= 0;
1017 _AtomicStatCounter stat_contains_size7
= 0;
1018 _AtomicStatCounter stat_contains_size8
= 0;
1019 _AtomicStatCounter stat_contains_size9
= 0;
1020 _AtomicStatCounter stat_contains_size10
= 0;
1021 _AtomicStatCounter stat_contains_size11
= 0;
1022 _AtomicStatCounter stat_contains_size12
= 0;
1023 _AtomicStatCounter stat_contains_size13_or_more
= 0;
1024 _AtomicStatCounter stat_contains_sizes
= 0;
1025 _AtomicStatCounter stat_grow_from_size0_to_1
= 0;
1026 _AtomicStatCounter stat_grow_from_size1_to_2
= 0;
1027 _AtomicStatCounter stat_double_the_number_of_buckets
= 0;
1028 _AtomicStatCounter stat_insert_found_hash_collision
= 0;
1029 _AtomicStatCounter stat_contains_in_non_trivial_set
= 0;
1030 _AtomicStatCounter stat_insert_key_that_was_already_present
= 0;
1032 /* Record statistics about the hash table sets, for debugging. */
1035 log_set_stats (void)
1038 if (set_log_fd
== -1)
1039 set_log_fd
= __vtv_open_log ("vtv_set_stats.log");
1041 __vtv_add_to_log (set_log_fd
, "---\n%s\n",
1042 insert_only_hash_tables_stats().c_str());
1046 /* Change the permissions on all the pages we have allocated for the
1047 data sets and all the ".vtable_map_var" sections in memory (which
1048 contain our vtable map variables). PERM indicates whether to make
1049 the permissions read-only or read-write. */
1051 extern "C" /* This is only being applied to __VLTChangePermission*/
1053 __VLTChangePermission (int perm
)
1055 if (debug_functions
)
1057 if (perm
== __VLTP_READ_WRITE
)
1058 fprintf (stdout
, "Changing VLT permisisons to Read-Write.\n");
1059 else if (perm
== __VLTP_READ_ONLY
)
1060 fprintf (stdout
, "Changing VLT permissions to Read-only.\n");
1063 fprintf (stdout
, "Unrecognized permissions value: %d\n", perm
);
1066 #ifndef __GTHREAD_MUTEX_INIT
1067 static __gthread_once_t mutex_once VTV_PROTECTED_VAR
= __GTHREAD_ONCE_INIT
;
1069 __gthread_once (&mutex_once
, initialize_change_permissions_mutexes
);
1072 /* Ordering of these unprotect/protect calls is very important.
1073 You first need to unprotect all the map vars and side
1074 structures before you do anything with the core data
1075 structures (hash_maps) */
1077 if (perm
== __VLTP_READ_WRITE
)
1079 /* TODO: Need to revisit this code for dlopen. It most probably
1080 is not unlocking the protected vtable vars after for load
1081 module that is not the first load module. */
1082 __gthread_mutex_lock (&change_permissions_lock
);
1084 vtv_unprotect_vtable_vars ();
1085 __vtv_malloc_init ();
1086 __vtv_malloc_unprotect ();
1089 else if (perm
== __VLTP_READ_ONLY
)
1094 __vtv_malloc_protect ();
1095 vtv_protect_vtable_vars ();
1097 __gthread_mutex_unlock (&change_permissions_lock
);
1101 /* This is the memory allocator used to create the hash table that
1102 maps from vtable map variable name to the data set that vtable map
1103 variable should point to. This is part of our vtable map variable
1104 symbol resolution, which is necessary because the same vtable map
1105 variable may be created by multiple compilation units and we need a
1106 method to make sure that all vtable map variables for a particular
1107 class point to the same data set at runtime. */
1109 struct insert_only_hash_map_allocator
1111 /* N is the number of bytes to allocate. */
1113 alloc (size_t n
) const
1115 return __vtv_malloc (n
);
1118 /* P points to the memory to be deallocated; N is the number of
1119 bytes to deallocate. */
1121 dealloc (void *p
, size_t) const
1127 /* Explicitly instantiate this class since this file is compiled with
1128 -fno-implicit-templates. These are for the hash table that is used
1129 to do vtable map variable symbol resolution. */
1130 template class insert_only_hash_map
<vtv_set_handle
*,
1131 insert_only_hash_map_allocator
>;
1132 typedef insert_only_hash_map
<vtv_set_handle
*,
1133 insert_only_hash_map_allocator
> s2s
;
1134 typedef const s2s::key_type vtv_symbol_key
;
1136 static s2s
* vtv_symbol_unification_map VTV_PROTECTED_VAR
= NULL
;
1138 const unsigned long SET_HANDLE_HANDLE_BIT
= 0x2;
1140 /* In the case where a vtable map variable is the only instance of the
1141 variable we have seen, it points directly to the set of valid
1142 vtable pointers. All subsequent instances of the 'same' vtable map
1143 variable point to the first vtable map variable. This function,
1144 given a vtable map variable PTR, checks a bit to see whether it's
1145 pointing directly to the data set or to the first vtable map
1149 is_set_handle_handle (void * ptr
)
1151 return ((uintptr_t) ptr
& SET_HANDLE_HANDLE_BIT
)
1152 == SET_HANDLE_HANDLE_BIT
;
1155 /* Returns the actual pointer value of a vtable map variable, PTR (see
1156 comments for is_set_handle_handle for more details). */
1158 static inline vtv_set_handle
*
1159 ptr_from_set_handle_handle (void * ptr
)
1161 return (vtv_set_handle
*) ((uintptr_t) ptr
& ~SET_HANDLE_HANDLE_BIT
);
1164 /* Given a vtable map variable, PTR, this function sets the bit that
1165 says this is the second (or later) instance of a vtable map
1168 static inline vtv_set_handle_handle
1169 set_handle_handle (vtv_set_handle
* ptr
)
1171 return (vtv_set_handle_handle
) ((uintptr_t) ptr
| SET_HANDLE_HANDLE_BIT
);
1175 register_set_common (void **set_handle_ptr
, size_t num_args
,
1176 void **vtable_ptr_array
, bool debug
)
1178 /* Now figure out what pointer to use for the set pointer, for the
1180 vtv_set_handle
*handle_ptr
= (vtv_set_handle
*) set_handle_ptr
;
1183 VTV_DEBUG_ASSERT (vtv_symbol_unification_map
!= NULL
);
1185 if (!is_set_handle_handle (*set_handle_ptr
))
1186 handle_ptr
= (vtv_set_handle
*) set_handle_ptr
;
1188 handle_ptr
= ptr_from_set_handle_handle (*set_handle_ptr
);
1190 /* Now we've got the set and it's initialized, add the vtable
1192 for (size_t index
= 0; index
< num_args
; ++index
)
1194 int_vptr vtbl_ptr
= (int_vptr
) vtable_ptr_array
[index
];
1195 vtv_sets::insert (vtbl_ptr
, handle_ptr
);
1200 register_pair_common (void **set_handle_ptr
, const void *vtable_ptr
,
1201 const char *set_symbol_name
, const char *vtable_name
,
1204 /* Now we've got the set and it's initialized, add the vtable
1205 pointer (assuming that it's not NULL...It may be NULL, as we may
1206 have called this function merely to initialize the set
1208 int_vptr vtbl_ptr
= (int_vptr
) vtable_ptr
;
1211 vtv_set_handle
*handle_ptr
= (vtv_set_handle
*) set_handle_ptr
;
1213 VTV_DEBUG_ASSERT (vtv_symbol_unification_map
!= NULL
);
1214 if (!is_set_handle_handle (*set_handle_ptr
))
1215 handle_ptr
= (vtv_set_handle
*) set_handle_ptr
;
1217 handle_ptr
= ptr_from_set_handle_handle (*set_handle_ptr
);
1219 vtv_sets::insert (vtbl_ptr
, handle_ptr
);
1222 if (debug
&& debug_init
)
1224 if (init_log_fd
== -1)
1225 init_log_fd
= __vtv_open_log("vtv_init.log");
1227 __vtv_add_to_log(init_log_fd
,
1228 "Registered %s : %s (%p) 2 level deref = %s\n",
1229 set_symbol_name
, vtable_name
, vtbl_ptr
,
1230 is_set_handle_handle(*set_handle_ptr
) ? "yes" : "no" );
1234 /* This routine initializes a set handle to a vtable set. It makes
1235 sure that there is only one set handle for a particular set by
1236 using a map from set name to pointer to set handle. Since there
1237 will be multiple copies of the pointer to the set handle (one per
1238 compilation unit that uses it), it makes sure to initialize all the
1239 pointers to the set handle so that the set handle is unique. To
1240 make this a little more efficient and avoid a level of indirection
1241 in some cases, the first pointer to handle for a particular handle
1242 becomes the handle itself and the other pointers will point to the
1243 set handle. This is the debug version of this function, so it
1244 outputs extra debugging messages and logging. SET_HANDLE_PTR is
1245 the address of the vtable map variable, SET_SYMBOL_KEY is the hash
1246 table key (containing the name of the map variable and the hash
1247 value) and SIZE_HINT is a guess for the best initial size for the
1248 set of vtable pointers that SET_HANDLE_POINTER will point to. */
1251 init_set_symbol_debug (void **set_handle_ptr
, const void *set_symbol_key
,
1254 VTV_DEBUG_ASSERT (set_handle_ptr
);
1256 if (vtv_symbol_unification_map
== NULL
)
1258 /* TODO: For now we have chosen 1024, but we need to come up with a
1259 better initial size for this. */
1260 vtv_symbol_unification_map
= s2s::create (1024);
1261 VTV_DEBUG_ASSERT(vtv_symbol_unification_map
);
1264 vtv_set_handle
*handle_ptr
= (vtv_set_handle
*) set_handle_ptr
;
1265 vtv_symbol_key
*symbol_key_ptr
= (vtv_symbol_key
*) set_symbol_key
;
1267 const s2s::value_type
* map_value_ptr
=
1268 vtv_symbol_unification_map
->get (symbol_key_ptr
);
1270 if (map_value_ptr
== NULL
)
1272 if (*handle_ptr
!= NULL
)
1274 snprintf (buffer
, sizeof (buffer
),
1275 "*** Found non-NULL local set ptr %p missing for symbol"
1277 *handle_ptr
, symbol_key_ptr
->n
, symbol_key_ptr
->bytes
);
1278 __vtv_log_verification_failure (buffer
, true);
1279 VTV_DEBUG_ASSERT (0);
1282 else if (*handle_ptr
!= NULL
&&
1283 (handle_ptr
!= *map_value_ptr
&&
1284 ptr_from_set_handle_handle (*handle_ptr
) != *map_value_ptr
))
1286 VTV_DEBUG_ASSERT (*map_value_ptr
!= NULL
);
1287 snprintf (buffer
, sizeof(buffer
),
1288 "*** Found diffence between local set ptr %p and set ptr %p"
1290 *handle_ptr
, *map_value_ptr
,
1291 symbol_key_ptr
->n
, symbol_key_ptr
->bytes
);
1292 __vtv_log_verification_failure (buffer
, true);
1293 VTV_DEBUG_ASSERT (0);
1295 else if (*handle_ptr
== NULL
)
1297 /* Execution should not reach this point. */
1300 if (*handle_ptr
!= NULL
)
1302 if (!is_set_handle_handle (*set_handle_ptr
))
1303 handle_ptr
= (vtv_set_handle
*) set_handle_ptr
;
1305 handle_ptr
= ptr_from_set_handle_handle (*set_handle_ptr
);
1306 vtv_sets::resize (size_hint
, handle_ptr
);
1310 VTV_DEBUG_ASSERT (*handle_ptr
== NULL
);
1311 if (map_value_ptr
!= NULL
)
1313 if (*map_value_ptr
== handle_ptr
)
1314 vtv_sets::resize (size_hint
, *map_value_ptr
);
1317 /* The one level handle to the set already exists. So, we
1318 are adding one level of indirection here and we will
1319 store a pointer to the one level handle here. */
1321 vtv_set_handle_handle
* handle_handle_ptr
=
1322 (vtv_set_handle_handle
*)handle_ptr
;
1323 *handle_handle_ptr
= set_handle_handle(*map_value_ptr
);
1324 VTV_DEBUG_ASSERT(*handle_handle_ptr
!= NULL
);
1326 /* The handle can itself be NULL if the set has only
1327 been initiazlied with size hint == 1. */
1328 vtv_sets::resize (size_hint
, *map_value_ptr
);
1333 /* We will create a new set. So, in this case handle_ptr is the
1334 one level pointer to the set handle. Create copy of map name
1335 in case the memory where this comes from gets unmapped by
1337 size_t map_key_len
= symbol_key_ptr
->n
+ sizeof (vtv_symbol_key
);
1338 void *map_key
= __vtv_malloc (map_key_len
);
1340 memcpy (map_key
, symbol_key_ptr
, map_key_len
);
1342 s2s::value_type
*value_ptr
;
1343 vtv_symbol_unification_map
=
1344 vtv_symbol_unification_map
->find_or_add_key ((vtv_symbol_key
*)map_key
,
1346 *value_ptr
= handle_ptr
;
1348 /* TODO: We should verify the return value. */
1349 vtv_sets::create (size_hint
, handle_ptr
);
1350 VTV_DEBUG_ASSERT (size_hint
<= 1 || *handle_ptr
!= NULL
);
1355 if (init_log_fd
== -1)
1356 init_log_fd
= __vtv_open_log ("vtv_init.log");
1358 __vtv_add_to_log (init_log_fd
,
1359 "Init handle:%p for symbol:%.*s hash:%u size_hint:%lu"
1360 "number of symbols:%lu \n",
1361 set_handle_ptr
, symbol_key_ptr
->n
,
1362 symbol_key_ptr
->bytes
, symbol_key_ptr
->hash
, size_hint
,
1363 vtv_symbol_unification_map
->size ());
1368 /* This routine initializes a set handle to a vtable set. It makes
1369 sure that there is only one set handle for a particular set by
1370 using a map from set name to pointer to set handle. Since there
1371 will be multiple copies of the pointer to the set handle (one per
1372 compilation unit that uses it), it makes sure to initialize all the
1373 pointers to the set handle so that the set handle is unique. To
1374 make this a little more efficient and avoid a level of indirection
1375 in some cases, the first pointer to handle for a particular handle
1376 becomes the handle itself and the other pointers will point to the
1377 set handle. This is the debug version of this function, so it
1378 outputs extra debugging messages and logging. SET_HANDLE_PTR is
1379 the address of the vtable map variable, SET_SYMBOL_KEY is the hash
1380 table key (containing the name of the map variable and the hash
1381 value) and SIZE_HINT is a guess for the best initial size for the
1382 set of vtable pointers that SET_HANDLE_POINTER will point to. */
1385 __VLTRegisterSetDebug (void **set_handle_ptr
, const void *set_symbol_key
,
1386 size_t size_hint
, size_t num_args
,
1387 void **vtable_ptr_array
)
1389 unsigned long long start
= get_cycle_count ();
1390 increment_num_calls (&num_calls_to_regset
);
1392 VTV_DEBUG_ASSERT(set_handle_ptr
!= NULL
);
1393 init_set_symbol_debug (set_handle_ptr
, set_symbol_key
, size_hint
);
1395 register_set_common (set_handle_ptr
, num_args
, vtable_ptr_array
, true);
1397 accumulate_cycle_count (®set_cycles
, start
);
1400 /* This function takes a the address of a vtable map variable
1401 (SET_HANDLE_PTR), a VTABLE_PTR to add to the data set, the name of
1402 the vtable map variable (SET_SYMBOL_NAME) and the name of the
1403 vtable (VTABLE_NAME) being pointed to. If the vtable map variable
1404 is NULL it creates a new data set and initializes the variable,
1405 otherwise it uses our symbol unification to find the right data
1406 set; in either case it then adds the vtable pointer to the set.
1407 The other two parameters are used for debugging information. */
1410 __VLTRegisterPairDebug (void **set_handle_ptr
, const void *set_symbol_key
,
1411 size_t size_hint
, const void *vtable_ptr
,
1412 const char *set_symbol_name
, const char *vtable_name
)
1414 unsigned long long start
= get_cycle_count ();
1415 increment_num_calls (&num_calls_to_regpair
);
1417 VTV_DEBUG_ASSERT(set_handle_ptr
!= NULL
);
1418 init_set_symbol_debug (set_handle_ptr
, set_symbol_key
, size_hint
);
1420 register_pair_common (set_handle_ptr
, vtable_ptr
, set_symbol_name
, vtable_name
,
1423 accumulate_cycle_count (®pair_cycles
, start
);
1427 /* This is the debug version of the verification function. It takes
1428 the address of a vtable map variable (SET_HANDLE_PTR) and a
1429 VTABLE_PTR to validate, as well as the name of the vtable map
1430 variable (SET_SYMBOL_NAME) and VTABLE_NAME, which are used for
1431 debugging messages. It checks to see if VTABLE_PTR is in the set
1432 pointed to by SET_HANDLE_PTR. If so, it returns VTABLE_PTR,
1433 otherwise it calls __vtv_verify_fail, which usually logs error
1434 messages and calls abort. */
1437 __VLTVerifyVtablePointerDebug (void **set_handle_ptr
, const void *vtable_ptr
,
1438 const char *set_symbol_name
,
1439 const char *vtable_name
)
1441 unsigned long long start
= get_cycle_count ();
1442 VTV_DEBUG_ASSERT (set_handle_ptr
!= NULL
&& *set_handle_ptr
!= NULL
);
1443 int_vptr vtbl_ptr
= (int_vptr
) vtable_ptr
;
1445 increment_num_calls (&num_calls_to_verify_vtable
);
1446 vtv_set_handle
*handle_ptr
;
1447 if (!is_set_handle_handle (*set_handle_ptr
))
1448 handle_ptr
= (vtv_set_handle
*) set_handle_ptr
;
1450 handle_ptr
= ptr_from_set_handle_handle (*set_handle_ptr
);
1452 if (vtv_sets::contains (vtbl_ptr
, handle_ptr
))
1454 if (debug_verify_vtable
)
1456 if (verify_vtable_log_fd
== -1)
1457 __vtv_open_log ("vtv_verify_vtable.log");
1458 __vtv_add_to_log (verify_vtable_log_fd
,
1459 "Verified %s %s value = %p\n",
1460 set_symbol_name
, vtable_name
, vtable_ptr
);
1465 /* We failed to find the vtable pointer in the set of valid
1466 pointers. Log the error data and call the failure
1468 snprintf (debug_log_message
, sizeof (debug_log_message
),
1469 "Looking for %s in %s\n", vtable_name
, set_symbol_name
);
1470 __vtv_verify_fail_debug (set_handle_ptr
, vtable_ptr
, debug_log_message
);
1472 /* Normally __vtv_verify_fail_debug will call abort, so we won't
1473 execute the return below. If we get this far, the assumption
1474 is that the programmer has replaced __vtv_verify_fail_debug
1475 with some kind of secondary verification AND this secondary
1476 verification succeeded, so the vtable pointer is valid. */
1478 accumulate_cycle_count (&verify_vtable_cycles
, start
);
1483 /* This routine initializes a set handle to a vtable set. It makes
1484 sure that there is only one set handle for a particular set by
1485 using a map from set name to pointer to set handle. Since there
1486 will be multiple copies of the pointer to the set handle (one per
1487 compilation unit that uses it), it makes sure to initialize all the
1488 pointers to the set handle so that the set handle is unique. To
1489 make this a little more efficient and avoid a level of indirection
1490 in some cases, the first pointer to handle for a particular handle
1491 becomes the handle itself and the other pointers will point to the
1492 set handle. SET_HANDLE_PTR is the address of the vtable map
1493 variable, SET_SYMBOL_KEY is the hash table key (containing the name
1494 of the map variable and the hash value) and SIZE_HINT is a guess
1495 for the best initial size for the set of vtable pointers that
1496 SET_HANDLE_POINTER will point to.*/
1499 init_set_symbol (void **set_handle_ptr
, const void *set_symbol_key
,
1502 vtv_set_handle
*handle_ptr
= (vtv_set_handle
*) set_handle_ptr
;
1504 if (*handle_ptr
!= NULL
)
1506 if (!is_set_handle_handle (*set_handle_ptr
))
1507 handle_ptr
= (vtv_set_handle
*) set_handle_ptr
;
1509 handle_ptr
= ptr_from_set_handle_handle (*set_handle_ptr
);
1510 vtv_sets::resize (size_hint
, handle_ptr
);
1514 if (vtv_symbol_unification_map
== NULL
)
1515 vtv_symbol_unification_map
= s2s::create (1024);
1517 vtv_symbol_key
*symbol_key_ptr
= (vtv_symbol_key
*) set_symbol_key
;
1518 const s2s::value_type
*map_value_ptr
=
1519 vtv_symbol_unification_map
->get (symbol_key_ptr
);
1521 if (map_value_ptr
!= NULL
)
1523 if (*map_value_ptr
== handle_ptr
)
1524 vtv_sets::resize (size_hint
, *map_value_ptr
);
1527 /* The one level handle to the set already exists. So, we
1528 are adding one level of indirection here and we will
1529 store a pointer to the one level pointer here. */
1530 vtv_set_handle_handle
*handle_handle_ptr
=
1531 (vtv_set_handle_handle
*) handle_ptr
;
1532 *handle_handle_ptr
= set_handle_handle (*map_value_ptr
);
1533 vtv_sets::resize (size_hint
, *map_value_ptr
);
1538 /* We will create a new set. So, in this case handle_ptr is the
1539 one level pointer to the set handle. Create copy of map name
1540 in case the memory where this comes from gets unmapped by
1542 size_t map_key_len
= symbol_key_ptr
->n
+ sizeof (vtv_symbol_key
);
1543 void * map_key
= __vtv_malloc (map_key_len
);
1544 memcpy (map_key
, symbol_key_ptr
, map_key_len
);
1546 s2s::value_type
* value_ptr
;
1547 vtv_symbol_unification_map
=
1548 vtv_symbol_unification_map
->find_or_add_key ((vtv_symbol_key
*)map_key
,
1551 *value_ptr
= handle_ptr
;
1553 /* TODO: We should verify the return value. */
1554 vtv_sets::create (size_hint
, handle_ptr
);
1558 /* This routine initializes a set handle to a vtable set. It makes
1559 sure that there is only one set handle for a particular set by
1560 using a map from set name to pointer to set handle. Since there
1561 will be multiple copies of the pointer to the set handle (one per
1562 compilation unit that uses it), it makes sure to initialize all the
1563 pointers to the set handle so that the set handle is unique. To
1564 make this a little more efficient and avoid a level of indirection
1565 in some cases, the first pointer to handle for a particular handle
1566 becomes the handle itself and the other pointers will point to the
1567 set handle. SET_HANDLE_PTR is the address of the vtable map
1568 variable, SET_SYMBOL_KEY is the hash table key (containing the name
1569 of the map variable and the hash value) and SIZE_HINT is a guess
1570 for the best initial size for the set of vtable pointers that
1571 SET_HANDLE_POINTER will point to.*/
1575 __VLTRegisterSet (void **set_handle_ptr
, const void *set_symbol_key
,
1576 size_t size_hint
, size_t num_args
, void **vtable_ptr_array
)
1578 unsigned long long start
= get_cycle_count ();
1579 increment_num_calls (&num_calls_to_regset
);
1581 init_set_symbol (set_handle_ptr
, set_symbol_key
, size_hint
);
1582 register_set_common (set_handle_ptr
, num_args
, vtable_ptr_array
, false);
1584 accumulate_cycle_count (®set_cycles
, start
);
1589 /* This function takes a the address of a vtable map variable
1590 (SET_HANDLE_PTR) and a VTABLE_PTR. If the vtable map variable is
1591 NULL it creates a new data set and initializes the variable,
1592 otherwise it uses our symbol unification to find the right data
1593 set; in either case it then adds the vtable pointer to the set. */
1596 __VLTRegisterPair (void **set_handle_ptr
, const void *set_symbol_key
,
1597 size_t size_hint
, const void *vtable_ptr
)
1599 unsigned long long start
= get_cycle_count ();
1600 increment_num_calls (&num_calls_to_regpair
);
1602 init_set_symbol (set_handle_ptr
, set_symbol_key
, size_hint
);
1603 register_pair_common (set_handle_ptr
, vtable_ptr
, NULL
, NULL
, false);
1605 accumulate_cycle_count (®pair_cycles
, start
);
1608 /* This is the main verification function. It takes the address of a
1609 vtable map variable (SET_HANDLE_PTR) and a VTABLE_PTR to validate.
1610 It checks to see if VTABLE_PTR is in the set pointed to by
1611 SET_HANDLE_PTR. If so, it returns VTABLE_PTR, otherwise it calls
1612 __vtv_verify_fail, which usually logs error messages and calls
1613 abort. Since this function gets called VERY frequently, it is
1614 important for it to be as efficient as possible. */
1617 __VLTVerifyVtablePointer (void ** set_handle_ptr
, const void * vtable_ptr
)
1619 unsigned long long start
= get_cycle_count ();
1620 int_vptr vtbl_ptr
= (int_vptr
) vtable_ptr
;
1622 vtv_set_handle
*handle_ptr
;
1623 increment_num_calls (&num_calls_to_verify_vtable
);
1624 if (!is_set_handle_handle (*set_handle_ptr
))
1625 handle_ptr
= (vtv_set_handle
*) set_handle_ptr
;
1627 handle_ptr
= ptr_from_set_handle_handle (*set_handle_ptr
);
1629 if (!vtv_sets::contains (vtbl_ptr
, handle_ptr
))
1631 __vtv_verify_fail ((void **) handle_ptr
, vtable_ptr
);
1632 /* Normally __vtv_verify_fail will call abort, so we won't
1633 execute the return below. If we get this far, the assumption
1634 is that the programmer has replaced __vtv_verify_fail with
1635 some kind of secondary verification AND this secondary
1636 verification succeeded, so the vtable pointer is valid. */
1638 accumulate_cycle_count (&verify_vtable_cycles
, start
);
1643 static int page_count_2
= 0;
1645 #if !defined (__CYGWIN__) && !defined (__MINGW32__)
1647 dl_iterate_phdr_count_pages (struct dl_phdr_info
*info
,
1648 size_t unused
__attribute__ ((__unused__
)),
1651 int *mprotect_flags
= (int *) data
;
1652 off_t map_sect_offset
= 0;
1653 ElfW (Word
) map_sect_len
= 0;
1654 const char *map_sect_name
= VTV_PROTECTED_VARS_SECTION
;
1656 /* Check to see if this is the record for the Linux Virtual Dynamic
1657 Shared Object (linux-vdso.so.1), which exists only in memory (and
1658 therefore cannot be read from disk). */
1660 if (strcmp (info
->dlpi_name
, "linux-vdso.so.1") == 0)
1663 if (strlen (info
->dlpi_name
) == 0
1664 && info
->dlpi_addr
!= 0)
1667 read_section_offset_and_length (info
, map_sect_name
, *mprotect_flags
,
1668 &map_sect_offset
, &map_sect_len
);
1670 /* See if we actually found the section. */
1672 page_count_2
+= (map_sect_len
+ VTV_PAGE_SIZE
- 1) / VTV_PAGE_SIZE
;
1679 count_all_pages (void)
1683 mprotect_flags
= PROT_READ
;
1686 #if defined (__CYGWIN__) || defined (__MINGW32__)
1687 iterate_modules ((void *) &mprotect_flags
);
1689 dl_iterate_phdr (dl_iterate_phdr_count_pages
, (void *) &mprotect_flags
);
1691 page_count_2
+= __vtv_count_mmapped_pages ();
1695 __VLTDumpStats (void)
1697 int log_fd
= __vtv_open_log ("vtv-runtime-stats.log");
1702 __vtv_add_to_log (log_fd
,
1703 "Calls: mprotect (%d) regset (%d) regpair (%d)"
1704 " verify_vtable (%d)\n",
1705 num_calls_to_mprotect
, num_calls_to_regset
,
1706 num_calls_to_regpair
, num_calls_to_verify_vtable
);
1707 __vtv_add_to_log (log_fd
,
1708 "Cycles: mprotect (%lld) regset (%lld) "
1709 "regpair (%lld) verify_vtable (%lld)\n",
1710 mprotect_cycles
, regset_cycles
, regpair_cycles
,
1711 verify_vtable_cycles
);
1712 __vtv_add_to_log (log_fd
,
1713 "Pages protected (1): %d\n", num_pages_protected
);
1714 __vtv_add_to_log (log_fd
, "Pages protected (2): %d\n", page_count_2
);
1720 /* This function is called from __VLTVerifyVtablePointerDebug; it
1721 sends as much debugging information as it can to the error log
1722 file, then calls __vtv_verify_fail. SET_HANDLE_PTR is the pointer
1723 to the set of valid vtable pointers, VTBL_PTR is the pointer that
1724 was not found in the set, and DEBUG_MSG is the message to be
1725 written to the log file before failing. n */
1728 __vtv_verify_fail_debug (void **set_handle_ptr
, const void *vtbl_ptr
,
1729 const char *debug_msg
)
1731 __vtv_log_verification_failure (debug_msg
, false);
1733 /* Call the public interface in case it has been overwritten by
1735 __vtv_verify_fail (set_handle_ptr
, vtbl_ptr
);
1737 __vtv_log_verification_failure ("Returned from __vtv_verify_fail."
1738 " Secondary verification succeeded.\n", false);
1741 /* This function calls __fortify_fail with a FAILURE_MSG and then
1745 __vtv_really_fail (const char *failure_msg
)
1747 __fortify_fail (failure_msg
);
1749 /* We should never get this far; __fortify_fail calls __libc_message
1750 which prints out a back trace and a memory dump and then is
1751 supposed to call abort, but let's play it safe anyway and call abort
1756 /* This function takes an error MSG, a vtable map variable
1757 (DATA_SET_PTR) and a vtable pointer (VTBL_PTR). It is called when
1758 an attempt to verify VTBL_PTR with the set pointed to by
1759 DATA_SET_PTR failed. It outputs a failure message with the
1760 addresses involved, and calls __vtv_really_fail. */
1763 vtv_fail (const char *msg
, void **data_set_ptr
, const void *vtbl_ptr
)
1767 const char *format_str
=
1768 "*** Unable to verify vtable pointer (%p) in set (%p) *** \n";
1770 snprintf (buffer
, sizeof (buffer
), format_str
, vtbl_ptr
,
1771 is_set_handle_handle(*data_set_ptr
) ?
1772 ptr_from_set_handle_handle (*data_set_ptr
) :
1774 buf_len
= strlen (buffer
);
1775 /* Send this to to stderr. */
1776 write (2, buffer
, buf_len
);
1778 #ifndef VTV_NO_ABORT
1779 __vtv_really_fail (msg
);
1783 /* Send information about what we were trying to do when verification
1784 failed to the error log, then call vtv_fail. This function can be
1785 overwritten/replaced by the user, to implement a secondary
1786 verification function instead. DATA_SET_PTR is the vtable map
1787 variable used for the failed verification, and VTBL_PTR is the
1788 vtable pointer that was not found in the set. */
1791 __vtv_verify_fail (void **data_set_ptr
, const void *vtbl_ptr
)
1794 snprintf (log_msg
, sizeof (log_msg
), "Looking for vtable %p in set %p.\n",
1796 is_set_handle_handle (*data_set_ptr
) ?
1797 ptr_from_set_handle_handle (*data_set_ptr
) :
1799 __vtv_log_verification_failure (log_msg
, false);
1801 const char *format_str
=
1802 "*** Unable to verify vtable pointer (%p) in set (%p) *** \n";
1803 snprintf (log_msg
, sizeof (log_msg
), format_str
, vtbl_ptr
, *data_set_ptr
);
1804 __vtv_log_verification_failure (log_msg
, false);
1805 __vtv_log_verification_failure (" Backtrace: \n", true);
1807 const char *fail_msg
= "Potential vtable pointer corruption detected!!\n";
1808 vtv_fail (fail_msg
, data_set_ptr
, vtbl_ptr
);