Account for prologue spills in reg_pressure scheduling
[official-gcc.git] / libsanitizer / sanitizer_common / sanitizer_coverage_libcdep.cc
blob9b02cd75017ce35a08c020e1ff9151ec02d7e566
1 //===-- sanitizer_coverage.cc ---------------------------------------------===//
2 //
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
5 //
6 //===----------------------------------------------------------------------===//
7 //
8 // Sanitizer Coverage.
9 // This file implements run-time support for a poor man's coverage tool.
11 // Compiler instrumentation:
12 // For every interesting basic block the compiler injects the following code:
13 // if (*Guard) {
14 // __sanitizer_cov();
15 // *Guard = 1;
16 // }
17 // It's fine to call __sanitizer_cov more than once for a given block.
19 // Run-time:
20 // - __sanitizer_cov(): record that we've executed the PC (GET_CALLER_PC).
21 // - __sanitizer_cov_dump: dump the coverage data to disk.
22 // For every module of the current process that has coverage data
23 // this will create a file module_name.PID.sancov. The file format is simple:
24 // it's just a sorted sequence of 4-byte offsets in the module.
26 // Eventually, this coverage implementation should be obsoleted by a more
27 // powerful general purpose Clang/LLVM coverage instrumentation.
28 // Consider this implementation as prototype.
30 // FIXME: support (or at least test with) dlclose.
31 //===----------------------------------------------------------------------===//
33 #include "sanitizer_allocator_internal.h"
34 #include "sanitizer_common.h"
35 #include "sanitizer_libc.h"
36 #include "sanitizer_mutex.h"
37 #include "sanitizer_procmaps.h"
38 #include "sanitizer_stacktrace.h"
39 #include "sanitizer_flags.h"
41 atomic_uint32_t dump_once_guard; // Ensure that CovDump runs only once.
43 // pc_array is the array containing the covered PCs.
44 // To make the pc_array thread- and async-signal-safe it has to be large enough.
45 // 128M counters "ought to be enough for anybody" (4M on 32-bit).
47 // With coverage_direct=1 in ASAN_OPTIONS, pc_array memory is mapped to a file.
48 // In this mode, __sanitizer_cov_dump does nothing, and CovUpdateMapping()
49 // dump current memory layout to another file.
51 static bool cov_sandboxed = false;
52 static int cov_fd = kInvalidFd;
53 static unsigned int cov_max_block_size = 0;
55 namespace __sanitizer {
57 class CoverageData {
58 public:
59 void Init();
60 void BeforeFork();
61 void AfterFork(int child_pid);
62 void Extend(uptr npcs);
63 void Add(uptr pc);
65 uptr *data();
66 uptr size();
68 private:
69 // Maximal size pc array may ever grow.
70 // We MmapNoReserve this space to ensure that the array is contiguous.
71 static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 27);
72 // The amount file mapping for the pc array is grown by.
73 static const uptr kPcArrayMmapSize = 64 * 1024;
75 // pc_array is allocated with MmapNoReserveOrDie and so it uses only as
76 // much RAM as it really needs.
77 uptr *pc_array;
78 // Index of the first available pc_array slot.
79 atomic_uintptr_t pc_array_index;
80 // Array size.
81 atomic_uintptr_t pc_array_size;
82 // Current file mapped size of the pc array.
83 uptr pc_array_mapped_size;
84 // Descriptor of the file mapped pc array.
85 int pc_fd;
86 StaticSpinMutex mu;
88 void DirectOpen();
89 void ReInit();
92 static CoverageData coverage_data;
94 void CoverageData::DirectOpen() {
95 InternalScopedString path(1024);
96 internal_snprintf((char *)path.data(), path.size(), "%s/%zd.sancov.raw",
97 common_flags()->coverage_dir, internal_getpid());
98 pc_fd = OpenFile(path.data(), true);
99 if (internal_iserror(pc_fd)) {
100 Report(" Coverage: failed to open %s for writing\n", path.data());
101 Die();
104 pc_array_mapped_size = 0;
105 CovUpdateMapping();
108 void CoverageData::Init() {
109 pc_array = reinterpret_cast<uptr *>(
110 MmapNoReserveOrDie(sizeof(uptr) * kPcArrayMaxSize, "CovInit"));
111 pc_fd = kInvalidFd;
112 if (common_flags()->coverage_direct) {
113 atomic_store(&pc_array_size, 0, memory_order_relaxed);
114 atomic_store(&pc_array_index, 0, memory_order_relaxed);
115 } else {
116 atomic_store(&pc_array_size, kPcArrayMaxSize, memory_order_relaxed);
117 atomic_store(&pc_array_index, 0, memory_order_relaxed);
121 void CoverageData::ReInit() {
122 internal_munmap(pc_array, sizeof(uptr) * kPcArrayMaxSize);
123 if (pc_fd != kInvalidFd) internal_close(pc_fd);
124 if (common_flags()->coverage_direct) {
125 // In memory-mapped mode we must extend the new file to the known array
126 // size.
127 uptr size = atomic_load(&pc_array_size, memory_order_relaxed);
128 Init();
129 if (size) Extend(size);
130 } else {
131 Init();
135 void CoverageData::BeforeFork() {
136 mu.Lock();
139 void CoverageData::AfterFork(int child_pid) {
140 // We are single-threaded so it's OK to release the lock early.
141 mu.Unlock();
142 if (child_pid == 0) ReInit();
145 // Extend coverage PC array to fit additional npcs elements.
146 void CoverageData::Extend(uptr npcs) {
147 if (!common_flags()->coverage_direct) return;
148 SpinMutexLock l(&mu);
150 if (pc_fd == kInvalidFd) DirectOpen();
151 CHECK_NE(pc_fd, kInvalidFd);
153 uptr size = atomic_load(&pc_array_size, memory_order_relaxed);
154 size += npcs * sizeof(uptr);
156 if (size > pc_array_mapped_size) {
157 uptr new_mapped_size = pc_array_mapped_size;
158 while (size > new_mapped_size) new_mapped_size += kPcArrayMmapSize;
160 // Extend the file and map the new space at the end of pc_array.
161 uptr res = internal_ftruncate(pc_fd, new_mapped_size);
162 int err;
163 if (internal_iserror(res, &err)) {
164 Printf("failed to extend raw coverage file: %d\n", err);
165 Die();
167 void *p = MapWritableFileToMemory(pc_array + pc_array_mapped_size,
168 new_mapped_size - pc_array_mapped_size,
169 pc_fd, pc_array_mapped_size);
170 CHECK_EQ(p, pc_array + pc_array_mapped_size);
171 pc_array_mapped_size = new_mapped_size;
174 atomic_store(&pc_array_size, size, memory_order_release);
177 // Simply add the pc into the vector under lock. If the function is called more
178 // than once for a given PC it will be inserted multiple times, which is fine.
179 void CoverageData::Add(uptr pc) {
180 if (!pc_array) return;
181 uptr idx = atomic_fetch_add(&pc_array_index, 1, memory_order_relaxed);
182 CHECK_LT(idx * sizeof(uptr),
183 atomic_load(&pc_array_size, memory_order_acquire));
184 pc_array[idx] = pc;
187 uptr *CoverageData::data() {
188 return pc_array;
191 uptr CoverageData::size() {
192 return atomic_load(&pc_array_index, memory_order_relaxed);
195 // Block layout for packed file format: header, followed by module name (no
196 // trailing zero), followed by data blob.
197 struct CovHeader {
198 int pid;
199 unsigned int module_name_length;
200 unsigned int data_length;
203 static void CovWritePacked(int pid, const char *module, const void *blob,
204 unsigned int blob_size) {
205 if (cov_fd < 0) return;
206 unsigned module_name_length = internal_strlen(module);
207 CovHeader header = {pid, module_name_length, blob_size};
209 if (cov_max_block_size == 0) {
210 // Writing to a file. Just go ahead.
211 internal_write(cov_fd, &header, sizeof(header));
212 internal_write(cov_fd, module, module_name_length);
213 internal_write(cov_fd, blob, blob_size);
214 } else {
215 // Writing to a socket. We want to split the data into appropriately sized
216 // blocks.
217 InternalScopedBuffer<char> block(cov_max_block_size);
218 CHECK_EQ((uptr)block.data(), (uptr)(CovHeader *)block.data());
219 uptr header_size_with_module = sizeof(header) + module_name_length;
220 CHECK_LT(header_size_with_module, cov_max_block_size);
221 unsigned int max_payload_size =
222 cov_max_block_size - header_size_with_module;
223 char *block_pos = block.data();
224 internal_memcpy(block_pos, &header, sizeof(header));
225 block_pos += sizeof(header);
226 internal_memcpy(block_pos, module, module_name_length);
227 block_pos += module_name_length;
228 char *block_data_begin = block_pos;
229 char *blob_pos = (char *)blob;
230 while (blob_size > 0) {
231 unsigned int payload_size = Min(blob_size, max_payload_size);
232 blob_size -= payload_size;
233 internal_memcpy(block_data_begin, blob_pos, payload_size);
234 blob_pos += payload_size;
235 ((CovHeader *)block.data())->data_length = payload_size;
236 internal_write(cov_fd, block.data(),
237 header_size_with_module + payload_size);
242 // If packed = false: <name>.<pid>.<sancov> (name = module name).
243 // If packed = true and name == 0: <pid>.<sancov>.<packed>.
244 // If packed = true and name != 0: <name>.<sancov>.<packed> (name is
245 // user-supplied).
246 static int CovOpenFile(bool packed, const char* name) {
247 InternalScopedBuffer<char> path(1024);
248 if (!packed) {
249 CHECK(name);
250 internal_snprintf((char *)path.data(), path.size(), "%s/%s.%zd.sancov",
251 common_flags()->coverage_dir, name, internal_getpid());
252 } else {
253 if (!name)
254 internal_snprintf((char *)path.data(), path.size(),
255 "%s/%zd.sancov.packed", common_flags()->coverage_dir,
256 internal_getpid());
257 else
258 internal_snprintf((char *)path.data(), path.size(), "%s/%s.sancov.packed",
259 common_flags()->coverage_dir, name);
261 uptr fd = OpenFile(path.data(), true);
262 if (internal_iserror(fd)) {
263 Report(" SanitizerCoverage: failed to open %s for writing\n", path.data());
264 return -1;
266 return fd;
269 // Dump the coverage on disk.
270 static void CovDump() {
271 if (!common_flags()->coverage || common_flags()->coverage_direct) return;
272 #if !SANITIZER_WINDOWS
273 if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed))
274 return;
275 uptr size = coverage_data.size();
276 InternalMmapVector<u32> offsets(size);
277 uptr *vb = coverage_data.data();
278 uptr *ve = vb + size;
279 SortArray(vb, size);
280 MemoryMappingLayout proc_maps(/*cache_enabled*/true);
281 uptr mb, me, off, prot;
282 InternalScopedBuffer<char> module(4096);
283 InternalScopedBuffer<char> path(4096 * 2);
284 for (int i = 0;
285 proc_maps.Next(&mb, &me, &off, module.data(), module.size(), &prot);
286 i++) {
287 if ((prot & MemoryMappingLayout::kProtectionExecute) == 0)
288 continue;
289 while (vb < ve && *vb < mb) vb++;
290 if (vb >= ve) break;
291 if (*vb < me) {
292 offsets.clear();
293 const uptr *old_vb = vb;
294 CHECK_LE(off, *vb);
295 for (; vb < ve && *vb < me; vb++) {
296 uptr diff = *vb - (i ? mb : 0) + off;
297 CHECK_LE(diff, 0xffffffffU);
298 offsets.push_back(static_cast<u32>(diff));
300 char *module_name = StripModuleName(module.data());
301 if (cov_sandboxed) {
302 if (cov_fd >= 0) {
303 CovWritePacked(internal_getpid(), module_name, offsets.data(),
304 offsets.size() * sizeof(u32));
305 VReport(1, " CovDump: %zd PCs written to packed file\n", vb - old_vb);
307 } else {
308 // One file per module per process.
309 internal_snprintf((char *)path.data(), path.size(), "%s/%s.%zd.sancov",
310 common_flags()->coverage_dir, module_name,
311 internal_getpid());
312 int fd = CovOpenFile(false /* packed */, module_name);
313 if (fd > 0) {
314 internal_write(fd, offsets.data(), offsets.size() * sizeof(u32));
315 internal_close(fd);
316 VReport(1, " CovDump: %s: %zd PCs written\n", path.data(),
317 vb - old_vb);
320 InternalFree(module_name);
323 if (cov_fd >= 0)
324 internal_close(cov_fd);
325 #endif // !SANITIZER_WINDOWS
328 void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
329 if (!args) return;
330 if (!common_flags()->coverage) return;
331 cov_sandboxed = args->coverage_sandboxed;
332 if (!cov_sandboxed) return;
333 cov_fd = args->coverage_fd;
334 cov_max_block_size = args->coverage_max_block_size;
335 if (cov_fd < 0)
336 // Pre-open the file now. The sandbox won't allow us to do it later.
337 cov_fd = CovOpenFile(true /* packed */, 0);
340 int MaybeOpenCovFile(const char *name) {
341 CHECK(name);
342 if (!common_flags()->coverage) return -1;
343 return CovOpenFile(true /* packed */, name);
346 void CovBeforeFork() {
347 coverage_data.BeforeFork();
350 void CovAfterFork(int child_pid) {
351 coverage_data.AfterFork(child_pid);
354 } // namespace __sanitizer
356 extern "C" {
357 SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov() {
358 coverage_data.Add(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()));
360 SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { CovDump(); }
361 SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() {
362 coverage_data.Init();
364 SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_module_init(uptr npcs) {
365 if (!common_flags()->coverage || !common_flags()->coverage_direct) return;
366 if (SANITIZER_ANDROID) {
367 // dlopen/dlclose interceptors do not work on Android, so we rely on
368 // Extend() calls to update .sancov.map.
369 CovUpdateMapping(GET_CALLER_PC());
371 coverage_data.Extend(npcs);
373 SANITIZER_INTERFACE_ATTRIBUTE
374 sptr __sanitizer_maybe_open_cov_file(const char *name) {
375 return MaybeOpenCovFile(name);
377 } // extern "C"