1 /* Profiling of shared libraries.
2 Copyright (C) 1997-2002, 2003, 2004 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
5 Based on the BSD mcount implementation.
7 The GNU C Library is free software; you can redistribute it and/or
8 modify it under the terms of the GNU Lesser General Public
9 License as published by the Free Software Foundation; either
10 version 2.1 of the License, or (at your option) any later version.
12 The GNU C Library is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Lesser General Public License for more details.
17 You should have received a copy of the GNU Lesser General Public
18 License along with the GNU C Library; if not, write to the Free
19 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
33 #include <sys/gmon_out.h>
35 #include <sys/param.h>
39 /* The LD_PROFILE feature has to be implemented different to the
40 normal profiling using the gmon/ functions. The problem is that an
41 arbitrary amount of processes simulataneously can be run using
42 profiling and all write the results in the same file. To provide
43 this mechanism one could implement a complicated mechanism to merge
44 the content of two profiling runs or one could extend the file
45 format to allow more than one data set. For the second solution we
46 would have the problem that the file can grow in size beyond any
47 limit and both solutions have the problem that the concurrency of
48 writing the results is a big problem.
50 Another much simpler method is to use mmap to map the same file in
51 all using programs and modify the data in the mmap'ed area and so
52 also automatically on the disk. Using the MAP_SHARED option of
53 mmap(2) this can be done without big problems in more than one
56 This approach is very different from the normal profiling. We have
57 to use the profiling data in exactly the way they are expected to
58 be written to disk. But the normal format used by gprof is not usable
59 to do this. It is optimized for size. It writes the tags as single
60 bytes but this means that the following 32/64 bit values are
63 Therefore we use a new format. This will look like this
65 0 1 2 3 <- byte is 32 bit word
67 0004 *version* <- GMON_SHOBJ_VERSION
72 0014 *tag* <- GMON_TAG_TIME_HIST
74 ?? ?? ?? ?? <- 32/64 bit LowPC
76 ?? ?? ?? ?? <- 32/64 bit HighPC
85 0030+2*A ?? ?? ?? ?? <- Count data
87 0030+2*A+K ?? ?? ?? ??
89 0030+2*A+K *tag* <- GMON_TAG_CG_ARC
91 0038+2*A+K ?? ?? ?? ??
92 ?? ?? ?? ?? <- FromPC#1
93 0038+3*A+K ?? ?? ?? ??
95 0038+4*A+K ?? ?? ?? ?? <- Count#1
97 0038+(2*(CN-1)+2)*A+(CN-1)*4+K ?? ?? ?? ??
98 ?? ?? ?? ?? <- FromPC#CGN
99 0038+(2*(CN-1)+3)*A+(CN-1)*4+K ?? ?? ?? ??
100 ?? ?? ?? ?? <- ToPC#CGN
101 0038+(2*CN+2)*A+(CN-1)*4+K ?? ?? ?? ?? <- Count#CGN
103 We put (for now?) no basic block information in the file since this would
104 introduce rase conditions among all the processes who want to write them.
106 `K' is the number of count entries which is computed as
108 textsize / HISTFRACTION
110 `CG' in the above table is the number of call graph arcs. Normally,
111 the table is sparse and the profiling code writes out only the those
112 entries which are really used in the program run. But since we must
113 not extend this table (the profiling file) we'll keep them all here.
114 So CN can be executed in advance as
116 MINARCS <= textsize*(ARCDENSITY/100) <= MAXARCS
118 Now the remaining question is: how to build the data structures we can
119 work with from this data. We need the from set and must associate the
120 froms with all the associated tos. We will do this by constructing this
121 data structures at the program start. To do this we'll simply visit all
122 entries in the call graph table and add it to the appropriate list. */
124 extern int __profile_frequency (void);
125 libc_hidden_proto (__profile_frequency
)
127 /* We define a special type to address the elements of the arc table.
128 This is basically the `gmon_cg_arc_record' format but it includes
129 the room for the tag and it uses real types. */
130 struct here_cg_arc_record
135 } __attribute__ ((packed
));
137 static struct here_cg_arc_record
*data
;
139 /* Nonzero if profiling is under way. */
142 /* This is the number of entry which have been incorporated in the toset. */
143 static uint32_t narcs
;
144 /* This is a pointer to the object representing the number of entries
145 currently in the mmaped file. At no point of time this has to be the
146 same as NARCS. If it is equal all entries from the file are in our
148 static volatile uint32_t *narcsp
;
150 static volatile uint16_t *kcount
;
151 static size_t kcountsize
;
153 struct here_fromstruct
155 struct here_cg_arc_record
volatile *here
;
159 static volatile uint16_t *tos
;
161 static struct here_fromstruct
*froms
;
162 static uint32_t fromlimit
;
163 static volatile uint32_t fromidx
;
165 static uintptr_t lowpc
;
166 static size_t textsize
;
167 static unsigned int hashfraction
;
168 static unsigned int log_hashfraction
;
172 /* Set up profiling data to profile object desribed by MAP. The output
173 file is found (or created) in OUTPUT_DIR. */
176 _dl_start_profile (void)
181 const ElfW(Phdr
) *ph
;
182 ElfW(Addr
) mapstart
= ~((ElfW(Addr
)) 0);
183 ElfW(Addr
) mapend
= 0;
184 struct gmon_hdr gmon_hdr
;
185 struct gmon_hist_hdr hist_hdr
;
191 struct gmon_hdr
*addr
= NULL
;
193 /* See profil(2) where this is described. */
195 #define SCALE_1_TO_1 0x10000L
197 /* Compute the size of the sections which contain program code. */
198 for (ph
= GL(dl_profile_map
)->l_phdr
;
199 ph
< &GL(dl_profile_map
)->l_phdr
[GL(dl_profile_map
)->l_phnum
]; ++ph
)
200 if (ph
->p_type
== PT_LOAD
&& (ph
->p_flags
& PF_X
))
202 ElfW(Addr
) start
= (ph
->p_vaddr
& ~(GLRO(dl_pagesize
) - 1));
203 ElfW(Addr
) end
= ((ph
->p_vaddr
+ ph
->p_memsz
+ GLRO(dl_pagesize
) - 1)
204 & ~(GLRO(dl_pagesize
) - 1));
206 if (start
< mapstart
)
212 /* Now we can compute the size of the profiling data. This is done
213 with the same formulars as in `monstartup' (see gmon.c). */
215 lowpc
= ROUNDDOWN (mapstart
+ GL(dl_profile_map
)->l_addr
,
216 HISTFRACTION
* sizeof (HISTCOUNTER
));
217 highpc
= ROUNDUP (mapend
+ GL(dl_profile_map
)->l_addr
,
218 HISTFRACTION
* sizeof (HISTCOUNTER
));
219 textsize
= highpc
- lowpc
;
220 kcountsize
= textsize
/ HISTFRACTION
;
221 hashfraction
= HASHFRACTION
;
222 if ((HASHFRACTION
& (HASHFRACTION
- 1)) == 0)
224 /* If HASHFRACTION is a power of two, mcount can use shifting
225 instead of integer division. Precompute shift amount.
227 This is a constant but the compiler cannot compile the
228 expression away since the __ffs implementation is not known
229 to the compiler. Help the compiler by precomputing the
231 assert (hashfraction
== 2);
233 if (sizeof (*froms
) == 8)
234 log_hashfraction
= 4;
235 else if (sizeof (*froms
) == 16)
236 log_hashfraction
= 5;
238 log_hashfraction
= __ffs (hashfraction
* sizeof (*froms
)) - 1;
241 log_hashfraction
= -1;
242 tossize
= textsize
/ HASHFRACTION
;
243 fromlimit
= textsize
* ARCDENSITY
/ 100;
244 if (fromlimit
< MINARCS
)
246 if (fromlimit
> MAXARCS
)
248 fromssize
= fromlimit
* sizeof (struct here_fromstruct
);
250 expected_size
= (sizeof (struct gmon_hdr
)
251 + 4 + sizeof (struct gmon_hist_hdr
) + kcountsize
252 + 4 + 4 + fromssize
* sizeof (struct here_cg_arc_record
));
254 /* Create the gmon_hdr we expect or write. */
255 memset (&gmon_hdr
, '\0', sizeof (struct gmon_hdr
));
256 memcpy (&gmon_hdr
.cookie
[0], GMON_MAGIC
, sizeof (gmon_hdr
.cookie
));
257 *(int32_t *) gmon_hdr
.version
= GMON_SHOBJ_VERSION
;
259 /* Create the hist_hdr we expect or write. */
260 *(char **) hist_hdr
.low_pc
= (char *) mapstart
;
261 *(char **) hist_hdr
.high_pc
= (char *) mapend
;
262 *(int32_t *) hist_hdr
.hist_size
= kcountsize
/ sizeof (HISTCOUNTER
);
263 *(int32_t *) hist_hdr
.prof_rate
= __profile_frequency ();
264 if (sizeof (hist_hdr
.dimen
) >= sizeof ("seconds"))
266 memcpy (hist_hdr
.dimen
, "seconds", sizeof ("seconds"));
267 memset (hist_hdr
.dimen
+ sizeof ("seconds"), '\0',
268 sizeof (hist_hdr
.dimen
) - sizeof ("seconds"));
271 strncpy (hist_hdr
.dimen
, "seconds", sizeof (hist_hdr
.dimen
));
272 hist_hdr
.dimen_abbrev
= 's';
274 /* First determine the output name. We write in the directory
275 OUTPUT_DIR and the name is composed from the shared objects
276 soname (or the file name) and the ending ".profile". */
277 filename
= (char *) alloca (strlen (GLRO(dl_profile_output
)) + 1
278 + strlen (GLRO(dl_profile
)) + sizeof ".profile");
279 cp
= __stpcpy (filename
, GLRO(dl_profile_output
));
281 __stpcpy (__stpcpy (cp
, GLRO(dl_profile
)), ".profile");
284 # define EXTRA_FLAGS | O_NOFOLLOW
288 fd
= __open (filename
, O_RDWR
| O_CREAT EXTRA_FLAGS
, DEFFILEMODE
);
291 /* We cannot write the profiling data so don't do anything. */
293 _dl_error_printf ("%s: cannot open file: %s\n", filename
,
294 __strerror_r (errno
, buf
, sizeof buf
));
298 if (__fxstat64 (_STAT_VER
, fd
, &st
) < 0 || !S_ISREG (st
.st_mode
))
300 /* Not stat'able or not a regular file => don't use it. */
304 _dl_error_printf ("%s: cannot stat file: %s\n", filename
,
305 __strerror_r (errnum
, buf
, sizeof buf
));
309 /* Test the size. If it does not match what we expect from the size
310 values in the map MAP we don't use it and warn the user. */
313 /* We have to create the file. */
314 char buf
[GLRO(dl_pagesize
)];
316 memset (buf
, '\0', GLRO(dl_pagesize
));
318 if (__lseek (fd
, expected_size
& ~(GLRO(dl_pagesize
) - 1), SEEK_SET
) == -1)
325 _dl_error_printf ("%s: cannot create file: %s\n", filename
,
326 __strerror_r (errnum
, buf
, sizeof buf
));
330 if (TEMP_FAILURE_RETRY (__libc_write (fd
, buf
, (expected_size
336 else if (st
.st_size
!= expected_size
)
342 __munmap ((void *) addr
, expected_size
);
344 _dl_error_printf ("%s: file is no correct profile data file for `%s'\n",
345 filename
, GLRO(dl_profile
));
349 addr
= (struct gmon_hdr
*) __mmap (NULL
, expected_size
, PROT_READ
|PROT_WRITE
,
350 MAP_SHARED
|MAP_FILE
, fd
, 0);
351 if (addr
== (struct gmon_hdr
*) MAP_FAILED
)
356 _dl_error_printf ("%s: cannot map file: %s\n", filename
,
357 __strerror_r (errnum
, buf
, sizeof buf
));
361 /* We don't need the file desriptor anymore. */
364 /* Pointer to data after the header. */
365 hist
= (char *) (addr
+ 1);
366 kcount
= (uint16_t *) ((char *) hist
+ sizeof (uint32_t)
367 + sizeof (struct gmon_hist_hdr
));
369 /* Compute pointer to array of the arc information. */
370 narcsp
= (uint32_t *) ((char *) kcount
+ kcountsize
+ sizeof (uint32_t));
371 data
= (struct here_cg_arc_record
*) ((char *) narcsp
+ sizeof (uint32_t));
375 /* Create the signature. */
376 memcpy (addr
, &gmon_hdr
, sizeof (struct gmon_hdr
));
378 *(uint32_t *) hist
= GMON_TAG_TIME_HIST
;
379 memcpy (hist
+ sizeof (uint32_t), &hist_hdr
,
380 sizeof (struct gmon_hist_hdr
));
382 narcsp
[-1] = GMON_TAG_CG_ARC
;
386 /* Test the signature in the file. */
387 if (memcmp (addr
, &gmon_hdr
, sizeof (struct gmon_hdr
)) != 0
388 || *(uint32_t *) hist
!= GMON_TAG_TIME_HIST
389 || memcmp (hist
+ sizeof (uint32_t), &hist_hdr
,
390 sizeof (struct gmon_hist_hdr
)) != 0
391 || narcsp
[-1] != GMON_TAG_CG_ARC
)
395 /* Allocate memory for the froms data and the pointer to the tos records. */
396 tos
= (uint16_t *) calloc (tossize
+ fromssize
, 1);
399 __munmap ((void *) addr
, expected_size
);
400 _dl_fatal_printf ("Out of memory while initializing profiler\n");
404 froms
= (struct here_fromstruct
*) ((char *) tos
+ tossize
);
407 /* Now we have to process all the arc count entries. BTW: it is
408 not critical whether the *NARCSP value changes meanwhile. Before
409 we enter a new entry in to toset we will check that everything is
410 available in TOS. This happens in _dl_mcount.
412 Loading the entries in reverse order should help to get the most
413 frequently used entries at the front of the list. */
414 for (idx
= narcs
= MIN (*narcsp
, fromlimit
); idx
> 0; )
419 to_index
= (data
[idx
].self_pc
/ (hashfraction
* sizeof (*tos
)));
420 newfromidx
= fromidx
++;
421 froms
[newfromidx
].here
= &data
[idx
];
422 froms
[newfromidx
].link
= tos
[to_index
];
423 tos
[to_index
] = newfromidx
;
426 /* Setup counting data. */
427 if (kcountsize
< highpc
- lowpc
)
430 s_scale
= ((double) kcountsize
/ (highpc
- lowpc
)) * SCALE_1_TO_1
;
432 size_t range
= highpc
- lowpc
;
433 size_t quot
= range
/ kcountsize
;
435 if (quot
>= SCALE_1_TO_1
)
437 else if (quot
>= SCALE_1_TO_1
/ 256)
438 s_scale
= SCALE_1_TO_1
/ quot
;
439 else if (range
> ULONG_MAX
/ 256)
440 s_scale
= (SCALE_1_TO_1
* 256) / (range
/ (kcountsize
/ 256));
442 s_scale
= (SCALE_1_TO_1
* 256) / ((range
* 256) / kcountsize
);
446 s_scale
= SCALE_1_TO_1
;
448 /* Start the profiler. */
449 __profil ((void *) kcount
, kcountsize
, lowpc
, s_scale
);
451 /* Turn on profiling. */
457 _dl_mcount (ElfW(Addr
) frompc
, ElfW(Addr
) selfpc
)
459 volatile uint16_t *topcindex
;
461 struct here_fromstruct
*fromp
;
466 /* Compute relative addresses. The shared object can be loaded at
467 any address. The value of frompc could be anything. We cannot
468 restrict it in any way, just set to a fixed value (0) in case it
469 is outside the allowed range. These calls show up as calls from
470 <external> in the gprof output. */
472 if (frompc
>= textsize
)
475 if (selfpc
>= textsize
)
478 /* Getting here we now have to find out whether the location was
479 already used. If yes we are lucky and only have to increment a
480 counter (this also has to be atomic). If the entry is new things
481 are getting complicated... */
483 /* Avoid integer divide if possible. */
484 if ((HASHFRACTION
& (HASHFRACTION
- 1)) == 0)
485 i
= selfpc
>> log_hashfraction
;
487 i
= selfpc
/ (hashfraction
* sizeof (*tos
));
490 fromindex
= *topcindex
;
493 goto check_new_or_add
;
495 fromp
= &froms
[fromindex
];
497 /* We have to look through the chain of arcs whether there is already
498 an entry for our arc. */
499 while (fromp
->here
->from_pc
!= frompc
)
501 if (fromp
->link
!= 0)
503 fromp
= &froms
[fromp
->link
];
504 while (fromp
->link
!= 0 && fromp
->here
->from_pc
!= frompc
);
506 if (fromp
->here
->from_pc
!= frompc
)
508 topcindex
= &fromp
->link
;
511 /* Our entry is not among the entries we read so far from the
512 data file. Now see whether we have to update the list. */
513 while (narcs
!= *narcsp
&& narcs
< fromlimit
)
517 to_index
= (data
[narcs
].self_pc
518 / (hashfraction
* sizeof (*tos
)));
519 newfromidx
= atomic_exchange_and_add (&fromidx
, 1) + 1;
520 froms
[newfromidx
].here
= &data
[narcs
];
521 froms
[newfromidx
].link
= tos
[to_index
];
522 tos
[to_index
] = newfromidx
;
523 atomic_increment (&narcs
);
526 /* If we still have no entry stop searching and insert. */
529 uint_fast32_t newarc
= atomic_exchange_and_add (narcsp
, 1);
531 /* In rare cases it could happen that all entries in FROMS are
532 occupied. So we cannot count this anymore. */
533 if (newarc
>= fromlimit
)
536 *topcindex
= atomic_exchange_and_add (&fromidx
, 1) + 1;
537 fromp
= &froms
[*topcindex
];
539 fromp
->here
= &data
[newarc
];
540 data
[newarc
].from_pc
= frompc
;
541 data
[newarc
].self_pc
= selfpc
;
542 data
[newarc
].count
= 0;
544 atomic_increment (&narcs
);
549 fromp
= &froms
[*topcindex
];
556 /* Increment the counter. */
557 atomic_increment (&fromp
->here
->count
);