2 +----------------------------------------------------------------------+
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
24 #include <folly/Bits.h>
25 #include <folly/Format.h>
26 #include <folly/portability/SysMman.h>
28 #include "hphp/util/arch.h"
29 #include "hphp/util/assertions.h"
34 constexpr int nosize
= 0;
35 constexpr int byte
= 1;
36 constexpr int word
= 2;
37 constexpr int dword
= 4;
38 constexpr int qword
= 8;
41 using Address
= uint8_t*;
42 using CodeAddress
= uint8_t*;
43 using ConstCodeAddress
= const uint8_t*;
45 struct DataBlockFull
: std::runtime_error
{
48 DataBlockFull(const std::string
& blockName
, const std::string msg
)
49 : std::runtime_error(msg
)
53 ~DataBlockFull() noexcept override
{}
57 * DataBlock is a simple bump-allocating wrapper around a chunk of memory, with
58 * basic tracking for unused memory and a simple interface to allocate it.
60 * Memory is allocated from the end of the block unless specifically allocated
63 * Unused memory can be freed using free(). If the memory is at the end of the
64 * block, the frontier will be moved back.
66 * Free memory is coalesced and allocation is done by best-fit.
69 DataBlock() = default;
71 DataBlock(DataBlock
&& other
) = default;
72 DataBlock
& operator=(DataBlock
&& other
) = default;
75 * Uses an existing chunk of memory.
77 * Addresses returned by DataBlock will be in the range [start, start + sz),
78 * while writes and reads will happen from the range [dest, dest + sz).
80 void init(Address start
, Address dest
, size_t sz
, size_t maxGrow
,
82 assertx(dest
!= start
|| sz
== maxGrow
);
84 m_base
= m_frontier
= start
;
91 void init(Address start
, Address dest
, size_t sz
, const char* name
) {
92 init(start
, dest
, sz
, sz
, name
);
95 void init(Address start
, size_t sz
, const char* name
) {
96 init(start
, start
, sz
, sz
, name
);
99 void alignFrontier(size_t align
) {
100 assertx(align
== 1 || align
== 2 || align
== 4 || align
== 8 || align
== 16);
102 auto const mask
= align
- 1;
103 auto const nf
= (uint8_t*)(((uintptr_t)m_frontier
+ mask
) & ~mask
);
104 assertCanEmit(nf
- m_frontier
);
112 * Simple bump allocator, supporting power-of-two alignment. alloc<T>() is a
113 * simple typed wrapper around allocRaw().
115 void* allocRaw(size_t sz
, size_t align
= 16) {
116 // Round frontier up to a multiple of align
117 alignFrontier(align
);
119 auto data
= m_frontier
;
121 assertx(m_frontier
<= m_base
+ m_size
);
125 template<typename T
> T
* alloc(size_t align
= 16, int n
= 1) {
126 return (T
*)allocRaw(sizeof(T
) * n
, align
);
129 bool canEmit(size_t nBytes
) {
130 assert(m_frontier
>= m_base
);
131 assert(m_frontier
<= m_base
+ m_size
);
132 return m_frontier
+ nBytes
<= m_base
+ m_size
;
135 bool grow(size_t nBytes
) {
136 if (m_maxGrow
== m_size
) return false;
137 assertx(m_destBase
!= m_base
);
139 auto const need
= nBytes
- available();
140 auto const amt
= std::min(std::max(m_size
+ need
, 2 * m_size
), m_maxGrow
);
141 if (amt
< m_size
+ need
) return false;
143 m_destBuf
.reset((Address
)::malloc(amt
));
144 ::memcpy(m_destBuf
.get(), m_destBase
, used());
146 m_destBuf
.reset((Address
)::realloc(m_destBuf
.release(), amt
));
148 if (!m_destBuf
) reportMallocError(amt
);
149 m_destBase
= m_destBuf
.get();
154 void assertCanEmit(size_t nBytes
) {
155 if (!canEmit(nBytes
) && !grow(nBytes
)) reportFull(nBytes
);
159 void reportFull(size_t nbytes
) const;
162 void reportMallocError(size_t nbytes
) const;
164 bool isValidAddress(const CodeAddress tca
) const {
165 return tca
>= m_base
&& tca
< (m_base
+ m_size
);
168 void byte(const uint8_t byte
) {
169 assertCanEmit(sz::byte
);
171 m_frontier
+= sz::byte
;
173 void word(const uint16_t word
) {
174 assertCanEmit(sz::word
);
175 *(uint16_t*)dest() = word
;
176 m_frontier
+= sz::word
;
178 void dword(const uint32_t dword
) {
179 assertCanEmit(sz::dword
);
180 *(uint32_t*)dest() = dword
;
181 m_frontier
+= sz::dword
;
183 void qword(const uint64_t qword
) {
184 assertCanEmit(sz::qword
);
185 *(uint64_t*)dest() = qword
;
186 m_frontier
+= sz::qword
;
189 void bytes(size_t n
, const uint8_t *bs
) {
191 if (n
<= 8 && canEmit(8) && m_destBase
== m_base
) {
192 // If it is a modest number of bytes, try executing in one machine
193 // store. This allows control-flow edges, including nop, to be
194 // appear idempotent on other CPUs. If m_destBase != m_base then the
195 // current block is a temporary buffer and this write is neither required
196 // nor safe, as we may override an adjacent buffer or write off the end
202 u
.qword
= *(uint64_t*)dest();
203 for (size_t i
= 0; i
< n
; ++i
) {
207 // If this address spans cache lines, on x64 this is not an atomic store.
208 // This being the case, use caution when overwriting code that is
209 // reachable by multiple threads: make sure it doesn't span cache lines.
210 *reinterpret_cast<uint64_t*>(dest()) = u
.qword
;
212 memcpy(dest(), bs
, n
);
217 void skip(size_t nbytes
) {
218 alloc
<uint8_t>(1, nbytes
);
221 Address
base() const { return m_base
; }
222 Address
frontier() const { return m_frontier
; }
223 size_t size() const { return m_size
; }
224 std::string
name() const { return m_name
; }
227 * DataBlock can emit into a range [A, B] while returning addresses in range
228 * [A', B']. This function will map an address in [A', B'] into [A, B], and
229 * it must be used before writing or reading from any address returned by
232 Address
toDestAddress(CodeAddress addr
) {
233 assertx(m_base
<= addr
&& addr
<= (m_base
+ m_size
));
234 return Address(m_destBase
+ (addr
- m_base
));
237 void setFrontier(Address addr
) {
238 assertx(m_base
<= addr
&& addr
<= (m_base
+ m_size
));
242 void moveFrontier(int64_t offset
) {
243 setFrontier(m_frontier
+ offset
);
246 size_t capacity() const {
250 size_t used() const {
251 return m_frontier
- m_base
;
254 size_t available() const {
255 return m_size
- (m_frontier
- m_base
);
258 bool contains(ConstCodeAddress addr
) const {
259 return addr
>= m_base
&& addr
< (m_base
+ m_size
);
262 bool contains(ConstCodeAddress start
, ConstCodeAddress end
) const {
263 return start
<= end
&&
264 start
>= m_base
&& end
<= (m_base
+ m_size
);
268 return m_base
== m_frontier
;
276 memset(m_destBase
, 0, m_frontier
- m_base
);
280 // Append address range to free list
281 void free(void* addr
, size_t len
);
283 // Attempt to allocate a range from within the free list
284 void* allocInner(size_t len
);
286 size_t numFrees() const { return m_nfree
; }
287 size_t numAllocs() const { return m_nalloc
; }
288 size_t bytesFree() const { return m_bytesFree
; }
289 size_t blocksFree() const { return m_freeRanges
.size(); }
291 void sync(Address begin
= nullptr, Address end
= nullptr) {
292 if (!begin
) begin
= m_base
;
293 if (!end
) end
= m_frontier
;
294 syncDirect(toDestAddress(begin
), toDestAddress(end
));
297 static void syncDirect(Address begin
, Address end
) {
298 if (arch() == Arch::ARM
&& begin
< end
) {
299 __builtin___clear_cache(reinterpret_cast<char*>(begin
),
300 reinterpret_cast<char*>(end
));
307 using Offset
= uint32_t;
308 using Size
= uint32_t;
310 // DataBlock can optionally be growable. The initial expansion of DataBlock
311 // will allocate a new buffer that is owned by the DataBlock, subsequent
312 // expansions will use realloc to expand this block until m_maxGrow has been
313 // reached. Only DataBlocks which have a different m_base from m_destBase may
314 // be grown, as expansion may move the location of m_destBase.
315 struct Deleter final
{ void operator()(uint8_t* a
) const { ::free(a
); } };
316 std::unique_ptr
<uint8_t, Deleter
> m_destBuf
{nullptr};
318 Address
dest() const { return m_destBase
+ (m_frontier
- m_base
); }
320 // DataBlock can track an alternate pseudo-frontier to support clients that
321 // wish to emit code to one location while keeping memory references relative
322 // to a separate location. The actual writes will be to m_dest.
323 Address m_destBase
{nullptr};
325 Address m_base
{nullptr};
326 Address m_frontier
{nullptr};
334 size_t m_bytesFree
{0};
335 std::unordered_map
<Offset
, int64_t> m_freeRanges
;
336 std::map
<Size
, std::unordered_set
<Offset
>> m_freeLists
;
339 using CodeBlock
= DataBlock
;
341 //////////////////////////////////////////////////////////////////////
344 explicit UndoMarker(CodeBlock
& cb
)
346 , m_oldFrontier(cb
.frontier()) {
350 m_cb
.setFrontier(m_oldFrontier
);
355 CodeAddress m_oldFrontier
;
359 * RAII bookmark for scoped rewinding of frontier.
361 struct CodeCursor
: UndoMarker
{
362 CodeCursor(CodeBlock
& cb
, CodeAddress newFrontier
) : UndoMarker(cb
) {
363 cb
.setFrontier(newFrontier
);
366 ~CodeCursor() { undo(); }