1 // wb.cc -- Add write barriers as needed.
3 // Copyright 2017 The Go Authors. All rights reserved.
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file.
10 #include "go-diagnostics.h"
14 #include "expressions.h"
15 #include "statements.h"
19 // Mark variables whose addresses are taken and do some other
20 // cleanups. This has to be done before the write barrier pass and
21 // after the escape analysis pass. It would be nice to do this
22 // elsewhere but there isn't an obvious place.
24 class Mark_address_taken
: public Traverse
27 Mark_address_taken(Gogo
* gogo
)
28 : Traverse(traverse_functions
30 | traverse_expressions
),
31 gogo_(gogo
), function_(NULL
)
35 function(Named_object
*);
38 statement(Block
*, size_t*, Statement
*);
41 expression(Expression
**);
46 // The function we are traversing.
47 Named_object
* function_
;
53 Mark_address_taken::function(Named_object
* no
)
55 go_assert(this->function_
== NULL
);
57 int t
= no
->func_value()->traverse(this);
58 this->function_
= NULL
;
60 if (t
== TRAVERSE_EXIT
)
62 return TRAVERSE_SKIP_COMPONENTS
;
65 // Traverse a statement.
68 Mark_address_taken::statement(Block
* block
, size_t* pindex
, Statement
* s
)
70 // If this is an assignment of the form s = append(s, ...), expand
71 // it now, so that we can assign it to the left hand side in the
72 // middle of the expansion and possibly skip a write barrier.
73 Assignment_statement
* as
= s
->assignment_statement();
74 if (as
!= NULL
&& !as
->lhs()->is_sink_expression())
76 Call_expression
* rce
= as
->rhs()->call_expression();
78 && rce
->builtin_call_expression() != NULL
79 && (rce
->builtin_call_expression()->code()
80 == Builtin_call_expression::BUILTIN_APPEND
)
81 && Expression::is_same_variable(as
->lhs(), rce
->args()->front()))
83 Statement_inserter inserter
= Statement_inserter(block
, pindex
);
85 rce
->builtin_call_expression()->flatten_append(this->gogo_
,
91 // That does the assignment, so remove this statement.
92 Expression
* e
= Expression::make_boolean(true, s
->location());
93 Statement
* dummy
= Statement::make_statement(e
, true);
94 block
->replace_statement(*pindex
, dummy
);
97 return TRAVERSE_CONTINUE
;
100 // Mark variable addresses taken.
103 Mark_address_taken::expression(Expression
** pexpr
)
105 Expression
* expr
= *pexpr
;
106 Unary_expression
* ue
= expr
->unary_expression();
108 ue
->check_operand_address_taken(this->gogo_
);
110 Array_index_expression
* aie
= expr
->array_index_expression();
112 && aie
->end() != NULL
113 && !aie
->array()->type()->is_slice_type())
115 // Slice of an array. The escape analysis models this with
116 // a child Node representing the address of the array.
117 bool escapes
= false;
118 Node
* n
= Node::make_node(expr
);
119 if (n
->child() == NULL
120 || (n
->child()->encoding() & ESCAPE_MASK
) != Node::ESCAPE_NONE
)
122 aie
->array()->address_taken(escapes
);
125 if (expr
->allocation_expression() != NULL
)
127 Node
* n
= Node::make_node(expr
);
128 if ((n
->encoding() & ESCAPE_MASK
) == Node::ESCAPE_NONE
)
129 expr
->allocation_expression()->set_allocate_on_stack();
131 if (expr
->heap_expression() != NULL
)
133 Node
* n
= Node::make_node(expr
);
134 if ((n
->encoding() & ESCAPE_MASK
) == Node::ESCAPE_NONE
)
135 expr
->heap_expression()->set_allocate_on_stack();
137 if (expr
->slice_literal() != NULL
)
139 Node
* n
= Node::make_node(expr
);
140 if ((n
->encoding() & ESCAPE_MASK
) == Node::ESCAPE_NONE
)
141 expr
->slice_literal()->set_storage_does_not_escape();
144 // Rewrite non-escaping makeslice with constant size to stack allocation.
145 Unsafe_type_conversion_expression
* uce
=
146 expr
->unsafe_conversion_expression();
148 && uce
->type()->is_slice_type()
149 && Node::make_node(uce
->expr())->encoding() == Node::ESCAPE_NONE
150 && uce
->expr()->call_expression() != NULL
)
152 Call_expression
* call
= uce
->expr()->call_expression();
153 if (call
->fn()->func_expression() != NULL
154 && call
->fn()->func_expression()->runtime_code() == Runtime::MAKESLICE
)
156 Expression
* len_arg
= call
->args()->at(1);
157 Expression
* cap_arg
= call
->args()->at(2);
158 Numeric_constant nclen
;
159 Numeric_constant nccap
;
162 if (len_arg
->numeric_constant_value(&nclen
)
163 && cap_arg
->numeric_constant_value(&nccap
)
164 && nclen
.to_unsigned_long(&vlen
) == Numeric_constant::NC_UL_VALID
165 && nccap
.to_unsigned_long(&vcap
) == Numeric_constant::NC_UL_VALID
)
167 // Turn it into a slice expression of an addressable array,
168 // which is allocated on stack.
169 Location loc
= expr
->location();
170 Type
* elmt_type
= expr
->type()->array_type()->element_type();
171 Expression
* len_expr
=
172 Expression::make_integer_ul(vcap
, cap_arg
->type(), loc
);
173 Type
* array_type
= Type::make_array_type(elmt_type
, len_expr
);
174 Expression
* alloc
= Expression::make_allocation(array_type
, loc
);
175 alloc
->allocation_expression()->set_allocate_on_stack();
176 Expression
* array
= Expression::make_unary(OPERATOR_MULT
, alloc
, loc
);
177 Expression
* zero
= Expression::make_integer_ul(0, len_arg
->type(), loc
);
179 Expression::make_array_index(array
, zero
, len_arg
, cap_arg
, loc
);
184 return TRAVERSE_CONTINUE
;
187 // Check variables and closures do not escape when compiling runtime.
189 class Check_escape
: public Traverse
192 Check_escape(Gogo
* gogo
)
193 : Traverse(traverse_expressions
| traverse_variables
),
198 expression(Expression
**);
201 variable(Named_object
*);
208 Check_escape::variable(Named_object
* no
)
210 if ((no
->is_variable() && no
->var_value()->is_in_heap())
211 || (no
->is_result_variable()
212 && no
->result_var_value()->is_in_heap()))
213 go_error_at(no
->location(),
214 "%s escapes to heap, not allowed in runtime",
215 no
->message_name().c_str());
216 return TRAVERSE_CONTINUE
;
220 Check_escape::expression(Expression
** pexpr
)
222 Expression
* expr
= *pexpr
;
223 Func_expression
* fe
= expr
->func_expression();
224 if (fe
!= NULL
&& fe
->closure() != NULL
)
226 Node
* n
= Node::make_node(expr
);
227 if (n
->encoding() == Node::ESCAPE_HEAP
)
228 go_error_at(expr
->location(),
229 "heap-allocated closure, not allowed in runtime");
231 return TRAVERSE_CONTINUE
;
234 // Add write barriers to the IR. This are required by the concurrent
235 // garbage collector. A write barrier is needed for any write of a
236 // pointer into memory controlled by the garbage collector. Write
237 // barriers are not required for writes to local variables that live
238 // on the stack. Write barriers are only required when the runtime
239 // enables them, which can be checked using a run time check on
240 // runtime.writeBarrier.enabled.
242 // Essentially, for each assignment A = B, where A is or contains a
243 // pointer, and where A is not, or at any rate may not be, a stack
244 // variable, we rewrite it into
245 // if runtime.writeBarrier.enabled {
246 // typedmemmove(typeof(A), &A, &B)
251 // The test of runtime.writeBarrier.Enabled is implemented by treating
252 // the variable as a *uint32, and testing *runtime.writeBarrier != 0.
253 // This is compatible with the definition in the runtime package.
255 // For types that are pointer shared (pointers, maps, chans, funcs),
256 // we replaced the call to typedmemmove with writebarrierptr(&A, B).
257 // As far as the GC is concerned, all pointers are the same, so it
258 // doesn't need the type descriptor.
260 // There are possible optimizations that are not implemented.
262 // runtime.writeBarrier can only change when the goroutine is
263 // preempted, which in practice means when a call is made into the
264 // runtime package, so we could optimize by only testing it once
265 // between function calls.
267 // A slice could be handled with a call to writebarrierptr plus two
270 // Traverse the IR adding write barriers.
272 class Write_barriers
: public Traverse
275 Write_barriers(Gogo
* gogo
)
276 : Traverse(traverse_functions
| traverse_variables
| traverse_statements
),
277 gogo_(gogo
), function_(NULL
), statements_added_()
281 function(Named_object
*);
284 variable(Named_object
*);
287 statement(Block
*, size_t* pindex
, Statement
*);
294 // Statements introduced.
295 Statement_inserter::Statements statements_added_
;
298 // Traverse a function. Just record it for later.
301 Write_barriers::function(Named_object
* no
)
303 go_assert(this->function_
== NULL
);
304 this->function_
= no
->func_value();
305 int t
= this->function_
->traverse(this);
306 this->function_
= NULL
;
308 if (t
== TRAVERSE_EXIT
)
310 return TRAVERSE_SKIP_COMPONENTS
;
313 // Insert write barriers for a global variable: ensure that variable
314 // initialization is handled correctly. This is rarely needed, since
315 // we currently don't enable background GC until after all global
316 // variables are initialized. But we do need this if an init function
320 Write_barriers::variable(Named_object
* no
)
322 // We handle local variables in the variable declaration statement.
323 // We only have to handle global variables here.
324 if (!no
->is_variable())
325 return TRAVERSE_CONTINUE
;
326 Variable
* var
= no
->var_value();
327 if (!var
->is_global())
328 return TRAVERSE_CONTINUE
;
330 // Nothing to do if there is no initializer.
331 Expression
* init
= var
->init();
333 return TRAVERSE_CONTINUE
;
335 // Nothing to do for variables that do not contain any pointers.
336 if (!var
->type()->has_pointer())
337 return TRAVERSE_CONTINUE
;
339 // Nothing to do if the initializer is static.
340 init
= Expression::make_cast(var
->type(), init
, var
->location());
341 if (!var
->has_pre_init() && init
->is_static_initializer())
342 return TRAVERSE_CONTINUE
;
344 // Nothing to do for a type that can not be in the heap, or a
345 // pointer to a type that can not be in the heap.
346 if (!var
->type()->in_heap())
347 return TRAVERSE_CONTINUE
;
348 if (var
->type()->points_to() != NULL
&& !var
->type()->points_to()->in_heap())
349 return TRAVERSE_CONTINUE
;
351 // Otherwise change the initializer into a pre_init assignment
352 // statement with a write barrier.
354 // We can't check for a dependency of the variable on itself after
355 // we make this change, because the preinit statement will always
356 // depend on the variable (since it assigns to it). So check for a
357 // self-dependency now.
358 this->gogo_
->check_self_dep(no
);
360 // Replace the initializer.
361 Location loc
= init
->location();
362 Expression
* ref
= Expression::make_var_reference(no
, loc
);
364 Statement_inserter
inserter(this->gogo_
, var
, &this->statements_added_
);
365 Statement
* s
= this->gogo_
->assign_with_write_barrier(NULL
, NULL
, &inserter
,
367 this->statements_added_
.insert(s
);
369 var
->add_preinit_statement(this->gogo_
, s
);
372 return TRAVERSE_CONTINUE
;
375 // Insert write barriers for statements.
378 Write_barriers::statement(Block
* block
, size_t* pindex
, Statement
* s
)
380 if (this->statements_added_
.find(s
) != this->statements_added_
.end())
381 return TRAVERSE_SKIP_COMPONENTS
;
383 switch (s
->classification())
388 case Statement::STATEMENT_VARIABLE_DECLARATION
:
390 Variable_declaration_statement
* vds
=
391 s
->variable_declaration_statement();
392 Named_object
* no
= vds
->var();
393 Variable
* var
= no
->var_value();
395 // We may need to emit a write barrier for the initialization
398 // Nothing to do for a variable with no initializer.
399 Expression
* init
= var
->init();
403 // Nothing to do if the variable is not in the heap. Only
404 // local variables get declaration statements, and local
405 // variables on the stack do not require write barriers.
406 if (!var
->is_in_heap())
409 // Nothing to do if the variable does not contain any pointers.
410 if (!var
->type()->has_pointer())
413 // Nothing to do for a type that can not be in the heap, or a
414 // pointer to a type that can not be in the heap.
415 if (!var
->type()->in_heap())
417 if (var
->type()->points_to() != NULL
418 && !var
->type()->points_to()->in_heap())
421 // Otherwise initialize the variable with a write barrier.
423 Function
* function
= this->function_
;
424 Location loc
= init
->location();
425 Statement_inserter
inserter(block
, pindex
, &this->statements_added_
);
427 // Insert the variable declaration statement with no
428 // initializer, so that the variable exists.
432 // Create a statement that initializes the variable with a
434 Expression
* ref
= Expression::make_var_reference(no
, loc
);
435 Statement
* assign
= this->gogo_
->assign_with_write_barrier(function
,
440 this->statements_added_
.insert(assign
);
442 // Replace the old variable declaration statement with the new
444 block
->replace_statement(*pindex
, assign
);
448 case Statement::STATEMENT_ASSIGNMENT
:
450 Assignment_statement
* as
= s
->assignment_statement();
452 if (as
->omit_write_barrier())
455 Expression
* lhs
= as
->lhs();
456 Expression
* rhs
= as
->rhs();
458 // We may need to emit a write barrier for the assignment.
460 if (!this->gogo_
->assign_needs_write_barrier(lhs
))
463 // Change the assignment to use a write barrier.
464 Function
* function
= this->function_
;
465 Location loc
= as
->location();
466 Statement_inserter inserter
=
467 Statement_inserter(block
, pindex
, &this->statements_added_
);
468 Statement
* assign
= this->gogo_
->assign_with_write_barrier(function
,
473 this->statements_added_
.insert(assign
);
474 block
->replace_statement(*pindex
, assign
);
479 return TRAVERSE_CONTINUE
;
482 // The write barrier pass.
485 Gogo::add_write_barriers()
490 Mark_address_taken
mat(this);
491 this->traverse(&mat
);
493 if (this->compiling_runtime() && this->package_name() == "runtime")
495 Check_escape
chk(this);
496 this->traverse(&chk
);
499 Write_barriers
wb(this);
503 // Return the runtime.writeBarrier variable.
506 Gogo::write_barrier_variable()
508 static Named_object
* write_barrier_var
;
509 if (write_barrier_var
== NULL
)
511 Location bloc
= Linemap::predeclared_location();
513 // We pretend that writeBarrier is a uint32, so that we do a
514 // 32-bit load. That is what the gc toolchain does.
515 Type
* uint32_type
= Type::lookup_integer_type("uint32");
516 Variable
* var
= new Variable(uint32_type
, NULL
, true, false, false,
520 Package
* package
= this->add_imported_package("runtime", "_", false,
521 "runtime", "runtime",
522 bloc
, &add_to_globals
);
523 write_barrier_var
= Named_object::make_variable("writeBarrier",
527 return write_barrier_var
;
530 // Return whether an assignment that sets LHS needs a write barrier.
533 Gogo::assign_needs_write_barrier(Expression
* lhs
)
535 // Nothing to do if the variable does not contain any pointers.
536 if (!lhs
->type()->has_pointer())
539 // An assignment to a field is handled like an assignment to the
543 // Nothing to do for a type that can not be in the heap, or a
544 // pointer to a type that can not be in the heap. We check this
545 // at each level of a struct.
546 if (!lhs
->type()->in_heap())
548 if (lhs
->type()->points_to() != NULL
549 && !lhs
->type()->points_to()->in_heap())
552 Field_reference_expression
* fre
= lhs
->field_reference_expression();
558 // Nothing to do for an assignment to a temporary.
559 if (lhs
->temporary_reference_expression() != NULL
)
562 // Nothing to do for an assignment to a sink.
563 if (lhs
->is_sink_expression())
566 // Nothing to do for an assignment to a local variable that is not
568 Var_expression
* ve
= lhs
->var_expression();
571 Named_object
* no
= ve
->named_object();
572 if (no
->is_variable())
574 Variable
* var
= no
->var_value();
575 if (!var
->is_global() && !var
->is_in_heap())
578 else if (no
->is_result_variable())
580 Result_variable
* rvar
= no
->result_var_value();
581 if (!rvar
->is_in_heap())
586 // For a struct assignment, we don't need a write barrier if all the
587 // pointer types can not be in the heap.
588 Struct_type
* st
= lhs
->type()->struct_type();
591 bool in_heap
= false;
592 const Struct_field_list
* fields
= st
->fields();
593 for (Struct_field_list::const_iterator p
= fields
->begin();
597 Type
* ft
= p
->type();
598 if (!ft
->has_pointer())
602 if (ft
->points_to() != NULL
&& !ft
->points_to()->in_heap())
611 // Write barrier needed in other cases.
615 // Return a statement that sets LHS to RHS using a write barrier.
616 // ENCLOSING is the enclosing block.
619 Gogo::assign_with_write_barrier(Function
* function
, Block
* enclosing
,
620 Statement_inserter
* inserter
, Expression
* lhs
,
621 Expression
* rhs
, Location loc
)
624 && ((function
->pragmas() & GOPRAGMA_NOWRITEBARRIER
) != 0
625 || (function
->pragmas() & GOPRAGMA_NOWRITEBARRIERREC
) != 0))
626 go_error_at(loc
, "write barrier prohibited");
628 Type
* type
= lhs
->type();
629 go_assert(type
->has_pointer());
632 if (lhs
->unary_expression() != NULL
633 && lhs
->unary_expression()->op() == OPERATOR_MULT
)
634 addr
= lhs
->unary_expression()->operand();
637 addr
= Expression::make_unary(OPERATOR_AND
, lhs
, loc
);
638 addr
->unary_expression()->set_does_not_escape();
640 Temporary_statement
* lhs_temp
= Statement::make_temporary(NULL
, addr
, loc
);
641 inserter
->insert(lhs_temp
);
642 lhs
= Expression::make_temporary_reference(lhs_temp
, loc
);
644 if (!Type::are_identical(type
, rhs
->type(), false, NULL
)
645 && rhs
->type()->interface_type() != NULL
646 && !rhs
->is_variable())
648 // May need a temporary for interface conversion.
649 Temporary_statement
* temp
= Statement::make_temporary(NULL
, rhs
, loc
);
650 inserter
->insert(temp
);
651 rhs
= Expression::make_temporary_reference(temp
, loc
);
653 rhs
= Expression::convert_for_assignment(this, type
, rhs
, loc
);
654 Temporary_statement
* rhs_temp
= NULL
;
655 if (!rhs
->is_variable() && !rhs
->is_constant())
657 rhs_temp
= Statement::make_temporary(NULL
, rhs
, loc
);
658 inserter
->insert(rhs_temp
);
659 rhs
= Expression::make_temporary_reference(rhs_temp
, loc
);
663 Expression::make_dereference(lhs
, Expression::NIL_CHECK_DEFAULT
, loc
);
664 Statement
* assign
= Statement::make_assignment(indir
, rhs
, loc
);
666 lhs
= Expression::make_temporary_reference(lhs_temp
, loc
);
667 if (rhs_temp
!= NULL
)
668 rhs
= Expression::make_temporary_reference(rhs_temp
, loc
);
670 Type
* unsafe_ptr_type
= Type::make_pointer_type(Type::make_void_type());
671 lhs
= Expression::make_unsafe_cast(unsafe_ptr_type
, lhs
, loc
);
674 switch (type
->base()->classification())
679 case Type::TYPE_ERROR
:
682 case Type::TYPE_POINTER
:
683 case Type::TYPE_FUNCTION
:
685 case Type::TYPE_CHANNEL
:
686 // These types are all represented by a single pointer.
687 call
= Runtime::make_call(Runtime::WRITEBARRIERPTR
, loc
, 2, lhs
, rhs
);
690 case Type::TYPE_STRING
:
691 case Type::TYPE_STRUCT
:
692 case Type::TYPE_ARRAY
:
693 case Type::TYPE_INTERFACE
:
695 rhs
= Expression::make_unary(OPERATOR_AND
, rhs
, loc
);
696 rhs
->unary_expression()->set_does_not_escape();
697 call
= Runtime::make_call(Runtime::TYPEDMEMMOVE
, loc
, 3,
698 Expression::make_type_descriptor(type
, loc
),
704 return this->check_write_barrier(enclosing
, assign
,
705 Statement::make_statement(call
, false));
708 // Return a statement that tests whether write barriers are enabled
709 // and executes either the efficient code or the write barrier
710 // function call, depending.
713 Gogo::check_write_barrier(Block
* enclosing
, Statement
* without
,
716 Location loc
= without
->location();
717 Named_object
* wb
= this->write_barrier_variable();
718 Expression
* ref
= Expression::make_var_reference(wb
, loc
);
719 Expression
* zero
= Expression::make_integer_ul(0, ref
->type(), loc
);
720 Expression
* cond
= Expression::make_binary(OPERATOR_EQEQ
, ref
, zero
, loc
);
722 Block
* then_block
= new Block(enclosing
, loc
);
723 then_block
->add_statement(without
);
725 Block
* else_block
= new Block(enclosing
, loc
);
726 else_block
->add_statement(with
);
728 return Statement::make_if_statement(cond
, then_block
, else_block
, loc
);