ractor_core.h

   1 #include "internal/gc.h"
   2 #include "ruby/ruby.h"
   3 #include "ruby/ractor.h"
   4 #include "vm_core.h"
   5 #include "id_table.h"
   6 #include "vm_debug.h"
   7
   8 #ifndef RACTOR_CHECK_MODE
   9 #define RACTOR_CHECK_MODE (VM_CHECK_MODE || RUBY_DEBUG) && (SIZEOF_UINT64_T == SIZEOF_VALUE)
  10 #endif
  11
  12 enum rb_ractor_basket_type {
  13     // basket is empty
  14     basket_type_none,
  15
  16     // value is available
  17     basket_type_ref,
  18     basket_type_copy,
  19     basket_type_move,
  20     basket_type_will,
  21
  22     // basket should be deleted
  23     basket_type_deleted,
  24
  25     // basket is reserved
  26     basket_type_reserved,
  27
  28     // take_basket is available
  29     basket_type_take_basket,
  30
  31     // basket is keeping by yielding ractor
  32     basket_type_yielding,
  33 };
  34
  35 // per ractor taking configuration
  36 struct rb_ractor_selector_take_config {
  37     bool closed;
  38     bool oneshot;
  39 };
  40
  41 struct rb_ractor_basket {
  42     union {
  43         enum rb_ractor_basket_type e;
  44         rb_atomic_t atomic;
  45     } type;
  46     VALUE sender;
  47
  48     union {
  49         struct {
  50             VALUE v;
  51             bool exception;
  52         } send;
  53
  54         struct {
  55             struct rb_ractor_basket *basket;
  56             struct rb_ractor_selector_take_config *config;
  57         } take;
  58     } p; // payload
  59 };
  60
  61 static inline bool
  62 basket_type_p(struct rb_ractor_basket *b, enum rb_ractor_basket_type type)
  63 {
  64     return b->type.e == type;
  65 }
  66
  67 static inline bool
  68 basket_none_p(struct rb_ractor_basket *b)
  69 {
  70     return basket_type_p(b, basket_type_none);
  71 }
  72
  73 struct rb_ractor_queue {
  74     struct rb_ractor_basket *baskets;
  75     int start;
  76     int cnt;
  77     int size;
  78     unsigned int serial;
  79     unsigned int reserved_cnt;
  80 };
  81
  82 enum rb_ractor_wait_status {
  83     wait_none      = 0x00,
  84     wait_receiving = 0x01,
  85     wait_taking    = 0x02,
  86     wait_yielding  = 0x04,
  87     wait_moving    = 0x08,
  88 };
  89
  90 enum rb_ractor_wakeup_status {
  91     wakeup_none,
  92     wakeup_by_send,
  93     wakeup_by_yield,
  94     wakeup_by_take,
  95     wakeup_by_close,
  96     wakeup_by_interrupt,
  97     wakeup_by_retry,
  98 };
  99
 100 struct rb_ractor_sync {
 101     // ractor lock
 102     rb_nativethread_lock_t lock;
 103 #if RACTOR_CHECK_MODE > 0
 104     VALUE locked_by;
 105 #endif
 106
 107     bool incoming_port_closed;
 108     bool outgoing_port_closed;
 109
 110     // All sent messages will be pushed into recv_queue
 111     struct rb_ractor_queue recv_queue;
 112
 113     // The following ractors waiting for the yielding by this ractor
 114     struct rb_ractor_queue takers_queue;
 115
 116     // Enabled if the ractor already terminated and not taken yet.
 117     struct rb_ractor_basket will_basket;
 118
 119     struct ractor_wait {
 120         enum rb_ractor_wait_status status;
 121         enum rb_ractor_wakeup_status wakeup_status;
 122         rb_thread_t *waiting_thread;
 123     } wait;
 124
 125 #ifndef RUBY_THREAD_PTHREAD_H
 126     rb_nativethread_cond_t cond;
 127 #endif
 128 };
 129
 130 // created
 131 //   | ready to run
 132 // ====================== inserted to vm->ractor
 133 //   v
 134 // blocking <---+ all threads are blocking
 135 //   |          |
 136 //   v          |
 137 // running -----+
 138 //   | all threads are terminated.
 139 // ====================== removed from vm->ractor
 140 //   v
 141 // terminated
 142 //
 143 // status is protected by VM lock (global state)
 144 enum ractor_status {
 145     ractor_created,
 146     ractor_running,
 147     ractor_blocking,
 148     ractor_terminated,
 149 };
 150
 151 struct rb_ractor_struct {
 152     struct rb_ractor_pub pub;
 153
 154     struct rb_ractor_sync sync;
 155     VALUE receiving_mutex;
 156
 157     // vm wide barrier synchronization
 158     rb_nativethread_cond_t barrier_wait_cond;
 159
 160     // thread management
 161     struct {
 162         struct ccan_list_head set;
 163         unsigned int cnt;
 164         unsigned int blocking_cnt;
 165         unsigned int sleeper;
 166         struct rb_thread_sched sched;
 167         rb_execution_context_t *running_ec;
 168         rb_thread_t *main;
 169     } threads;
 170     VALUE thgroup_default;
 171
 172     VALUE name;
 173     VALUE loc;
 174
 175     enum ractor_status status_;
 176
 177     struct ccan_list_node vmlr_node;
 178
 179     // ractor local data
 180
 181     st_table *local_storage;
 182     struct rb_id_table *idkey_local_storage;
 183
 184     VALUE r_stdin;
 185     VALUE r_stdout;
 186     VALUE r_stderr;
 187     VALUE verbose;
 188     VALUE debug;
 189
 190     rb_ractor_newobj_cache_t newobj_cache;
 191
 192     // gc.c rb_objspace_reachable_objects_from
 193     struct gc_mark_func_data_struct {
 194         void *data;
 195         void (*mark_func)(VALUE v, void *data);
 196     } *mfd;
 197 }; // rb_ractor_t is defined in vm_core.h
 198
 199
 200 static inline VALUE
 201 rb_ractor_self(const rb_ractor_t *r)
 202 {
 203     return r->pub.self;
 204 }
 205
 206 rb_ractor_t *rb_ractor_main_alloc(void);
 207 void rb_ractor_main_setup(rb_vm_t *vm, rb_ractor_t *main_ractor, rb_thread_t *main_thread);
 208 void rb_ractor_atexit(rb_execution_context_t *ec, VALUE result);
 209 void rb_ractor_atexit_exception(rb_execution_context_t *ec);
 210 void rb_ractor_teardown(rb_execution_context_t *ec);
 211 void rb_ractor_receive_parameters(rb_execution_context_t *ec, rb_ractor_t *g, int len, VALUE *ptr);
 212 void rb_ractor_send_parameters(rb_execution_context_t *ec, rb_ractor_t *g, VALUE args);
 213
 214 VALUE rb_thread_create_ractor(rb_ractor_t *g, VALUE args, VALUE proc); // defined in thread.c
 215
 216 int rb_ractor_living_thread_num(const rb_ractor_t *);
 217 VALUE rb_ractor_thread_list(void);
 218 bool rb_ractor_p(VALUE rv);
 219
 220 void rb_ractor_living_threads_init(rb_ractor_t *r);
 221 void rb_ractor_living_threads_insert(rb_ractor_t *r, rb_thread_t *th);
 222 void rb_ractor_living_threads_remove(rb_ractor_t *r, rb_thread_t *th);
 223 void rb_ractor_blocking_threads_inc(rb_ractor_t *r, const char *file, int line); // TODO: file, line only for RUBY_DEBUG_LOG
 224 void rb_ractor_blocking_threads_dec(rb_ractor_t *r, const char *file, int line); // TODO: file, line only for RUBY_DEBUG_LOG
 225
 226 void rb_ractor_vm_barrier_interrupt_running_thread(rb_ractor_t *r);
 227 void rb_ractor_terminate_interrupt_main_thread(rb_ractor_t *r);
 228 void rb_ractor_terminate_all(void);
 229 bool rb_ractor_main_p_(void);
 230 void rb_ractor_finish_marking(void);
 231 void rb_ractor_atfork(rb_vm_t *vm, rb_thread_t *th);
 232
 233 VALUE rb_ractor_ensure_shareable(VALUE obj, VALUE name);
 234
 235 RUBY_SYMBOL_EXPORT_BEGIN
 236 bool rb_ractor_shareable_p_continue(VALUE obj);
 237
 238 // THIS FUNCTION SHOULD NOT CALL WHILE INCREMENTAL MARKING!!
 239 // This function is for T_DATA::free_func
 240 void rb_ractor_local_storage_delkey(rb_ractor_local_key_t key);
 241
 242 RUBY_SYMBOL_EXPORT_END
 243
 244 static inline bool
 245 rb_ractor_main_p(void)
 246 {
 247     if (ruby_single_main_ractor) {
 248         return true;
 249     }
 250     else {
 251         return rb_ractor_main_p_();
 252     }
 253 }
 254
 255 static inline bool
 256 rb_ractor_status_p(rb_ractor_t *r, enum ractor_status status)
 257 {
 258     return r->status_ == status;
 259 }
 260
 261 static inline void
 262 rb_ractor_sleeper_threads_inc(rb_ractor_t *r)
 263 {
 264     r->threads.sleeper++;
 265 }
 266
 267 static inline void
 268 rb_ractor_sleeper_threads_dec(rb_ractor_t *r)
 269 {
 270     r->threads.sleeper--;
 271 }
 272
 273 static inline void
 274 rb_ractor_sleeper_threads_clear(rb_ractor_t *r)
 275 {
 276     r->threads.sleeper = 0;
 277 }
 278
 279 static inline int
 280 rb_ractor_sleeper_thread_num(rb_ractor_t *r)
 281 {
 282     return r->threads.sleeper;
 283 }
 284
 285 static inline void
 286 rb_ractor_thread_switch(rb_ractor_t *cr, rb_thread_t *th)
 287 {
 288     RUBY_DEBUG_LOG("th:%d->%u%s",
 289                    cr->threads.running_ec ? (int)rb_th_serial(cr->threads.running_ec->thread_ptr) : -1,
 290                    rb_th_serial(th), cr->threads.running_ec == th->ec ? " (same)" : "");
 291
 292     if (cr->threads.running_ec != th->ec) {
 293         if (0) {
 294             ruby_debug_printf("rb_ractor_thread_switch ec:%p->%p\n",
 295                               (void *)cr->threads.running_ec, (void *)th->ec);
 296         }
 297     }
 298     else {
 299         return;
 300     }
 301
 302     if (cr->threads.running_ec != th->ec) {
 303         th->running_time_us = 0;
 304     }
 305
 306     cr->threads.running_ec = th->ec;
 307
 308     VM_ASSERT(cr == GET_RACTOR());
 309 }
 310
 311 #define rb_ractor_set_current_ec(cr, ec) rb_ractor_set_current_ec_(cr, ec, __FILE__, __LINE__)
 312
 313 static inline void
 314 rb_ractor_set_current_ec_(rb_ractor_t *cr, rb_execution_context_t *ec, const char *file, int line)
 315 {
 316 #ifdef RB_THREAD_LOCAL_SPECIFIER
 317
 318 # ifdef __APPLE__
 319     rb_current_ec_set(ec);
 320 # else
 321     ruby_current_ec = ec;
 322 # endif
 323
 324 #else
 325     native_tls_set(ruby_current_ec_key, ec);
 326 #endif
 327     RUBY_DEBUG_LOG2(file, line, "ec:%p->%p", (void *)cr->threads.running_ec, (void *)ec);
 328     VM_ASSERT(ec == NULL || cr->threads.running_ec != ec);
 329     cr->threads.running_ec = ec;
 330 }
 331
 332 void rb_vm_ractor_blocking_cnt_inc(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line);
 333 void rb_vm_ractor_blocking_cnt_dec(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line);
 334
 335 static inline uint32_t
 336 rb_ractor_id(const rb_ractor_t *r)
 337 {
 338     return r->pub.id;
 339 }
 340
 341 #if RACTOR_CHECK_MODE > 0
 342 # define RACTOR_BELONGING_ID(obj) (*(uint32_t *)(((uintptr_t)(obj)) + rb_gc_obj_slot_size(obj)))
 343
 344 uint32_t rb_ractor_current_id(void);
 345
 346 static inline void
 347 rb_ractor_setup_belonging_to(VALUE obj, uint32_t rid)
 348 {
 349     RACTOR_BELONGING_ID(obj) = rid;
 350 }
 351
 352 static inline void
 353 rb_ractor_setup_belonging(VALUE obj)
 354 {
 355     rb_ractor_setup_belonging_to(obj, rb_ractor_current_id());
 356 }
 357
 358 static inline uint32_t
 359 rb_ractor_belonging(VALUE obj)
 360 {
 361     if (SPECIAL_CONST_P(obj) || RB_OBJ_SHAREABLE_P(obj)) {
 362         return 0;
 363     }
 364     else {
 365         return RACTOR_BELONGING_ID(obj);
 366     }
 367 }
 368
 369 static inline VALUE
 370 rb_ractor_confirm_belonging(VALUE obj)
 371 {
 372     uint32_t id = rb_ractor_belonging(obj);
 373
 374     if (id == 0) {
 375         if (UNLIKELY(!rb_ractor_shareable_p(obj))) {
 376             rp(obj);
 377             rb_bug("id == 0 but not shareable");
 378         }
 379     }
 380     else if (UNLIKELY(id != rb_ractor_current_id())) {
 381         if (rb_ractor_shareable_p(obj)) {
 382             // ok
 383         }
 384         else {
 385             rp(obj);
 386             rb_bug("rb_ractor_confirm_belonging object-ractor id:%u, current-ractor id:%u", id, rb_ractor_current_id());
 387         }
 388     }
 389     return obj;
 390 }
 391 #else
 392 #define rb_ractor_confirm_belonging(obj) obj
 393 #endif