18 #include "playout/elo.h"
19 #include "playout/moggy.h"
20 #include "playout/light.h"
24 #include "uct/internal.h"
25 #include "uct/prior.h"
30 struct uct_policy
*policy_ucb1_init(struct uct
*u
, char *arg
);
31 struct uct_policy
*policy_ucb1amaf_init(struct uct
*u
, char *arg
);
32 static void uct_pondering_stop(struct uct
*u
);
35 /* Default number of simulations to perform per move.
36 * Note that this is now in total over all threads! (Unless TM_ROOT.) */
37 #define MC_GAMES 80000
38 #define MC_GAMELEN MAX_GAMELEN
39 static const struct time_info default_ti
= {
42 .len
= { .games
= MC_GAMES
},
45 /* How big proportion of ownermap counts must be of one color to consider
48 /* How many games to consider at minimum before judging groups. */
49 #define GJ_MINGAMES 500
51 /* How often to inspect the tree from the main thread to check for playout
52 * stop, progress reports, etc. (in seconds) */
53 #define TREE_BUSYWAIT_INTERVAL 0.1 /* 100ms */
55 /* Once per how many simulations (per thread) to show a progress report line. */
56 #define TREE_SIMPROGRESS_INTERVAL 10000
58 /* When terminating uct_search() early, the safety margin to add to the
59 * remaining playout number estimate when deciding whether the result can
61 #define PLAYOUT_DELTA_SAFEMARGIN 1000
65 setup_state(struct uct
*u
, struct board
*b
, enum stone color
)
67 u
->t
= tree_init(b
, color
, u
->fast_alloc
? u
->max_tree_size
: 0);
69 fast_srandom(u
->force_seed
);
71 fprintf(stderr
, "Fresh board with random seed %lu\n", fast_getseed());
72 //board_print(b, stderr);
73 if (!u
->no_book
&& b
->moves
== 0) {
74 assert(color
== S_BLACK
);
80 reset_state(struct uct
*u
)
83 tree_done(u
->t
); u
->t
= NULL
;
87 setup_dynkomi(struct uct
*u
, struct board
*b
, enum stone to_play
)
89 if (u
->dynkomi
> b
->moves
&& u
->t
->use_extra_komi
)
90 u
->t
->extra_komi
= uct_get_extra_komi(u
, b
);
96 prepare_move(struct engine
*e
, struct board
*b
, enum stone color
)
98 struct uct
*u
= e
->data
;
101 /* Verify that we have sane state. */
103 assert(u
->t
&& b
->moves
);
104 if (color
!= stone_other(u
->t
->root_color
)) {
105 fprintf(stderr
, "Fatal: Non-alternating play detected %d %d\n",
106 color
, u
->t
->root_color
);
111 /* We need fresh state. */
113 setup_state(u
, b
, color
);
116 u
->ownermap
.playouts
= 0;
117 memset(u
->ownermap
.map
, 0, board_size2(b
) * sizeof(u
->ownermap
.map
[0]));
121 dead_group_list(struct uct
*u
, struct board
*b
, struct move_queue
*mq
)
123 struct group_judgement gj
;
125 gj
.gs
= alloca(board_size2(b
) * sizeof(gj
.gs
[0]));
126 board_ownermap_judge_group(b
, &u
->ownermap
, &gj
);
127 groups_of_status(b
, &gj
, GS_DEAD
, mq
);
131 uct_pass_is_safe(struct uct
*u
, struct board
*b
, enum stone color
, bool pass_all_alive
)
133 if (u
->ownermap
.playouts
< GJ_MINGAMES
)
136 struct move_queue mq
= { .moves
= 0 };
138 dead_group_list(u
, b
, &mq
);
139 return pass_is_safe(b
, color
, &mq
);
144 uct_printhook_ownermap(struct board
*board
, coord_t c
, FILE *f
)
146 struct uct
*u
= board
->es
;
148 const char chr
[] = ":XO,"; // dame, black, white, unclear
149 const char chm
[] = ":xo,";
150 char ch
= chr
[board_ownermap_judge_point(&u
->ownermap
, c
, GJ_THRES
)];
151 if (ch
== ',') { // less precise estimate then?
152 ch
= chm
[board_ownermap_judge_point(&u
->ownermap
, c
, 0.67)];
154 fprintf(f
, "%c ", ch
);
158 uct_notify_play(struct engine
*e
, struct board
*b
, struct move
*m
)
160 struct uct
*u
= e
->data
;
162 /* No state, create one - this is probably game beginning
163 * and we need to load the opening book right now. */
164 prepare_move(e
, b
, m
->color
);
168 /* Stop pondering. */
169 /* XXX: If we are about to receive multiple 'play' commands,
170 * e.g. in a rengo, we will not ponder during the rest of them. */
171 uct_pondering_stop(u
);
173 if (is_resign(m
->coord
)) {
179 /* Promote node of the appropriate move to the tree root. */
181 if (!tree_promote_at(u
->t
, b
, m
->coord
)) {
183 fprintf(stderr
, "Warning: Cannot promote move node! Several play commands in row?\n");
187 /* Setting up dynkomi is not necessary here, probably, but we
188 * better do it anyway for consistency reasons. */
189 setup_dynkomi(u
, b
, stone_other(m
->color
));
194 uct_chat(struct engine
*e
, struct board
*b
, char *cmd
)
196 struct uct
*u
= e
->data
;
197 static char reply
[1024];
199 cmd
+= strspn(cmd
, " \n\t");
200 if (!strncasecmp(cmd
, "winrate", 7)) {
202 return "no game context (yet?)";
203 enum stone color
= u
->t
->root_color
;
204 struct tree_node
*n
= u
->t
->root
;
205 snprintf(reply
, 1024, "In %d playouts at %d threads, %s %s can win with %.2f%% probability",
206 n
->u
.playouts
, u
->threads
, stone2str(color
), coord2sstr(n
->coord
, b
),
207 tree_node_get_value(u
->t
, -1, n
->u
.value
) * 100);
208 if (u
->t
->use_extra_komi
&& abs(u
->t
->extra_komi
) >= 0.5) {
209 sprintf(reply
+ strlen(reply
), ", while self-imposing extra komi %.1f",
219 uct_dead_group_list(struct engine
*e
, struct board
*b
, struct move_queue
*mq
)
221 struct uct
*u
= e
->data
;
223 /* This means the game is probably over, no use pondering on. */
224 uct_pondering_stop(u
);
226 if (u
->pass_all_alive
)
227 return; // no dead groups
229 bool mock_state
= false;
232 /* No state, but we cannot just back out - we might
233 * have passed earlier, only assuming some stones are
234 * dead, and then re-connected, only to lose counting
235 * when all stones are assumed alive. */
236 /* Mock up some state and seed the ownermap by few
238 prepare_move(e
, b
, S_BLACK
); assert(u
->t
);
239 for (int i
= 0; i
< GJ_MINGAMES
; i
++)
240 uct_playout(u
, b
, S_BLACK
, u
->t
);
244 dead_group_list(u
, b
, mq
);
247 /* Clean up the mock state in case we will receive
248 * a genmove; we could get a non-alternating-move
249 * error from prepare_move() in that case otherwise. */
255 playout_policy_done(struct playout_policy
*p
)
257 if (p
->done
) p
->done(p
);
258 if (p
->data
) free(p
->data
);
263 uct_done(struct engine
*e
)
265 /* This is called on engine reset, especially when clear_board
266 * is received and new game should begin. */
267 struct uct
*u
= e
->data
;
268 uct_pondering_stop(u
);
269 if (u
->t
) reset_state(u
);
270 free(u
->ownermap
.map
);
273 free(u
->random_policy
);
274 playout_policy_done(u
->playout
);
275 uct_prior_done(u
->prior
);
279 /* Pachi threading structure (if uct_playouts_parallel() is used):
282 * | main(), GTP communication, ...
283 * | starts and stops the search managed by thread_manager
286 * | spawns and collects worker threads
292 * uct_playouts() loop, doing descend-playout until uct_halt
294 * Another way to look at it is by functions (lines denote thread boundaries):
297 * | uct_search() (uct_search_start() .. uct_search_stop())
298 * | -----------------------
299 * | spawn_thread_manager()
300 * | -----------------------
302 * V uct_playouts() */
304 /* Set in thread manager in case the workers should stop. */
305 volatile sig_atomic_t uct_halt
= 0;
306 /* ID of the running worker thread. */
307 __thread
int thread_id
= -1;
308 /* ID of the thread manager. */
309 static pthread_t thread_manager
;
310 static bool thread_manager_running
;
312 static pthread_mutex_t finish_mutex
= PTHREAD_MUTEX_INITIALIZER
;
313 static pthread_cond_t finish_cond
= PTHREAD_COND_INITIALIZER
;
314 static volatile int finish_thread
;
315 static pthread_mutex_t finish_serializer
= PTHREAD_MUTEX_INITIALIZER
;
328 spawn_worker(void *ctx_
)
330 struct spawn_ctx
*ctx
= ctx_
;
332 fast_srandom(ctx
->seed
);
333 thread_id
= ctx
->tid
;
335 ctx
->games
= uct_playouts(ctx
->u
, ctx
->b
, ctx
->color
, ctx
->t
);
337 pthread_mutex_lock(&finish_serializer
);
338 pthread_mutex_lock(&finish_mutex
);
339 finish_thread
= ctx
->tid
;
340 pthread_cond_signal(&finish_cond
);
341 pthread_mutex_unlock(&finish_mutex
);
345 /* Thread manager, controlling worker threads. It must be called with
346 * finish_mutex lock held, but it will unlock it itself before exiting;
347 * this is necessary to be completely deadlock-free. */
348 /* The finish_cond can be signalled for it to stop; in that case,
349 * the caller should set finish_thread = -1. */
350 /* After it is started, it will update mctx->t to point at some tree
351 * used for the actual search (matters only for TM_ROOT), on return
352 * it will set mctx->games to the number of performed simulations. */
354 spawn_thread_manager(void *ctx_
)
356 /* In thread_manager, we use only some of the ctx fields. */
357 struct spawn_ctx
*mctx
= ctx_
;
358 struct uct
*u
= mctx
->u
;
359 struct tree
*t
= mctx
->t
;
360 bool shared_tree
= u
->parallel_tree
;
361 fast_srandom(mctx
->seed
);
363 int played_games
= 0;
364 pthread_t threads
[u
->threads
];
369 /* Spawn threads... */
370 for (int ti
= 0; ti
< u
->threads
; ti
++) {
371 struct spawn_ctx
*ctx
= malloc(sizeof(*ctx
));
372 ctx
->u
= u
; ctx
->b
= mctx
->b
; ctx
->color
= mctx
->color
;
373 mctx
->t
= ctx
->t
= shared_tree
? t
: tree_copy(t
);
374 ctx
->tid
= ti
; ctx
->seed
= fast_random(65536) + ti
;
375 pthread_create(&threads
[ti
], NULL
, spawn_worker
, ctx
);
377 fprintf(stderr
, "Spawned worker %d\n", ti
);
380 /* ...and collect them back: */
381 while (joined
< u
->threads
) {
382 /* Wait for some thread to finish... */
383 pthread_cond_wait(&finish_cond
, &finish_mutex
);
384 if (finish_thread
< 0) {
385 /* Stop-by-caller. Tell the workers to wrap up. */
389 /* ...and gather its remnants. */
390 struct spawn_ctx
*ctx
;
391 pthread_join(threads
[finish_thread
], (void **) &ctx
);
392 played_games
+= ctx
->games
;
395 if (ctx
->t
== mctx
->t
) mctx
->t
= t
;
396 tree_merge(t
, ctx
->t
);
401 fprintf(stderr
, "Joined worker %d\n", finish_thread
);
402 pthread_mutex_unlock(&finish_serializer
);
405 pthread_mutex_unlock(&finish_mutex
);
408 tree_normalize(mctx
->t
, u
->threads
);
410 mctx
->games
= played_games
;
414 static struct spawn_ctx
*
415 uct_search_start(struct uct
*u
, struct board
*b
, enum stone color
, struct tree
*t
)
417 assert(u
->threads
> 0);
418 assert(!thread_manager_running
);
420 struct spawn_ctx ctx
= { .u
= u
, .b
= b
, .color
= color
, .t
= t
, .seed
= fast_random(65536) };
421 static struct spawn_ctx mctx
; mctx
= ctx
;
422 pthread_mutex_lock(&finish_mutex
);
423 pthread_create(&thread_manager
, NULL
, spawn_thread_manager
, &mctx
);
424 thread_manager_running
= true;
428 static struct spawn_ctx
*
429 uct_search_stop(void)
431 assert(thread_manager_running
);
433 /* Signal thread manager to stop the workers. */
434 pthread_mutex_lock(&finish_mutex
);
436 pthread_cond_signal(&finish_cond
);
437 pthread_mutex_unlock(&finish_mutex
);
439 /* Collect the thread manager. */
440 struct spawn_ctx
*pctx
;
441 thread_manager_running
= false;
442 pthread_join(thread_manager
, (void **) &pctx
);
447 /* Determine whether we should terminate the search early. */
449 uct_search_stop_early(struct uct
*u
, struct tree
*t
, struct board
*b
,
450 struct time_info
*ti
, struct time_stop
*stop
,
451 struct tree_node
*best
, struct tree_node
*best2
,
452 int base_playouts
, int i
)
454 /* Early break in won situation. */
455 if (best
->u
.playouts
>= 2000 && tree_node_get_value(t
, 1, best
->u
.value
) >= u
->loss_threshold
)
457 /* Earlier break in super-won situation. */
458 if (best
->u
.playouts
>= 500 && tree_node_get_value(t
, 1, best
->u
.value
) >= 0.95)
461 /* Break early if we estimate the second-best move cannot
462 * catch up in assigned time anymore. We use all our time
463 * if we are in byoyomi with single stone remaining in our
464 * period, however - it's better to pre-ponder. */
465 bool time_indulgent
= (!ti
->len
.t
.main_time
&& ti
->len
.t
.byoyomi_stones
== 1);
466 if (best2
&& ti
->dim
== TD_WALLTIME
&& !time_indulgent
) {
467 double elapsed
= time_now() - ti
->len
.t
.timer_start
;
468 double remaining
= stop
->worst
.time
- elapsed
;
469 double pps
= ((double)i
- base_playouts
) / elapsed
;
470 double estplayouts
= remaining
* pps
+ PLAYOUT_DELTA_SAFEMARGIN
;
471 if (best
->u
.playouts
> best2
->u
.playouts
+ estplayouts
) {
473 fprintf(stderr
, "Early stop, result cannot change: "
474 "best %d, best2 %d, estimated %f simulations to go\n",
475 best
->u
.playouts
, best2
->u
.playouts
, estplayouts
);
483 /* Determine whether we should terminate the search later. */
485 uct_search_keep_looking(struct uct
*u
, struct tree
*t
, struct board
*b
,
486 struct tree_node
*best
, struct tree_node
*best2
,
487 struct tree_node
*bestr
, struct tree_node
*winner
, int i
)
491 fprintf(stderr
, "Did not find best move, still trying...\n");
495 if (u
->best2_ratio
> 0) {
496 /* Check best/best2 simulations ratio. If the
497 * two best moves give very similar results,
498 * keep simulating. */
499 if (best2
&& best2
->u
.playouts
500 && (double)best
->u
.playouts
/ best2
->u
.playouts
< u
->best2_ratio
) {
502 fprintf(stderr
, "Best2 ratio %f < threshold %f\n",
503 (double)best
->u
.playouts
/ best2
->u
.playouts
,
509 if (u
->bestr_ratio
> 0) {
510 /* Check best, best_best value difference. If the best move
511 * and its best child do not give similar enough results,
512 * keep simulating. */
513 if (bestr
&& bestr
->u
.playouts
514 && fabs((double)best
->u
.value
- bestr
->u
.value
) > u
->bestr_ratio
) {
516 fprintf(stderr
, "Bestr delta %f > threshold %f\n",
517 fabs((double)best
->u
.value
- bestr
->u
.value
),
523 if (winner
&& winner
!= best
) {
524 /* Keep simulating if best explored
525 * does not have also highest value. */
527 fprintf(stderr
, "[%d] best %3s [%d] %f != winner %3s [%d] %f\n", i
,
528 coord2sstr(best
->coord
, t
->board
),
529 best
->u
.playouts
, tree_node_get_value(t
, 1, best
->u
.value
),
530 coord2sstr(winner
->coord
, t
->board
),
531 winner
->u
.playouts
, tree_node_get_value(t
, 1, winner
->u
.value
));
535 /* No reason to keep simulating, bye. */
539 /* Run time-limited MCTS search on foreground. */
541 uct_search(struct uct
*u
, struct board
*b
, struct time_info
*ti
, enum stone color
, struct tree
*t
)
543 int base_playouts
= u
->t
->root
->u
.playouts
;
544 if (UDEBUGL(2) && base_playouts
> 0)
545 fprintf(stderr
, "<pre-simulated %d games skipped>\n", base_playouts
);
547 /* Set up time conditions. */
548 if (ti
->period
== TT_NULL
) *ti
= default_ti
;
549 struct time_stop stop
;
550 time_stop_conditions(ti
, b
, u
->fuseki_end
, u
->yose_start
, &stop
);
552 /* Number of last game with progress print. */
553 int last_print
= t
->root
->u
.playouts
;
554 /* Number of simulations to wait before next print. */
555 int print_interval
= TREE_SIMPROGRESS_INTERVAL
* (u
->thread_model
== TM_ROOT
? 1 : u
->threads
);
556 /* Printed notification about full memory? */
557 bool print_fullmem
= false;
559 struct spawn_ctx
*ctx
= uct_search_start(u
, b
, color
, t
);
561 /* The search tree is ctx->t. This is normally == t, but in case of
562 * TM_ROOT, it is one of the trees belonging to the independent
563 * workers. It is important to reference ctx->t directly since the
564 * thread manager will swap the tree pointer asynchronously. */
565 /* XXX: This means TM_ROOT support is suboptimal since single stalled
566 * thread can stall the others in case of limiting the search by game
567 * count. However, TM_ROOT just does not deserve any more extra code
570 struct tree_node
*best
= NULL
;
571 struct tree_node
*best2
= NULL
; // Second-best move.
572 struct tree_node
*bestr
= NULL
; // best's best child.
573 struct tree_node
*winner
= NULL
;
575 double busywait_interval
= TREE_BUSYWAIT_INTERVAL
;
577 /* Now, just periodically poll the search tree. */
579 time_sleep(busywait_interval
);
580 /* busywait_interval should never be less than desired time, or the
581 * time control is broken. But if it happens to be less, we still search
582 * at least 100ms otherwise the move is completely random. */
584 int i
= ctx
->t
->root
->u
.playouts
;
586 /* Print progress? */
587 if (i
- last_print
> print_interval
) {
588 last_print
+= print_interval
; // keep the numbers tidy
589 uct_progress_status(u
, ctx
->t
, color
, last_print
);
591 if (!print_fullmem
&& ctx
->t
->nodes_size
> u
->max_tree_size
) {
593 fprintf(stderr
, "memory limit hit (%ld > %lu)\n", ctx
->t
->nodes_size
, u
->max_tree_size
);
594 print_fullmem
= true;
597 best
= u
->policy
->choose(u
->policy
, ctx
->t
->root
, b
, color
, resign
);
598 if (best
) best2
= u
->policy
->choose(u
->policy
, ctx
->t
->root
, b
, color
, best
->coord
);
600 /* Possibly stop search early if it's no use to try on. */
601 if (best
&& uct_search_stop_early(u
, ctx
->t
, b
, ti
, &stop
, best
, best2
, base_playouts
, i
))
604 /* Check against time settings. */
605 bool desired_done
= false;
606 if (ti
->dim
== TD_WALLTIME
) {
607 double elapsed
= time_now() - ti
->len
.t
.timer_start
;
608 if (elapsed
> stop
.worst
.time
) break;
609 desired_done
= elapsed
> stop
.desired
.time
;
611 } else { assert(ti
->dim
== TD_GAMES
);
612 if (i
> stop
.worst
.playouts
) break;
613 desired_done
= i
> stop
.desired
.playouts
;
616 /* We want to stop simulating, but are willing to keep trying
617 * if we aren't completely sure about the winner yet. */
619 if (u
->policy
->winner
&& u
->policy
->evaluate
)
620 winner
= u
->policy
->winner(u
->policy
, ctx
->t
, ctx
->t
->root
);
622 bestr
= u
->policy
->choose(u
->policy
, best
, b
, stone_other(color
), resign
);
623 if (!uct_search_keep_looking(u
, ctx
->t
, b
, best
, best2
, bestr
, winner
, i
))
627 /* TODO: Early break if best->variance goes under threshold and we already
628 * have enough playouts (possibly thanks to book or to pondering)? */
631 ctx
= uct_search_stop();
634 tree_dump(t
, u
->dumpthres
);
636 uct_progress_status(u
, t
, color
, ctx
->games
);
642 /* Start pondering background with @color to play. */
644 uct_pondering_start(struct uct
*u
, struct board
*b0
, struct tree
*t
, enum stone color
)
647 fprintf(stderr
, "Starting to ponder with color %s\n", stone2str(stone_other(color
)));
650 /* We need a local board copy to ponder upon. */
651 struct board
*b
= malloc(sizeof(*b
)); board_copy(b
, b0
);
653 /* *b0 did not have the genmove'd move played yet. */
654 struct move m
= { t
->root
->coord
, t
->root_color
};
655 int res
= board_play(b
, &m
);
657 setup_dynkomi(u
, b
, stone_other(m
.color
));
659 /* Start MCTS manager thread "headless". */
660 uct_search_start(u
, b
, color
, t
);
663 /* uct_search_stop() frontend for the pondering (non-genmove) mode. */
665 uct_pondering_stop(struct uct
*u
)
667 u
->pondering
= false;
668 if (!thread_manager_running
)
671 /* Stop the thread manager. */
672 struct spawn_ctx
*ctx
= uct_search_stop();
674 fprintf(stderr
, "(pondering) ");
675 uct_progress_status(u
, ctx
->t
, ctx
->color
, ctx
->games
);
682 uct_genmove(struct engine
*e
, struct board
*b
, struct time_info
*ti
, enum stone color
, bool pass_all_alive
)
684 double start_time
= time_now();
685 struct uct
*u
= e
->data
;
687 if (b
->superko_violation
) {
688 fprintf(stderr
, "!!! WARNING: SUPERKO VIOLATION OCCURED BEFORE THIS MOVE\n");
689 fprintf(stderr
, "Maybe you play with situational instead of positional superko?\n");
690 fprintf(stderr
, "I'm going to ignore the violation, but note that I may miss\n");
691 fprintf(stderr
, "some moves valid under this ruleset because of this.\n");
692 b
->superko_violation
= false;
696 uct_pondering_stop(u
);
697 prepare_move(e
, b
, color
);
700 /* How to decide whether to use dynkomi in this game? Since we use
701 * pondering, it's not simple "who-to-play" matter. Decide based on
702 * the last genmove issued. */
703 u
->t
->use_extra_komi
= !!(u
->dynkomi_mask
& color
);
704 setup_dynkomi(u
, b
, color
);
706 /* Make pessimistic assumption about komi for Japanese rules to
707 * avoid losing by 0.5 when winning by 0.5 with Chinese rules.
708 * The rules usually give the same winner if the integer part of komi
709 * is odd so we adjust the komi only if it is even (for a board of
710 * odd size). We are not trying to get an exact evaluation for rare
711 * cases of seki. For details see http://home.snafu.de/jasiek/parity.html
712 * TODO: Support the kgs-rules command once available. */
713 if (u
->territory_scoring
&& (((int)floor(b
->komi
) + b
->size
) & 1)) {
714 b
->komi
+= (color
== S_BLACK
? 1.0 : -1.0);
716 fprintf(stderr
, "Setting komi to %.1f assuming Japanese rules\n",
720 /* Perform the Monte Carlo Tree Search! */
721 int played_games
= uct_search(u
, b
, ti
, color
, u
->t
);
723 /* Choose the best move from the tree. */
724 struct tree_node
*best
= u
->policy
->choose(u
->policy
, u
->t
->root
, b
, color
, resign
);
727 return coord_copy(pass
);
730 fprintf(stderr
, "*** WINNER is %s (%d,%d) with score %1.4f (%d/%d:%d games)\n",
731 coord2sstr(best
->coord
, b
), coord_x(best
->coord
, b
), coord_y(best
->coord
, b
),
732 tree_node_get_value(u
->t
, 1, best
->u
.value
),
733 best
->u
.playouts
, u
->t
->root
->u
.playouts
, played_games
);
735 /* Do not resign if we're so short of time that evaluation of best move is completely
736 * unreliable, we might be winning actually. In this case best is almost random but
737 * still better than resign. */
738 if (tree_node_get_value(u
->t
, 1, best
->u
.value
) < u
->resign_ratio
&& !is_pass(best
->coord
)
739 && best
->u
.playouts
> GJ_MINGAMES
) {
741 return coord_copy(resign
);
744 /* If the opponent just passed and we win counting, always
746 if (b
->moves
> 1 && is_pass(b
->last_move
.coord
)) {
747 /* Make sure enough playouts are simulated. */
748 while (u
->ownermap
.playouts
< GJ_MINGAMES
)
749 uct_playout(u
, b
, color
, u
->t
);
750 if (uct_pass_is_safe(u
, b
, color
, u
->pass_all_alive
|| pass_all_alive
)) {
752 fprintf(stderr
, "<Will rather pass, looks safe enough.>\n");
757 tree_promote_node(u
->t
, &best
);
758 /* After a pass, pondering is harmful for two reasons:
759 * (i) We might keep pondering even when the game is over.
760 * Of course this is the case for opponent resign as well.
761 * (ii) More importantly, the ownermap will get skewed since
762 * the UCT will start cutting off any playouts. */
763 if (u
->pondering_opt
&& !is_pass(best
->coord
)) {
764 uct_pondering_start(u
, b
, u
->t
, stone_other(color
));
767 double time
= time_now() - start_time
+ 0.000001; /* avoid divide by zero */
768 fprintf(stderr
, "genmove in %0.2fs (%d games/s, %d games/s/thread)\n",
769 time
, (int)(played_games
/time
), (int)(played_games
/time
/u
->threads
));
771 return coord_copy(best
->coord
);
776 uct_genbook(struct engine
*e
, struct board
*b
, struct time_info
*ti
, enum stone color
)
778 struct uct
*u
= e
->data
;
779 if (!u
->t
) prepare_move(e
, b
, color
);
782 if (ti
->dim
== TD_GAMES
) {
783 /* Don't count in games that already went into the book. */
784 ti
->len
.games
+= u
->t
->root
->u
.playouts
;
786 uct_search(u
, b
, ti
, color
, u
->t
);
788 assert(ti
->dim
== TD_GAMES
);
789 tree_save(u
->t
, b
, ti
->len
.games
/ 100);
795 uct_dumpbook(struct engine
*e
, struct board
*b
, enum stone color
)
797 struct uct
*u
= e
->data
;
798 struct tree
*t
= tree_init(b
, color
, u
->fast_alloc
? u
->max_tree_size
: 0);
806 uct_state_init(char *arg
, struct board
*b
)
808 struct uct
*u
= calloc(1, sizeof(struct uct
));
811 u
->gamelen
= MC_GAMELEN
;
815 u
->playout_amaf
= true;
816 u
->playout_amaf_nakade
= false;
817 u
->amaf_prior
= false;
818 u
->max_tree_size
= 3072ULL * 1048576;
820 if (board_size(b
) - 2 >= 19)
822 u
->dynkomi_mask
= S_BLACK
;
823 u
->handicap_value
= 7;
826 u
->thread_model
= TM_TREEVL
;
827 u
->parallel_tree
= true;
828 u
->virtual_loss
= true;
829 u
->fuseki_end
= 20; // max time at 361*20% = 72 moves (our 36th move, still 99 to play)
830 u
->yose_start
= 40; // (100-40-25)*361/100/2 = 63 moves still to play by us then
832 u
->val_scale
= 0.04; u
->val_points
= 40;
835 char *optspec
, *next
= arg
;
838 next
+= strcspn(next
, ",");
839 if (*next
) { *next
++ = 0; } else { *next
= 0; }
841 char *optname
= optspec
;
842 char *optval
= strchr(optspec
, '=');
843 if (optval
) *optval
++ = 0;
845 if (!strcasecmp(optname
, "debug")) {
847 u
->debug_level
= atoi(optval
);
850 } else if (!strcasecmp(optname
, "mercy") && optval
) {
851 /* Minimal difference of black/white captures
852 * to stop playout - "Mercy Rule". Speeds up
853 * hopeless playouts at the expense of some
855 u
->mercymin
= atoi(optval
);
856 } else if (!strcasecmp(optname
, "gamelen") && optval
) {
857 u
->gamelen
= atoi(optval
);
858 } else if (!strcasecmp(optname
, "expand_p") && optval
) {
859 u
->expand_p
= atoi(optval
);
860 } else if (!strcasecmp(optname
, "dumpthres") && optval
) {
861 u
->dumpthres
= atoi(optval
);
862 } else if (!strcasecmp(optname
, "best2_ratio") && optval
) {
863 /* If set, prolong simulating while
864 * first_best/second_best playouts ratio
865 * is less than best2_ratio. */
866 u
->best2_ratio
= atof(optval
);
867 } else if (!strcasecmp(optname
, "bestr_ratio") && optval
) {
868 /* If set, prolong simulating while
869 * best,best_best_child values delta
870 * is more than bestr_ratio. */
871 u
->bestr_ratio
= atof(optval
);
872 } else if (!strcasecmp(optname
, "playout_amaf")) {
873 /* Whether to include random playout moves in
874 * AMAF as well. (Otherwise, only tree moves
875 * are included in AMAF. Of course makes sense
876 * only in connection with an AMAF policy.) */
877 /* with-without: 55.5% (+-4.1) */
878 if (optval
&& *optval
== '0')
879 u
->playout_amaf
= false;
881 u
->playout_amaf
= true;
882 } else if (!strcasecmp(optname
, "playout_amaf_nakade")) {
883 /* Whether to include nakade moves from playouts
884 * in the AMAF statistics; this tends to nullify
885 * the playout_amaf effect by adding too much
887 if (optval
&& *optval
== '0')
888 u
->playout_amaf_nakade
= false;
890 u
->playout_amaf_nakade
= true;
891 } else if (!strcasecmp(optname
, "playout_amaf_cutoff") && optval
) {
892 /* Keep only first N% of playout stage AMAF
894 u
->playout_amaf_cutoff
= atoi(optval
);
895 } else if ((!strcasecmp(optname
, "policy") || !strcasecmp(optname
, "random_policy")) && optval
) {
896 char *policyarg
= strchr(optval
, ':');
897 struct uct_policy
**p
= !strcasecmp(optname
, "policy") ? &u
->policy
: &u
->random_policy
;
900 if (!strcasecmp(optval
, "ucb1")) {
901 *p
= policy_ucb1_init(u
, policyarg
);
902 } else if (!strcasecmp(optval
, "ucb1amaf")) {
903 *p
= policy_ucb1amaf_init(u
, policyarg
);
905 fprintf(stderr
, "UCT: Invalid tree policy %s\n", optval
);
908 } else if (!strcasecmp(optname
, "playout") && optval
) {
909 char *playoutarg
= strchr(optval
, ':');
912 if (!strcasecmp(optval
, "moggy")) {
913 u
->playout
= playout_moggy_init(playoutarg
, b
);
914 } else if (!strcasecmp(optval
, "light")) {
915 u
->playout
= playout_light_init(playoutarg
, b
);
916 } else if (!strcasecmp(optval
, "elo")) {
917 u
->playout
= playout_elo_init(playoutarg
, b
);
919 fprintf(stderr
, "UCT: Invalid playout policy %s\n", optval
);
922 } else if (!strcasecmp(optname
, "prior") && optval
) {
923 u
->prior
= uct_prior_init(optval
, b
);
924 } else if (!strcasecmp(optname
, "amaf_prior") && optval
) {
925 u
->amaf_prior
= atoi(optval
);
926 } else if (!strcasecmp(optname
, "threads") && optval
) {
927 /* By default, Pachi will run with only single
928 * tree search thread! */
929 u
->threads
= atoi(optval
);
930 } else if (!strcasecmp(optname
, "thread_model") && optval
) {
931 if (!strcasecmp(optval
, "root")) {
932 /* Root parallelization - each thread
933 * does independent search, trees are
934 * merged at the end. */
935 u
->thread_model
= TM_ROOT
;
936 u
->parallel_tree
= false;
937 u
->virtual_loss
= false;
938 } else if (!strcasecmp(optval
, "tree")) {
939 /* Tree parallelization - all threads
940 * grind on the same tree. */
941 u
->thread_model
= TM_TREE
;
942 u
->parallel_tree
= true;
943 u
->virtual_loss
= false;
944 } else if (!strcasecmp(optval
, "treevl")) {
945 /* Tree parallelization, but also
946 * with virtual losses - this discou-
947 * rages most threads choosing the
948 * same tree branches to read. */
949 u
->thread_model
= TM_TREEVL
;
950 u
->parallel_tree
= true;
951 u
->virtual_loss
= true;
953 fprintf(stderr
, "UCT: Invalid thread model %s\n", optval
);
956 } else if (!strcasecmp(optname
, "pondering")) {
957 /* Keep searching even during opponent's turn. */
958 u
->pondering_opt
= !optval
|| atoi(optval
);
959 } else if (!strcasecmp(optname
, "fuseki_end") && optval
) {
960 /* At the very beginning it's not worth thinking
961 * too long because the playout evaluations are
962 * very noisy. So gradually increase the thinking
963 * time up to maximum when fuseki_end percent
964 * of the board has been played.
965 * This only applies if we are not in byoyomi. */
966 u
->fuseki_end
= atoi(optval
);
967 } else if (!strcasecmp(optname
, "yose_start") && optval
) {
968 /* When yose_start percent of the board has been
969 * played, or if we are in byoyomi, stop spending
970 * more time and spread the remaining time
972 * Between fuseki_end and yose_start, we spend
973 * a constant proportion of the remaining time
974 * on each move. (yose_start should actually
975 * be much earlier than when real yose start,
976 * but "yose" is a good short name to convey
978 u
->yose_start
= atoi(optval
);
979 } else if (!strcasecmp(optname
, "force_seed") && optval
) {
980 u
->force_seed
= atoi(optval
);
981 } else if (!strcasecmp(optname
, "no_book")) {
983 } else if (!strcasecmp(optname
, "dynkomi")) {
984 /* Dynamic komi in handicap game; linearly
985 * decreases to basic settings until move
987 u
->dynkomi
= optval
? atoi(optval
) : 200;
988 } else if (!strcasecmp(optname
, "dynkomi_mask") && optval
) {
989 /* Bitmask of colors the player must be
990 * for dynkomi be applied; you may want
991 * to use dynkomi_mask=3 to allow dynkomi
992 * even in games where Pachi is white. */
993 u
->dynkomi_mask
= atoi(optval
);
994 } else if (!strcasecmp(optname
, "handicap_value") && optval
) {
995 /* Point value of single handicap stone,
996 * for dynkomi computation. */
997 u
->handicap_value
= atoi(optval
);
998 } else if (!strcasecmp(optname
, "val_scale") && optval
) {
999 /* How much of the game result value should be
1000 * influenced by win size. Zero means it isn't. */
1001 u
->val_scale
= atof(optval
);
1002 } else if (!strcasecmp(optname
, "val_points") && optval
) {
1003 /* Maximum size of win to be scaled into game
1004 * result value. Zero means boardsize^2. */
1005 u
->val_points
= atoi(optval
) * 2; // result values are doubled
1006 } else if (!strcasecmp(optname
, "val_extra")) {
1007 /* If false, the score coefficient will be simply
1008 * added to the value, instead of scaling the result
1009 * coefficient because of it. */
1010 u
->val_extra
= !optval
|| atoi(optval
);
1011 } else if (!strcasecmp(optname
, "root_heuristic") && optval
) {
1012 /* Whether to bias exploration by root node values
1013 * (must be supported by the used policy).
1015 * 1: Do, value = result.
1016 * Try to temper the result:
1017 * 2: Do, value = 0.5+(result-expected)/2.
1018 * 3: Do, value = 0.5+bzz((result-expected)^2). */
1019 u
->root_heuristic
= atoi(optval
);
1020 } else if (!strcasecmp(optname
, "pass_all_alive")) {
1021 /* Whether to consider all stones alive at the game
1022 * end instead of marking dead groupd. */
1023 u
->pass_all_alive
= !optval
|| atoi(optval
);
1024 } else if (!strcasecmp(optname
, "territory_scoring")) {
1025 /* Use territory scoring (default is area scoring).
1026 * An explicit kgs-rules command overrides this. */
1027 u
->territory_scoring
= !optval
|| atoi(optval
);
1028 } else if (!strcasecmp(optname
, "random_policy_chance") && optval
) {
1029 /* If specified (N), with probability 1/N, random_policy policy
1030 * descend is used instead of main policy descend; useful
1031 * if specified policy (e.g. UCB1AMAF) can make unduly biased
1032 * choices sometimes, you can fall back to e.g.
1033 * random_policy=UCB1. */
1034 u
->random_policy_chance
= atoi(optval
);
1035 } else if (!strcasecmp(optname
, "max_tree_size") && optval
) {
1036 /* Maximum amount of memory [MiB] consumed by the move tree.
1037 * Default is 3072 (3 GiB). Note that if you use TM_ROOT,
1038 * this limits size of only one of the trees, not all of them
1040 u
->max_tree_size
= atol(optval
) * 1048576;
1041 } else if (!strcasecmp(optname
, "fast_alloc")) {
1042 u
->fast_alloc
= !optval
|| atoi(optval
);
1043 } else if (!strcasecmp(optname
, "banner") && optval
) {
1044 /* Additional banner string. This must come as the
1045 * last engine parameter. */
1046 if (*next
) *--next
= ',';
1047 u
->banner
= strdup(optval
);
1050 fprintf(stderr
, "uct: Invalid engine argument %s or missing value\n", optname
);
1056 u
->resign_ratio
= 0.2; /* Resign when most games are lost. */
1057 u
->loss_threshold
= 0.85; /* Stop reading if after at least 5000 playouts this is best value. */
1059 u
->policy
= policy_ucb1amaf_init(u
, NULL
);
1061 if (!!u
->random_policy_chance
^ !!u
->random_policy
) {
1062 fprintf(stderr
, "uct: Only one of random_policy and random_policy_chance is set\n");
1066 if (u
->fast_alloc
&& !u
->parallel_tree
) {
1067 fprintf(stderr
, "fast_alloc not supported with root parallelization.\n");
1072 u
->prior
= uct_prior_init(NULL
, b
);
1075 u
->playout
= playout_moggy_init(NULL
, b
);
1076 u
->playout
->debug_level
= u
->debug_level
;
1078 u
->ownermap
.map
= malloc(board_size2(b
) * sizeof(u
->ownermap
.map
[0]));
1080 /* Some things remain uninitialized for now - the opening book
1081 * is not loaded and the tree not set up. */
1082 /* This will be initialized in setup_state() at the first move
1083 * received/requested. This is because right now we are not aware
1084 * about any komi or handicap setup and such. */
1090 engine_uct_init(char *arg
, struct board
*b
)
1092 struct uct
*u
= uct_state_init(arg
, b
);
1093 struct engine
*e
= calloc(1, sizeof(struct engine
));
1094 e
->name
= "UCT Engine";
1095 e
->printhook
= uct_printhook_ownermap
;
1096 e
->notify_play
= uct_notify_play
;
1098 e
->genmove
= uct_genmove
;
1099 e
->dead_group_list
= uct_dead_group_list
;
1103 const char banner
[] = "I'm playing UCT. When I'm losing, I will resign, "
1104 "if I think I win, I play until you pass. "
1105 "Anyone can send me 'winrate' in private chat to get my assessment of the position.";
1106 if (!u
->banner
) u
->banner
= "";
1107 e
->comment
= malloc(sizeof(banner
) + strlen(u
->banner
) + 1);
1108 sprintf(e
->comment
, "%s %s", banner
, u
->banner
);