Patterns: Use unsigned where appropriate
[pachi.git] / uct / walk.c
blobfb5683b3fa63ecc5bdc285c0a093298fd68698fa
1 #include <assert.h>
2 #include <math.h>
3 #include <pthread.h>
4 #include <signal.h>
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <string.h>
9 #define DEBUG
11 #include "debug.h"
12 #include "board.h"
13 #include "move.h"
14 #include "playout.h"
15 #include "probdist.h"
16 #include "random.h"
17 #include "tactics/util.h"
18 #include "uct/dynkomi.h"
19 #include "uct/internal.h"
20 #include "uct/search.h"
21 #include "uct/tree.h"
22 #include "uct/uct.h"
23 #include "uct/walk.h"
25 #define DESCENT_DLEN 512
28 void
29 uct_progress_text(struct uct *u, struct tree *t, enum stone color, int playouts, bool final)
31 if (!UDEBUGL(0))
32 return;
34 /* Best move */
35 struct tree_node *best = u->policy->choose(u->policy, t->root, t->board, color, resign);
36 if (!best) {
37 fprintf(stderr, "... No moves left\n");
38 return;
40 fprintf(stderr, "[%d] ", playouts);
41 fprintf(stderr, "best %f ", tree_node_get_value(t, 1, best->u.value));
43 /* Dynamic komi */
44 if (t->use_extra_komi)
45 fprintf(stderr, "komi %.1f ", t->extra_komi);
47 /* Best sequence */
48 fprintf(stderr, "| seq ");
49 for (int depth = 0; depth < 4; depth++) {
50 if (best && best->u.playouts >= 25) {
51 fprintf(stderr, "%3s ", coord2sstr(node_coord(best), t->board));
52 best = u->policy->choose(u->policy, best, t->board, color, resign);
53 } else {
54 fprintf(stderr, " ");
58 /* Best candidates */
59 fprintf(stderr, "| can ");
60 int cans = 4;
61 struct tree_node *can[cans];
62 memset(can, 0, sizeof(can));
63 best = t->root->children;
64 while (best) {
65 int c = 0;
66 while ((!can[c] || best->u.playouts > can[c]->u.playouts) && ++c < cans);
67 for (int d = 0; d < c; d++) can[d] = can[d + 1];
68 if (c > 0) can[c - 1] = best;
69 best = best->sibling;
71 while (--cans >= 0) {
72 if (can[cans]) {
73 fprintf(stderr, "%3s(%.3f) ",
74 coord2sstr(node_coord(can[cans]), t->board),
75 tree_node_get_value(t, 1, can[cans]->u.value));
76 } else {
77 fprintf(stderr, " ");
81 fprintf(stderr, "\n");
84 void
85 uct_progress_json(struct uct *u, struct tree *t, enum stone color, int playouts, bool final, bool big)
87 /* Prefix indicating JSON line. */
88 fprintf(stderr, "{\"%s\": {", final ? "move" : "frame");
90 /* Plaout count */
91 fprintf(stderr, "\"playouts\": %d", playouts);
93 /* Dynamic komi */
94 if (t->use_extra_komi)
95 fprintf(stderr, ", \"extrakomi\": %.1f", t->extra_komi);
97 struct tree_node *best = u->policy->choose(u->policy, t->root, t->board, color, resign);
98 if (best) {
99 /* Best move */
100 fprintf(stderr, ", \"best\": {\"%s\": %f}",
101 coord2sstr(best->coord, t->board),
102 tree_node_get_value(t, 1, best->u.value));
105 /* Best candidates */
106 int cans = 4;
107 struct tree_node *can[cans];
108 memset(can, 0, sizeof(can));
109 best = t->root->children;
110 while (best) {
111 int c = 0;
112 while ((!can[c] || best->u.playouts > can[c]->u.playouts) && ++c < cans);
113 for (int d = 0; d < c; d++) can[d] = can[d + 1];
114 if (c > 0) can[c - 1] = best;
115 best = best->sibling;
117 fprintf(stderr, ", \"can\": [");
118 while (--cans >= 0) {
119 if (!can[cans]) break;
120 /* Best sequence */
121 fprintf(stderr, "%s[", cans < 3 ? ", " : "");
122 best = can[cans];
123 for (int depth = 0; depth < 4; depth++) {
124 if (!best || best->u.playouts < 25) break;
125 fprintf(stderr, "%s{\"%s\":%.3f}", depth > 0 ? "," : "",
126 coord2sstr(best->coord, t->board),
127 tree_node_get_value(t, 1, best->u.value));
128 best = u->policy->choose(u->policy, best, t->board, color, resign);
130 fprintf(stderr, "]");
132 fprintf(stderr, "]");
134 if (big) {
135 /* Average score. */
136 if (t->avg_score.playouts > 0)
137 fprintf(stderr, ", \"avg\": {\"score\": %.3f}", t->avg_score.value);
138 /* Per-intersection information. */
139 fprintf(stderr, ", \"boards\": {");
140 /* Position coloring information. */
141 fprintf(stderr, "\"colors\": [");
142 int f = 0;
143 foreach_point(t->board) {
144 if (board_at(t->board, c) == S_OFFBOARD) continue;
145 fprintf(stderr, "%s%d", f++ > 0 ? "," : "", board_at(t->board, c));
146 } foreach_point_end;
147 fprintf(stderr, "]");
148 /* Ownership statistics. Value (0..1000) for each possible
149 * point describes likelihood of this point becoming black.
150 * Normally, white rate is 1000-value; exception are possible
151 * seki points, but these should be rare. */
152 fprintf(stderr, ", \"territory\": [");
153 f = 0;
154 foreach_point(t->board) {
155 if (board_at(t->board, c) == S_OFFBOARD) continue;
156 int rate = u->ownermap.map[c][S_BLACK] * 1000 / u->ownermap.playouts;
157 fprintf(stderr, "%s%d", f++ > 0 ? "," : "", rate);
158 } foreach_point_end;
159 fprintf(stderr, "]");
160 fprintf(stderr, "}");
163 fprintf(stderr, "}}\n");
166 void
167 uct_progress_status(struct uct *u, struct tree *t, enum stone color, int playouts, bool final)
169 switch (u->reporting) {
170 case UR_TEXT:
171 uct_progress_text(u, t, color, playouts, final);
172 break;
173 case UR_JSON:
174 case UR_JSON_BIG:
175 uct_progress_json(u, t, color, playouts, final,
176 u->reporting == UR_JSON_BIG);
177 break;
178 default: assert(0);
183 static inline void
184 record_amaf_move(struct playout_amafmap *amaf, coord_t coord)
186 assert(amaf->gamelen < MAX_GAMELEN);
187 amaf->game[amaf->gamelen++] = coord;
191 struct uct_playout_callback {
192 struct uct *uct;
193 struct tree *tree;
194 struct tree_node *lnode;
198 static coord_t
199 uct_playout_hook(struct playout_policy *playout, struct playout_setup *setup, struct board *b, enum stone color, int mode)
201 /* XXX: This is used in some non-master branches. */
202 return pass;
205 static coord_t
206 uct_playout_prepolicy(struct playout_policy *playout, struct playout_setup *setup, struct board *b, enum stone color)
208 return uct_playout_hook(playout, setup, b, color, 0);
211 static coord_t
212 uct_playout_postpolicy(struct playout_policy *playout, struct playout_setup *setup, struct board *b, enum stone color)
214 return uct_playout_hook(playout, setup, b, color, 1);
218 static int
219 uct_leaf_node(struct uct *u, struct board *b, enum stone player_color,
220 struct playout_amafmap *amaf,
221 struct uct_descent *descent, int *dlen,
222 struct tree_node *significant[2],
223 struct tree *t, struct tree_node *n, enum stone node_color,
224 char *spaces)
226 enum stone next_color = stone_other(node_color);
227 int parity = (next_color == player_color ? 1 : -1);
229 if (UDEBUGL(7))
230 fprintf(stderr, "%s*-- UCT playout #%d start [%s] %f\n",
231 spaces, n->u.playouts, coord2sstr(node_coord(n), t->board),
232 tree_node_get_value(t, parity, n->u.value));
234 struct uct_playout_callback upc = {
235 .uct = u,
236 .tree = t,
237 /* TODO: Don't necessarily restart the sequence walk when
238 * entering playout. */
239 .lnode = NULL,
242 struct playout_setup ps = {
243 .gamelen = u->gamelen,
244 .mercymin = u->mercymin,
245 .prepolicy_hook = uct_playout_prepolicy,
246 .postpolicy_hook = uct_playout_postpolicy,
247 .hook_data = &upc,
249 int result = play_random_game(&ps, b, next_color,
250 u->playout_amaf ? amaf : NULL,
251 &u->ownermap, u->playout);
252 if (next_color == S_WHITE) {
253 /* We need the result from black's perspective. */
254 result = - result;
256 if (UDEBUGL(7))
257 fprintf(stderr, "%s -- [%d..%d] %s random playout result %d\n",
258 spaces, player_color, next_color, coord2sstr(node_coord(n), t->board), result);
260 return result;
263 static floating_t
264 scale_value(struct uct *u, struct board *b, int result)
266 floating_t rval = result > 0 ? 1.0 : result < 0 ? 0.0 : 0.5;
267 if (u->val_scale && result != 0) {
268 int vp = u->val_points;
269 if (!vp) {
270 vp = board_size(b) - 1; vp *= vp; vp *= 2;
273 floating_t sval = (floating_t) abs(result) / vp;
274 sval = sval > 1 ? 1 : sval;
275 if (result < 0) sval = 1 - sval;
276 if (u->val_extra)
277 rval += u->val_scale * sval;
278 else
279 rval = (1 - u->val_scale) * rval + u->val_scale * sval;
280 // fprintf(stderr, "score %d => sval %f, rval %f\n", result, sval, rval);
282 return rval;
285 static double
286 local_value(struct uct *u, struct board *b, coord_t coord, enum stone color)
288 /* Tactical evaluation of move @coord by color @color, given
289 * simulation end position @b. I.e., a move is tactically good
290 * if the resulting group stays on board until the game end. */
291 /* We can also take into account surrounding stones, e.g. to
292 * encourage taking off external liberties during a semeai. */
293 double val = board_local_value(u->local_tree_neival, b, coord, color);
294 return (color == S_WHITE) ? 1.f - val : val;
297 static void
298 record_local_sequence(struct uct *u, struct tree *t, struct board *endb,
299 struct uct_descent *descent, int dlen, int di,
300 enum stone seq_color)
302 #define LTREE_DEBUG if (UDEBUGL(6))
304 /* Ignore pass sequences. */
305 if (is_pass(node_coord(descent[di].node)))
306 return;
308 LTREE_DEBUG board_print(endb, stderr);
309 LTREE_DEBUG fprintf(stderr, "recording local %s sequence: ",
310 stone2str(seq_color));
312 /* Sequences starting deeper are less relevant in general. */
313 int pval = LTREE_PLAYOUTS_MULTIPLIER;
314 if (u->local_tree && u->local_tree_depth_decay > 0)
315 pval = ((floating_t) pval) / pow(u->local_tree_depth_decay, di - 1);
316 if (!pval) {
317 LTREE_DEBUG fprintf(stderr, "too deep @%d\n", di);
318 return;
321 /* Pick the right local tree root... */
322 struct tree_node *lnode = seq_color == S_BLACK ? t->ltree_black : t->ltree_white;
323 lnode->u.playouts++;
325 /* ...determine the sequence value... */
326 double sval = 0.5;
327 if (u->local_tree_eval != LTE_EACH) {
328 sval = local_value(u, endb, node_coord(descent[di].node), seq_color);
329 LTREE_DEBUG fprintf(stderr, "(goal %s[%s %1.3f][%d]) ",
330 coord2sstr(node_coord(descent[di].node), t->board),
331 stone2str(seq_color), sval, descent[di].node->d);
333 if (u->local_tree_eval == LTE_TOTAL) {
334 int di0 = di;
335 while (di < dlen && (di == di0 || descent[di].node->d < u->tenuki_d)) {
336 enum stone color = (di - di0) % 2 ? stone_other(seq_color) : seq_color;
337 double rval = local_value(u, endb, node_coord(descent[di].node), color);
338 if ((di - di0) % 2)
339 rval = 1 - rval;
340 sval += rval;
341 di++;
343 sval /= (di - di0 + 1);
344 di = di0;
348 /* ...and record the sequence. */
349 int di0 = di;
350 while (di < dlen && !is_pass(node_coord(descent[di].node))
351 && (di == di0 || descent[di].node->d < u->tenuki_d)) {
352 enum stone color = (di - di0) % 2 ? stone_other(seq_color) : seq_color;
353 double rval;
354 if (u->local_tree_eval != LTE_EACH)
355 rval = sval;
356 else
357 rval = local_value(u, endb, node_coord(descent[di].node), color);
358 LTREE_DEBUG fprintf(stderr, "%s[%s %1.3f][%d] ",
359 coord2sstr(node_coord(descent[di].node), t->board),
360 stone2str(color), rval, descent[di].node->d);
361 lnode = tree_get_node(t, lnode, node_coord(descent[di++].node), true);
362 assert(lnode);
363 stats_add_result(&lnode->u, rval, pval);
366 /* Add lnode for tenuki (pass) if we descended further. */
367 if (di < dlen) {
368 double rval = u->local_tree_eval != LTE_EACH ? sval : 0.5;
369 LTREE_DEBUG fprintf(stderr, "pass ");
370 lnode = tree_get_node(t, lnode, pass, true);
371 assert(lnode);
372 stats_add_result(&lnode->u, rval, pval);
375 LTREE_DEBUG fprintf(stderr, "\n");
380 uct_playout(struct uct *u, struct board *b, enum stone player_color, struct tree *t)
382 struct board b2;
383 board_copy(&b2, b);
385 struct playout_amafmap amaf;
386 amaf.gamelen = amaf.game_baselen = 0;
388 /* Walk the tree until we find a leaf, then expand it and do
389 * a random playout. */
390 struct tree_node *n = t->root;
391 enum stone node_color = stone_other(player_color);
392 assert(node_color == t->root_color);
394 /* Make sure the root node is expanded. */
395 if (tree_leaf_node(n) && !__sync_lock_test_and_set(&n->is_expanded, 1))
396 tree_expand_node(t, n, &b2, player_color, u, 1);
398 /* Tree descent history. */
399 /* XXX: This is somewhat messy since @n and descent[dlen-1].node are
400 * redundant. */
401 struct uct_descent descent[DESCENT_DLEN];
402 descent[0].node = n; descent[0].lnode = NULL;
403 int dlen = 1;
404 /* Total value of the sequence. */
405 struct move_stats seq_value = { .playouts = 0 };
406 /* The last "significant" node along the descent (i.e. node
407 * with higher than configured number of playouts). For black
408 * and white. */
409 struct tree_node *significant[2] = { NULL, NULL };
410 if (n->u.playouts >= u->significant_threshold)
411 significant[node_color - 1] = n;
413 int result;
414 int pass_limit = (board_size(&b2) - 2) * (board_size(&b2) - 2) / 2;
415 int passes = is_pass(b->last_move.coord) && b->moves > 0;
417 /* debug */
418 static char spaces[] = "\0 ";
419 /* /debug */
420 if (UDEBUGL(8))
421 fprintf(stderr, "--- UCT walk with color %d\n", player_color);
423 while (!tree_leaf_node(n) && passes < 2) {
424 spaces[dlen - 1] = ' '; spaces[dlen] = 0;
427 /*** Choose a node to descend to: */
429 /* Parity is chosen already according to the child color, since
430 * it is applied to children. */
431 node_color = stone_other(node_color);
432 int parity = (node_color == player_color ? 1 : -1);
434 assert(dlen < DESCENT_DLEN);
435 descent[dlen] = descent[dlen - 1];
436 if (u->local_tree && (!descent[dlen].lnode || descent[dlen].node->d >= u->tenuki_d)) {
437 /* Start new local sequence. */
438 /* Remember that node_color already holds color of the
439 * to-be-found child. */
440 descent[dlen].lnode = node_color == S_BLACK ? t->ltree_black : t->ltree_white;
443 if (!u->random_policy_chance || fast_random(u->random_policy_chance))
444 u->policy->descend(u->policy, t, &descent[dlen], parity, b2.moves > pass_limit);
445 else
446 u->random_policy->descend(u->random_policy, t, &descent[dlen], parity, b2.moves > pass_limit);
449 /*** Perform the descent: */
451 if (descent[dlen].node->u.playouts >= u->significant_threshold) {
452 significant[node_color - 1] = descent[dlen].node;
455 seq_value.playouts += descent[dlen].value.playouts;
456 seq_value.value += descent[dlen].value.value * descent[dlen].value.playouts;
457 n = descent[dlen++].node;
458 assert(n == t->root || n->parent);
459 if (UDEBUGL(7))
460 fprintf(stderr, "%s+-- UCT sent us to [%s:%d] %d,%f\n",
461 spaces, coord2sstr(node_coord(n), t->board),
462 node_coord(n), n->u.playouts,
463 tree_node_get_value(t, parity, n->u.value));
465 /* Add virtual loss if we need to; this is used to discourage
466 * other threads from visiting this node in case of multiple
467 * threads doing the tree search. */
468 if (u->virtual_loss)
469 stats_add_result(&n->u, node_color == S_BLACK ? 0.0 : 1.0, u->virtual_loss);
471 assert(node_coord(n) >= -1);
472 record_amaf_move(&amaf, node_coord(n));
474 struct move m = { node_coord(n), node_color };
475 int res = board_play(&b2, &m);
477 if (res < 0 || (!is_pass(m.coord) && !group_at(&b2, m.coord)) /* suicide */
478 || b2.superko_violation) {
479 if (UDEBUGL(4)) {
480 for (struct tree_node *ni = n; ni; ni = ni->parent)
481 fprintf(stderr, "%s<%"PRIhash"> ", coord2sstr(node_coord(ni), t->board), ni->hash);
482 fprintf(stderr, "marking invalid %s node %d,%d res %d group %d spk %d\n",
483 stone2str(node_color), coord_x(node_coord(n),b), coord_y(node_coord(n),b),
484 res, group_at(&b2, m.coord), b2.superko_violation);
486 n->hints |= TREE_HINT_INVALID;
487 result = 0;
488 goto end;
491 if (is_pass(node_coord(n)))
492 passes++;
493 else
494 passes = 0;
496 enum stone next_color = stone_other(node_color);
497 /* We need to make sure only one thread expands the node. If
498 * we are unlucky enough for two threads to meet in the same
499 * node, the latter one will simply do another simulation from
500 * the node itself, no big deal. t->nodes_size may exceed
501 * the maximum in multi-threaded case but not by much so it's ok.
502 * The size test must be before the test&set not after, to allow
503 * expansion of the node later if enough nodes have been freed. */
504 if (tree_leaf_node(n)
505 && n->u.playouts - u->virtual_loss >= u->expand_p && t->nodes_size < u->max_tree_size
506 && !__sync_lock_test_and_set(&n->is_expanded, 1))
507 tree_expand_node(t, n, &b2, next_color, u, -parity);
510 amaf.game_baselen = amaf.gamelen;
512 if (t->use_extra_komi && u->dynkomi->persim) {
513 b2.komi += round(u->dynkomi->persim(u->dynkomi, &b2, t, n));
516 if (passes >= 2) {
517 /* XXX: No dead groups support. */
518 floating_t score = board_official_score(&b2, NULL);
519 /* Result from black's perspective (no matter who
520 * the player; black's perspective is always
521 * what the tree stores. */
522 result = - (score * 2);
524 if (UDEBUGL(5))
525 fprintf(stderr, "[%d..%d] %s p-p scoring playout result %d (W %f)\n",
526 player_color, node_color, coord2sstr(node_coord(n), t->board), result, score);
527 if (UDEBUGL(6))
528 board_print(&b2, stderr);
530 board_ownermap_fill(&u->ownermap, &b2);
532 } else { // assert(tree_leaf_node(n));
533 /* In case of parallel tree search, the assertion might
534 * not hold if two threads chew on the same node. */
535 result = uct_leaf_node(u, &b2, player_color, &amaf, descent, &dlen, significant, t, n, node_color, spaces);
538 if (u->policy->wants_amaf && u->playout_amaf_cutoff) {
539 unsigned int cutoff = amaf.game_baselen;
540 cutoff += (amaf.gamelen - amaf.game_baselen) * u->playout_amaf_cutoff / 100;
541 amaf.gamelen = cutoff;
544 /* Record the result. */
546 assert(n == t->root || n->parent);
547 floating_t rval = scale_value(u, b, result);
548 u->policy->update(u->policy, t, n, node_color, player_color, &amaf, &b2, rval);
550 stats_add_result(&t->avg_score, result / 2, 1);
551 if (t->use_extra_komi) {
552 stats_add_result(&u->dynkomi->score, result / 2, 1);
553 stats_add_result(&u->dynkomi->value, rval, 1);
556 if (u->local_tree && n->parent && !is_pass(node_coord(n)) && dlen > 0) {
557 /* Get the local sequences and record them in ltree. */
558 /* We will look for sequence starts in our descent
559 * history, then run record_local_sequence() for each
560 * found sequence start; record_local_sequence() may
561 * pick longer sequences from descent history then,
562 * which is expected as it will create new lnodes. */
563 enum stone seq_color = player_color;
564 /* First move always starts a sequence. */
565 record_local_sequence(u, t, &b2, descent, dlen, 1, seq_color);
566 seq_color = stone_other(seq_color);
567 for (int dseqi = 2; dseqi < dlen; dseqi++, seq_color = stone_other(seq_color)) {
568 if (u->local_tree_allseq) {
569 /* We are configured to record all subsequences. */
570 record_local_sequence(u, t, &b2, descent, dlen, dseqi, seq_color);
571 continue;
573 if (descent[dseqi].node->d >= u->tenuki_d) {
574 /* Tenuki! Record the fresh sequence. */
575 record_local_sequence(u, t, &b2, descent, dlen, dseqi, seq_color);
576 continue;
578 if (descent[dseqi].lnode && !descent[dseqi].lnode) {
579 /* Record result for in-descent picked sequence. */
580 record_local_sequence(u, t, &b2, descent, dlen, dseqi, seq_color);
581 continue;
586 end:
587 /* We need to undo the virtual loss we added during descend. */
588 if (u->virtual_loss) {
589 floating_t loss = node_color == S_BLACK ? 0.0 : 1.0;
590 for (; n->parent; n = n->parent) {
591 stats_rm_result(&n->u, loss, u->virtual_loss);
592 loss = 1.0 - loss;
596 board_done_noalloc(&b2);
597 return result;
601 uct_playouts(struct uct *u, struct board *b, enum stone color, struct tree *t, struct time_info *ti)
603 int i;
604 if (ti && ti->dim == TD_GAMES) {
605 for (i = 0; t->root->u.playouts <= ti->len.games && !uct_halt; i++)
606 uct_playout(u, b, color, t);
607 } else {
608 for (i = 0; !uct_halt; i++)
609 uct_playout(u, b, color, t);
611 return i;