Fix parsing of mqprob, moves & handicap_value parameters
[pachi/peepo.git] / uct / walk.c
blobf622d4788c2c35c8db3dedc43b599eec85e4d0a8
1 #include <assert.h>
2 #include <math.h>
3 #include <pthread.h>
4 #include <signal.h>
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <string.h>
9 #define DEBUG
11 #include "debug.h"
12 #include "board.h"
13 #include "move.h"
14 #include "playout.h"
15 #include "probdist.h"
16 #include "random.h"
17 #include "uct/dynkomi.h"
18 #include "uct/internal.h"
19 #include "uct/search.h"
20 #include "uct/tree.h"
21 #include "uct/uct.h"
22 #include "uct/walk.h"
24 #define DESCENT_DLEN 512
26 void
27 uct_progress_status(struct uct *u, struct tree *t, enum stone color, int playouts)
29 if (!UDEBUGL(0))
30 return;
32 /* Best move */
33 struct tree_node *best = u->policy->choose(u->policy, t->root, t->board, color, resign);
34 if (!best) {
35 fprintf(stderr, "... No moves left\n");
36 return;
38 fprintf(stderr, "[%d] ", playouts);
39 fprintf(stderr, "best %f ", tree_node_get_value(t, 1, best->u.value));
41 /* Dynamic komi */
42 if (t->use_extra_komi)
43 fprintf(stderr, "komi %.1f ", t->extra_komi);
45 /* Best sequence */
46 fprintf(stderr, "| seq ");
47 for (int depth = 0; depth < 4; depth++) {
48 if (best && best->u.playouts >= 25) {
49 fprintf(stderr, "%3s ", coord2sstr(best->coord, t->board));
50 best = u->policy->choose(u->policy, best, t->board, color, resign);
51 } else {
52 fprintf(stderr, " ");
56 /* Best candidates */
57 fprintf(stderr, "| can ");
58 int cans = 4;
59 struct tree_node *can[cans];
60 memset(can, 0, sizeof(can));
61 best = t->root->children;
62 while (best) {
63 int c = 0;
64 while ((!can[c] || best->u.playouts > can[c]->u.playouts) && ++c < cans);
65 for (int d = 0; d < c; d++) can[d] = can[d + 1];
66 if (c > 0) can[c - 1] = best;
67 best = best->sibling;
69 while (--cans >= 0) {
70 if (can[cans]) {
71 fprintf(stderr, "%3s(%.3f) ",
72 coord2sstr(can[cans]->coord, t->board),
73 tree_node_get_value(t, 1, can[cans]->u.value));
74 } else {
75 fprintf(stderr, " ");
79 fprintf(stderr, "\n");
83 static void
84 record_amaf_move(struct playout_amafmap *amaf, coord_t coord, enum stone color)
86 if (amaf->map[coord] == S_NONE || amaf->map[coord] == color) {
87 amaf->map[coord] = color;
88 } else { // XXX: Respect amaf->record_nakade
89 amaf_op(amaf->map[coord], +);
91 amaf->game[amaf->gamelen].coord = coord;
92 amaf->game[amaf->gamelen].color = color;
93 amaf->gamelen++;
94 assert(amaf->gamelen < sizeof(amaf->game) / sizeof(amaf->game[0]));
98 struct uct_playout_callback {
99 struct uct *uct;
100 struct tree *tree;
101 struct tree_node *lnode;
105 static coord_t
106 uct_playout_hook(struct playout_policy *playout, struct playout_setup *setup, struct board *b, enum stone color, int mode)
108 /* XXX: This is used in some non-master branches. */
109 return pass;
112 static coord_t
113 uct_playout_prepolicy(struct playout_policy *playout, struct playout_setup *setup, struct board *b, enum stone color)
115 return uct_playout_hook(playout, setup, b, color, 0);
118 static coord_t
119 uct_playout_postpolicy(struct playout_policy *playout, struct playout_setup *setup, struct board *b, enum stone color)
121 return uct_playout_hook(playout, setup, b, color, 1);
125 static int
126 uct_leaf_node(struct uct *u, struct board *b, enum stone player_color,
127 struct playout_amafmap *amaf,
128 struct uct_descent *descent, int *dlen,
129 struct tree_node *significant[2],
130 struct tree *t, struct tree_node *n, enum stone node_color,
131 char *spaces)
133 enum stone next_color = stone_other(node_color);
134 int parity = (next_color == player_color ? 1 : -1);
136 /* We need to make sure only one thread expands the node. If
137 * we are unlucky enough for two threads to meet in the same
138 * node, the latter one will simply do another simulation from
139 * the node itself, no big deal. t->nodes_size may exceed
140 * the maximum in multi-threaded case but not by much so it's ok.
141 * The size test must be before the test&set not after, to allow
142 * expansion of the node later if enough nodes have been freed. */
143 if (n->u.playouts >= u->expand_p && t->nodes_size < u->max_tree_size
144 && !__sync_lock_test_and_set(&n->is_expanded, 1)) {
145 tree_expand_node(t, n, b, next_color, u, parity);
147 if (UDEBUGL(7))
148 fprintf(stderr, "%s*-- UCT playout #%d start [%s] %f\n",
149 spaces, n->u.playouts, coord2sstr(n->coord, t->board),
150 tree_node_get_value(t, parity, n->u.value));
152 struct uct_playout_callback upc = {
153 .uct = u,
154 .tree = t,
155 /* TODO: Don't necessarily restart the sequence walk when
156 * entering playout. */
157 .lnode = NULL,
160 struct playout_setup ps = {
161 .gamelen = u->gamelen,
162 .mercymin = u->mercymin,
163 .prepolicy_hook = uct_playout_prepolicy,
164 .postpolicy_hook = uct_playout_postpolicy,
165 .hook_data = &upc,
167 int result = play_random_game(&ps, b, next_color,
168 u->playout_amaf ? amaf : NULL,
169 &u->ownermap, u->playout);
170 if (next_color == S_WHITE) {
171 /* We need the result from black's perspective. */
172 result = - result;
174 if (UDEBUGL(7))
175 fprintf(stderr, "%s -- [%d..%d] %s random playout result %d\n",
176 spaces, player_color, next_color, coord2sstr(n->coord, t->board), result);
178 return result;
181 static floating_t
182 scale_value(struct uct *u, struct board *b, int result)
184 floating_t rval = result > 0 ? 1.0 : result < 0 ? 0.0 : 0.5;
185 if (u->val_scale && result != 0) {
186 int vp = u->val_points;
187 if (!vp) {
188 vp = board_size(b) - 1; vp *= vp; vp *= 2;
191 floating_t sval = (floating_t) abs(result) / vp;
192 sval = sval > 1 ? 1 : sval;
193 if (result < 0) sval = 1 - sval;
194 if (u->val_extra)
195 rval += u->val_scale * sval;
196 else
197 rval = (1 - u->val_scale) * rval + u->val_scale * sval;
198 // fprintf(stderr, "score %d => sval %f, rval %f\n", result, sval, rval);
200 return rval;
203 static void
204 record_local_sequence(struct uct *u, struct tree *t,
205 struct uct_descent *descent, int dlen, int di,
206 enum stone seq_color, floating_t rval)
208 #define LTREE_DEBUG if (UDEBUGL(6))
210 /* Ignore pass sequences. */
211 if (is_pass(descent[di].node->coord))
212 return;
214 /* Transform the rval appropriately, based on the expected
215 * result at the root of the sequence. */
216 if (u->local_tree_rootseqval) {
217 float expval = descent[di - 1].value.value;
218 rval = stats_temper_value(rval, expval, u->local_tree);
221 LTREE_DEBUG fprintf(stderr, "recording result %f in local %s sequence: ",
222 rval, stone2str(seq_color));
224 /* Sequences starting deeper are less relevant in general. */
225 int pval = LTREE_PLAYOUTS_MULTIPLIER;
226 if (u->local_tree && u->local_tree_depth_decay > 0)
227 pval = ((floating_t) pval) / pow(u->local_tree_depth_decay, di - 1);
228 if (!pval) {
229 LTREE_DEBUG fprintf(stderr, "too deep @%d\n", di);
230 return;
233 /* Pick the right local tree root... */
234 struct tree_node *lnode = seq_color == S_BLACK ? t->ltree_black : t->ltree_white;
235 lnode->u.playouts++;
237 /* ...and record the sequence. */
238 int di0 = di;
239 while (di < dlen && (di == di0 || descent[di].node->d < u->tenuki_d)) {
240 LTREE_DEBUG fprintf(stderr, "%s[%d] ",
241 coord2sstr(descent[di].node->coord, t->board),
242 descent[di].node->d);
243 lnode = tree_get_node(t, lnode, descent[di++].node->coord, true);
244 assert(lnode);
245 stats_add_result(&lnode->u, rval, pval);
248 /* Add lnode for tenuki (pass) if we descended further. */
249 if (di < dlen) {
250 LTREE_DEBUG fprintf(stderr, "pass ");
251 lnode = tree_get_node(t, lnode, pass, true);
252 assert(lnode);
253 stats_add_result(&lnode->u, rval, pval);
256 LTREE_DEBUG fprintf(stderr, "\n");
261 uct_playout(struct uct *u, struct board *b, enum stone player_color, struct tree *t)
263 struct board b2;
264 board_copy(&b2, b);
266 struct playout_amafmap *amaf = NULL;
267 if (u->policy->wants_amaf) {
268 amaf = calloc2(1, sizeof(*amaf));
269 amaf->map = calloc2(board_size2(&b2) + 1, sizeof(*amaf->map));
270 amaf->map++; // -1 is pass
273 /* Walk the tree until we find a leaf, then expand it and do
274 * a random playout. */
275 struct tree_node *n = t->root;
276 enum stone node_color = stone_other(player_color);
277 assert(node_color == t->root_color);
279 /* Tree descent history. */
280 /* XXX: This is somewhat messy since @n and descent[dlen-1].node are
281 * redundant. */
282 struct uct_descent descent[DESCENT_DLEN];
283 descent[0].node = n; descent[0].lnode = NULL;
284 int dlen = 1;
285 /* Total value of the sequence. */
286 struct move_stats seq_value = { .playouts = 0 };
287 /* The last "significant" node along the descent (i.e. node
288 * with higher than configured number of playouts). For black
289 * and white. */
290 struct tree_node *significant[2] = { NULL, NULL };
291 if (n->u.playouts >= u->significant_threshold)
292 significant[node_color - 1] = n;
294 int result;
295 int pass_limit = (board_size(&b2) - 2) * (board_size(&b2) - 2) / 2;
296 int passes = is_pass(b->last_move.coord) && b->moves > 0;
298 /* debug */
299 static char spaces[] = "\0 ";
300 /* /debug */
301 if (UDEBUGL(8))
302 fprintf(stderr, "--- UCT walk with color %d\n", player_color);
304 while (!tree_leaf_node(n) && passes < 2) {
305 spaces[dlen - 1] = ' '; spaces[dlen] = 0;
308 /*** Choose a node to descend to: */
310 /* Parity is chosen already according to the child color, since
311 * it is applied to children. */
312 node_color = stone_other(node_color);
313 int parity = (node_color == player_color ? 1 : -1);
315 assert(dlen < DESCENT_DLEN);
316 descent[dlen] = descent[dlen - 1];
317 if (u->local_tree && (!descent[dlen].lnode || descent[dlen].node->d >= u->tenuki_d)) {
318 /* Start new local sequence. */
319 /* Remember that node_color already holds color of the
320 * to-be-found child. */
321 descent[dlen].lnode = node_color == S_BLACK ? t->ltree_black : t->ltree_white;
324 if (!u->random_policy_chance || fast_random(u->random_policy_chance))
325 u->policy->descend(u->policy, t, &descent[dlen], parity, b2.moves > pass_limit);
326 else
327 u->random_policy->descend(u->random_policy, t, &descent[dlen], parity, b2.moves > pass_limit);
330 /*** Perform the descent: */
332 if (descent[dlen].node->u.playouts >= u->significant_threshold) {
333 significant[node_color - 1] = descent[dlen].node;
336 seq_value.playouts += descent[dlen].value.playouts;
337 seq_value.value += descent[dlen].value.value * descent[dlen].value.playouts;
338 n = descent[dlen++].node;
339 assert(n == t->root || n->parent);
340 if (UDEBUGL(7))
341 fprintf(stderr, "%s+-- UCT sent us to [%s:%d] %d,%f\n",
342 spaces, coord2sstr(n->coord, t->board),
343 n->coord, n->u.playouts,
344 tree_node_get_value(t, parity, n->u.value));
346 /* Add virtual loss if we need to; this is used to discourage
347 * other threads from visiting this node in case of multiple
348 * threads doing the tree search. */
349 if (u->virtual_loss)
350 stats_add_result(&n->u, node_color == S_BLACK ? 0.0 : 1.0, u->virtual_loss);
352 assert(n->coord >= -1);
353 if (amaf && !is_pass(n->coord))
354 record_amaf_move(amaf, n->coord, node_color);
356 struct move m = { n->coord, node_color };
357 int res = board_play(&b2, &m);
359 if (res < 0 || (!is_pass(m.coord) && !group_at(&b2, m.coord)) /* suicide */
360 || b2.superko_violation) {
361 if (UDEBUGL(4)) {
362 for (struct tree_node *ni = n; ni; ni = ni->parent)
363 fprintf(stderr, "%s<%"PRIhash"> ", coord2sstr(ni->coord, t->board), ni->hash);
364 fprintf(stderr, "marking invalid %s node %d,%d res %d group %d spk %d\n",
365 stone2str(node_color), coord_x(n->coord,b), coord_y(n->coord,b),
366 res, group_at(&b2, m.coord), b2.superko_violation);
368 n->hints |= TREE_HINT_INVALID;
369 result = 0;
370 goto end;
373 if (is_pass(n->coord))
374 passes++;
375 else
376 passes = 0;
379 if (amaf) {
380 amaf->game_baselen = amaf->gamelen;
381 amaf->record_nakade = u->playout_amaf_nakade;
384 if (t->use_extra_komi && u->dynkomi->persim) {
385 b2.komi += round(u->dynkomi->persim(u->dynkomi, &b2, t, n));
388 if (passes >= 2) {
389 /* XXX: No dead groups support. */
390 floating_t score = board_official_score(&b2, NULL);
391 /* Result from black's perspective (no matter who
392 * the player; black's perspective is always
393 * what the tree stores. */
394 result = - (score * 2);
396 if (UDEBUGL(5))
397 fprintf(stderr, "[%d..%d] %s p-p scoring playout result %d (W %f)\n",
398 player_color, node_color, coord2sstr(n->coord, t->board), result, score);
399 if (UDEBUGL(6))
400 board_print(&b2, stderr);
402 board_ownermap_fill(&u->ownermap, &b2);
404 } else { // assert(tree_leaf_node(n));
405 /* In case of parallel tree search, the assertion might
406 * not hold if two threads chew on the same node. */
407 result = uct_leaf_node(u, &b2, player_color, amaf, descent, &dlen, significant, t, n, node_color, spaces);
410 if (amaf && u->playout_amaf_cutoff) {
411 unsigned int cutoff = amaf->game_baselen;
412 cutoff += (amaf->gamelen - amaf->game_baselen) * u->playout_amaf_cutoff / 100;
413 /* Now, reconstruct the amaf map. */
414 memset(amaf->map, 0, board_size2(&b2) * sizeof(*amaf->map));
415 for (unsigned int i = 0; i < cutoff; i++) {
416 coord_t coord = amaf->game[i].coord;
417 enum stone color = amaf->game[i].color;
418 if (amaf->map[coord] == S_NONE || amaf->map[coord] == color) {
419 amaf->map[coord] = color;
420 /* Nakade always recorded for in-tree part */
421 } else if (amaf->record_nakade || i <= amaf->game_baselen) {
422 amaf_op(amaf->map[n->coord], +);
427 /* Record the result. */
429 assert(n == t->root || n->parent);
430 floating_t rval = scale_value(u, b, result);
431 u->policy->update(u->policy, t, n, node_color, player_color, amaf, &b2, rval);
433 if (t->use_extra_komi) {
434 stats_add_result(&u->dynkomi->score, result / 2, 1);
435 stats_add_result(&u->dynkomi->value, rval, 1);
438 if (u->local_tree && n->parent && !is_pass(n->coord) && dlen > 0) {
439 /* Possibly transform the rval appropriately. */
440 if (!u->local_tree_rootseqval) {
441 floating_t expval = seq_value.value / seq_value.playouts;
442 rval = stats_temper_value(rval, expval, u->local_tree);
445 /* Get the local sequences and record them in ltree. */
446 /* We will look for sequence starts in our descent
447 * history, then run record_local_sequence() for each
448 * found sequence start; record_local_sequence() may
449 * pick longer sequences from descent history then,
450 * which is expected as it will create new lnodes. */
451 enum stone seq_color = player_color;
452 /* First move always starts a sequence. */
453 record_local_sequence(u, t, descent, dlen, 1, seq_color, rval);
454 seq_color = stone_other(seq_color);
455 for (int dseqi = 2; dseqi < dlen; dseqi++, seq_color = stone_other(seq_color)) {
456 if (u->local_tree_allseq) {
457 /* We are configured to record all subsequences. */
458 record_local_sequence(u, t, descent, dlen, dseqi, seq_color, rval);
459 continue;
461 if (descent[dseqi].node->d >= u->tenuki_d) {
462 /* Tenuki! Record the fresh sequence. */
463 record_local_sequence(u, t, descent, dlen, dseqi, seq_color, rval);
464 continue;
466 if (descent[dseqi].lnode && !descent[dseqi].lnode) {
467 /* Record result for in-descent picked sequence. */
468 record_local_sequence(u, t, descent, dlen, dseqi, seq_color, rval);
469 continue;
474 end:
475 /* We need to undo the virtual loss we added during descend. */
476 if (u->virtual_loss) {
477 floating_t loss = node_color == S_BLACK ? 0.0 : 1.0;
478 for (; n->parent; n = n->parent) {
479 stats_rm_result(&n->u, loss, u->virtual_loss);
480 loss = 1.0 - loss;
484 if (amaf) {
485 free(amaf->map - 1);
486 free(amaf);
488 board_done_noalloc(&b2);
489 return result;
493 uct_playouts(struct uct *u, struct board *b, enum stone color, struct tree *t, struct time_info *ti)
495 int i;
496 if (ti && ti->dim == TD_GAMES) {
497 for (i = 0; t->root->u.playouts <= ti->len.games; i++)
498 uct_playout(u, b, color, t);
499 } else {
500 for (i = 0; !uct_halt; i++)
501 uct_playout(u, b, color, t);
503 return i;