uct/walk.c

   1 #include <assert.h>
   2 #include <math.h>
   3 #include <pthread.h>
   4 #include <signal.h>
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8
   9 #define DEBUG
  10
  11 #include "debug.h"
  12 #include "board.h"
  13 #include "move.h"
  14 #include "playout.h"
  15 #include "probdist.h"
  16 #include "random.h"
  17 #include "tactics/util.h"
  18 #include "uct/dynkomi.h"
  19 #include "uct/internal.h"
  20 #include "uct/search.h"
  21 #include "uct/tree.h"
  22 #include "uct/uct.h"
  23 #include "uct/walk.h"
  24
  25 #define DESCENT_DLEN 512
  26
  27 void
  28 uct_progress_status(struct uct *u, struct tree *t, enum stone color, int playouts)
  29 {
  30         if (!UDEBUGL(0))
  31                 return;
  32
  33         /* Best move */
  34         struct tree_node *best = u->policy->choose(u->policy, t->root, t->board, color, resign);
  35         if (!best) {
  36                 fprintf(stderr, "... No moves left\n");
  37                 return;
  38         }
  39         fprintf(stderr, "[%d] ", playouts);
  40         fprintf(stderr, "best %f ", tree_node_get_value(t, 1, best->u.value));
  41
  42         /* Dynamic komi */
  43         if (t->use_extra_komi)
  44                 fprintf(stderr, "komi %.1f ", t->extra_komi);
  45
  46         /* Best sequence */
  47         fprintf(stderr, "| seq ");
  48         for (int depth = 0; depth < 4; depth++) {
  49                 if (best && best->u.playouts >= 25) {
  50                         fprintf(stderr, "%3s ", coord2sstr(node_coord(best), t->board));
  51                         best = u->policy->choose(u->policy, best, t->board, color, resign);
  52                 } else {
  53                         fprintf(stderr, "    ");
  54                 }
  55         }
  56
  57         /* Best candidates */
  58         fprintf(stderr, "| can ");
  59         int cans = 4;
  60         struct tree_node *can[cans];
  61         memset(can, 0, sizeof(can));
  62         best = t->root->children;
  63         while (best) {
  64                 int c = 0;
  65                 while ((!can[c] || best->u.playouts > can[c]->u.playouts) && ++c < cans);
  66                 for (int d = 0; d < c; d++) can[d] = can[d + 1];
  67                 if (c > 0) can[c - 1] = best;
  68                 best = best->sibling;
  69         }
  70         while (--cans >= 0) {
  71                 if (can[cans]) {
  72                         fprintf(stderr, "%3s(%.3f) ",
  73                                 coord2sstr(node_coord(can[cans]), t->board),
  74                                 tree_node_get_value(t, 1, can[cans]->u.value));
  75                 } else {
  76                         fprintf(stderr, "           ");
  77                 }
  78         }
  79
  80         fprintf(stderr, "\n");
  81 }
  82
  83
  84 static void
  85 record_amaf_move(struct playout_amafmap *amaf, coord_t coord, enum stone color)
  86 {
  87         if (amaf->map[coord] == S_NONE || amaf->map[coord] == color) {
  88                 amaf->map[coord] = color;
  89         } else { // XXX: Respect amaf->record_nakade
  90                 amaf_op(amaf->map[coord], +);
  91         }
  92         amaf->game[amaf->gamelen].coord = coord;
  93         amaf->game[amaf->gamelen].color = color;
  94         amaf->gamelen++;
  95         assert(amaf->gamelen < sizeof(amaf->game) / sizeof(amaf->game[0]));
  96 }
  97
  98
  99 struct uct_playout_callback {
 100         struct uct *uct;
 101         struct tree *tree;
 102         struct tree_node *lnode;
 103 };
 104
 105
 106 static coord_t
 107 uct_playout_hook(struct playout_policy *playout, struct playout_setup *setup, struct board *b, enum stone color, int mode)
 108 {
 109         /* XXX: This is used in some non-master branches. */
 110         return pass;
 111 }
 112
 113 static coord_t
 114 uct_playout_prepolicy(struct playout_policy *playout, struct playout_setup *setup, struct board *b, enum stone color)
 115 {
 116         return uct_playout_hook(playout, setup, b, color, 0);
 117 }
 118
 119 static coord_t
 120 uct_playout_postpolicy(struct playout_policy *playout, struct playout_setup *setup, struct board *b, enum stone color)
 121 {
 122         return uct_playout_hook(playout, setup, b, color, 1);
 123 }
 124
 125
 126 static int
 127 uct_leaf_node(struct uct *u, struct board *b, enum stone player_color,
 128               struct playout_amafmap *amaf,
 129               struct uct_descent *descent, int *dlen,
 130               struct tree_node *significant[2],
 131               struct tree *t, struct tree_node *n, enum stone node_color,
 132               char *spaces)
 133 {
 134         enum stone next_color = stone_other(node_color);
 135         int parity = (next_color == player_color ? 1 : -1);
 136
 137         if (UDEBUGL(7))
 138                 fprintf(stderr, "%s*-- UCT playout #%d start [%s] %f\n",
 139                         spaces, n->u.playouts, coord2sstr(node_coord(n), t->board),
 140                         tree_node_get_value(t, parity, n->u.value));
 141
 142         struct uct_playout_callback upc = {
 143                 .uct = u,
 144                 .tree = t,
 145                 /* TODO: Don't necessarily restart the sequence walk when
 146                  * entering playout. */
 147                 .lnode = NULL,
 148         };
 149
 150         struct playout_setup ps = {
 151                 .gamelen = u->gamelen,
 152                 .mercymin = u->mercymin,
 153                 .prepolicy_hook = uct_playout_prepolicy,
 154                 .postpolicy_hook = uct_playout_postpolicy,
 155                 .hook_data = &upc,
 156         };
 157         int result = play_random_game(&ps, b, next_color,
 158                                       u->playout_amaf ? amaf : NULL,
 159                                       &u->ownermap, u->playout);
 160         if (next_color == S_WHITE) {
 161                 /* We need the result from black's perspective. */
 162                 result = - result;
 163         }
 164         if (UDEBUGL(7))
 165                 fprintf(stderr, "%s -- [%d..%d] %s random playout result %d\n",
 166                         spaces, player_color, next_color, coord2sstr(node_coord(n), t->board), result);
 167
 168         return result;
 169 }
 170
 171 static floating_t
 172 scale_value(struct uct *u, struct board *b, int result)
 173 {
 174         if (result == 0) return 0.5;
 175         floating_t rval = result > 0 ? 1.0 : 0.0;
 176
 177         floating_t scale = u->val_scale;
 178         /* Give more weight to territory when winning big (maximize win). This reduces
 179          * the number of silly moves and makes the game more enjoyable for humans. */
 180         if (u->t->root->u.playouts > GJ_MINGAMES &&
 181             tree_node_get_value(u->t, -1, u->t->root->u.value) >= u->sure_win_threshold) {
 182                 scale = u->val_scale_max;
 183         }
 184         if (scale == 0) return rval;
 185
 186         int vp = u->val_points;
 187         /* By default do not try to win by more than 44 points on 19x19,
 188          * 12 points on 9x9. Remember that result here is twice the score. */
 189         if (!vp) vp = board_size2(b) / 5;
 190
 191         floating_t sval = (floating_t) abs(result) / vp;
 192         sval = sval > 1 ? 1 : sval;
 193         if (result < 0) sval = 1 - sval;
 194         if (u->val_extra)
 195                 rval += scale * sval;
 196         else
 197                 rval = (1 - scale) * rval + scale * sval;
 198         // fprintf(stderr, "score %d => sval %f, rval %f\n", result, sval, rval);
 199         return rval;
 200 }
 201
 202 static double
 203 local_value(struct uct *u, struct board *b, coord_t coord, enum stone color)
 204 {
 205         /* Tactical evaluation of move @coord by color @color, given
 206          * simulation end position @b. I.e., a move is tactically good
 207          * if the resulting group stays on board until the game end. */
 208         /* We can also take into account surrounding stones, e.g. to
 209          * encourage taking off external liberties during a semeai. */
 210         double val = board_local_value(u->local_tree_neival, b, coord, color);
 211         return (color == S_WHITE) ? 1.f - val : val;
 212 }
 213
 214 static void
 215 record_local_sequence(struct uct *u, struct tree *t, struct board *endb,
 216                       struct uct_descent *descent, int dlen, int di,
 217                       enum stone seq_color)
 218 {
 219 #define LTREE_DEBUG if (UDEBUGL(6))
 220
 221         /* Ignore pass sequences. */
 222         if (is_pass(node_coord(descent[di].node)))
 223                 return;
 224
 225         LTREE_DEBUG board_print(endb, stderr);
 226         LTREE_DEBUG fprintf(stderr, "recording local %s sequence: ",
 227                 stone2str(seq_color));
 228
 229         /* Sequences starting deeper are less relevant in general. */
 230         int pval = LTREE_PLAYOUTS_MULTIPLIER;
 231         if (u->local_tree && u->local_tree_depth_decay > 0)
 232                 pval = ((floating_t) pval) / pow(u->local_tree_depth_decay, di - 1);
 233         if (!pval) {
 234                 LTREE_DEBUG fprintf(stderr, "too deep @%d\n", di);
 235                 return;
 236         }
 237
 238         /* Pick the right local tree root... */
 239         struct tree_node *lnode = seq_color == S_BLACK ? t->ltree_black : t->ltree_white;
 240         lnode->u.playouts++;
 241
 242         double sval = 0.5;
 243         if (u->local_tree_rootgoal) {
 244                 sval = local_value(u, endb, node_coord(descent[di].node), seq_color);
 245                 LTREE_DEBUG fprintf(stderr, "(goal %s[%s %1.3f][%d]) ",
 246                         coord2sstr(node_coord(descent[di].node), t->board),
 247                         stone2str(seq_color), sval, descent[di].node->d);
 248         }
 249
 250         /* ...and record the sequence. */
 251         int di0 = di;
 252         while (di < dlen && (di == di0 || descent[di].node->d < u->tenuki_d)) {
 253                 enum stone color = (di - di0) % 2 ? stone_other(seq_color) : seq_color;
 254                 double rval;
 255                 if (u->local_tree_rootgoal)
 256                         rval = sval;
 257                 else
 258                         rval = local_value(u, endb, node_coord(descent[di].node), color);
 259                 LTREE_DEBUG fprintf(stderr, "%s[%s %1.3f][%d] ",
 260                         coord2sstr(node_coord(descent[di].node), t->board),
 261                         stone2str(color), rval, descent[di].node->d);
 262                 lnode = tree_get_node(t, lnode, node_coord(descent[di++].node), true);
 263                 assert(lnode);
 264                 stats_add_result(&lnode->u, rval, pval);
 265         }
 266
 267         /* Add lnode for tenuki (pass) if we descended further. */
 268         if (di < dlen) {
 269                 double rval = u->local_tree_rootgoal ? sval : 0.5;
 270                 LTREE_DEBUG fprintf(stderr, "pass ");
 271                 lnode = tree_get_node(t, lnode, pass, true);
 272                 assert(lnode);
 273                 stats_add_result(&lnode->u, rval, pval);
 274         }
 275
 276         LTREE_DEBUG fprintf(stderr, "\n");
 277 }
 278
 279
 280 int
 281 uct_playout(struct uct *u, struct board *b, enum stone player_color, struct tree *t)
 282 {
 283         struct board b2;
 284         board_copy(&b2, b);
 285
 286         struct playout_amafmap *amaf = NULL;
 287         if (u->policy->wants_amaf) {
 288                 amaf = calloc2(1, sizeof(*amaf));
 289                 amaf->map = calloc2(board_size2(&b2) + 1, sizeof(*amaf->map));
 290                 amaf->map++; // -1 is pass
 291         }
 292
 293         /* Walk the tree until we find a leaf, then expand it and do
 294          * a random playout. */
 295         struct tree_node *n = t->root;
 296         enum stone node_color = stone_other(player_color);
 297         assert(node_color == t->root_color);
 298
 299         /* Make sure the root node is expanded. */
 300         if (tree_leaf_node(n) && !__sync_lock_test_and_set(&n->is_expanded, 1))
 301                 tree_expand_node(t, n, &b2, player_color, u, 1);
 302
 303         /* Tree descent history. */
 304         /* XXX: This is somewhat messy since @n and descent[dlen-1].node are
 305          * redundant. */
 306         struct uct_descent descent[DESCENT_DLEN];
 307         descent[0].node = n; descent[0].lnode = NULL;
 308         int dlen = 1;
 309         /* Total value of the sequence. */
 310         struct move_stats seq_value = { .playouts = 0 };
 311         /* The last "significant" node along the descent (i.e. node
 312          * with higher than configured number of playouts). For black
 313          * and white. */
 314         struct tree_node *significant[2] = { NULL, NULL };
 315         if (n->u.playouts >= u->significant_threshold)
 316                 significant[node_color - 1] = n;
 317
 318         int result;
 319         int pass_limit = (board_size(&b2) - 2) * (board_size(&b2) - 2) / 2;
 320         int passes = is_pass(b->last_move.coord) && b->moves > 0;
 321
 322         /* debug */
 323         static char spaces[] = "\0                                                      ";
 324         /* /debug */
 325         if (UDEBUGL(8))
 326                 fprintf(stderr, "--- UCT walk with color %d\n", player_color);
 327
 328         while (!tree_leaf_node(n) && passes < 2) {
 329                 spaces[dlen - 1] = ' '; spaces[dlen] = 0;
 330
 331
 332                 /*** Choose a node to descend to: */
 333
 334                 /* Parity is chosen already according to the child color, since
 335                  * it is applied to children. */
 336                 node_color = stone_other(node_color);
 337                 int parity = (node_color == player_color ? 1 : -1);
 338
 339                 assert(dlen < DESCENT_DLEN);
 340                 descent[dlen] = descent[dlen - 1];
 341                 if (u->local_tree && (!descent[dlen].lnode || descent[dlen].node->d >= u->tenuki_d)) {
 342                         /* Start new local sequence. */
 343                         /* Remember that node_color already holds color of the
 344                          * to-be-found child. */
 345                         descent[dlen].lnode = node_color == S_BLACK ? t->ltree_black : t->ltree_white;
 346                 }
 347
 348                 if (!u->random_policy_chance || fast_random(u->random_policy_chance))
 349                         u->policy->descend(u->policy, t, &descent[dlen], parity, b2.moves > pass_limit);
 350                 else
 351                         u->random_policy->descend(u->random_policy, t, &descent[dlen], parity, b2.moves > pass_limit);
 352
 353
 354                 /*** Perform the descent: */
 355
 356                 if (descent[dlen].node->u.playouts >= u->significant_threshold) {
 357                         significant[node_color - 1] = descent[dlen].node;
 358                 }
 359
 360                 seq_value.playouts += descent[dlen].value.playouts;
 361                 seq_value.value += descent[dlen].value.value * descent[dlen].value.playouts;
 362                 n = descent[dlen++].node;
 363                 assert(n == t->root || n->parent);
 364                 if (UDEBUGL(7))
 365                         fprintf(stderr, "%s+-- UCT sent us to [%s:%d] %d,%f\n",
 366                                 spaces, coord2sstr(node_coord(n), t->board),
 367                                 node_coord(n), n->u.playouts,
 368                                 tree_node_get_value(t, parity, n->u.value));
 369
 370                 /* Add virtual loss if we need to; this is used to discourage
 371                  * other threads from visiting this node in case of multiple
 372                  * threads doing the tree search. */
 373                 if (u->virtual_loss)
 374                         stats_add_result(&n->u, node_color == S_BLACK ? 0.0 : 1.0, u->virtual_loss);
 375
 376                 assert(node_coord(n) >= -1);
 377                 if (amaf && !is_pass(node_coord(n)))
 378                         record_amaf_move(amaf, node_coord(n), node_color);
 379
 380                 struct move m = { node_coord(n), node_color };
 381                 int res = board_play(&b2, &m);
 382
 383                 if (res < 0 || (!is_pass(m.coord) && !group_at(&b2, m.coord)) /* suicide */
 384                     || b2.superko_violation) {
 385                         if (UDEBUGL(4)) {
 386                                 for (struct tree_node *ni = n; ni; ni = ni->parent)
 387                                         fprintf(stderr, "%s<%"PRIhash"> ", coord2sstr(node_coord(ni), t->board), ni->hash);
 388                                 fprintf(stderr, "marking invalid %s node %d,%d res %d group %d spk %d\n",
 389                                         stone2str(node_color), coord_x(node_coord(n),b), coord_y(node_coord(n),b),
 390                                         res, group_at(&b2, m.coord), b2.superko_violation);
 391                         }
 392                         n->hints |= TREE_HINT_INVALID;
 393                         result = 0;
 394                         goto end;
 395                 }
 396
 397                 if (is_pass(node_coord(n)))
 398                         passes++;
 399                 else
 400                         passes = 0;
 401
 402                 enum stone next_color = stone_other(node_color);
 403                 /* We need to make sure only one thread expands the node. If
 404                  * we are unlucky enough for two threads to meet in the same
 405                  * node, the latter one will simply do another simulation from
 406                  * the node itself, no big deal. t->nodes_size may exceed
 407                  * the maximum in multi-threaded case but not by much so it's ok.
 408                  * The size test must be before the test&set not after, to allow
 409                  * expansion of the node later if enough nodes have been freed. */
 410                 if (tree_leaf_node(n)
 411                     && n->u.playouts - u->virtual_loss >= u->expand_p && t->nodes_size < u->max_tree_size
 412                     && !__sync_lock_test_and_set(&n->is_expanded, 1))
 413                         tree_expand_node(t, n, &b2, next_color, u, -parity);
 414         }
 415
 416         if (amaf) {
 417                 amaf->game_baselen = amaf->gamelen;
 418                 amaf->record_nakade = u->playout_amaf_nakade;
 419         }
 420
 421         if (t->use_extra_komi && u->dynkomi->persim) {
 422                 b2.komi += round(u->dynkomi->persim(u->dynkomi, &b2, t, n));
 423         }
 424
 425         if (passes >= 2) {
 426                 /* XXX: No dead groups support. */
 427                 floating_t score = board_official_score(&b2, NULL);
 428                 /* Result from black's perspective (no matter who
 429                  * the player; black's perspective is always
 430                  * what the tree stores. */
 431                 result = - (score * 2);
 432
 433                 if (UDEBUGL(5))
 434                         fprintf(stderr, "[%d..%d] %s p-p scoring playout result %d (W %f)\n",
 435                                 player_color, node_color, coord2sstr(node_coord(n), t->board), result, score);
 436                 if (UDEBUGL(6))
 437                         board_print(&b2, stderr);
 438
 439                 board_ownermap_fill(&u->ownermap, &b2);
 440
 441         } else { // assert(tree_leaf_node(n));
 442                 /* In case of parallel tree search, the assertion might
 443                  * not hold if two threads chew on the same node. */
 444                 result = uct_leaf_node(u, &b2, player_color, amaf, descent, &dlen, significant, t, n, node_color, spaces);
 445         }
 446
 447         if (amaf && u->playout_amaf_cutoff) {
 448                 unsigned int cutoff = amaf->game_baselen;
 449                 cutoff += (amaf->gamelen - amaf->game_baselen) * u->playout_amaf_cutoff / 100;
 450                 /* Now, reconstruct the amaf map. */
 451                 memset(amaf->map, 0, board_size2(&b2) * sizeof(*amaf->map));
 452                 for (unsigned int i = 0; i < cutoff; i++) {
 453                         coord_t coord = amaf->game[i].coord;
 454                         enum stone color = amaf->game[i].color;
 455                         if (amaf->map[coord] == S_NONE || amaf->map[coord] == color) {
 456                                 amaf->map[coord] = color;
 457                         /* Nakade always recorded for in-tree part */
 458                         } else if (amaf->record_nakade || i <= amaf->game_baselen) {
 459                                 amaf_op(amaf->map[node_coord(n)], +);
 460                         }
 461                 }
 462         }
 463
 464         /* Record the result. */
 465
 466         assert(n == t->root || n->parent);
 467         floating_t rval = scale_value(u, b, result);
 468         u->policy->update(u->policy, t, n, node_color, player_color, amaf, &b2, rval);
 469
 470         if (t->use_extra_komi) {
 471                 stats_add_result(&u->dynkomi->score, result / 2, 1);
 472                 stats_add_result(&u->dynkomi->value, rval, 1);
 473         }
 474
 475         if (u->local_tree && n->parent && !is_pass(node_coord(n)) && dlen > 0) {
 476                 /* Get the local sequences and record them in ltree. */
 477                 /* We will look for sequence starts in our descent
 478                  * history, then run record_local_sequence() for each
 479                  * found sequence start; record_local_sequence() may
 480                  * pick longer sequences from descent history then,
 481                  * which is expected as it will create new lnodes. */
 482                 enum stone seq_color = player_color;
 483                 /* First move always starts a sequence. */
 484                 record_local_sequence(u, t, &b2, descent, dlen, 1, seq_color);
 485                 seq_color = stone_other(seq_color);
 486                 for (int dseqi = 2; dseqi < dlen; dseqi++, seq_color = stone_other(seq_color)) {
 487                         if (u->local_tree_allseq) {
 488                                 /* We are configured to record all subsequences. */
 489                                 record_local_sequence(u, t, &b2, descent, dlen, dseqi, seq_color);
 490                                 continue;
 491                         }
 492                         if (descent[dseqi].node->d >= u->tenuki_d) {
 493                                 /* Tenuki! Record the fresh sequence. */
 494                                 record_local_sequence(u, t, &b2, descent, dlen, dseqi, seq_color);
 495                                 continue;
 496                         }
 497                         if (descent[dseqi].lnode && !descent[dseqi].lnode) {
 498                                 /* Record result for in-descent picked sequence. */
 499                                 record_local_sequence(u, t, &b2, descent, dlen, dseqi, seq_color);
 500                                 continue;
 501                         }
 502                 }
 503         }
 504
 505 end:
 506         /* We need to undo the virtual loss we added during descend. */
 507         if (u->virtual_loss) {
 508                 floating_t loss = node_color == S_BLACK ? 0.0 : 1.0;
 509                 for (; n->parent; n = n->parent) {
 510                         stats_rm_result(&n->u, loss, u->virtual_loss);
 511                         loss = 1.0 - loss;
 512                 }
 513         }
 514
 515         if (amaf) {
 516                 free(amaf->map - 1);
 517                 free(amaf);
 518         }
 519         board_done_noalloc(&b2);
 520         return result;
 521 }
 522
 523 int
 524 uct_playouts(struct uct *u, struct board *b, enum stone color, struct tree *t, struct time_info *ti)
 525 {
 526         int i;
 527         if (ti && ti->dim == TD_GAMES) {
 528                 for (i = 0; t->root->u.playouts <= ti->len.games && !uct_halt; i++)
 529                         uct_playout(u, b, color, t);
 530         } else {
 531                 for (i = 0; !uct_halt; i++)
 532                         uct_playout(u, b, color, t);
 533         }
 534         return i;
 535 }