uct/walk.c

   1 #include <assert.h>
   2 #include <math.h>
   3 #include <pthread.h>
   4 #include <signal.h>
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8
   9 #define DEBUG
  10
  11 #include "debug.h"
  12 #include "board.h"
  13 #include "move.h"
  14 #include "playout.h"
  15 #include "probdist.h"
  16 #include "random.h"
  17 #include "uct/dynkomi.h"
  18 #include "uct/internal.h"
  19 #include "uct/search.h"
  20 #include "uct/tree.h"
  21 #include "uct/uct.h"
  22 #include "uct/walk.h"
  23
  24 #define DESCENT_DLEN 512
  25
  26 void
  27 uct_progress_status(struct uct *u, struct tree *t, enum stone color, int playouts)
  28 {
  29         if (!UDEBUGL(0))
  30                 return;
  31
  32         /* Best move */
  33         struct tree_node *best = u->policy->choose(u->policy, t->root, t->board, color, resign);
  34         if (!best) {
  35                 fprintf(stderr, "... No moves left\n");
  36                 return;
  37         }
  38         fprintf(stderr, "[%d] size=%lu/%lu ", playouts, t->nodes_size, t->max_tree_size);
  39         fprintf(stderr, "best %f ", tree_node_get_value(t, 1, best->u.value));
  40
  41         /* Dynamic komi */
  42         if (t->use_extra_komi)
  43                 fprintf(stderr, "komi %.1f ", t->extra_komi);
  44
  45         /* Best sequence */
  46         fprintf(stderr, "| seq ");
  47         for (int depth = 0; depth < 4; depth++) {
  48                 if (best && best->u.playouts >= 25) {
  49                         fprintf(stderr, "%3s ", coord2sstr(best->coord, t->board));
  50                         best = u->policy->choose(u->policy, best, t->board, color, resign);
  51                 } else {
  52                         fprintf(stderr, "    ");
  53                 }
  54         }
  55
  56         /* Best candidates */
  57         fprintf(stderr, "| can ");
  58         int cans = 4;
  59         struct tree_node *can[cans];
  60         memset(can, 0, sizeof(can));
  61         best = t->root->children;
  62         while (best) {
  63                 int c = 0;
  64                 while ((!can[c] || best->u.playouts > can[c]->u.playouts) && ++c < cans);
  65                 for (int d = 0; d < c; d++) can[d] = can[d + 1];
  66                 if (c > 0) can[c - 1] = best;
  67                 best = best->sibling;
  68         }
  69         while (--cans >= 0) {
  70                 if (can[cans]) {
  71                         fprintf(stderr, "%3s(%.3f) ",
  72                                 coord2sstr(can[cans]->coord, t->board),
  73                                 tree_node_get_value(t, 1, can[cans]->u.value));
  74                 } else {
  75                         fprintf(stderr, "           ");
  76                 }
  77         }
  78
  79         fprintf(stderr, "\n");
  80 }
  81
  82
  83 static void
  84 record_amaf_move(struct playout_amafmap *amaf, coord_t coord, enum stone color)
  85 {
  86         if (amaf->map[coord] == S_NONE || amaf->map[coord] == color) {
  87                 amaf->map[coord] = color;
  88         } else { // XXX: Respect amaf->record_nakade
  89                 amaf_op(amaf->map[coord], +);
  90         }
  91         amaf->game[amaf->gamelen].coord = coord;
  92         amaf->game[amaf->gamelen].color = color;
  93         amaf->gamelen++;
  94         assert(amaf->gamelen < sizeof(amaf->game) / sizeof(amaf->game[0]));
  95 }
  96
  97
  98 struct uct_playout_callback {
  99         struct uct *uct;
 100         struct tree *tree;
 101         struct tree_node *lnode;
 102 };
 103
 104
 105 static coord_t
 106 uct_playout_hook(struct playout_policy *playout, struct playout_setup *setup, struct board *b, enum stone color, int mode)
 107 {
 108         /* XXX: This is used in some non-master branches. */
 109         return pass;
 110 }
 111
 112 static coord_t
 113 uct_playout_prepolicy(struct playout_policy *playout, struct playout_setup *setup, struct board *b, enum stone color)
 114 {
 115         return uct_playout_hook(playout, setup, b, color, 0);
 116 }
 117
 118 static coord_t
 119 uct_playout_postpolicy(struct playout_policy *playout, struct playout_setup *setup, struct board *b, enum stone color)
 120 {
 121         return uct_playout_hook(playout, setup, b, color, 1);
 122 }
 123
 124
 125 static int
 126 uct_leaf_node(struct uct *u, struct board *b, enum stone player_color,
 127               struct playout_amafmap *amaf,
 128               struct uct_descent *descent, int *dlen,
 129               struct tree_node *significant[2],
 130               struct tree *t, struct tree_node *n, enum stone node_color,
 131               char *spaces)
 132 {
 133         enum stone next_color = stone_other(node_color);
 134         int parity = (next_color == player_color ? 1 : -1);
 135
 136         /* We need to make sure only one thread expands the node. If
 137          * we are unlucky enough for two threads to meet in the same
 138          * node, the latter one will simply do another simulation from
 139          * the node itself, no big deal. t->nodes_size may exceed
 140          * the maximum in multi-threaded case but not by much so it's ok.
 141          * The size test must be before the test&set not after, to allow
 142          * expansion of the node later if enough nodes have been freed. */
 143         if (n->u.playouts >= u->expand_p && t->nodes_size < u->max_tree_size
 144             && !__sync_lock_test_and_set(&n->is_expanded, 1)) {
 145                 tree_expand_node(t, n, b, next_color, u, parity);
 146         }
 147         if (UDEBUGL(7))
 148                 fprintf(stderr, "%s*-- UCT playout #%d start [%s] %f\n",
 149                         spaces, n->u.playouts, coord2sstr(n->coord, t->board),
 150                         tree_node_get_value(t, parity, n->u.value));
 151
 152         struct uct_playout_callback upc = {
 153                 .uct = u,
 154                 .tree = t,
 155                 /* TODO: Don't necessarily restart the sequence walk when
 156                  * entering playout. */
 157                 .lnode = NULL,
 158         };
 159
 160         struct playout_setup ps = {
 161                 .gamelen = u->gamelen,
 162                 .mercymin = u->mercymin,
 163                 .prepolicy_hook = uct_playout_prepolicy,
 164                 .postpolicy_hook = uct_playout_postpolicy,
 165                 .hook_data = &upc,
 166         };
 167         int result = play_random_game(&ps, b, next_color,
 168                                       u->playout_amaf ? amaf : NULL,
 169                                       &u->ownermap, u->playout);
 170         if (next_color == S_WHITE) {
 171                 /* We need the result from black's perspective. */
 172                 result = - result;
 173         }
 174         if (UDEBUGL(7))
 175                 fprintf(stderr, "%s -- [%d..%d] %s random playout result %d\n",
 176                         spaces, player_color, next_color, coord2sstr(n->coord, t->board), result);
 177
 178         return result;
 179 }
 180
 181 static floating_t
 182 scale_value(struct uct *u, struct board *b, int result)
 183 {
 184         floating_t rval = result > 0 ? 1.0 : result < 0 ? 0.0 : 0.5;
 185         if (u->val_scale && result != 0) {
 186                 int vp = u->val_points;
 187                 if (!vp) {
 188                         vp = board_size(b) - 1; vp *= vp; vp *= 2;
 189                 }
 190
 191                 floating_t sval = (floating_t) abs(result) / vp;
 192                 sval = sval > 1 ? 1 : sval;
 193                 if (result < 0) sval = 1 - sval;
 194                 if (u->val_extra)
 195                         rval += u->val_scale * sval;
 196                 else
 197                         rval = (1 - u->val_scale) * rval + u->val_scale * sval;
 198                 // fprintf(stderr, "score %d => sval %f, rval %f\n", result, sval, rval);
 199         }
 200         return rval;
 201 }
 202
 203 static void
 204 record_local_sequence(struct uct *u, struct tree *t,
 205                       struct uct_descent *descent, int dlen, int di,
 206                       enum stone seq_color, floating_t rval)
 207 {
 208 #define LTREE_DEBUG if (UDEBUGL(6))
 209
 210         /* Ignore pass sequences. */
 211         if (is_pass(descent[di].node->coord))
 212                 return;
 213
 214         /* Transform the rval appropriately, based on the expected
 215          * result at the root of the sequence. */
 216         if (u->local_tree_rootseqval) {
 217                 float expval = descent[di - 1].value.value;
 218                 rval = stats_temper_value(rval, expval, u->local_tree);
 219         }
 220
 221         LTREE_DEBUG fprintf(stderr, "recording result %f in local %s sequence: ",
 222                 rval, stone2str(seq_color));
 223
 224         /* Sequences starting deeper are less relevant in general. */
 225         int pval = LTREE_PLAYOUTS_MULTIPLIER;
 226         if (u->local_tree && u->local_tree_depth_decay > 0)
 227                 pval = ((floating_t) pval) / pow(u->local_tree_depth_decay, di - 1);
 228         if (!pval) {
 229                 LTREE_DEBUG fprintf(stderr, "too deep @%d\n", di);
 230                 return;
 231         }
 232
 233         /* Pick the right local tree root... */
 234         struct tree_node *lnode = seq_color == S_BLACK ? t->ltree_black : t->ltree_white;
 235         lnode->u.playouts++;
 236
 237         /* ...and record the sequence. */
 238         int di0 = di;
 239         while (di < dlen && (di == di0 || descent[di].node->d < u->tenuki_d)) {
 240                 LTREE_DEBUG fprintf(stderr, "%s[%d] ",
 241                         coord2sstr(descent[di].node->coord, t->board),
 242                         descent[di].node->d);
 243                 lnode = tree_get_node(t, lnode, descent[di++].node->coord, true);
 244                 assert(lnode);
 245                 stats_add_result(&lnode->u, rval, pval);
 246         }
 247
 248         /* Add lnode for tenuki (pass) if we descended further. */
 249         if (di < dlen) {
 250                 LTREE_DEBUG fprintf(stderr, "pass ");
 251                 lnode = tree_get_node(t, lnode, pass, true);
 252                 assert(lnode);
 253                 stats_add_result(&lnode->u, rval, pval);
 254         }
 255
 256         LTREE_DEBUG fprintf(stderr, "\n");
 257 }
 258
 259
 260 int
 261 uct_playout(struct uct *u, struct board *b, enum stone player_color, struct tree *t)
 262 {
 263         struct board b2;
 264         board_copy(&b2, b);
 265
 266         struct playout_amafmap *amaf = NULL;
 267         if (u->policy->wants_amaf) {
 268                 amaf = calloc2(1, sizeof(*amaf));
 269                 amaf->map = calloc2(board_size2(&b2) + 1, sizeof(*amaf->map));
 270                 amaf->map++; // -1 is pass
 271         }
 272
 273         /* Walk the tree until we find a leaf, then expand it and do
 274          * a random playout. */
 275         struct tree_node *n = t->root;
 276         enum stone node_color = stone_other(player_color);
 277         assert(node_color == t->root_color);
 278
 279         /* Tree descent history. */
 280         /* XXX: This is somewhat messy since @n and descent[dlen-1].node are
 281          * redundant. */
 282         struct uct_descent descent[DESCENT_DLEN];
 283         descent[0].node = n; descent[0].lnode = NULL;
 284         int dlen = 1;
 285         /* Total value of the sequence. */
 286         struct move_stats seq_value = { .playouts = 0 };
 287         /* The last "significant" node along the descent (i.e. node
 288          * with higher than configured number of playouts). For black
 289          * and white. */
 290         struct tree_node *significant[2] = { NULL, NULL };
 291         if (n->u.playouts >= u->significant_threshold)
 292                 significant[node_color - 1] = n;
 293
 294         int result;
 295         int pass_limit = (board_size(&b2) - 2) * (board_size(&b2) - 2) / 2;
 296         int passes = is_pass(b->last_move.coord) && b->moves > 0;
 297
 298         /* debug */
 299         static char spaces[DESCENT_DLEN];
 300         spaces[0] = 0;
 301         /* /debug */
 302         if (UDEBUGL(8))
 303                 fprintf(stderr, "--- UCT walk with color %d\n", player_color);
 304
 305         while (!tree_leaf_node(n) && passes < 2) {
 306                 /*** Choose a node to descend to: */
 307
 308                 /* Parity is chosen already according to the child color, since
 309                  * it is applied to children. */
 310                 node_color = stone_other(node_color);
 311                 int parity = (node_color == player_color ? 1 : -1);
 312
 313                 assert(dlen < DESCENT_DLEN);
 314                 spaces[dlen - 1] = ' '; spaces[dlen] = 0;
 315                 descent[dlen] = descent[dlen - 1];
 316                 if (u->local_tree && (!descent[dlen].lnode || descent[dlen].node->d >= u->tenuki_d)) {
 317                         /* Start new local sequence. */
 318                         /* Remember that node_color already holds color of the
 319                          * to-be-found child. */
 320                         descent[dlen].lnode = node_color == S_BLACK ? t->ltree_black : t->ltree_white;
 321                 }
 322
 323                 if (!u->random_policy_chance || fast_random(u->random_policy_chance))
 324                         u->policy->descend(u->policy, t, &descent[dlen], parity, b2.moves > pass_limit);
 325                 else
 326                         u->random_policy->descend(u->random_policy, t, &descent[dlen], parity, b2.moves > pass_limit);
 327
 328
 329                 /*** Perform the descent: */
 330
 331                 if (descent[dlen].node->u.playouts >= u->significant_threshold) {
 332                         significant[node_color - 1] = descent[dlen].node;
 333                 }
 334
 335                 seq_value.playouts += descent[dlen].value.playouts;
 336                 seq_value.value += descent[dlen].value.value * descent[dlen].value.playouts;
 337                 n = descent[dlen++].node;
 338                 assert(n == t->root || n->parent);
 339                 if (UDEBUGL(7))
 340                         fprintf(stderr, "%s+-- UCT sent us to [%s:%d] %d,%f\n",
 341                                 spaces, coord2sstr(n->coord, t->board),
 342                                 n->coord, n->u.playouts,
 343                                 tree_node_get_value(t, parity, n->u.value));
 344
 345                 /* Add virtual loss if we need to; this is used to discourage
 346                  * other threads from visiting this node in case of multiple
 347                  * threads doing the tree search. */
 348                 if (u->virtual_loss)
 349                         stats_add_result(&n->u, node_color == S_BLACK ? 0.0 : 1.0, u->virtual_loss);
 350
 351                 assert(n->coord >= -1);
 352                 if (amaf && !is_pass(n->coord))
 353                         record_amaf_move(amaf, n->coord, node_color);
 354
 355                 struct move m = { n->coord, node_color };
 356                 int res = board_play(&b2, &m);
 357
 358                 if (res < 0 || (!is_pass(m.coord) && !group_at(&b2, m.coord)) /* suicide */
 359                     || b2.superko_violation) {
 360                         if (UDEBUGL(4)) {
 361                                 for (struct tree_node *ni = n; ni; ni = ni->parent)
 362                                         fprintf(stderr, "%s<%"PRIhash"> ", coord2sstr(ni->coord, t->board), ni->hash);
 363                                 fprintf(stderr, "marking invalid %s node %d,%d res %d group %d spk %d\n",
 364                                         stone2str(node_color), coord_x(n->coord,b), coord_y(n->coord,b),
 365                                         res, group_at(&b2, m.coord), b2.superko_violation);
 366                         }
 367                         n->hints |= TREE_HINT_INVALID;
 368                         result = 0;
 369                         goto end;
 370                 }
 371
 372                 if (is_pass(n->coord))
 373                         passes++;
 374                 else
 375                         passes = 0;
 376         }
 377
 378         if (amaf) {
 379                 amaf->game_baselen = amaf->gamelen;
 380                 amaf->record_nakade = u->playout_amaf_nakade;
 381         }
 382
 383         if (t->use_extra_komi && u->dynkomi->persim) {
 384                 b2.komi += round(u->dynkomi->persim(u->dynkomi, &b2, t, n));
 385         }
 386
 387         if (passes >= 2) {
 388                 /* XXX: No dead groups support. */
 389                 floating_t score = board_official_score(&b2, NULL);
 390                 /* Result from black's perspective (no matter who
 391                  * the player; black's perspective is always
 392                  * what the tree stores. */
 393                 result = - (score * 2);
 394
 395                 if (UDEBUGL(5))
 396                         fprintf(stderr, "[%d..%d] %s p-p scoring playout result %d (W %f)\n",
 397                                 player_color, node_color, coord2sstr(n->coord, t->board), result, score);
 398                 if (UDEBUGL(6))
 399                         board_print(&b2, stderr);
 400
 401                 board_ownermap_fill(&u->ownermap, &b2);
 402
 403         } else { // assert(tree_leaf_node(n));
 404                 /* In case of parallel tree search, the assertion might
 405                  * not hold if two threads chew on the same node. */
 406                 result = uct_leaf_node(u, &b2, player_color, amaf, descent, &dlen, significant, t, n, node_color, spaces);
 407         }
 408
 409         if (amaf && u->playout_amaf_cutoff) {
 410                 unsigned int cutoff = amaf->game_baselen;
 411                 cutoff += (amaf->gamelen - amaf->game_baselen) * u->playout_amaf_cutoff / 100;
 412                 /* Now, reconstruct the amaf map. */
 413                 memset(amaf->map, 0, board_size2(&b2) * sizeof(*amaf->map));
 414                 for (unsigned int i = 0; i < cutoff; i++) {
 415                         coord_t coord = amaf->game[i].coord;
 416                         enum stone color = amaf->game[i].color;
 417                         if (amaf->map[coord] == S_NONE || amaf->map[coord] == color) {
 418                                 amaf->map[coord] = color;
 419                         /* Nakade always recorded for in-tree part */
 420                         } else if (amaf->record_nakade || i <= amaf->game_baselen) {
 421                                 amaf_op(amaf->map[n->coord], +);
 422                         }
 423                 }
 424         }
 425
 426         /* Record the result. */
 427
 428         assert(n == t->root || n->parent);
 429         floating_t rval = scale_value(u, b, result);
 430         u->policy->update(u->policy, t, n, node_color, player_color, amaf, &b2, rval);
 431
 432         if (t->use_extra_komi) {
 433                 stats_add_result(&u->dynkomi->score, (floating_t) 0.5 * result, 1);
 434                 stats_add_result(&u->dynkomi->value, rval, 1);
 435         }
 436
 437         if (u->local_tree && n->parent && !is_pass(n->coord) && dlen > 0) {
 438                 /* Possibly transform the rval appropriately. */
 439                 if (!u->local_tree_rootseqval) {
 440                         floating_t expval = seq_value.value / seq_value.playouts;
 441                         rval = stats_temper_value(rval, expval, u->local_tree);
 442                 }
 443
 444                 /* Get the local sequences and record them in ltree. */
 445                 /* We will look for sequence starts in our descent
 446                  * history, then run record_local_sequence() for each
 447                  * found sequence start; record_local_sequence() may
 448                  * pick longer sequences from descent history then,
 449                  * which is expected as it will create new lnodes. */
 450                 enum stone seq_color = player_color;
 451                 /* First move always starts a sequence. */
 452                 record_local_sequence(u, t, descent, dlen, 1, seq_color, rval);
 453                 seq_color = stone_other(seq_color);
 454                 for (int dseqi = 2; dseqi < dlen; dseqi++, seq_color = stone_other(seq_color)) {
 455                         if (u->local_tree_allseq) {
 456                                 /* We are configured to record all subsequences. */
 457                                 record_local_sequence(u, t, descent, dlen, dseqi, seq_color, rval);
 458                                 continue;
 459                         }
 460                         if (descent[dseqi].node->d >= u->tenuki_d) {
 461                                 /* Tenuki! Record the fresh sequence. */
 462                                 record_local_sequence(u, t, descent, dlen, dseqi, seq_color, rval);
 463                                 continue;
 464                         }
 465                         if (descent[dseqi].lnode && !descent[dseqi].lnode) {
 466                                 /* Record result for in-descent picked sequence. */
 467                                 record_local_sequence(u, t, descent, dlen, dseqi, seq_color, rval);
 468                                 continue;
 469                         }
 470                 }
 471         }
 472
 473 end:
 474         /* We need to undo the virtual loss we added during descend. */
 475         if (u->virtual_loss) {
 476                 floating_t loss = node_color == S_BLACK ? 0.0 : 1.0;
 477                 for (; n->parent; n = n->parent) {
 478                         stats_rm_result(&n->u, loss, u->virtual_loss);
 479                         loss = 1.0 - loss;
 480                 }
 481         }
 482
 483         if (amaf) {
 484                 free(amaf->map - 1);
 485                 free(amaf);
 486         }
 487         board_done_noalloc(&b2);
 488         return result;
 489 }
 490
 491 int
 492 uct_playouts(struct uct *u, struct board *b, enum stone color, struct tree *t, struct time_info *ti)
 493 {
 494         int i;
 495         if (ti && ti->dim == TD_GAMES) {
 496                 /* We must halt if uct_search_check_stop() decides that it is time to stop.  For
 497                  * example, if memory gets full, we may well have to stop early; in any case since
 498                  * uct_search_progress() is no longer called to update the dynkomi information and
 499                  * show progress information, continuing might well give strange results.  Normally
 500                  * uct_search_check_stop() will stop at s->stop.worst.playouts, which is equal to
 501                  * ti->len.games according to timeinfo.c:time_stop_conditions(). */
 502                 for (i = 0; !uct_halt && t->root->u.playouts <= ti->len.games; i++)
 503                         uct_playout(u, b, color, t);
 504         } else {
 505                 for (i = 0; !uct_halt; i++)
 506                         uct_playout(u, b, color, t);
 507         }
 508         return i;
 509 }