uct/walk.c

   1 #include <assert.h>
   2 #include <pthread.h>
   3 #include <signal.h>
   4 #include <stdio.h>
   5 #include <stdlib.h>
   6 #include <string.h>
   7
   8 #define DEBUG
   9
  10 #include "debug.h"
  11 #include "board.h"
  12 #include "move.h"
  13 #include "playout.h"
  14 #include "playout/elo.h"
  15 #include "probdist.h"
  16 #include "random.h"
  17 #include "uct/dynkomi.h"
  18 #include "uct/internal.h"
  19 #include "uct/tree.h"
  20 #include "uct/uct.h"
  21 #include "uct/walk.h"
  22
  23 void
  24 uct_progress_status(struct uct *u, struct tree *t, enum stone color, int playouts)
  25 {
  26         if (!UDEBUGL(0))
  27                 return;
  28
  29         /* Best move */
  30         struct tree_node *best = u->policy->choose(u->policy, t->root, t->board, color, resign);
  31         if (!best) {
  32                 fprintf(stderr, "... No moves left\n");
  33                 return;
  34         }
  35         fprintf(stderr, "[%d] ", playouts);
  36         fprintf(stderr, "best %f ", tree_node_get_value(t, 1, best->u.value));
  37
  38         /* Max depth */
  39         fprintf(stderr, "deepest % 2d ", t->max_depth - t->root->depth);
  40
  41         /* Best sequence */
  42         fprintf(stderr, "| seq ");
  43         for (int depth = 0; depth < 6; depth++) {
  44                 if (best && best->u.playouts >= 25) {
  45                         fprintf(stderr, "%3s ", coord2sstr(best->coord, t->board));
  46                         best = u->policy->choose(u->policy, best, t->board, color, resign);
  47                 } else {
  48                         fprintf(stderr, "    ");
  49                 }
  50         }
  51
  52         /* Best candidates */
  53         fprintf(stderr, "| can ");
  54         int cans = 4;
  55         struct tree_node *can[cans];
  56         memset(can, 0, sizeof(can));
  57         best = t->root->children;
  58         while (best) {
  59                 int c = 0;
  60                 while ((!can[c] || best->u.playouts > can[c]->u.playouts) && ++c < cans);
  61                 for (int d = 0; d < c; d++) can[d] = can[d + 1];
  62                 if (c > 0) can[c - 1] = best;
  63                 best = best->sibling;
  64         }
  65         while (--cans >= 0) {
  66                 if (can[cans]) {
  67                         fprintf(stderr, "%3s(%.3f) ",
  68                                 coord2sstr(can[cans]->coord, t->board),
  69                                 tree_node_get_value(t, 1, can[cans]->u.value));
  70                 } else {
  71                         fprintf(stderr, "           ");
  72                 }
  73         }
  74
  75         fprintf(stderr, "\n");
  76 }
  77
  78
  79 struct uct_playout_callback {
  80         struct uct *uct;
  81         struct tree *tree;
  82         struct tree_node *lnode;
  83 };
  84
  85 static void
  86 uct_playout_probdist(void *data, struct board *b, enum stone to_play, struct probdist *pd)
  87 {
  88         /* Create probability distribution according to found local tree
  89          * sequence. */
  90         struct uct_playout_callback *upc = data;
  91         assert(upc && upc->tree && pd && b);
  92         coord_t c = b->last_move.coord;
  93         enum stone color = b->last_move.color;
  94
  95         if (is_pass(c)) {
  96                 /* Break local sequence. */
  97                 upc->lnode = NULL;
  98         } else if (upc->lnode) {
  99                 /* Try to follow local sequence. */
 100                 upc->lnode = tree_get_node(upc->tree, upc->lnode, c, false);
 101         }
 102
 103         if (!upc->lnode || !upc->lnode->children) {
 104                 /* There's no local sequence, start new one! */
 105                 upc->lnode = color == S_BLACK ? upc->tree->ltree_black : upc->tree->ltree_white;
 106                 upc->lnode = tree_get_node(upc->tree, upc->lnode, c, false);
 107         }
 108
 109         if (!upc->lnode || !upc->lnode->children) {
 110                 /* We have no local sequence and we cannot find any starting
 111                  * by node corresponding to last move. */
 112                 if (!upc->uct->local_tree_pseqroot) {
 113                         /* Give up then, we have nothing to contribute. */
 114                         return;
 115                 }
 116                 /* Construct probability distribution from possible first
 117                  * sequence move. Remember that @color is color of the
 118                  * *last* move. */
 119                 upc->lnode = color == S_BLACK ? upc->tree->ltree_white : upc->tree->ltree_black;
 120                 if (!upc->lnode->children) {
 121                         /* We don't even have anything in our tree yet. */
 122                         return;
 123                 }
 124         }
 125
 126         /* The probdist has the right structure only if BOARD_GAMMA is defined. */
 127 #ifndef BOARD_GAMMA
 128         assert(0);
 129 #endif
 130
 131         /* Construct probability distribution from lnode children. */
 132         /* XXX: How to derive the appropriate gamma? */
 133         #define li_value(color, li) (li->u.playouts * (color == S_BLACK ? li->u.value : (1 - li->u.value)))
 134         #define li_gamma(color, li) (0.5 + li_value(color, li))
 135         struct tree_node *li = upc->lnode->children;
 136         assert(li);
 137         if (is_pass(li->coord)) {
 138                 /* Tenuki. */
 139                 /* TODO: Spread tenuki gamma over all moves we don't touch. */
 140                 li = li->sibling;
 141         }
 142         for (; li; li = li->sibling) {
 143                 if (board_at(b, li->coord) != S_NONE)
 144                         continue;
 145                 probdist_set(pd, li->coord, pd->items[li->coord] * li_gamma(to_play, li));
 146         }
 147 }
 148
 149
 150 static int
 151 uct_leaf_node(struct uct *u, struct board *b, enum stone player_color,
 152               struct playout_amafmap *amaf,
 153               struct tree *t, struct tree_node *n, enum stone node_color,
 154               char *spaces)
 155 {
 156         enum stone next_color = stone_other(node_color);
 157         int parity = (next_color == player_color ? 1 : -1);
 158
 159         /* If we don't anticipate well the opponent move during pondering
 160          * (the played move has few playouts) we still need more memory
 161          * during genmove to explore the tree actually played.
 162          * For fast_alloc, the tree compaction will free enough memory
 163          * immediately. */
 164         unsigned long max_tree_size = u->max_tree_size;
 165         if (u->pondering && !u->fast_alloc)
 166                 max_tree_size = (max_tree_size * (100 - MIN_FREE_MEM_PERCENT)) / 100;
 167
 168         /* We need to make sure only one thread expands the node. If
 169          * we are unlucky enough for two threads to meet in the same
 170          * node, the latter one will simply do another simulation from
 171          * the node itself, no big deal. t->nodes_size may exceed
 172          * the maximum in multi-threaded case but not by much so it's ok.
 173          * The size test must be before the test&set not after, to allow
 174          * expansion of the node later if enough nodes have been freed. */
 175         if (n->u.playouts >= u->expand_p && t->nodes_size < max_tree_size
 176             && !__sync_lock_test_and_set(&n->is_expanded, 1)) {
 177                 tree_expand_node(t, n, b, next_color, u, parity);
 178         }
 179         if (UDEBUGL(7))
 180                 fprintf(stderr, "%s*-- UCT playout #%d start [%s] %f\n",
 181                         spaces, n->u.playouts, coord2sstr(n->coord, t->board),
 182                         tree_node_get_value(t, parity, n->u.value));
 183
 184         /* TODO: Don't necessarily restart the sequence walk when entering
 185          * playout. */
 186         struct uct_playout_callback upc = { .uct = u, .tree = t, .lnode = NULL };
 187         if (u->local_tree_playout) {
 188                 /* N.B.: We know this is ELO playout. */
 189                 playout_elo_callback(u->playout, uct_playout_probdist, &upc);
 190         }
 191
 192         struct playout_setup ps = { .gamelen = u->gamelen, .mercymin = u->mercymin };
 193         int result = play_random_game(&ps, b, next_color,
 194                                       u->playout_amaf ? amaf : NULL,
 195                                       &u->ownermap, u->playout);
 196         if (next_color == S_WHITE) {
 197                 /* We need the result from black's perspective. */
 198                 result = - result;
 199         }
 200         if (UDEBUGL(7))
 201                 fprintf(stderr, "%s -- [%d..%d] %s random playout result %d\n",
 202                         spaces, player_color, next_color, coord2sstr(n->coord, t->board), result);
 203
 204         return result;
 205 }
 206
 207 static float
 208 scale_value(struct uct *u, struct board *b, int result)
 209 {
 210         float rval = result > 0;
 211         if (u->val_scale) {
 212                 int vp = u->val_points;
 213                 if (!vp) {
 214                         vp = board_size(b) - 1; vp *= vp; vp *= 2;
 215                 }
 216
 217                 float sval = (float) abs(result) / vp;
 218                 sval = sval > 1 ? 1 : sval;
 219                 if (result < 0) sval = 1 - sval;
 220                 if (u->val_extra)
 221                         rval += u->val_scale * sval;
 222                 else
 223                         rval = (1 - u->val_scale) * rval + u->val_scale * sval;
 224                 // fprintf(stderr, "score %d => sval %f, rval %f\n", result, sval, rval);
 225         }
 226         return rval;
 227 }
 228
 229 static void
 230 record_local_sequence(struct uct *u, struct tree *t,
 231                       struct uct_descent *descent, int dlen, int di,
 232                       enum stone seq_color, float rval)
 233 {
 234         /* Ignore pass sequences. */
 235         if (is_pass(descent[di].node->coord))
 236                 return;
 237
 238 #define LTREE_DEBUG if (UDEBUGL(6))
 239         LTREE_DEBUG fprintf(stderr, "recording result %f in local %s sequence: ",
 240                 rval, stone2str(seq_color));
 241         int di0 = di;
 242
 243         /* Pick the right local tree root... */
 244         struct tree_node *lnode = seq_color == S_BLACK ? t->ltree_black : t->ltree_white;
 245         lnode->u.playouts++;
 246
 247         /* ...and record the sequence. */
 248         while (di < dlen && (di == di0 || descent[di].node->d < u->tenuki_d)) {
 249                 LTREE_DEBUG fprintf(stderr, "%s[%d] ",
 250                         coord2sstr(descent[di].node->coord, t->board),
 251                         descent[di].node->d);
 252                 lnode = tree_get_node(t, lnode, descent[di++].node->coord, true);
 253                 assert(lnode);
 254                 stats_add_result(&lnode->u, rval, 1);
 255         }
 256
 257         /* Add lnode for tenuki (pass) if we descended further. */
 258         if (di < dlen) {
 259                 LTREE_DEBUG fprintf(stderr, "pass ");
 260                 lnode = tree_get_node(t, lnode, pass, true);
 261                 assert(lnode);
 262                 stats_add_result(&lnode->u, rval, 1);
 263         }
 264
 265         LTREE_DEBUG fprintf(stderr, "\n");
 266 }
 267
 268
 269 int
 270 uct_playout(struct uct *u, struct board *b, enum stone player_color, struct tree *t)
 271 {
 272         struct board b2;
 273         board_copy(&b2, b);
 274
 275         struct playout_amafmap *amaf = NULL;
 276         if (u->policy->wants_amaf) {
 277                 amaf = calloc(1, sizeof(*amaf));
 278                 amaf->map = calloc(board_size2(&b2) + 1, sizeof(*amaf->map));
 279                 amaf->map++; // -1 is pass
 280         }
 281
 282         /* Walk the tree until we find a leaf, then expand it and do
 283          * a random playout. */
 284         struct tree_node *n = t->root;
 285         enum stone node_color = stone_other(player_color);
 286         assert(node_color == t->root_color);
 287
 288         /* Tree descent history. */
 289         /* XXX: This is somewhat messy since @n and descent[dlen-1].node are
 290          * redundant. */
 291         #define DLEN 512
 292         struct uct_descent descent[DLEN];
 293         descent[0].node = n; descent[0].lnode = NULL;
 294         int dlen = 1;
 295         /* Total value of the sequence. */
 296         struct move_stats seq_value = { .playouts = 0 };
 297
 298         int result;
 299         int pass_limit = (board_size(&b2) - 2) * (board_size(&b2) - 2) / 2;
 300         int passes = is_pass(b->last_move.coord) && b->moves > 0;
 301
 302         /* debug */
 303         int depth = 0;
 304         static char spaces[] = "\0                                                      ";
 305         /* /debug */
 306         if (UDEBUGL(8))
 307                 fprintf(stderr, "--- UCT walk with color %d\n", player_color);
 308
 309         while (!tree_leaf_node(n) && passes < 2) {
 310                 spaces[depth++] = ' '; spaces[depth] = 0;
 311
 312
 313                 /*** Choose a node to descend to: */
 314
 315                 /* Parity is chosen already according to the child color, since
 316                  * it is applied to children. */
 317                 node_color = stone_other(node_color);
 318                 int parity = (node_color == player_color ? 1 : -1);
 319
 320                 assert(dlen < DLEN);
 321                 descent[dlen] = descent[dlen - 1];
 322                 if (u->local_tree && (!descent[dlen].lnode || descent[dlen].node->d >= u->tenuki_d)) {
 323                         /* Start new local sequence. */
 324                         /* Remember that node_color already holds color of the
 325                          * to-be-found child. */
 326                         descent[dlen].lnode = node_color == S_BLACK ? t->ltree_black : t->ltree_white;
 327                 }
 328
 329                 if (!u->random_policy_chance || fast_random(u->random_policy_chance))
 330                         u->policy->descend(u->policy, t, &descent[dlen], parity, b2.moves > pass_limit);
 331                 else
 332                         u->random_policy->descend(u->random_policy, t, &descent[dlen], parity, b2.moves > pass_limit);
 333
 334
 335                 /*** Perform the descent: */
 336
 337                 seq_value.playouts += descent[dlen].value.playouts;
 338                 seq_value.value += descent[dlen].value.value * descent[dlen].value.playouts;
 339                 n = descent[dlen++].node;
 340                 assert(n == t->root || n->parent);
 341                 if (UDEBUGL(7))
 342                         fprintf(stderr, "%s+-- UCT sent us to [%s:%d] %f\n",
 343                                 spaces, coord2sstr(n->coord, t->board), n->coord,
 344                                 tree_node_get_value(t, parity, n->u.value));
 345
 346                 /* Add virtual loss if we need to; this is used to discourage
 347                  * other threads from visiting this node in case of multiple
 348                  * threads doing the tree search. */
 349                 if (u->virtual_loss)
 350                         stats_add_result(&n->u, tree_parity(t, parity) > 0 ? 0 : 1, 1);
 351
 352                 assert(n->coord >= -1);
 353                 if (amaf && !is_pass(n->coord)) {
 354                         if (amaf->map[n->coord] == S_NONE || amaf->map[n->coord] == node_color) {
 355                                 amaf->map[n->coord] = node_color;
 356                         } else { // XXX: Respect amaf->record_nakade
 357                                 amaf_op(amaf->map[n->coord], +);
 358                         }
 359                         amaf->game[amaf->gamelen].coord = n->coord;
 360                         amaf->game[amaf->gamelen].color = node_color;
 361                         amaf->gamelen++;
 362                         assert(amaf->gamelen < sizeof(amaf->game) / sizeof(amaf->game[0]));
 363                 }
 364
 365                 struct move m = { n->coord, node_color };
 366                 int res = board_play(&b2, &m);
 367
 368                 if (res < 0 || (!is_pass(m.coord) && !group_at(&b2, m.coord)) /* suicide */
 369                     || b2.superko_violation) {
 370                         if (UDEBUGL(3)) {
 371                                 for (struct tree_node *ni = n; ni; ni = ni->parent)
 372                                         fprintf(stderr, "%s<%"PRIhash"> ", coord2sstr(ni->coord, t->board), ni->hash);
 373                                 fprintf(stderr, "marking invalid %s node %d,%d res %d group %d spk %d\n",
 374                                         stone2str(node_color), coord_x(n->coord,b), coord_y(n->coord,b),
 375                                         res, group_at(&b2, m.coord), b2.superko_violation);
 376                         }
 377                         n->hints |= TREE_HINT_INVALID;
 378                         result = 0;
 379                         goto end;
 380                 }
 381
 382                 if (is_pass(n->coord))
 383                         passes++;
 384                 else
 385                         passes = 0;
 386         }
 387
 388         if (amaf) {
 389                 amaf->game_baselen = amaf->gamelen;
 390                 amaf->record_nakade = u->playout_amaf_nakade;
 391         }
 392
 393         if (t->use_extra_komi && u->dynkomi->persim) {
 394                 b2.komi += u->dynkomi->persim(u->dynkomi, &b2, t, n);
 395         }
 396
 397         if (passes >= 2) {
 398                 /* XXX: No dead groups support. */
 399                 float score = board_official_score(&b2, NULL);
 400                 /* Result from black's perspective (no matter who
 401                  * the player; black's perspective is always
 402                  * what the tree stores. */
 403                 result = - (score * 2);
 404
 405                 if (UDEBUGL(5))
 406                         fprintf(stderr, "[%d..%d] %s p-p scoring playout result %d (W %f)\n",
 407                                 player_color, node_color, coord2sstr(n->coord, t->board), result, score);
 408                 if (UDEBUGL(6))
 409                         board_print(&b2, stderr);
 410
 411                 board_ownermap_fill(&u->ownermap, &b2);
 412
 413         } else { assert(u->parallel_tree || tree_leaf_node(n));
 414                 /* In case of parallel tree search, the assertion might
 415                  * not hold if two threads chew on the same node. */
 416                 result = uct_leaf_node(u, &b2, player_color, amaf, t, n, node_color, spaces);
 417         }
 418
 419         if (amaf && u->playout_amaf_cutoff) {
 420                 int cutoff = amaf->game_baselen;
 421                 cutoff += (amaf->gamelen - amaf->game_baselen) * u->playout_amaf_cutoff / 100;
 422                 /* Now, reconstruct the amaf map. */
 423                 memset(amaf->map, 0, board_size2(&b2) * sizeof(*amaf->map));
 424                 for (int i = 0; i < cutoff; i++) {
 425                         coord_t coord = amaf->game[i].coord;
 426                         enum stone color = amaf->game[i].color;
 427                         if (amaf->map[coord] == S_NONE || amaf->map[coord] == color) {
 428                                 amaf->map[coord] = color;
 429                         /* Nakade always recorded for in-tree part */
 430                         } else if (amaf->record_nakade || i <= amaf->game_baselen) {
 431                                 amaf_op(amaf->map[n->coord], +);
 432                         }
 433                 }
 434         }
 435
 436         assert(n == t->root || n->parent);
 437         if (result != 0) {
 438                 stats_add_result(&t->score, result, 1);
 439
 440                 float rval = scale_value(u, b, result);
 441                 u->policy->update(u->policy, t, n, node_color, player_color, amaf, rval);
 442
 443                 if (u->local_tree && n->parent && !is_pass(n->coord) && dlen > 0) {
 444                         /* Possibly transform the rval appropriately. */
 445                         float expval = seq_value.value / seq_value.playouts;
 446                         rval = stats_temper_value(rval, expval, u->local_tree);
 447
 448                         /* Get the local sequences and record them in ltree. */
 449                         /* We will look for sequence starts in our descent
 450                          * history, then run record_local_sequence() for each
 451                          * found sequence start; record_local_sequence() may
 452                          * pick longer sequences from descent history then,
 453                          * which is expected as it will create new lnodes. */
 454                         enum stone seq_color = player_color;
 455                         /* First move always starts a sequence. */
 456                         record_local_sequence(u, t, descent, dlen, 1, seq_color, rval);
 457                         seq_color = stone_other(seq_color);
 458                         for (int dseqi = 2; dseqi < dlen; dseqi++, seq_color = stone_other(seq_color)) {
 459                                 if (u->local_tree_allseq) {
 460                                         /* We are configured to record all subsequences. */
 461                                         record_local_sequence(u, t, descent, dlen, dseqi, seq_color, rval);
 462                                         continue;
 463                                 }
 464                                 if (descent[dseqi].node->d >= u->tenuki_d) {
 465                                         /* Tenuki! Record the fresh sequence. */
 466                                         record_local_sequence(u, t, descent, dlen, dseqi, seq_color, rval);
 467                                         continue;
 468                                 }
 469                                 if (descent[dseqi].lnode && !descent[dseqi].lnode) {
 470                                         /* Record result for in-descent picked sequence. */
 471                                         record_local_sequence(u, t, descent, dlen, dseqi, seq_color, rval);
 472                                         continue;
 473                                 }
 474                         }
 475                 }
 476         }
 477
 478 end:
 479         /* We need to undo the virtual loss we added during descend. */
 480         if (u->virtual_loss) {
 481                 int parity = (node_color == player_color ? 1 : -1);
 482                 for (; n->parent; n = n->parent) {
 483                         stats_rm_result(&n->u, tree_parity(t, parity) > 0 ? 0 : 1, 1);
 484                         parity = -parity;
 485                 }
 486         }
 487
 488         if (amaf) {
 489                 free(amaf->map - 1);
 490                 free(amaf);
 491         }
 492         board_done_noalloc(&b2);
 493         return result;
 494 }
 495
 496 int
 497 uct_playouts(struct uct *u, struct board *b, enum stone color, struct tree *t)
 498 {
 499         int i;
 500         for (i = 0; !uct_halt; i++)
 501                 uct_playout(u, b, color, t);
 502         return i;
 503 }