uct/walk.c

   1 #include <assert.h>
   2 #include <pthread.h>
   3 #include <signal.h>
   4 #include <stdio.h>
   5 #include <stdlib.h>
   6 #include <string.h>
   7
   8 #define DEBUG
   9
  10 #include "debug.h"
  11 #include "board.h"
  12 #include "move.h"
  13 #include "playout.h"
  14 #include "probdist.h"
  15 #include "random.h"
  16 #include "tactics.h"
  17 #include "uct/internal.h"
  18 #include "uct/tree.h"
  19 #include "uct/uct.h"
  20 #include "uct/walk.h"
  21
  22
  23 float
  24 uct_get_extra_komi(struct uct *u, struct board *b)
  25 {
  26         float extra_komi = board_effective_handicap(b) * (u->dynkomi - b->moves) / u->dynkomi;
  27         return extra_komi;
  28 }
  29
  30 void
  31 uct_progress_status(struct uct *u, struct tree *t, enum stone color, int playouts)
  32 {
  33         if (!UDEBUGL(0))
  34                 return;
  35
  36         /* Best move */
  37         struct tree_node *best = u->policy->choose(u->policy, t->root, t->board, color);
  38         if (!best) {
  39                 fprintf(stderr, "... No moves left\n");
  40                 return;
  41         }
  42         fprintf(stderr, "[%d] ", playouts);
  43         fprintf(stderr, "best %f ", tree_node_get_value(t, 1, best->u.value));
  44
  45         /* Max depth */
  46         fprintf(stderr, "deepest % 2d ", t->max_depth - t->root->depth);
  47
  48         /* Best sequence */
  49         fprintf(stderr, "| seq ");
  50         for (int depth = 0; depth < 6; depth++) {
  51                 if (best && best->u.playouts >= 25) {
  52                         fprintf(stderr, "%3s ", coord2sstr(best->coord, t->board));
  53                         best = u->policy->choose(u->policy, best, t->board, color);
  54                 } else {
  55                         fprintf(stderr, "    ");
  56                 }
  57         }
  58
  59         /* Best candidates */
  60         fprintf(stderr, "| can ");
  61         int cans = 4;
  62         struct tree_node *can[cans];
  63         memset(can, 0, sizeof(can));
  64         best = t->root->children;
  65         while (best) {
  66                 int c = 0;
  67                 while ((!can[c] || best->u.playouts > can[c]->u.playouts) && ++c < cans);
  68                 for (int d = 0; d < c; d++) can[d] = can[d + 1];
  69                 if (c > 0) can[c - 1] = best;
  70                 best = best->sibling;
  71         }
  72         while (--cans >= 0) {
  73                 if (can[cans]) {
  74                         fprintf(stderr, "%3s(%.3f) ",
  75                                 coord2sstr(can[cans]->coord, t->board),
  76                                 tree_node_get_value(t, 1, can[cans]->u.value));
  77                 } else {
  78                         fprintf(stderr, "           ");
  79                 }
  80         }
  81
  82         fprintf(stderr, "\n");
  83 }
  84
  85
  86 static int
  87 uct_leaf_node(struct uct *u, struct board *b, enum stone player_color,
  88               struct playout_amafmap *amaf,
  89               struct tree *t, struct tree_node *n, enum stone node_color,
  90               char *spaces)
  91 {
  92         enum stone next_color = stone_other(node_color);
  93         int parity = (next_color == player_color ? 1 : -1);
  94         if (n->u.playouts >= u->expand_p) {
  95                 // fprintf(stderr, "expanding %s (%p ^-%p)\n", coord2sstr(n->coord, b), n, n->parent);
  96                 if (!u->parallel_tree) {
  97                         /* Single-threaded, life is easy. */
  98                         tree_expand_node(t, n, b, next_color, u, parity);
  99                 } else {
 100                         /* We need to make sure only one thread expands
 101                          * the node. If we are unlucky enough for two
 102                          * threads to meet in the same node, the latter
 103                          * one will simply do another simulation from
 104                          * the node itself, no big deal. */
 105                         pthread_mutex_lock(&t->expansion_mutex);
 106                         if (tree_leaf_node(n)) {
 107                                 tree_expand_node(t, n, b, next_color, u, parity);
 108                         } else {
 109                                 // fprintf(stderr, "cancelling expansion, thread collision\n");
 110                         }
 111                         pthread_mutex_unlock(&t->expansion_mutex);
 112                 }
 113         }
 114         if (UDEBUGL(7))
 115                 fprintf(stderr, "%s*-- UCT playout #%d start [%s] %f\n",
 116                         spaces, n->u.playouts, coord2sstr(n->coord, t->board),
 117                         tree_node_get_value(t, parity, n->u.value));
 118
 119         struct uct_board *ub = b->es; assert(ub);
 120         struct playout_setup ps = { .gamelen = u->gamelen };
 121         int result = play_random_game(&ps, b, next_color,
 122                                       u->playout_amaf ? amaf : NULL,
 123                                       &ub->ownermap, u->playout);
 124         if (next_color == S_WHITE) {
 125                 /* We need the result from black's perspective. */
 126                 result = - result;
 127         }
 128         if (UDEBUGL(7))
 129                 fprintf(stderr, "%s -- [%d..%d] %s random playout result %d\n",
 130                         spaces, player_color, next_color, coord2sstr(n->coord, t->board), result);
 131
 132         return result;
 133 }
 134
 135 static float
 136 scale_value(struct uct *u, struct board *b, int result)
 137 {
 138         float rval = result > 0;
 139         if (u->val_scale) {
 140                 int vp = u->val_points;
 141                 if (!vp) {
 142                         vp = board_size(b) - 1; vp *= vp; vp *= 2;
 143                 }
 144
 145                 float sval = (float) abs(result) / vp;
 146                 sval = sval > 1 ? 1 : sval;
 147                 if (result < 0) sval = 1 - sval;
 148                 if (u->val_extra)
 149                         rval += u->val_scale * sval;
 150                 else
 151                         rval = (1 - u->val_scale) * rval + u->val_scale * sval;
 152                 // fprintf(stderr, "score %d => sval %f, rval %f\n", result, sval, rval);
 153         }
 154         return rval;
 155 }
 156
 157
 158 int
 159 uct_playout(struct uct *u, struct board *b, enum stone player_color, struct tree *t)
 160 {
 161         struct board b2;
 162         board_copy(&b2, b);
 163
 164         struct playout_amafmap *amaf = NULL;
 165         if (u->policy->wants_amaf) {
 166                 amaf = calloc(1, sizeof(*amaf));
 167                 amaf->map = calloc(board_size2(&b2) + 1, sizeof(*amaf->map));
 168                 amaf->map++; // -1 is pass
 169         }
 170
 171         /* Walk the tree until we find a leaf, then expand it and do
 172          * a random playout. */
 173         struct tree_node *n = t->root;
 174         enum stone node_color = stone_other(player_color);
 175         assert(node_color == t->root_color);
 176
 177         void *dstate = NULL, *dstater = NULL;
 178
 179         int result;
 180         int pass_limit = (board_size(&b2) - 2) * (board_size(&b2) - 2) / 2;
 181         int passes = is_pass(b->last_move.coord) && b->moves > 0;
 182
 183         /* debug */
 184         int depth = 0;
 185         static char spaces[] = "\0                                                      ";
 186         /* /debug */
 187         if (UDEBUGL(8))
 188                 fprintf(stderr, "--- UCT walk with color %d\n", player_color);
 189
 190         while (!tree_leaf_node(n) && passes < 2) {
 191                 spaces[depth++] = ' '; spaces[depth] = 0;
 192
 193                 /* Parity is chosen already according to the child color, since
 194                  * it is applied to children. */
 195                 node_color = stone_other(node_color);
 196                 int parity = (node_color == player_color ? 1 : -1);
 197                 n = (!u->random_policy_chance || fast_random(u->random_policy_chance))
 198                         ? u->policy->descend(u->policy, &dstate, t, n, parity, pass_limit)
 199                         : u->random_policy->descend(u->random_policy, &dstater, t, n, parity, pass_limit);
 200
 201                 assert(n == t->root || n->parent);
 202                 if (UDEBUGL(7))
 203                         fprintf(stderr, "%s+-- UCT sent us to [%s:%d] %f\n",
 204                                 spaces, coord2sstr(n->coord, t->board), n->coord,
 205                                 tree_node_get_value(t, parity, n->u.value));
 206
 207                 /* Add virtual loss if we need to; this is used to discourage
 208                  * other threads from visiting this node in case of multiple
 209                  * threads doing the tree search. */
 210                 if (u->virtual_loss)
 211                         stats_add_result(&n->u, tree_parity(t, parity) > 0 ? 0 : 1, 1);
 212
 213                 assert(n->coord >= -1);
 214                 if (amaf && !is_pass(n->coord)) {
 215                         if (amaf->map[n->coord] == S_NONE || amaf->map[n->coord] == node_color) {
 216                                 amaf->map[n->coord] = node_color;
 217                         } else { // XXX: Respect amaf->record_nakade
 218                                 amaf_op(amaf->map[n->coord], +);
 219                         }
 220                         amaf->game[amaf->gamelen].coord = n->coord;
 221                         amaf->game[amaf->gamelen].color = node_color;
 222                         amaf->gamelen++;
 223                         assert(amaf->gamelen < sizeof(amaf->game) / sizeof(amaf->game[0]));
 224                 }
 225
 226                 struct move m = { n->coord, node_color };
 227                 int res = board_play(&b2, &m);
 228
 229                 if (res < 0 || (!is_pass(m.coord) && !group_at(&b2, m.coord)) /* suicide */
 230                     || b2.superko_violation) {
 231                         if (UDEBUGL(3)) {
 232                                 for (struct tree_node *ni = n; ni; ni = ni->parent)
 233                                         fprintf(stderr, "%s<%"PRIhash"> ", coord2sstr(ni->coord, t->board), ni->hash);
 234                                 fprintf(stderr, "marking invalid %s node %d,%d res %d group %d spk %d\n",
 235                                         stone2str(node_color), coord_x(n->coord,b), coord_y(n->coord,b),
 236                                         res, group_at(&b2, m.coord), b2.superko_violation);
 237                         }
 238                         n->hints |= TREE_HINT_INVALID;
 239                         result = 0;
 240                         goto end;
 241                 }
 242
 243                 if (is_pass(n->coord))
 244                         passes++;
 245                 else
 246                         passes = 0;
 247         }
 248
 249         if (amaf) {
 250                 amaf->game_baselen = amaf->gamelen;
 251                 amaf->record_nakade = u->playout_amaf_nakade;
 252         }
 253
 254         if (u->dynkomi > b2.moves && (player_color & u->dynkomi_mask))
 255                 b2.komi += uct_get_extra_komi(u, &b2);
 256
 257         if (passes >= 2) {
 258                 /* XXX: No dead groups support. */
 259                 float score = board_official_score(&b2, NULL);
 260                 /* Result from black's perspective (no matter who
 261                  * the player; black's perspective is always
 262                  * what the tree stores. */
 263                 result = - (score * 2);
 264
 265                 if (UDEBUGL(5))
 266                         fprintf(stderr, "[%d..%d] %s p-p scoring playout result %d (W %f)\n",
 267                                 player_color, node_color, coord2sstr(n->coord, t->board), result, score);
 268                 if (UDEBUGL(6))
 269                         board_print(&b2, stderr);
 270
 271                 struct uct_board *ub = b->es; assert(ub);
 272                 board_ownermap_fill(&ub->ownermap, &b2);
 273
 274         } else { assert(u->parallel_tree || tree_leaf_node(n));
 275                 /* In case of parallel tree search, the assertion might
 276                  * not hold if two threads chew on the same node. */
 277                 result = uct_leaf_node(u, &b2, player_color, amaf, t, n, node_color, spaces);
 278         }
 279
 280         if (amaf && u->playout_amaf_cutoff) {
 281                 int cutoff = amaf->game_baselen;
 282                 cutoff += (amaf->gamelen - amaf->game_baselen) * u->playout_amaf_cutoff / 100;
 283                 /* Now, reconstruct the amaf map. */
 284                 memset(amaf->map, 0, board_size2(&b2) * sizeof(*amaf->map));
 285                 for (int i = 0; i < cutoff; i++) {
 286                         coord_t coord = amaf->game[i].coord;
 287                         enum stone color = amaf->game[i].color;
 288                         if (amaf->map[coord] == S_NONE || amaf->map[coord] == color) {
 289                                 amaf->map[coord] = color;
 290                         /* Nakade always recorded for in-tree part */
 291                         } else if (amaf->record_nakade || i <= amaf->game_baselen) {
 292                                 amaf_op(amaf->map[n->coord], +);
 293                         }
 294                 }
 295         }
 296
 297         assert(n == t->root || n->parent);
 298         if (result != 0) {
 299                 float rval = scale_value(u, b, result);
 300
 301                 u->policy->update(u->policy, t, n, node_color, player_color, amaf, rval);
 302
 303                 if (u->root_heuristic && n->parent) {
 304                         if (!t->chvals) {
 305                                 t->chvals = calloc(board_size2(b), sizeof(t->chvals[0]));
 306                                 t->chchvals = calloc(board_size2(b), sizeof(t->chchvals[0]));
 307                         }
 308
 309                         /* Possibly transform the rval appropriately. */
 310                         rval = stats_temper_value(rval, n->parent->u.value, u->root_heuristic);
 311
 312                         struct tree_node *ni = n;
 313                         while (ni->parent->parent && ni->parent->parent->parent)
 314                                 ni = ni->parent;
 315                         if (ni->parent->parent) {
 316                                 if (likely(!is_pass(ni->coord)))
 317                                         stats_add_result(&t->chchvals[ni->coord], rval, 1);
 318                                 ni = ni->parent;
 319                         }
 320                         assert(ni->parent && !ni->parent->parent);
 321
 322                         if (likely(!is_pass(ni->coord)))
 323                                 stats_add_result(&t->chvals[ni->coord], rval, 1);
 324                 }
 325         }
 326
 327 end:
 328         /* We need to undo the virtual loss we added during descend. */
 329         if (u->virtual_loss) {
 330                 int parity = (node_color == player_color ? 1 : -1);
 331                 for (; n->parent; n = n->parent) {
 332                         stats_rm_result(&n->u, tree_parity(t, parity) > 0 ? 0 : 1, 1);
 333                         parity = -parity;
 334                 }
 335         }
 336
 337         if (dstater) free(dstater);
 338         if (dstate) free(dstate);
 339         if (amaf) {
 340                 free(amaf->map - 1);
 341                 free(amaf);
 342         }
 343         board_done_noalloc(&b2);
 344         return result;
 345 }
 346
 347 int
 348 uct_playouts(struct uct *u, struct board *b, enum stone color, struct tree *t, int games)
 349 {
 350         /* Should we print progress info? In case all threads work on the same
 351          * tree, only the first thread does. */
 352         #define ok_to_talk (!u->parallel_tree || !thread_id)
 353
 354         int i;
 355         for (i = 0; i < games; i++) {
 356                 int result = uct_playout(u, b, color, t);
 357                 if (result == 0) {
 358                         /* Tree descent has hit invalid move. */
 359                         continue;
 360                 }
 361
 362                 if (unlikely(ok_to_talk && i > 0 && !(i % 10000))) {
 363                         uct_progress_status(u, t, color, i);
 364                 }
 365
 366                 if (i > 0 && !(i % 500)) {
 367                         struct tree_node *best = u->policy->choose(u->policy, t->root, b, color);
 368                         if (best && ((best->u.playouts >= 2000 && tree_node_get_value(t, 1, best->u.value) >= u->loss_threshold)
 369                                      || (best->u.playouts >= 500 && tree_node_get_value(t, 1, best->u.value) >= 0.95)))
 370                                 break;
 371                 }
 372
 373                 if (uct_halt) {
 374                         if (UDEBUGL(2))
 375                                 fprintf(stderr, "<halting early, %d games skipped>\n", games - i);
 376                         break;
 377                 }
 378         }
 379
 380         if (ok_to_talk) {
 381                 uct_progress_status(u, t, color, i);
 382                 if (UDEBUGL(3))
 383                         tree_dump(t, u->dumpthres);
 384         }
 385         return i;
 386 }