release/src/router/nginx/src/http/ngx_http_upstream_round_robin.c

   1
   2 /*
   3  * Copyright (C) Igor Sysoev
   4  * Copyright (C) Nginx, Inc.
   5  */
   6
   7
   8 #include <ngx_config.h>
   9 #include <ngx_core.h>
  10 #include <ngx_http.h>
  11
  12
  13 static ngx_http_upstream_rr_peer_t *ngx_http_upstream_get_peer(
  14     ngx_http_upstream_rr_peer_data_t *rrp);
  15
  16 #if (NGX_HTTP_SSL)
  17
  18 static ngx_int_t ngx_http_upstream_empty_set_session(ngx_peer_connection_t *pc,
  19     void *data);
  20 static void ngx_http_upstream_empty_save_session(ngx_peer_connection_t *pc,
  21     void *data);
  22
  23 #endif
  24
  25
  26 ngx_int_t
  27 ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
  28     ngx_http_upstream_srv_conf_t *us)
  29 {
  30     ngx_url_t                      u;
  31     ngx_uint_t                     i, j, n, w;
  32     ngx_http_upstream_server_t    *server;
  33     ngx_http_upstream_rr_peers_t  *peers, *backup;
  34
  35     us->peer.init = ngx_http_upstream_init_round_robin_peer;
  36
  37     if (us->servers) {
  38         server = us->servers->elts;
  39
  40         n = 0;
  41         w = 0;
  42
  43         for (i = 0; i < us->servers->nelts; i++) {
  44             if (server[i].backup) {
  45                 continue;
  46             }
  47
  48             n += server[i].naddrs;
  49             w += server[i].naddrs * server[i].weight;
  50         }
  51
  52         if (n == 0) {
  53             ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
  54                           "no servers in upstream \"%V\" in %s:%ui",
  55                           &us->host, us->file_name, us->line);
  56             return NGX_ERROR;
  57         }
  58
  59         peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
  60                               + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
  61         if (peers == NULL) {
  62             return NGX_ERROR;
  63         }
  64
  65         peers->single = (n == 1);
  66         peers->number = n;
  67         peers->weighted = (w != n);
  68         peers->total_weight = w;
  69         peers->name = &us->host;
  70
  71         n = 0;
  72
  73         for (i = 0; i < us->servers->nelts; i++) {
  74             for (j = 0; j < server[i].naddrs; j++) {
  75                 if (server[i].backup) {
  76                     continue;
  77                 }
  78
  79                 peers->peer[n].sockaddr = server[i].addrs[j].sockaddr;
  80                 peers->peer[n].socklen = server[i].addrs[j].socklen;
  81                 peers->peer[n].name = server[i].addrs[j].name;
  82                 peers->peer[n].max_fails = server[i].max_fails;
  83                 peers->peer[n].fail_timeout = server[i].fail_timeout;
  84                 peers->peer[n].down = server[i].down;
  85                 peers->peer[n].weight = server[i].weight;
  86                 peers->peer[n].effective_weight = server[i].weight;
  87                 peers->peer[n].current_weight = 0;
  88                 n++;
  89             }
  90         }
  91
  92         us->peer.data = peers;
  93
  94         /* backup servers */
  95
  96         n = 0;
  97         w = 0;
  98
  99         for (i = 0; i < us->servers->nelts; i++) {
 100             if (!server[i].backup) {
 101                 continue;
 102             }
 103
 104             n += server[i].naddrs;
 105             w += server[i].naddrs * server[i].weight;
 106         }
 107
 108         if (n == 0) {
 109             return NGX_OK;
 110         }
 111
 112         backup = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
 113                               + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
 114         if (backup == NULL) {
 115             return NGX_ERROR;
 116         }
 117
 118         peers->single = 0;
 119         backup->single = 0;
 120         backup->number = n;
 121         backup->weighted = (w != n);
 122         backup->total_weight = w;
 123         backup->name = &us->host;
 124
 125         n = 0;
 126
 127         for (i = 0; i < us->servers->nelts; i++) {
 128             for (j = 0; j < server[i].naddrs; j++) {
 129                 if (!server[i].backup) {
 130                     continue;
 131                 }
 132
 133                 backup->peer[n].sockaddr = server[i].addrs[j].sockaddr;
 134                 backup->peer[n].socklen = server[i].addrs[j].socklen;
 135                 backup->peer[n].name = server[i].addrs[j].name;
 136                 backup->peer[n].weight = server[i].weight;
 137                 backup->peer[n].effective_weight = server[i].weight;
 138                 backup->peer[n].current_weight = 0;
 139                 backup->peer[n].max_fails = server[i].max_fails;
 140                 backup->peer[n].fail_timeout = server[i].fail_timeout;
 141                 backup->peer[n].down = server[i].down;
 142                 n++;
 143             }
 144         }
 145
 146         peers->next = backup;
 147
 148         return NGX_OK;
 149     }
 150
 151
 152     /* an upstream implicitly defined by proxy_pass, etc. */
 153
 154     if (us->port == 0) {
 155         ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
 156                       "no port in upstream \"%V\" in %s:%ui",
 157                       &us->host, us->file_name, us->line);
 158         return NGX_ERROR;
 159     }
 160
 161     ngx_memzero(&u, sizeof(ngx_url_t));
 162
 163     u.host = us->host;
 164     u.port = us->port;
 165
 166     if (ngx_inet_resolve_host(cf->pool, &u) != NGX_OK) {
 167         if (u.err) {
 168             ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
 169                           "%s in upstream \"%V\" in %s:%ui",
 170                           u.err, &us->host, us->file_name, us->line);
 171         }
 172
 173         return NGX_ERROR;
 174     }
 175
 176     n = u.naddrs;
 177
 178     peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
 179                               + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
 180     if (peers == NULL) {
 181         return NGX_ERROR;
 182     }
 183
 184     peers->single = (n == 1);
 185     peers->number = n;
 186     peers->weighted = 0;
 187     peers->total_weight = n;
 188     peers->name = &us->host;
 189
 190     for (i = 0; i < u.naddrs; i++) {
 191         peers->peer[i].sockaddr = u.addrs[i].sockaddr;
 192         peers->peer[i].socklen = u.addrs[i].socklen;
 193         peers->peer[i].name = u.addrs[i].name;
 194         peers->peer[i].weight = 1;
 195         peers->peer[i].effective_weight = 1;
 196         peers->peer[i].current_weight = 0;
 197         peers->peer[i].max_fails = 1;
 198         peers->peer[i].fail_timeout = 10;
 199     }
 200
 201     us->peer.data = peers;
 202
 203     /* implicitly defined upstream has no backup servers */
 204
 205     return NGX_OK;
 206 }
 207
 208
 209 ngx_int_t
 210 ngx_http_upstream_init_round_robin_peer(ngx_http_request_t *r,
 211     ngx_http_upstream_srv_conf_t *us)
 212 {
 213     ngx_uint_t                         n;
 214     ngx_http_upstream_rr_peer_data_t  *rrp;
 215
 216     rrp = r->upstream->peer.data;
 217
 218     if (rrp == NULL) {
 219         rrp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_rr_peer_data_t));
 220         if (rrp == NULL) {
 221             return NGX_ERROR;
 222         }
 223
 224         r->upstream->peer.data = rrp;
 225     }
 226
 227     rrp->peers = us->peer.data;
 228     rrp->current = 0;
 229
 230     n = rrp->peers->number;
 231
 232     if (rrp->peers->next && rrp->peers->next->number > n) {
 233         n = rrp->peers->next->number;
 234     }
 235
 236     if (n <= 8 * sizeof(uintptr_t)) {
 237         rrp->tried = &rrp->data;
 238         rrp->data = 0;
 239
 240     } else {
 241         n = (n + (8 * sizeof(uintptr_t) - 1)) / (8 * sizeof(uintptr_t));
 242
 243         rrp->tried = ngx_pcalloc(r->pool, n * sizeof(uintptr_t));
 244         if (rrp->tried == NULL) {
 245             return NGX_ERROR;
 246         }
 247     }
 248
 249     r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
 250     r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
 251     r->upstream->peer.tries = rrp->peers->number;
 252 #if (NGX_HTTP_SSL)
 253     r->upstream->peer.set_session =
 254                                ngx_http_upstream_set_round_robin_peer_session;
 255     r->upstream->peer.save_session =
 256                                ngx_http_upstream_save_round_robin_peer_session;
 257 #endif
 258
 259     return NGX_OK;
 260 }
 261
 262
 263 ngx_int_t
 264 ngx_http_upstream_create_round_robin_peer(ngx_http_request_t *r,
 265     ngx_http_upstream_resolved_t *ur)
 266 {
 267     u_char                            *p;
 268     size_t                             len;
 269     ngx_uint_t                         i, n;
 270     struct sockaddr_in                *sin;
 271     ngx_http_upstream_rr_peers_t      *peers;
 272     ngx_http_upstream_rr_peer_data_t  *rrp;
 273
 274     rrp = r->upstream->peer.data;
 275
 276     if (rrp == NULL) {
 277         rrp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_rr_peer_data_t));
 278         if (rrp == NULL) {
 279             return NGX_ERROR;
 280         }
 281
 282         r->upstream->peer.data = rrp;
 283     }
 284
 285     peers = ngx_pcalloc(r->pool, sizeof(ngx_http_upstream_rr_peers_t)
 286                      + sizeof(ngx_http_upstream_rr_peer_t) * (ur->naddrs - 1));
 287     if (peers == NULL) {
 288         return NGX_ERROR;
 289     }
 290
 291     peers->single = (ur->naddrs == 1);
 292     peers->number = ur->naddrs;
 293     peers->name = &ur->host;
 294
 295     if (ur->sockaddr) {
 296         peers->peer[0].sockaddr = ur->sockaddr;
 297         peers->peer[0].socklen = ur->socklen;
 298         peers->peer[0].name = ur->host;
 299         peers->peer[0].weight = 1;
 300         peers->peer[0].effective_weight = 1;
 301         peers->peer[0].current_weight = 0;
 302         peers->peer[0].max_fails = 1;
 303         peers->peer[0].fail_timeout = 10;
 304
 305     } else {
 306
 307         for (i = 0; i < ur->naddrs; i++) {
 308
 309             len = NGX_INET_ADDRSTRLEN + sizeof(":65536") - 1;
 310
 311             p = ngx_pnalloc(r->pool, len);
 312             if (p == NULL) {
 313                 return NGX_ERROR;
 314             }
 315
 316             len = ngx_inet_ntop(AF_INET, &ur->addrs[i], p, NGX_INET_ADDRSTRLEN);
 317             len = ngx_sprintf(&p[len], ":%d", ur->port) - p;
 318
 319             sin = ngx_pcalloc(r->pool, sizeof(struct sockaddr_in));
 320             if (sin == NULL) {
 321                 return NGX_ERROR;
 322             }
 323
 324             sin->sin_family = AF_INET;
 325             sin->sin_port = htons(ur->port);
 326             sin->sin_addr.s_addr = ur->addrs[i];
 327
 328             peers->peer[i].sockaddr = (struct sockaddr *) sin;
 329             peers->peer[i].socklen = sizeof(struct sockaddr_in);
 330             peers->peer[i].name.len = len;
 331             peers->peer[i].name.data = p;
 332             peers->peer[i].weight = 1;
 333             peers->peer[i].effective_weight = 1;
 334             peers->peer[i].current_weight = 0;
 335             peers->peer[i].max_fails = 1;
 336             peers->peer[i].fail_timeout = 10;
 337         }
 338     }
 339
 340     rrp->peers = peers;
 341     rrp->current = 0;
 342
 343     if (rrp->peers->number <= 8 * sizeof(uintptr_t)) {
 344         rrp->tried = &rrp->data;
 345         rrp->data = 0;
 346
 347     } else {
 348         n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
 349                 / (8 * sizeof(uintptr_t));
 350
 351         rrp->tried = ngx_pcalloc(r->pool, n * sizeof(uintptr_t));
 352         if (rrp->tried == NULL) {
 353             return NGX_ERROR;
 354         }
 355     }
 356
 357     r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
 358     r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
 359     r->upstream->peer.tries = rrp->peers->number;
 360 #if (NGX_HTTP_SSL)
 361     r->upstream->peer.set_session = ngx_http_upstream_empty_set_session;
 362     r->upstream->peer.save_session = ngx_http_upstream_empty_save_session;
 363 #endif
 364
 365     return NGX_OK;
 366 }
 367
 368
 369 ngx_int_t
 370 ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
 371 {
 372     ngx_http_upstream_rr_peer_data_t  *rrp = data;
 373
 374     ngx_int_t                      rc;
 375     ngx_uint_t                     i, n;
 376     ngx_http_upstream_rr_peer_t   *peer;
 377     ngx_http_upstream_rr_peers_t  *peers;
 378
 379     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
 380                    "get rr peer, try: %ui", pc->tries);
 381
 382     /* ngx_lock_mutex(rrp->peers->mutex); */
 383
 384     pc->cached = 0;
 385     pc->connection = NULL;
 386
 387     if (rrp->peers->single) {
 388         peer = &rrp->peers->peer[0];
 389
 390         if (peer->down) {
 391             goto failed;
 392         }
 393
 394     } else {
 395
 396         /* there are several peers */
 397
 398         peer = ngx_http_upstream_get_peer(rrp);
 399
 400         if (peer == NULL) {
 401             goto failed;
 402         }
 403
 404         ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
 405                        "get rr peer, current: %ui %i",
 406                        rrp->current, peer->current_weight);
 407     }
 408
 409     pc->sockaddr = peer->sockaddr;
 410     pc->socklen = peer->socklen;
 411     pc->name = &peer->name;
 412
 413     /* ngx_unlock_mutex(rrp->peers->mutex); */
 414
 415     if (pc->tries == 1 && rrp->peers->next) {
 416         pc->tries += rrp->peers->next->number;
 417     }
 418
 419     return NGX_OK;
 420
 421 failed:
 422
 423     peers = rrp->peers;
 424
 425     if (peers->next) {
 426
 427         /* ngx_unlock_mutex(peers->mutex); */
 428
 429         ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "backup servers");
 430
 431         rrp->peers = peers->next;
 432         pc->tries = rrp->peers->number;
 433
 434         n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
 435                 / (8 * sizeof(uintptr_t));
 436
 437         for (i = 0; i < n; i++) {
 438              rrp->tried[i] = 0;
 439         }
 440
 441         rc = ngx_http_upstream_get_round_robin_peer(pc, rrp);
 442
 443         if (rc != NGX_BUSY) {
 444             return rc;
 445         }
 446
 447         /* ngx_lock_mutex(peers->mutex); */
 448     }
 449
 450     /* all peers failed, mark them as live for quick recovery */
 451
 452     for (i = 0; i < peers->number; i++) {
 453         peers->peer[i].fails = 0;
 454     }
 455
 456     /* ngx_unlock_mutex(peers->mutex); */
 457
 458     pc->name = peers->name;
 459
 460     return NGX_BUSY;
 461 }
 462
 463
 464 static ngx_http_upstream_rr_peer_t *
 465 ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp)
 466 {
 467     time_t                        now;
 468     uintptr_t                     m;
 469     ngx_int_t                     total;
 470     ngx_uint_t                    i, n;
 471     ngx_http_upstream_rr_peer_t  *peer, *best;
 472
 473     now = ngx_time();
 474
 475     best = NULL;
 476     total = 0;
 477
 478     for (i = 0; i < rrp->peers->number; i++) {
 479
 480         n = i / (8 * sizeof(uintptr_t));
 481         m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
 482
 483         if (rrp->tried[n] & m) {
 484             continue;
 485         }
 486
 487         peer = &rrp->peers->peer[i];
 488
 489         if (peer->down) {
 490             continue;
 491         }
 492
 493         if (peer->max_fails
 494             && peer->fails >= peer->max_fails
 495             && now - peer->checked <= peer->fail_timeout)
 496         {
 497             continue;
 498         }
 499
 500         peer->current_weight += peer->effective_weight;
 501         total += peer->effective_weight;
 502
 503         if (peer->effective_weight < peer->weight) {
 504             peer->effective_weight++;
 505         }
 506
 507         if (best == NULL || peer->current_weight > best->current_weight) {
 508             best = peer;
 509         }
 510     }
 511
 512     if (best == NULL) {
 513         return NULL;
 514     }
 515
 516     i = best - &rrp->peers->peer[0];
 517
 518     rrp->current = i;
 519
 520     n = i / (8 * sizeof(uintptr_t));
 521     m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
 522
 523     rrp->tried[n] |= m;
 524
 525     best->current_weight -= total;
 526     best->checked = now;
 527
 528     return best;
 529 }
 530
 531
 532 void
 533 ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
 534     ngx_uint_t state)
 535 {
 536     ngx_http_upstream_rr_peer_data_t  *rrp = data;
 537
 538     time_t                       now;
 539     ngx_http_upstream_rr_peer_t  *peer;
 540
 541     ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
 542                    "free rr peer %ui %ui", pc->tries, state);
 543
 544     /* TODO: NGX_PEER_KEEPALIVE */
 545
 546     if (rrp->peers->single) {
 547         pc->tries = 0;
 548         return;
 549     }
 550
 551     peer = &rrp->peers->peer[rrp->current];
 552
 553     if (state & NGX_PEER_FAILED) {
 554         now = ngx_time();
 555
 556         /* ngx_lock_mutex(rrp->peers->mutex); */
 557
 558         peer->fails++;
 559         peer->accessed = now;
 560         peer->checked = now;
 561
 562         if (peer->max_fails) {
 563             peer->effective_weight -= peer->weight / peer->max_fails;
 564         }
 565
 566         ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
 567                        "free rr peer failed: %ui %i",
 568                        rrp->current, peer->effective_weight);
 569
 570         if (peer->effective_weight < 0) {
 571             peer->effective_weight = 0;
 572         }
 573
 574         /* ngx_unlock_mutex(rrp->peers->mutex); */
 575
 576     } else {
 577
 578         /* mark peer live if check passed */
 579
 580         if (peer->accessed < peer->checked) {
 581             peer->fails = 0;
 582         }
 583     }
 584
 585     if (pc->tries) {
 586         pc->tries--;
 587     }
 588
 589     /* ngx_unlock_mutex(rrp->peers->mutex); */
 590 }
 591
 592
 593 #if (NGX_HTTP_SSL)
 594
 595 ngx_int_t
 596 ngx_http_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc,
 597     void *data)
 598 {
 599     ngx_http_upstream_rr_peer_data_t  *rrp = data;
 600
 601     ngx_int_t                     rc;
 602     ngx_ssl_session_t            *ssl_session;
 603     ngx_http_upstream_rr_peer_t  *peer;
 604
 605     peer = &rrp->peers->peer[rrp->current];
 606
 607     /* TODO: threads only mutex */
 608     /* ngx_lock_mutex(rrp->peers->mutex); */
 609
 610     ssl_session = peer->ssl_session;
 611
 612     rc = ngx_ssl_set_session(pc->connection, ssl_session);
 613
 614     ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
 615                    "set session: %p:%d",
 616                    ssl_session, ssl_session ? ssl_session->references : 0);
 617
 618     /* ngx_unlock_mutex(rrp->peers->mutex); */
 619
 620     return rc;
 621 }
 622
 623
 624 void
 625 ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc,
 626     void *data)
 627 {
 628     ngx_http_upstream_rr_peer_data_t  *rrp = data;
 629
 630     ngx_ssl_session_t            *old_ssl_session, *ssl_session;
 631     ngx_http_upstream_rr_peer_t  *peer;
 632
 633     ssl_session = ngx_ssl_get_session(pc->connection);
 634
 635     if (ssl_session == NULL) {
 636         return;
 637     }
 638
 639     ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
 640                    "save session: %p:%d", ssl_session, ssl_session->references);
 641
 642     peer = &rrp->peers->peer[rrp->current];
 643
 644     /* TODO: threads only mutex */
 645     /* ngx_lock_mutex(rrp->peers->mutex); */
 646
 647     old_ssl_session = peer->ssl_session;
 648     peer->ssl_session = ssl_session;
 649
 650     /* ngx_unlock_mutex(rrp->peers->mutex); */
 651
 652     if (old_ssl_session) {
 653
 654         ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
 655                        "old session: %p:%d",
 656                        old_ssl_session, old_ssl_session->references);
 657
 658         /* TODO: may block */
 659
 660         ngx_ssl_free_session(old_ssl_session);
 661     }
 662 }
 663
 664
 665 static ngx_int_t
 666 ngx_http_upstream_empty_set_session(ngx_peer_connection_t *pc, void *data)
 667 {
 668     return NGX_OK;
 669 }
 670
 671
 672 static void
 673 ngx_http_upstream_empty_save_session(ngx_peer_connection_t *pc, void *data)
 674 {
 675     return;
 676 }
 677
 678 #endif