python/samba/tests/prefork_restart.py

   1 # Tests for process restarting in the pre-fork process model
   2 #
   3 # Copyright (C) Andrew Bartlett <abartlet@samba.org> 2018
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 #
  18 """Tests process restarting in the pre-fork process model.
  19    NOTE: As this test kills samba processes it won't play nicely with other
  20          tests, so needs to be run in it's own environment.
  21 """
  22
  23
  24 import os
  25 import signal
  26 import time
  27
  28 import samba
  29 from samba.tests import TestCase, delete_force
  30 from samba.dcerpc import echo, netlogon
  31 from samba.messaging import Messaging
  32 from samba.samdb import SamDB
  33 from samba.credentials import Credentials, DONT_USE_KERBEROS
  34 from samba.common import get_string
  35 from samba.dsdb import (
  36     UF_WORKSTATION_TRUST_ACCOUNT,
  37     UF_PASSWD_NOTREQD)
  38 from samba.dcerpc.misc import SEC_CHAN_WKSTA
  39 from samba.auth import system_session
  40
  41 NUM_WORKERS = 4
  42 MACHINE_NAME = "PFRS"
  43
  44
  45 class PreforkProcessRestartTests(TestCase):
  46
  47     def setUp(self):
  48         super(PreforkProcessRestartTests, self).setUp()
  49         lp_ctx = self.get_loadparm()
  50         self.msg_ctx = Messaging(lp_ctx=lp_ctx)
  51
  52     def tearDown(self):
  53         super(PreforkProcessRestartTests, self).tearDown()
  54
  55     def get_process_data(self):
  56         services = self.msg_ctx.irpc_all_servers()
  57
  58         processes = []
  59         for service in services:
  60             for id in service.ids:
  61                 processes.append((service.name, id.pid))
  62         return processes
  63
  64     def get_process(self, name):
  65         processes = self.get_process_data()
  66         for pname, pid in processes:
  67             if name == pname:
  68                 return pid
  69         return None
  70
  71     def get_worker_pids(self, name, workers):
  72         pids = []
  73         for x in range(workers):
  74             process_name = "prefork-worker-{0}-{1}".format(name, x)
  75             pids.append(self.get_process(process_name))
  76             self.assertIsNotNone(pids[x])
  77         return pids
  78
  79     def wait_for_workers(self, name, workers):
  80         num_workers = len(workers)
  81         for x in range(num_workers):
  82             process_name = "prefork-worker-{0}-{1}".format(name, x)
  83             self.wait_for_process(process_name, workers[x], 0, 1, 30)
  84
  85     def wait_for_process(self, name, pid, initial_delay, wait, timeout):
  86         time.sleep(initial_delay)
  87         delay = initial_delay
  88         while delay < timeout:
  89             p = self.get_process(name)
  90             if p is not None and p != pid:
  91                 # process has restarted
  92                 return
  93             time.sleep(wait)
  94             delay += wait
  95         self.fail("Times out after {0} seconds waiting for {1} to restart".
  96                   format(delay, name))
  97
  98     def check_for_duplicate_processes(self):
  99             processes = self.get_process_data()
 100             process_map = {}
 101             for name, p in processes:
 102                 if (name.startswith("prefork-") or
 103                     name.endswith("_server") or
 104                     name.endswith("srv")):
 105
 106                     if name in process_map:
 107                         if p != process_map[name]:
 108                             self.fail(
 109                                 "Duplicate process for {0}, pids {1} and {2}".
 110                                 format(name, p, process_map[name]))
 111
 112     def simple_bind(self):
 113         creds = self.insta_creds(template=self.get_credentials())
 114         creds.set_bind_dn("%s\\%s" % (creds.get_domain(),
 115                                       creds.get_username()))
 116
 117         self.samdb = SamDB(url="ldaps://%s" % os.environ["SERVER"],
 118                            lp=self.get_loadparm(),
 119                            credentials=creds)
 120
 121     def rpc_echo(self):
 122         conn = echo.rpcecho("ncalrpc:", self.get_loadparm())
 123         self.assertEqual([1, 2, 3], conn.EchoData([1, 2, 3]))
 124
 125     def netlogon(self):
 126         server = os.environ["SERVER"]
 127         host = os.environ["SERVER_IP"]
 128         lp = self.get_loadparm()
 129
 130         credentials = self.get_credentials()
 131
 132         session = system_session()
 133         ldb = SamDB(url="ldap://%s" % host,
 134                     session_info=session,
 135                     credentials=credentials,
 136                     lp=lp)
 137         machine_pass = samba.generate_random_password(32, 32)
 138         machine_name = MACHINE_NAME
 139         machine_dn = "cn=%s,%s" % (machine_name, ldb.domain_dn())
 140
 141         delete_force(ldb, machine_dn)
 142
 143         utf16pw = ('"%s"' % get_string(machine_pass)).encode('utf-16-le')
 144         ldb.add({
 145             "dn": machine_dn,
 146             "objectclass": "computer",
 147             "sAMAccountName": "%s$" % machine_name,
 148             "userAccountControl":
 149                 str(UF_WORKSTATION_TRUST_ACCOUNT | UF_PASSWD_NOTREQD),
 150             "unicodePwd": utf16pw})
 151
 152         machine_creds = Credentials()
 153         machine_creds.guess(lp)
 154         machine_creds.set_secure_channel_type(SEC_CHAN_WKSTA)
 155         machine_creds.set_kerberos_state(DONT_USE_KERBEROS)
 156         machine_creds.set_password(machine_pass)
 157         machine_creds.set_username(machine_name + "$")
 158         machine_creds.set_workstation(machine_name)
 159
 160         netlogon.netlogon(
 161             "ncacn_ip_tcp:%s[schannel,seal]" % server,
 162             lp,
 163             machine_creds)
 164
 165         delete_force(ldb, machine_dn)
 166
 167     def test_ldap_master_restart(self):
 168         # check ldap connection, do a simple bind
 169         self.simple_bind()
 170
 171         # get ldap master process
 172         pid = self.get_process("prefork-master-ldap")
 173         self.assertIsNotNone(pid)
 174
 175         # Get the worker processes
 176         workers = self.get_worker_pids("ldap", NUM_WORKERS)
 177
 178         # kill it
 179         os.kill(pid, signal.SIGTERM)
 180
 181         # wait for the process to restart
 182         self.wait_for_process("prefork-master-ldap", pid, 1, 1, 30)
 183
 184         # restarting the master restarts the workers as well, so make sure
 185         # they have finished restarting
 186         self.wait_for_workers("ldap", workers)
 187
 188         # get ldap master process
 189         new_pid = self.get_process("prefork-master-ldap")
 190         self.assertIsNotNone(new_pid)
 191
 192         # check that the pid has changed
 193         self.assertNotEquals(pid, new_pid)
 194
 195         # check that the worker processes have restarted
 196         new_workers = self.get_worker_pids("ldap", NUM_WORKERS)
 197         for x in range(NUM_WORKERS):
 198             self.assertNotEquals(workers[x], new_workers[x])
 199
 200         # check that the previous server entries have been removed.
 201         self.check_for_duplicate_processes()
 202
 203         # check ldap connection, another simple bind
 204         self.simple_bind()
 205
 206     def test_ldap_worker_restart(self):
 207         # check ldap connection, do a simple bind
 208         self.simple_bind()
 209
 210         # get ldap master process
 211         pid = self.get_process("prefork-master-ldap")
 212         self.assertIsNotNone(pid)
 213
 214         # Get the worker processes
 215         workers = self.get_worker_pids("ldap", NUM_WORKERS)
 216
 217         # kill worker 0
 218         os.kill(workers[0], signal.SIGTERM)
 219
 220         # wait for the process to restart
 221         self.wait_for_process("prefork-worker-ldap-0", pid, 1, 1, 30)
 222
 223         # get ldap master process
 224         new_pid = self.get_process("prefork-master-ldap")
 225         self.assertIsNotNone(new_pid)
 226
 227         # check that the pid has not changed
 228         self.assertEqual(pid, new_pid)
 229
 230         # check that the worker processes have restarted
 231         new_workers = self.get_worker_pids("ldap", NUM_WORKERS)
 232         # process 0 should have a new pid the others should be unchanged
 233         self.assertNotEquals(workers[0], new_workers[0])
 234         self.assertEqual(workers[1], new_workers[1])
 235         self.assertEqual(workers[2], new_workers[2])
 236         self.assertEqual(workers[3], new_workers[3])
 237
 238         # check that the previous server entries have been removed.
 239         self.check_for_duplicate_processes()
 240
 241         # check ldap connection, another simple bind
 242         self.simple_bind()
 243
 244     #
 245     # Kill all the ldap worker processes and ensure that they are restarted
 246     # correctly
 247     #
 248     def test_ldap_all_workers_restart(self):
 249         # check ldap connection, do a simple bind
 250         self.simple_bind()
 251
 252         # get ldap master process
 253         pid = self.get_process("prefork-master-ldap")
 254         self.assertIsNotNone(pid)
 255
 256         # Get the worker processes
 257         workers = self.get_worker_pids("ldap", NUM_WORKERS)
 258
 259         # kill all the worker processes
 260         for x in workers:
 261             os.kill(x, signal.SIGTERM)
 262
 263         # wait for the worker processes to restart
 264         self.wait_for_workers("ldap", workers)
 265
 266         # get ldap master process
 267         new_pid = self.get_process("prefork-master-ldap")
 268         self.assertIsNotNone(new_pid)
 269
 270         # check that the pid has not changed
 271         self.assertEqual(pid, new_pid)
 272
 273         # check that the worker processes have restarted
 274         new_workers = self.get_worker_pids("ldap", NUM_WORKERS)
 275         for x in range(NUM_WORKERS):
 276             self.assertNotEquals(workers[x], new_workers[x])
 277
 278         # check that the previous server entries have been removed.
 279         self.check_for_duplicate_processes()
 280
 281         # check ldap connection, another simple bind
 282         self.simple_bind()
 283
 284     def test_rpc_master_restart(self):
 285         # check rpc connection, make a rpc echo request
 286         self.rpc_echo()
 287
 288         # get rpc master process
 289         pid = self.get_process("prefork-master-rpc")
 290         self.assertIsNotNone(pid)
 291
 292         # Get the worker processes
 293         workers = self.get_worker_pids("rpc", NUM_WORKERS)
 294
 295         # kill it
 296         os.kill(pid, signal.SIGTERM)
 297
 298         # wait for the process to restart
 299         self.wait_for_process("prefork-master-rpc", pid, 1, 1, 30)
 300
 301         # wait for workers to restart as well
 302         self.wait_for_workers("rpc", workers)
 303
 304         # get ldap master process
 305         new_pid = self.get_process("prefork-master-rpc")
 306         self.assertIsNotNone(new_pid)
 307
 308         # check that the pid has changed
 309         self.assertNotEquals(pid, new_pid)
 310
 311         # check that the worker processes have restarted
 312         new_workers = self.get_worker_pids("rpc", NUM_WORKERS)
 313         for x in range(NUM_WORKERS):
 314             self.assertNotEquals(workers[x], new_workers[x])
 315
 316         # check that the previous server entries have been removed.
 317         self.check_for_duplicate_processes()
 318
 319         # check rpc connection, another rpc echo request
 320         self.rpc_echo()
 321
 322     def test_rpc_worker_zero_restart(self):
 323         # check rpc connection, make a rpc echo request and a netlogon request
 324         self.rpc_echo()
 325         self.netlogon()
 326
 327         # get rpc master process
 328         pid = self.get_process("prefork-master-rpc")
 329         self.assertIsNotNone(pid)
 330
 331         # Get the worker processes
 332         workers = self.get_worker_pids("rpc", NUM_WORKERS)
 333
 334         # kill worker 0
 335         os.kill(workers[0], signal.SIGTERM)
 336
 337         # wait for the process to restart
 338         self.wait_for_process("prefork-worker-rpc-0", workers[0], 1, 1, 30)
 339
 340         # get rpc master process
 341         new_pid = self.get_process("prefork-master-rpc")
 342         self.assertIsNotNone(new_pid)
 343
 344         # check that the pid has not changed
 345         self.assertEqual(pid, new_pid)
 346
 347         # check that the worker processes have restarted
 348         new_workers = self.get_worker_pids("rpc", NUM_WORKERS)
 349         # process 0 should have a new pid the others should be unchanged
 350         self.assertNotEquals(workers[0], new_workers[0])
 351         self.assertEqual(workers[1], new_workers[1])
 352         self.assertEqual(workers[2], new_workers[2])
 353         self.assertEqual(workers[3], new_workers[3])
 354
 355         # check that the previous server entries have been removed.
 356         self.check_for_duplicate_processes()
 357
 358         # check rpc connection, another rpc echo request, and netlogon request
 359         self.rpc_echo()
 360         self.netlogon()
 361
 362     def test_rpc_all_workers_restart(self):
 363         # check rpc connection, make a rpc echo request, and a netlogon request
 364         self.rpc_echo()
 365         self.netlogon()
 366
 367         # get rpc master process
 368         pid = self.get_process("prefork-master-rpc")
 369         self.assertIsNotNone(pid)
 370
 371         # Get the worker processes
 372         workers = self.get_worker_pids("rpc", NUM_WORKERS)
 373
 374         # kill all the worker processes
 375         for x in workers:
 376             os.kill(x, signal.SIGTERM)
 377
 378         # wait for the worker processes to restart
 379         for x in range(NUM_WORKERS):
 380             self.wait_for_process(
 381                 "prefork-worker-rpc-{0}".format(x), workers[x], 0, 1, 30)
 382
 383         # get rpc master process
 384         new_pid = self.get_process("prefork-master-rpc")
 385         self.assertIsNotNone(new_pid)
 386
 387         # check that the pid has not changed
 388         self.assertEqual(pid, new_pid)
 389
 390         # check that the worker processes have restarted
 391         new_workers = self.get_worker_pids("rpc", NUM_WORKERS)
 392         for x in range(NUM_WORKERS):
 393             self.assertNotEquals(workers[x], new_workers[x])
 394
 395         # check that the previous server entries have been removed.
 396         self.check_for_duplicate_processes()
 397
 398         # check rpc connection, another rpc echo request and netlogon
 399         self.rpc_echo()
 400         self.netlogon()
 401
 402     def test_master_restart_backoff(self):
 403
 404         # get kdc master process
 405         pid = self.get_process("prefork-master-echo")
 406         self.assertIsNotNone(pid)
 407
 408         #
 409         # Check that the processes get backed off as expected
 410         #
 411         # have prefork backoff increment = 5
 412         #      prefork maximum backoff   = 10
 413         backoff_increment = 5
 414         for expected in [0, 5, 10, 10]:
 415             # Get the worker processes
 416             workers = self.get_worker_pids("kdc", NUM_WORKERS)
 417
 418             process = self.get_process("prefork-master-echo")
 419             os.kill(process, signal.SIGTERM)
 420             # wait for the process to restart
 421             start = time.time()
 422             self.wait_for_process("prefork-master-echo", process, 0, 1, 30)
 423             # wait for the workers to restart as well
 424             self.wait_for_workers("echo", workers)
 425             end = time.time()
 426             duration = end - start
 427
 428             # process restart will take some time. Check that the elapsed
 429             # duration falls somewhere in the expected range, i.e. we haven't
 430             # taken longer than the backoff increment
 431             self.assertLess(duration, expected + backoff_increment)
 432             self.assertGreaterEqual(duration, expected)
 433
 434         # check that the worker processes have restarted
 435         new_workers = self.get_worker_pids("echo", NUM_WORKERS)
 436         for x in range(NUM_WORKERS):
 437             self.assertNotEquals(workers[x], new_workers[x])
 438
 439         # check that the previous server entries have been removed.
 440         self.check_for_duplicate_processes()
 441
 442     def test_worker_restart_backoff(self):
 443         #
 444         # Check that the processes get backed off as expected
 445         #
 446         # have prefork backoff increment = 5
 447         #      prefork maximum backoff   = 10
 448         backoff_increment = 5
 449         for expected in [0, 5, 10, 10]:
 450             process = self.get_process("prefork-worker-echo-2")
 451             self.assertIsNotNone(process)
 452             os.kill(process, signal.SIGTERM)
 453             # wait for the process to restart
 454             start = time.time()
 455             self.wait_for_process("prefork-worker-echo-2", process, 0, 1, 30)
 456             end = time.time()
 457             duration = end - start
 458
 459             # process restart will take some time. Check that the elapsed
 460             # duration falls somewhere in the expected range, i.e. we haven't
 461             # taken longer than the backoff increment
 462             self.assertLess(duration, expected + backoff_increment)
 463             self.assertGreaterEqual(duration, expected)
 464
 465         self.check_for_duplicate_processes()