netrender/client.py

   1 # ##### BEGIN GPL LICENSE BLOCK #####
   2 #
   3 #  This program is free software; you can redistribute it and/or
   4 #  modify it under the terms of the GNU General Public License
   5 #  as published by the Free Software Foundation; either version 2
   6 #  of the License, or (at your option) any later version.
   7 #
   8 #  This program is distributed in the hope that it will be useful,
   9 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 #  GNU General Public License for more details.
  12 #
  13 #  You should have received a copy of the GNU General Public License
  14 #  along with this program; if not, write to the Free Software Foundation,
  15 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16 #
  17 # ##### END GPL LICENSE BLOCK #####
  18
  19 import bpy
  20 import os, re
  21 import http, http.client, http.server
  22 import time
  23 import json
  24
  25 import netrender
  26 import netrender.model
  27 import netrender.slave as slave
  28 import netrender.master as master
  29 from netrender.utils import *
  30
  31 def addFluidFiles(job, path):
  32     if os.path.exists(path):
  33         pattern = re.compile("fluidsurface_(final|preview)_([0-9]+)\.(bobj|bvel)\.gz")
  34
  35         for fluid_file in sorted(os.listdir(path)):
  36             match = pattern.match(fluid_file)
  37
  38             if match:
  39                 # fluid frames starts at 0, which explains the +1
  40                 # This is stupid
  41                 current_frame = int(match.groups()[1]) + 1
  42                 job.addFile(os.path.join(path, fluid_file), current_frame, current_frame)
  43
  44 def addPointCache(job, ob, point_cache, default_path):
  45     if not point_cache.use_disk_cache:
  46         return
  47
  48
  49     name = cacheName(ob, point_cache)
  50
  51     cache_path = bpy.path.abspath(point_cache.filepath) if point_cache.use_external else default_path
  52
  53     index = "%02i" % point_cache.index
  54
  55     if os.path.exists(cache_path):
  56         pattern = re.compile(name + "_([0-9]+)_" + index + "\.bphys")
  57
  58         cache_files = []
  59
  60         for cache_file in sorted(os.listdir(cache_path)):
  61             match = pattern.match(cache_file)
  62
  63             if match:
  64                 cache_frame = int(match.groups()[0])
  65                 cache_files.append((cache_frame, cache_file))
  66
  67         cache_files.sort()
  68
  69         if len(cache_files) == 1:
  70             cache_frame, cache_file = cache_files[0]
  71             job.addFile(os.path.join(cache_path, cache_file), cache_frame, cache_frame)
  72         else:
  73             for i in range(len(cache_files)):
  74                 current_item = cache_files[i]
  75                 next_item = cache_files[i+1] if i + 1 < len(cache_files) else None
  76                 previous_item = cache_files[i - 1] if i > 0 else None
  77
  78                 current_frame, current_file = current_item
  79
  80                 if not next_item and not previous_item:
  81                     job.addFile(os.path.join(cache_path, current_file), current_frame, current_frame)
  82                 elif next_item and not previous_item:
  83                     next_frame = next_item[0]
  84                     job.addFile(os.path.join(cache_path, current_file), current_frame, next_frame)
  85                 elif not next_item and previous_item:
  86                     previous_frame = previous_item[0]
  87                     job.addFile(os.path.join(cache_path, current_file), previous_frame, current_frame)
  88                 else:
  89                     next_frame = next_item[0]
  90                     previous_frame = previous_item[0]
  91                     job.addFile(os.path.join(cache_path, current_file), previous_frame, next_frame)
  92
  93 def fillCommonJobSettings(job, job_name, netsettings):
  94     job.name = job_name
  95     job.category = netsettings.job_category
  96
  97     for bad_slave in netrender.blacklist:
  98         job.blacklist.append(bad_slave.id)
  99
 100     job.chunks = netsettings.chunks
 101     job.priority = netsettings.priority
 102
 103     if netsettings.job_render_engine == "OTHER":
 104         job.render = netsettings.job_render_engine_other
 105     else:
 106         job.render = netsettings.job_render_engine
 107
 108     if netsettings.job_tags:
 109         job.tags.update(netsettings.job_tags.split(";"))
 110
 111     if netsettings.job_type == "JOB_BLENDER":
 112         job.type = netrender.model.JOB_BLENDER
 113     elif netsettings.job_type == "JOB_PROCESS":
 114         job.type = netrender.model.JOB_PROCESS
 115     elif netsettings.job_type == "JOB_VCS":
 116         job.type = netrender.model.JOB_VCS
 117
 118 def sendJob(conn, scene, anim = False, can_save = True):
 119     netsettings = scene.network_render
 120     if netsettings.job_type == "JOB_BLENDER":
 121         return sendJobBlender(conn, scene, anim, can_save)
 122     elif netsettings.job_type == "JOB_VCS":
 123         return sendJobVCS(conn, scene, anim)
 124
 125 def sendJobVCS(conn, scene, anim = False):
 126     netsettings = scene.network_render
 127     job = netrender.model.RenderJob()
 128
 129     if anim:
 130         for f in range(scene.frame_start, scene.frame_end + 1):
 131             job.addFrame(f)
 132     else:
 133         job.addFrame(scene.frame_current)
 134
 135     filename = bpy.data.filepath
 136
 137     if not filename.startswith(netsettings.vcs_wpath):
 138         # this is an error, need better way to handle this
 139         return
 140
 141     filename = filename[len(netsettings.vcs_wpath):]
 142
 143     if filename[0] in {os.sep, os.altsep}:
 144         filename = filename[1:]
 145
 146     job.addFile(filename, signed=False)
 147
 148     job_name = netsettings.job_name
 149     path, name = os.path.split(filename)
 150     if job_name == "[default]":
 151         job_name = name
 152
 153
 154     fillCommonJobSettings(job, job_name, netsettings)
 155
 156     # VCS Specific code
 157     job.version_info = netrender.model.VersioningInfo()
 158     job.version_info.system = netsettings.vcs_system
 159     job.version_info.wpath = netsettings.vcs_wpath
 160     job.version_info.rpath = netsettings.vcs_rpath
 161     job.version_info.revision = netsettings.vcs_revision
 162
 163     job.tags.add(netrender.model.TAG_RENDER)
 164
 165     # try to send path first
 166     with ConnectionContext():
 167         conn.request("POST", "/job", json.dumps(job.serialize()))
 168     response = conn.getresponse()
 169     response.read()
 170
 171     job_id = response.getheader("job-id")
 172
 173     # a VCS job is always good right now, need error handling
 174
 175     return job_id
 176
 177 def sendJobBaking(conn, scene, can_save = True):
 178     netsettings = scene.network_render
 179     job = netrender.model.RenderJob()
 180
 181     filename = bpy.data.filepath
 182
 183     if not os.path.exists(filename):
 184         raise RuntimeError("Current file path not defined\nSave your file before sending a job")
 185
 186     if can_save and netsettings.save_before_job:
 187         bpy.ops.wm.save_mainfile(filepath=filename, check_existing=False)
 188
 189     job.addFile(filename)
 190
 191     job_name = netsettings.job_name
 192     path, name = os.path.split(filename)
 193     if job_name == "[default]":
 194         job_name = name
 195
 196     ###############################
 197     # LIBRARIES (needed for baking)
 198     ###############################
 199     for lib in bpy.data.libraries:
 200         file_path = bpy.path.abspath(lib.filepath)
 201         if os.path.exists(file_path):
 202             job.addFile(file_path)
 203
 204     tasks = set()
 205
 206     ####################################
 207     # FLUID + POINT CACHE (what we bake)
 208     ####################################
 209     def pointCacheFunc(object, owner, point_cache):
 210         if type(owner) == bpy.types.ParticleSystem:
 211             index = [index for index, data in enumerate(object.particle_systems) if data == owner][0]
 212             tasks.add(("PARTICLE_SYSTEM", object.name, str(index)))
 213         else: # owner is modifier
 214             index = [index for index, data in enumerate(object.modifiers) if data == owner][0]
 215             tasks.add((owner.type, object.name, str(index)))
 216
 217     def fluidFunc(object, modifier, cache_path):
 218         pass
 219
 220     def multiresFunc(object, modifier, cache_path):
 221         pass
 222
 223     processObjectDependencies(pointCacheFunc, fluidFunc, multiresFunc)
 224
 225     fillCommonJobSettings(job, job_name, netsettings)
 226
 227     job.tags.add(netrender.model.TAG_BAKING)
 228     job.subtype = netrender.model.JOB_SUB_BAKING
 229     job.chunks = 1 # No chunking for baking
 230
 231     for i, task in enumerate(tasks):
 232         job.addFrame(i + 1)
 233         job.frames[-1].command = netrender.baking.taskToCommand(task)
 234
 235     # try to send path first
 236     with ConnectionContext():
 237         conn.request("POST", "/job", json.dumps(job.serialize()))
 238     response = conn.getresponse()
 239     response.read()
 240
 241     job_id = response.getheader("job-id")
 242
 243     # if not ACCEPTED (but not processed), send files
 244     if response.status == http.client.ACCEPTED:
 245         for rfile in job.files:
 246             f = open(rfile.filepath, "rb")
 247             with ConnectionContext():
 248                 conn.request("PUT", fileURL(job_id, rfile.index), f)
 249             f.close()
 250             response = conn.getresponse()
 251             response.read()
 252
 253     # server will reply with ACCEPTED until all files are found
 254
 255     return job_id
 256
 257 def sendJobBlender(conn, scene, anim = False, can_save = True):
 258     netsettings = scene.network_render
 259     job = netrender.model.RenderJob()
 260
 261     if anim:
 262         for f in range(scene.frame_start, scene.frame_end + 1):
 263             job.addFrame(f)
 264     else:
 265         job.addFrame(scene.frame_current)
 266
 267     filename = bpy.data.filepath
 268
 269     if not os.path.exists(filename):
 270         raise RuntimeError("Current file path not defined\nSave your file before sending a job")
 271
 272     if can_save and netsettings.save_before_job:
 273         bpy.ops.wm.save_mainfile(filepath=filename, check_existing=False)
 274
 275     job.addFile(filename)
 276
 277     job_name = netsettings.job_name
 278     path, name = os.path.split(filename)
 279     if job_name == "[default]":
 280         job_name = name
 281
 282     ###########################
 283     # LIBRARIES
 284     ###########################
 285     for lib in bpy.data.libraries:
 286         file_path = bpy.path.abspath(lib.filepath)
 287         if os.path.exists(file_path):
 288             job.addFile(file_path)
 289
 290     ###########################
 291     # IMAGES
 292     ###########################
 293     for image in bpy.data.images:
 294         if image.source == "FILE" and not image.packed_file:
 295             file_path = bpy.path.abspath(image.filepath)
 296             if os.path.exists(file_path):
 297                 job.addFile(file_path)
 298
 299                 tex_path = os.path.splitext(file_path)[0] + ".tex"
 300                 if os.path.exists(tex_path):
 301                     job.addFile(tex_path)
 302
 303     ###########################
 304     # FLUID + POINT CACHE
 305     ###########################
 306     default_path = cachePath(filename)
 307
 308     def pointCacheFunc(object, owner, point_cache):
 309         addPointCache(job, object, point_cache, default_path)
 310
 311     def fluidFunc(object, modifier, cache_path):
 312         addFluidFiles(job, cache_path)
 313
 314     def multiresFunc(object, modifier, cache_path):
 315         job.addFile(cache_path)
 316
 317     processObjectDependencies(pointCacheFunc, fluidFunc, multiresFunc)
 318
 319     #print(job.files)
 320
 321     fillCommonJobSettings(job, job_name, netsettings)
 322
 323     job.tags.add(netrender.model.TAG_RENDER)
 324
 325     # try to send path first
 326     with ConnectionContext():
 327         conn.request("POST", "/job", json.dumps(job.serialize()))
 328     response = conn.getresponse()
 329     response.read()
 330
 331     job_id = response.getheader("job-id")
 332
 333     # if not ACCEPTED (but not processed), send files
 334     if response.status == http.client.ACCEPTED:
 335         for rfile in job.files:
 336             f = open(rfile.filepath, "rb")
 337             with ConnectionContext():
 338                 conn.request("PUT", fileURL(job_id, rfile.index), f)
 339             f.close()
 340             response = conn.getresponse()
 341             response.read()
 342
 343     # server will reply with ACCEPTED until all files are found
 344
 345     return job_id
 346
 347 def requestResult(conn, job_id, frame):
 348     with ConnectionContext():
 349         conn.request("GET", renderURL(job_id, frame))
 350
 351 class NetworkRenderEngine(bpy.types.RenderEngine):
 352     bl_idname = 'NET_RENDER'
 353     bl_label = "Network Render"
 354     bl_use_postprocess = False
 355     def render(self, scene):
 356         try:
 357             if scene.network_render.mode == "RENDER_CLIENT":
 358                 self.render_client(scene)
 359             elif scene.network_render.mode == "RENDER_SLAVE":
 360                 self.render_slave(scene)
 361             elif scene.network_render.mode == "RENDER_MASTER":
 362                 self.render_master(scene)
 363             else:
 364                 print("UNKNOWN OPERATION MODE")
 365         except Exception as e:
 366             self.report({'ERROR'}, str(e))
 367             raise e
 368
 369     def render_master(self, scene):
 370         netsettings = scene.network_render
 371
 372         address = "" if netsettings.server_address == "[default]" else netsettings.server_address
 373
 374         master.runMaster(address = (address, netsettings.server_port),
 375                          broadcast = netsettings.use_master_broadcast,
 376                          clear = netsettings.use_master_clear,
 377                          force = netsettings.use_master_force_upload,
 378                          path = bpy.path.abspath(netsettings.path),
 379                          update_stats = self.update_stats,
 380                          test_break = self.test_break,
 381                          use_ssl=netsettings.use_ssl,
 382                          cert_path=netsettings.cert_path,
 383                          key_path=netsettings.key_path)
 384
 385
 386     def render_slave(self, scene):
 387         slave.render_slave(self, scene.network_render, scene.render.threads)
 388
 389     def render_client(self, scene):
 390         netsettings = scene.network_render
 391         self.update_stats("", "Network render client initiation")
 392
 393
 394         conn = clientConnection(netsettings)
 395
 396         if conn:
 397             # Sending file
 398
 399             self.update_stats("", "Network render exporting")
 400
 401             new_job = False
 402
 403             job_id = netsettings.job_id
 404
 405             # reading back result
 406
 407             self.update_stats("", "Network render waiting for results")
 408
 409
 410             requestResult(conn, job_id, scene.frame_current)
 411             response = conn.getresponse()
 412             buf = response.read()
 413
 414             if response.status == http.client.NO_CONTENT:
 415                 new_job = True
 416                 netsettings.job_id = sendJob(conn, scene, can_save = False)
 417                 job_id = netsettings.job_id
 418
 419                 requestResult(conn, job_id, scene.frame_current)
 420                 response = conn.getresponse()
 421                 buf = response.read()
 422
 423             while response.status == http.client.ACCEPTED and not self.test_break():
 424                 time.sleep(1)
 425                 requestResult(conn, job_id, scene.frame_current)
 426                 response = conn.getresponse()
 427                 buf = response.read()
 428
 429             # cancel new jobs (animate on network) on break
 430             if self.test_break() and new_job:
 431                 with ConnectionContext():
 432                     conn.request("POST", cancelURL(job_id))
 433                 response = conn.getresponse()
 434                 response.read()
 435                 print( response.status, response.reason )
 436                 netsettings.job_id = 0
 437
 438             if response.status != http.client.OK:
 439                 conn.close()
 440                 return
 441
 442             r = scene.render
 443             x= int(r.resolution_x*r.resolution_percentage*0.01)
 444             y= int(r.resolution_y*r.resolution_percentage*0.01)
 445
 446             result_path = os.path.join(bpy.path.abspath(netsettings.path), "output.exr")
 447
 448             folder = os.path.split(result_path)[0]
 449             verifyCreateDir(folder)
 450
 451             f = open(result_path, "wb")
 452
 453             f.write(buf)
 454
 455             f.close()
 456
 457             result = self.begin_result(0, 0, x, y)
 458             result.load_from_file(result_path)
 459             self.end_result(result)
 460
 461             conn.close()
 462
 463 def compatible(module):
 464     module = __import__("bl_ui." + module)
 465     for subclass in module.__dict__.values():
 466         try:            subclass.COMPAT_ENGINES.add('NET_RENDER')
 467         except: pass
 468     del module
 469
 470 compatible("properties_world")
 471 compatible("properties_material")
 472 compatible("properties_data_mesh")
 473 compatible("properties_data_camera")
 474 compatible("properties_texture")