Cleanup: trailing space
[blender-addons.git] / blenderkit / download.py
blob26ae0745e6510e18c7bcde47598b3913e641f5dc
1 # ##### BEGIN GPL LICENSE BLOCK #####
3 # This program is free software; you can redistribute it and/or
4 # modify it under the terms of the GNU General Public License
5 # as published by the Free Software Foundation; either version 2
6 # of the License, or (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License
14 # along with this program; if not, write to the Free Software Foundation,
15 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 # ##### END GPL LICENSE BLOCK #####
20 from blenderkit import paths, append_link, utils, ui, colors, tasks_queue, rerequests, resolutions
22 import threading
23 import time
24 import requests
25 import shutil, sys, os
26 import uuid
27 import copy
28 import logging
30 bk_logger = logging.getLogger('blenderkit')
32 import bpy
33 from bpy.props import (
34 IntProperty,
35 FloatProperty,
36 FloatVectorProperty,
37 StringProperty,
38 EnumProperty,
39 BoolProperty,
40 PointerProperty,
42 from bpy.app.handlers import persistent
44 download_threads = []
47 def check_missing():
48 '''checks for missing files, and possibly starts re-download of these into the scene'''
49 s = bpy.context.scene
50 # missing libs:
51 # TODO: put these into a panel and let the user decide if these should be downloaded.
52 missing = []
53 for l in bpy.data.libraries:
54 fp = l.filepath
55 if fp.startswith('//'):
56 fp = bpy.path.abspath(fp)
57 if not os.path.exists(fp) and l.get('asset_data') is not None:
58 missing.append(l)
60 # print('missing libraries', missing)
62 for l in missing:
63 asset_data = l['asset_data']
65 downloaded = check_existing(asset_data, resolution=asset_data['resolution'])
66 if downloaded:
67 try:
68 l.reload()
69 except:
70 download(l['asset_data'], redownload=True)
71 else:
72 download(l['asset_data'], redownload=True)
75 def check_unused():
76 '''find assets that have been deleted from scene but their library is still present.'''
77 # this is obviously broken. Blender should take care of the extra data automaticlaly
78 return;
79 used_libs = []
80 for ob in bpy.data.objects:
81 if ob.instance_collection is not None and ob.instance_collection.library is not None:
82 # used_libs[ob.instance_collection.name] = True
83 if ob.instance_collection.library not in used_libs:
84 used_libs.append(ob.instance_collection.library)
86 for ps in ob.particle_systems:
87 set = ps.settings
88 if ps.settings.render_type == 'GROUP' \
89 and ps.settings.instance_collection is not None \
90 and ps.settings.instance_collection.library not in used_libs:
91 used_libs.append(ps.settings.instance_collection)
93 for l in bpy.data.libraries:
94 if l not in used_libs and l.getn('asset_data'):
95 print('attempt to remove this library: ', l.filepath)
96 # have to unlink all groups, since the file is a 'user' even if the groups aren't used at all...
97 for user_id in l.users_id:
98 if type(user_id) == bpy.types.Collection:
99 bpy.data.collections.remove(user_id)
100 l.user_clear()
103 @persistent
104 def scene_save(context):
105 ''' does cleanup of blenderkit props and sends a message to the server about assets used.'''
106 # TODO this can be optimized by merging these 2 functions, since both iterate over all objects.
107 if not bpy.app.background:
108 check_unused()
109 report_usages()
112 @persistent
113 def scene_load(context):
114 '''restart broken downloads on scene load'''
115 t = time.time()
116 s = bpy.context.scene
117 global download_threads
118 download_threads = []
120 # commenting this out - old restore broken download on scene start. Might come back if downloads get recorded in scene
121 # reset_asset_ids = {}
122 # reset_obs = {}
123 # for ob in bpy.context.scene.collection.objects:
124 # if ob.name[:12] == 'downloading ':
125 # obn = ob.name
127 # asset_data = ob['asset_data']
129 # # obn.replace('#', '')
130 # # if asset_data['id'] not in reset_asset_ids:
132 # if reset_obs.get(asset_data['id']) is None:
133 # reset_obs[asset_data['id']] = [obn]
134 # reset_asset_ids[asset_data['id']] = asset_data
135 # else:
136 # reset_obs[asset_data['id']].append(obn)
137 # for asset_id in reset_asset_ids:
138 # asset_data = reset_asset_ids[asset_id]
139 # done = False
140 # if check_existing(asset_data, resolution = should be here):
141 # for obname in reset_obs[asset_id]:
142 # downloader = s.collection.objects[obname]
143 # done = try_finished_append(asset_data,
144 # model_location=downloader.location,
145 # model_rotation=downloader.rotation_euler)
147 # if not done:
148 # downloading = check_downloading(asset_data)
149 # if not downloading:
150 # print('redownloading %s' % asset_data['name'])
151 # download(asset_data, downloaders=reset_obs[asset_id], delete=True)
153 # check for group users that have been deleted, remove the groups /files from the file...
154 # TODO scenes fixing part... download the assets not present on drive,
155 # and erase from scene linked files that aren't used in the scene.
156 # print('continue downlaods ', time.time() - t)
157 t = time.time()
158 check_missing()
159 # print('missing check', time.time() - t)
162 def get_scene_id():
163 '''gets scene id and possibly also generates a new one'''
164 bpy.context.scene['uuid'] = bpy.context.scene.get('uuid', str(uuid.uuid4()))
165 return bpy.context.scene['uuid']
168 def report_usages():
169 '''report the usage of assets to the server.'''
170 mt = time.time()
171 user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
172 api_key = user_preferences.api_key
173 sid = get_scene_id()
174 headers = utils.get_headers(api_key)
175 url = paths.get_api_url() + paths.BLENDERKIT_REPORT_URL
177 assets = {}
178 asset_obs = []
179 scene = bpy.context.scene
180 asset_usages = {}
182 for ob in scene.collection.objects:
183 if ob.get('asset_data') != None:
184 asset_obs.append(ob)
186 for ob in asset_obs:
187 asset_data = ob['asset_data']
188 abid = asset_data['assetBaseId']
190 if assets.get(abid) is None:
191 asset_usages[abid] = {'count': 1}
192 assets[abid] = asset_data
193 else:
194 asset_usages[abid]['count'] += 1
196 # brushes
197 for b in bpy.data.brushes:
198 if b.get('asset_data') != None:
199 abid = b['asset_data']['assetBaseId']
200 asset_usages[abid] = {'count': 1}
201 assets[abid] = b['asset_data']
202 # materials
203 for ob in scene.collection.objects:
204 for ms in ob.material_slots:
205 m = ms.material
207 if m is not None and m.get('asset_data') is not None:
209 abid = m['asset_data']['assetBaseId']
210 if assets.get(abid) is None:
211 asset_usages[abid] = {'count': 1}
212 assets[abid] = m['asset_data']
213 else:
214 asset_usages[abid]['count'] += 1
216 assets_list = []
217 assets_reported = scene.get('assets reported', {})
219 new_assets_count = 0
220 for k in asset_usages.keys():
221 if k not in assets_reported.keys():
222 data = asset_usages[k]
223 list_item = {
224 'asset': k,
225 'usageCount': data['count'],
226 'proximitySet': data.get('proximity', [])
228 assets_list.append(list_item)
229 new_assets_count += 1
230 if k not in assets_reported.keys():
231 assets_reported[k] = True
233 scene['assets reported'] = assets_reported
235 if new_assets_count == 0:
236 bk_logger.debug('no new assets were added')
237 return;
238 usage_report = {
239 'scene': sid,
240 'reportType': 'save',
241 'assetusageSet': assets_list
244 au = scene.get('assets used', {})
245 ad = scene.get('assets deleted', {})
247 ak = assets.keys()
248 for k in au.keys():
249 if k not in ak:
250 ad[k] = au[k]
251 else:
252 if k in ad:
253 ad.pop(k)
255 # scene['assets used'] = {}
256 for k in ak: # rewrite assets used.
257 scene['assets used'][k] = assets[k]
259 ###########check ratings herer too:
260 scene['assets rated'] = scene.get('assets rated', {})
261 for k in assets.keys():
262 scene['assets rated'][k] = scene['assets rated'].get(k, False)
263 thread = threading.Thread(target=utils.requests_post_thread, args=(url, usage_report, headers))
264 thread.start()
265 mt = time.time() - mt
266 # print('report generation: ', mt)
269 def udpate_asset_data_in_dicts(asset_data):
271 updates asset data in all relevant dictionaries, after a threaded download task \
272 - where the urls were retrieved, and now they can be reused
273 Parameters
274 ----------
275 asset_data - data coming back from thread, thus containing also download urls
277 scene = bpy.context.scene
278 scene['assets used'] = scene.get('assets used', {})
279 scene['assets used'][asset_data['assetBaseId']] = asset_data.copy()
281 scene['assets rated'] = scene.get('assets rated', {})
282 id = asset_data['assetBaseId']
283 scene['assets rated'][id] = scene['assets rated'].get(id, False)
284 sr = bpy.context.window_manager['search results']
285 if not sr:
286 return;
287 for i, r in enumerate(sr):
288 if r['assetBaseId'] == asset_data['assetBaseId']:
289 for f in asset_data['files']:
290 if f.get('url'):
291 for f1 in r['files']:
292 if f1['fileType'] == f['fileType']:
293 f1['url'] = f['url']
296 def append_asset(asset_data, **kwargs): # downloaders=[], location=None,
297 '''Link asset to the scene.
302 file_names = paths.get_download_filepaths(asset_data, kwargs['resolution'])
303 props = None
304 #####
305 # how to do particle drop:
306 # link the group we are interested in( there are more groups in File!!!! , have to get the correct one!)
307 s = bpy.context.scene
309 user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
311 if user_preferences.api_key == '':
312 user_preferences.asset_counter += 1
314 if asset_data['assetType'] == 'scene':
315 sprops = s.blenderkit_scene
317 scene = append_link.append_scene(file_names[0], link=sprops.append_link == 'LINK', fake_user=False)
318 print('scene appended')
319 if scene is not None:
320 props = scene.blenderkit
321 asset_main = scene
322 print(sprops.switch_after_append)
323 if sprops.switch_after_append:
324 bpy.context.window_manager.windows[0].scene = scene
326 if asset_data['assetType'] == 'hdr':
327 hdr = append_link.load_HDR(file_name=file_names[0], name=asset_data['name'])
328 props = hdr.blenderkit
329 asset_main = hdr
332 if asset_data['assetType'] == 'model':
333 downloaders = kwargs.get('downloaders')
334 sprops = s.blenderkit_models
335 # TODO this is here because combinations of linking objects or appending groups are rather not-usefull
336 if sprops.append_method == 'LINK_COLLECTION':
337 sprops.append_link = 'LINK'
338 sprops.import_as = 'GROUP'
339 else:
340 sprops.append_link = 'APPEND'
341 sprops.import_as = 'INDIVIDUAL'
343 # copy for override
344 al = sprops.append_link
345 # set consistency for objects already in scene, otherwise this literally breaks blender :)
346 ain, resolution = asset_in_scene(asset_data)
347 # this is commented out since it already happens in start_download function.
348 # if resolution:
349 # kwargs['resolution'] = resolution
350 # override based on history
351 if ain is not False:
352 if ain == 'LINKED':
353 al = 'LINK'
354 else:
355 al = 'APPEND'
356 if asset_data['assetType'] == 'model':
357 source_parent = get_asset_in_scene(asset_data)
358 if source_parent:
359 asset_main, new_obs = duplicate_asset(source=source_parent, **kwargs)
360 asset_main.location = kwargs['model_location']
361 asset_main.rotation_euler = kwargs['model_rotation']
362 # this is a case where asset is already in scene and should be duplicated instead.
363 # there is a big chance that the duplication wouldn't work perfectly(hidden or unselectable objects)
364 # so here we need to check and return if there was success
365 # also, if it was successful, no other operations are needed , basically all asset data is already ready from the original asset
366 if new_obs:
367 # update here assets rated/used because there might be new download urls?
368 udpate_asset_data_in_dicts(asset_data)
369 bpy.ops.wm.undo_push_context(message='add %s to scene' % asset_data['name'])
371 return
373 # first get conditions for append link
374 link = al == 'LINK'
375 # then append link
376 if downloaders:
377 for downloader in downloaders:
378 # this cares for adding particle systems directly to target mesh, but I had to block it now,
379 # because of the sluggishnes of it. Possibly re-enable when it's possible to do this faster?
380 if 'particle_plants' in asset_data['tags']:
381 append_link.append_particle_system(file_names[-1],
382 target_object=kwargs['target_object'],
383 rotation=downloader['rotation'],
384 link=False,
385 name=asset_data['name'])
386 return
388 if link:
389 asset_main, new_obs = append_link.link_collection(file_names[-1],
390 location=downloader['location'],
391 rotation=downloader['rotation'],
392 link=link,
393 name=asset_data['name'],
394 parent=kwargs.get('parent'))
396 else:
398 asset_main, new_obs = append_link.append_objects(file_names[-1],
399 location=downloader['location'],
400 rotation=downloader['rotation'],
401 link=link,
402 name=asset_data['name'],
403 parent=kwargs.get('parent'))
404 if asset_main.type == 'EMPTY' and link:
405 bmin = asset_data['bbox_min']
406 bmax = asset_data['bbox_max']
407 size_min = min(1.0, (bmax[0] - bmin[0] + bmax[1] - bmin[1] + bmax[2] - bmin[2]) / 3)
408 asset_main.empty_display_size = size_min
410 elif kwargs.get('model_location') is not None:
411 if link:
412 asset_main, new_obs = append_link.link_collection(file_names[-1],
413 location=kwargs['model_location'],
414 rotation=kwargs['model_rotation'],
415 link=link,
416 name=asset_data['name'],
417 parent=kwargs.get('parent'))
418 else:
419 asset_main, new_obs = append_link.append_objects(file_names[-1],
420 location=kwargs['model_location'],
421 rotation=kwargs['model_rotation'],
422 link=link,
423 name=asset_data['name'],
424 parent=kwargs.get('parent'))
426 # scale Empty for assets, so they don't clutter the scene.
427 if asset_main.type == 'EMPTY' and link:
428 bmin = asset_data['bbox_min']
429 bmax = asset_data['bbox_max']
430 size_min = min(1.0, (bmax[0] - bmin[0] + bmax[1] - bmin[1] + bmax[2] - bmin[2]) / 3)
431 asset_main.empty_display_size = size_min
433 if link:
434 group = asset_main.instance_collection
436 lib = group.library
437 lib['asset_data'] = asset_data
439 elif asset_data['assetType'] == 'brush':
441 # TODO if already in scene, should avoid reappending.
442 inscene = False
443 for b in bpy.data.brushes:
445 if b.blenderkit.id == asset_data['id']:
446 inscene = True
447 brush = b
448 break;
449 if not inscene:
450 brush = append_link.append_brush(file_names[-1], link=False, fake_user=False)
452 thumbnail_name = asset_data['thumbnail'].split(os.sep)[-1]
453 tempdir = paths.get_temp_dir('brush_search')
454 thumbpath = os.path.join(tempdir, thumbnail_name)
455 asset_thumbs_dir = paths.get_download_dirs('brush')[0]
456 asset_thumb_path = os.path.join(asset_thumbs_dir, thumbnail_name)
457 shutil.copy(thumbpath, asset_thumb_path)
458 brush.icon_filepath = asset_thumb_path
460 if bpy.context.view_layer.objects.active.mode == 'SCULPT':
461 bpy.context.tool_settings.sculpt.brush = brush
462 elif bpy.context.view_layer.objects.active.mode == 'TEXTURE_PAINT': # could be just else, but for future possible more types...
463 bpy.context.tool_settings.image_paint.brush = brush
464 # TODO set brush by by asset data(user can be downloading while switching modes.)
466 # bpy.context.tool_settings.image_paint.brush = brush
467 props = brush.blenderkit
468 asset_main = brush
470 elif asset_data['assetType'] == 'material':
471 inscene = False
472 sprops = s.blenderkit_mat
474 for m in bpy.data.materials:
475 if m.blenderkit.id == asset_data['id']:
476 inscene = True
477 material = m
478 break;
479 if not inscene:
480 link = sprops.append_method == 'LINK'
481 material = append_link.append_material(file_names[-1], link=link, fake_user=False)
482 target_object = bpy.data.objects[kwargs['target_object']]
484 if len(target_object.material_slots) == 0:
485 target_object.data.materials.append(material)
486 else:
487 target_object.material_slots[kwargs['material_target_slot']].material = material
489 asset_main = material
491 asset_data['resolution'] = kwargs['resolution']
492 udpate_asset_data_in_dicts(asset_data)
494 asset_main['asset_data'] = asset_data # TODO remove this??? should write to blenderkit Props?
495 bpy.ops.wm.undo_push_context(message='add %s to scene' % asset_data['name'])
496 # moving reporting to on save.
497 # report_use_success(asset_data['id'])
500 def replace_resolution_linked(file_paths, asset_data):
501 # replace one asset resolution for another.
502 # this is the much simpler case
503 # - find the library,
504 # - replace the path and name of the library, reload.
505 file_name = os.path.basename(file_paths[-1])
507 for l in bpy.data.libraries:
508 if not l.get('asset_data'):
509 continue;
510 if not l['asset_data']['assetBaseId'] == asset_data['assetBaseId']:
511 continue;
513 bk_logger.debug('try to re-link library')
515 if not os.path.isfile(file_paths[-1]):
516 bk_logger.debug('library file doesnt exist')
517 break;
518 l.filepath = os.path.join(os.path.dirname(l.filepath), file_name)
519 l.name = file_name
520 udpate_asset_data_in_dicts(asset_data)
523 def replace_resolution_appended(file_paths, asset_data, resolution):
524 # In this case the texture paths need to be replaced.
525 # Find the file path pattern that is present in texture paths
526 # replace the pattern with the new one.
527 file_name = os.path.basename(file_paths[-1])
529 new_filename_pattern = os.path.splitext(file_name)[0]
530 all_patterns = []
531 for suff in paths.resolution_suffix.values():
532 pattern = f"{asset_data['id']}{os.sep}textures{suff}{os.sep}"
533 all_patterns.append(pattern)
534 new_pattern = f"{asset_data['id']}{os.sep}textures{paths.resolution_suffix[resolution]}{os.sep}"
536 # replace the pattern with the new one.
537 # print(existing_filename_patterns)
538 # print(new_filename_pattern)
539 # print('existing images:')
540 for i in bpy.data.images:
542 for old_pattern in all_patterns:
543 if i.filepath.find(old_pattern) > -1:
544 fp = i.filepath.replace(old_pattern, new_pattern)
545 fpabs = bpy.path.abspath(fp)
546 if not os.path.exists(fpabs):
547 # this currently handles .png's that have been swapped to .jpg's during resolution generation process.
548 # should probably also handle .exr's and similar others.
549 # bk_logger.debug('need to find a replacement')
550 base, ext = os.path.splitext(fp)
551 if resolution == 'blend' and i.get('original_extension'):
552 fp = base + i.get('original_extension')
553 elif ext in ('.png', '.PNG'):
554 fp = base + '.jpg'
555 i.filepath = fp
556 i.filepath_raw = fp # bpy.path.abspath(fp)
557 for pf in i.packed_files:
558 pf.filepath = fp
559 i.reload()
560 udpate_asset_data_in_dicts(asset_data)
563 # @bpy.app.handlers.persistent
564 def timer_update():
565 # TODO might get moved to handle all blenderkit stuff, not to slow down.
567 check for running and finished downloads.
568 Running downloads get checked for progress which is passed to UI.
569 Finished downloads are processed and linked/appended to scene.
571 global download_threads
572 # bk_logger.debug('timer download')
574 if len(download_threads) == 0:
575 return 2.0
576 s = bpy.context.scene
577 for threaddata in download_threads:
578 t = threaddata[0]
579 asset_data = threaddata[1]
580 tcom = threaddata[2]
582 progress_bars = []
583 downloaders = []
585 if t.is_alive(): # set downloader size
586 sr = bpy.context.window_manager.get('search results')
587 if sr is not None:
588 for r in sr:
589 if asset_data['id'] == r['id']:
590 r['downloaded'] = tcom.progress
592 if not t.is_alive():
593 if tcom.error:
594 sprops = utils.get_search_props()
595 sprops.report = tcom.report
596 download_threads.remove(threaddata)
597 return
598 file_paths = paths.get_download_filepaths(asset_data, tcom.passargs['resolution'])
600 if len(file_paths) == 0:
601 bk_logger.debug('library names not found in asset data after download')
602 download_threads.remove(threaddata)
603 break;
605 wm = bpy.context.window_manager
607 at = asset_data['assetType']
608 if ((bpy.context.mode == 'OBJECT' and \
609 (at == 'model' or at == 'material'))) \
610 or ((at == 'brush') \
611 and wm.get('appendable') == True) or at == 'scene' or at == 'hdr':
612 # don't do this stuff in editmode and other modes, just wait...
613 download_threads.remove(threaddata)
615 # duplicate file if the global and subdir are used in prefs
616 if len(file_paths) == 2: # todo this should try to check if both files exist and are ok.
617 utils.copy_asset(file_paths[0], file_paths[1])
618 # shutil.copyfile(file_paths[0], file_paths[1])
620 bk_logger.debug('appending asset')
621 # progress bars:
623 # we need to check if mouse isn't down, which means an operator can be running.
624 # Especially for sculpt mode, where appending a brush during a sculpt stroke causes crasehes
627 if tcom.passargs.get('redownload'):
628 # handle lost libraries here:
629 for l in bpy.data.libraries:
630 if l.get('asset_data') is not None and l['asset_data']['id'] == asset_data['id']:
631 l.filepath = file_paths[-1]
632 l.reload()
634 if tcom.passargs.get('replace_resolution'):
635 # try to relink first.
637 ain, resolution = asset_in_scene(asset_data)
639 if ain == 'LINKED':
640 replace_resolution_linked(file_paths, asset_data)
643 elif ain == 'APPENDED':
644 replace_resolution_appended(file_paths, asset_data, tcom.passargs['resolution'])
648 else:
649 done = try_finished_append(asset_data, **tcom.passargs)
650 if not done:
651 at = asset_data['assetType']
652 tcom.passargs['retry_counter'] = tcom.passargs.get('retry_counter', 0) + 1
653 download(asset_data, **tcom.passargs)
655 if bpy.context.window_manager['search results'] is not None and done:
656 for sres in bpy.context.window_manager['search results']:
657 if asset_data['id'] == sres['id']:
658 sres['downloaded'] = 100
660 bk_logger.debug('finished download thread')
661 return .5
664 def delete_unfinished_file(file_name):
666 Deletes download if it wasn't finished. If the folder it's containing is empty, it also removes the directory
667 Parameters
668 ----------
669 file_name
671 Returns
672 -------
673 None
675 try:
676 os.remove(file_name)
677 except Exception as e:
678 print(e)
679 asset_dir = os.path.dirname(file_name)
680 if len(os.listdir(asset_dir)) == 0:
681 os.rmdir(asset_dir)
682 return
685 def download_file(asset_data, resolution='blend'):
686 # this is a simple non-threaded way to download files for background resolution genenration tool
687 file_name = paths.get_download_filepaths(asset_data, resolution)[0] # prefer global dir if possible.
689 if check_existing(asset_data, resolution=resolution):
690 # this sends the thread for processing, where another check should occur, since the file might be corrupted.
691 bk_logger.debug('not downloading, already in db')
692 return file_name
693 preferences = bpy.context.preferences.addons['blenderkit'].preferences
694 api_key = preferences.api_key
696 download_canceled = False
698 with open(file_name, "wb") as f:
699 print("Downloading %s" % file_name)
700 headers = utils.get_headers(api_key)
701 res_file_info, resolution = paths.get_res_file(asset_data, resolution)
702 response = requests.get(res_file_info['url'], stream=True)
703 total_length = response.headers.get('Content-Length')
705 if total_length is None or int(total_length) < 1000: # no content length header
706 download_canceled = True
707 print(response.content)
708 else:
709 total_length = int(total_length)
710 dl = 0
711 last_percent = 0
712 percent = 0
713 for data in response.iter_content(chunk_size=4096 * 10):
714 dl += len(data)
716 # the exact output you're looking for:
717 fs_str = utils.files_size_to_text(total_length)
719 percent = int(dl * 100 / total_length)
720 if percent > last_percent:
721 last_percent = percent
722 # sys.stdout.write('\r')
723 # sys.stdout.write(f'Downloading {asset_data['name']} {fs_str} {percent}% ') # + int(dl * 50 / total_length) * 'x')
724 print(
725 f'Downloading {asset_data["name"]} {fs_str} {percent}% ') # + int(dl * 50 / total_length) * 'x')
726 # sys.stdout.flush()
728 # print(int(dl*50/total_length)*'x'+'\r')
729 f.write(data)
730 if download_canceled:
731 delete_unfinished_file(file_name)
732 return None
734 return file_name
737 class Downloader(threading.Thread):
738 def __init__(self, asset_data, tcom, scene_id, api_key, resolution='blend'):
739 super(Downloader, self).__init__()
740 self.asset_data = asset_data
741 self.tcom = tcom
742 self.scene_id = scene_id
743 self.api_key = api_key
744 self.resolution = resolution
745 self._stop_event = threading.Event()
747 def stop(self):
748 self._stop_event.set()
750 def stopped(self):
751 return self._stop_event.is_set()
753 # def main_download_thread(asset_data, tcom, scene_id, api_key):
754 def run(self):
755 '''try to download file from blenderkit'''
756 asset_data = self.asset_data
757 tcom = self.tcom
758 scene_id = self.scene_id
759 api_key = self.api_key
760 tcom.report = 'Looking for asset'
761 # TODO get real link here...
762 has_url = get_download_url(asset_data, scene_id, api_key, resolution=self.resolution, tcom=tcom)
764 if not has_url:
765 tasks_queue.add_task(
766 (ui.add_report, ('Failed to obtain download URL for %s.' % asset_data['name'], 5, colors.RED)))
767 return;
768 if tcom.error:
769 return
770 # only now we can check if the file already exists. This should have 2 levels, for materials and for brushes
771 # different than for the non free content. delete is here when called after failed append tries.
773 if check_existing(asset_data, resolution=self.resolution) and not tcom.passargs.get('delete'):
774 # this sends the thread for processing, where another check should occur, since the file might be corrupted.
775 tcom.downloaded = 100
776 bk_logger.debug('not downloading, trying to append again')
777 return
779 file_name = paths.get_download_filepaths(asset_data, self.resolution)[0] # prefer global dir if possible.
780 # for k in asset_data:
781 # print(asset_data[k])
782 if self.stopped():
783 bk_logger.debug('stopping download: ' + asset_data['name'])
784 return
786 download_canceled = False
787 with open(file_name, "wb") as f:
788 bk_logger.debug("Downloading %s" % file_name)
789 headers = utils.get_headers(api_key)
790 res_file_info, self.resolution = paths.get_res_file(asset_data, self.resolution)
791 response = requests.get(res_file_info['url'], stream=True)
792 total_length = response.headers.get('Content-Length')
793 if total_length is None: # no content length header
794 print('no content length')
795 print(response.content)
796 tcom.report = response.content
797 download_canceled = True
798 else:
799 # bk_logger.debug(total_length)
800 if int(total_length) < 1000: # means probably no file returned.
801 tasks_queue.add_task((ui.add_report, (response.content, 20, colors.RED)))
803 tcom.report = response.content
805 tcom.file_size = int(total_length)
806 fsmb = tcom.file_size // (1024 * 1024)
807 fskb = tcom.file_size % 1024
808 if fsmb == 0:
809 t = '%iKB' % fskb
810 else:
811 t = ' %iMB' % fsmb
812 tcom.report = f'Downloading {t} {self.resolution}'
814 dl = 0
815 totdata = []
816 for data in response.iter_content(chunk_size=4096 * 32): # crashed here... why? investigate:
817 dl += len(data)
818 tcom.downloaded = dl
819 tcom.progress = int(100 * tcom.downloaded / tcom.file_size)
820 f.write(data)
821 if self.stopped():
822 bk_logger.debug('stopping download: ' + asset_data['name'])
823 download_canceled = True
824 break
826 if download_canceled:
827 delete_unfinished_file(file_name)
828 return
829 # unpack the file immediately after download
831 tcom.report = f'Unpacking files'
832 self.asset_data['resolution'] = self.resolution
833 resolutions.send_to_bg(self.asset_data, file_name, command='unpack')
836 class ThreadCom: # object passed to threads to read background process stdout info
837 def __init__(self):
838 self.file_size = 1000000000000000 # property that gets written to.
839 self.downloaded = 0
840 self.lasttext = ''
841 self.error = False
842 self.report = ''
843 self.progress = 0.0
844 self.passargs = {}
847 def download(asset_data, **kwargs):
848 '''start the download thread'''
849 user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
850 api_key = user_preferences.api_key
851 scene_id = get_scene_id()
853 tcom = ThreadCom()
854 tcom.passargs = kwargs
856 if kwargs.get('retry_counter', 0) > 3:
857 sprops = utils.get_search_props()
858 report = f"Maximum retries exceeded for {asset_data['name']}"
859 sprops.report = report
860 ui.add_report(report, 5, colors.RED)
862 bk_logger.debug(sprops.report)
863 return
865 # incoming data can be either directly dict from python, or blender id property
866 # (recovering failed downloads on reload)
867 if type(asset_data) == dict:
868 asset_data = copy.deepcopy(asset_data)
869 else:
870 asset_data = asset_data.to_dict()
871 readthread = Downloader(asset_data, tcom, scene_id, api_key, resolution=kwargs['resolution'])
872 readthread.start()
874 global download_threads
875 download_threads.append(
876 [readthread, asset_data, tcom])
879 def check_downloading(asset_data, **kwargs):
880 ''' check if an asset is already downloading, if yes, just make a progress bar with downloader object.'''
881 global download_threads
883 downloading = False
885 for p in download_threads:
886 p_asset_data = p[1]
887 if p_asset_data['id'] == asset_data['id']:
888 at = asset_data['assetType']
889 if at in ('model', 'material'):
890 downloader = {'location': kwargs['model_location'],
891 'rotation': kwargs['model_rotation']}
892 p[2].passargs['downloaders'].append(downloader)
893 downloading = True
895 return downloading
898 def check_existing(asset_data, resolution='blend', can_return_others=False):
899 ''' check if the object exists on the hard drive'''
900 fexists = False
902 if asset_data.get('files') == None:
903 # this is because of some very odl files where asset data had no files structure.
904 return False
906 file_names = paths.get_download_filepaths(asset_data, resolution, can_return_others=can_return_others)
908 bk_logger.debug('check if file already exists' + str(file_names))
909 if len(file_names) == 2:
910 # TODO this should check also for failed or running downloads.
911 # If download is running, assign just the running thread. if download isn't running but the file is wrong size,
912 # delete file and restart download (or continue downoad? if possible.)
913 if os.path.isfile(file_names[0]): # and not os.path.isfile(file_names[1])
914 utils.copy_asset(file_names[0], file_names[1])
915 elif not os.path.isfile(file_names[0]) and os.path.isfile(
916 file_names[1]): # only in case of changed settings or deleted/moved global dict.
917 utils.copy_asset(file_names[1], file_names[0])
919 if len(file_names) > 0 and os.path.isfile(file_names[0]):
920 fexists = True
921 return fexists
924 def try_finished_append(asset_data, **kwargs): # location=None, material_target=None):
925 ''' try to append asset, if not successfully delete source files.
926 This means probably wrong download, so download should restart'''
927 file_names = paths.get_download_filepaths(asset_data, kwargs['resolution'])
928 done = False
929 bk_logger.debug('try to append already existing asset')
930 if len(file_names) > 0:
931 if os.path.isfile(file_names[-1]):
932 kwargs['name'] = asset_data['name']
933 append_asset(asset_data, **kwargs)
934 done = True
935 return done
936 try:
937 append_asset(asset_data, **kwargs)
938 done = True
939 except Exception as e:
940 # TODO: this should distinguis if the appending failed (wrong file)
941 # or something else happened(shouldn't delete the files)
942 print(e)
943 done = False
944 for f in file_names:
945 try:
946 os.remove(f)
947 except Exception as e:
948 # e = sys.exc_info()[0]
949 print(e)
950 pass;
951 return done
953 return done
956 def get_asset_in_scene(asset_data):
957 '''tries to find an appended copy of particular asset and duplicate it - so it doesn't have to be appended again.'''
958 scene = bpy.context.scene
959 for ob in bpy.context.scene.objects:
960 ad1 = ob.get('asset_data')
961 if not ad1:
962 continue
963 if ad1.get('assetBaseId') == asset_data['assetBaseId']:
964 return ob
965 return None
968 def check_all_visible(obs):
969 '''checks all objects are visible, so they can be manipulated/copied.'''
970 for ob in obs:
971 if not ob.visible_get():
972 return False
973 return True
976 def check_selectible(obs):
977 '''checks if all objects can be selected and selects them if possible.
978 this isn't only select_hide, but all possible combinations of collections e.t.c. so hard to check otherwise.'''
979 for ob in obs:
980 ob.select_set(True)
981 if not ob.select_get():
982 return False
983 return True
986 def duplicate_asset(source, **kwargs):
988 Duplicate asset when it's already appended in the scene,
989 so that blender's append doesn't create duplicated data.
991 bk_logger.debug('duplicate asset instead')
992 # we need to save selection
993 sel = utils.selection_get()
994 bpy.ops.object.select_all(action='DESELECT')
996 # check visibility
997 obs = utils.get_hierarchy(source)
998 if not check_all_visible(obs):
999 return None
1000 # check selectability and select in one run
1001 if not check_selectible(obs):
1002 return None
1004 # duplicate the asset objects
1005 bpy.ops.object.duplicate(linked=True)
1007 nobs = bpy.context.selected_objects[:]
1008 # get asset main object
1009 for ob in nobs:
1010 if ob.parent not in nobs:
1011 asset_main = ob
1012 break
1014 # in case of replacement,there might be a paarent relationship that can be restored
1015 if kwargs.get('parent'):
1016 parent = bpy.data.objects[kwargs['parent']]
1017 asset_main.parent = parent # even if parent is None this is ok without if condition
1018 else:
1019 asset_main.parent = None
1020 # restore original selection
1021 utils.selection_set(sel)
1022 return asset_main, nobs
1025 def asset_in_scene(asset_data):
1026 '''checks if the asset is already in scene. If yes, modifies asset data so the asset can be reached again.'''
1027 scene = bpy.context.scene
1028 au = scene.get('assets used', {})
1030 id = asset_data['assetBaseId']
1031 print(id)
1032 if id in au.keys():
1033 ad = au[id]
1034 if ad.get('files'):
1035 for fi in ad['files']:
1036 if fi.get('file_name') != None:
1038 for fi1 in asset_data['files']:
1039 if fi['fileType'] == fi1['fileType']:
1040 fi1['file_name'] = fi['file_name']
1041 fi1['url'] = fi['url']
1043 # browse all collections since linked collections can have same name.
1044 if asset_data['assetType'] == 'MODEL':
1045 for c in bpy.data.collections:
1046 if c.name == ad['name']:
1047 # there can also be more linked collections with same name, we need to check id.
1048 if c.library and c.library.get('asset_data') and c.library['asset_data'][
1049 'assetBaseId'] == id:
1050 print('asset linked')
1051 return 'LINKED', ad.get('resolution')
1052 elif asset_data['assetType'] == 'MATERIAL':
1053 for m in bpy.data.materials:
1054 if not m.get('asset_data'):
1055 continue
1056 if m['asset_data']['assetBaseId'] == asset_data[
1057 'assetBaseId'] and bpy.context.active_object.active_material.library:
1058 return 'LINKED', ad.get('resolution')
1060 print('asset appended')
1061 return 'APPENDED', ad.get('resolution')
1062 return False, None
1065 def fprint(text):
1066 print('###################################################################################')
1067 print('\n\n\n')
1068 print(text)
1069 print('\n\n\n')
1070 print('###################################################################################')
1073 def get_download_url(asset_data, scene_id, api_key, tcom=None, resolution='blend'):
1074 ''''retrieves the download url. The server checks if user can download the item.'''
1075 mt = time.time()
1076 utils.pprint('getting download url')
1078 headers = utils.get_headers(api_key)
1080 data = {
1081 'scene_uuid': scene_id
1083 r = None
1085 res_file_info, resolution = paths.get_res_file(asset_data, resolution)
1087 try:
1088 r = rerequests.get(res_file_info['downloadUrl'], params=data, headers=headers)
1089 except Exception as e:
1090 print(e)
1091 if tcom is not None:
1092 tcom.error = True
1093 if r == None:
1094 if tcom is not None:
1095 tcom.report = 'Connection Error'
1096 tcom.error = True
1097 return 'Connection Error'
1099 if r.status_code < 400:
1100 data = r.json()
1101 url = data['filePath']
1103 res_file_info['url'] = url
1104 res_file_info['file_name'] = paths.extract_filename_from_url(url)
1106 # print(res_file_info, url)
1107 print(url)
1108 return True
1110 # let's print it into UI
1111 tasks_queue.add_task((ui.add_report, (str(r), 10, colors.RED)))
1113 if r.status_code == 403:
1114 r = 'You need Full plan to get this item.'
1115 # r1 = 'All materials and brushes are available for free. Only users registered to Standard plan can use all models.'
1116 # tasks_queue.add_task((ui.add_report, (r1, 5, colors.RED)))
1117 if tcom is not None:
1118 tcom.report = r
1119 tcom.error = True
1121 if r.status_code == 404:
1122 r = 'Url not found - 404.'
1123 # r1 = 'All materials and brushes are available for free. Only users registered to Standard plan can use all models.'
1124 if tcom is not None:
1125 tcom.report = r
1126 tcom.error = True
1128 elif r.status_code >= 500:
1129 # bk_logger.debug(r.text)
1130 if tcom is not None:
1131 tcom.report = 'Server error'
1132 tcom.error = True
1133 return False
1136 def start_download(asset_data, **kwargs):
1138 check if file isn't downloading or doesn't exist, then start new download
1140 # first check if the asset is already in scene. We can use that asset without checking with server
1141 ain, resolution = asset_in_scene(asset_data)
1142 # quota_ok = ain is not False
1144 # if resolution:
1145 # kwargs['resolution'] = resolution
1146 # otherwise, check on server
1148 s = bpy.context.scene
1149 done = False
1150 # is the asseet being currently downloaded?
1151 downloading = check_downloading(asset_data, **kwargs)
1152 if not downloading:
1153 # check if there are files already. This check happens 2x once here(for free assets),
1154 # once in thread(for non-free)
1155 fexists = check_existing(asset_data, resolution=kwargs['resolution'])
1156 bk_logger.debug('does file exist?' + str(fexists))
1157 bk_logger.debug('asset is in scene' + str(ain))
1158 if ain and not kwargs.get('replace_resolution'):
1159 # this goes to appending asset - where it should duplicate the original asset already in scene.
1160 done = try_finished_append(asset_data, **kwargs)
1161 # else:
1162 # props = utils.get_search_props()
1163 # props.report = str('asset ')
1164 if not done:
1165 at = asset_data['assetType']
1166 if at in ('model', 'material'):
1167 downloader = {'location': kwargs['model_location'],
1168 'rotation': kwargs['model_rotation']}
1169 download(asset_data, downloaders=[downloader], **kwargs)
1171 else:
1172 download(asset_data, **kwargs)
1175 asset_types = (
1176 ('MODEL', 'Model', 'set of objects'),
1177 ('SCENE', 'Scene', 'scene'),
1178 ('HDR', 'Hdr', 'hdr'),
1179 ('MATERIAL', 'Material', 'any .blend Material'),
1180 ('TEXTURE', 'Texture', 'a texture, or texture set'),
1181 ('BRUSH', 'Brush', 'brush, can be any type of blender brush'),
1182 ('ADDON', 'Addon', 'addnon'),
1186 class BlenderkitKillDownloadOperator(bpy.types.Operator):
1187 """Kill a download"""
1188 bl_idname = "scene.blenderkit_download_kill"
1189 bl_label = "BlenderKit Kill Asset Download"
1190 bl_options = {'REGISTER', 'INTERNAL'}
1192 thread_index: IntProperty(name="Thread index", description='index of the thread to kill', default=-1)
1194 def execute(self, context):
1195 global download_threads
1196 td = download_threads[self.thread_index]
1197 download_threads.remove(td)
1198 td[0].stop()
1199 return {'FINISHED'}
1202 def available_resolutions_callback(self, context):
1204 Returns
1205 checks active asset for available resolutions and offers only those available
1206 TODO: this currently returns always the same list of resolutions, make it actually work
1208 # print('callback called', self.asset_data)
1209 pat_items = (
1210 ('512', '512', '', 1),
1211 ('1024', '1024', '', 2),
1212 ('2048', '2048', '', 3),
1213 ('4096', '4096', '', 4),
1214 ('8192', '8192', '', 5),
1216 items = []
1217 for item in pat_items:
1218 if int(self.max_resolution) >= int(item[0]):
1219 items.append(item)
1220 items.append(('ORIGINAL', 'Original', '', 6))
1221 return items
1224 def show_enum_values(obj, prop_name):
1225 print([item.identifier for item in obj.bl_rna.properties[prop_name].enum_items])
1228 class BlenderkitDownloadOperator(bpy.types.Operator):
1229 """Download and link asset to scene. Only link if asset already available locally"""
1230 bl_idname = "scene.blenderkit_download"
1231 bl_label = "BlenderKit Asset Download"
1232 bl_options = {'REGISTER', 'UNDO', 'INTERNAL'}
1234 # asset_type: EnumProperty(
1235 # name="Type",
1236 # items=asset_types,
1237 # description="Type of download",
1238 # default="MODEL",
1240 asset_index: IntProperty(name="Asset Index", description='asset index in search results', default=-1)
1242 asset_base_id: StringProperty(
1243 name="Asset base Id",
1244 description="Asset base id, used instead of search result index",
1245 default="")
1247 target_object: StringProperty(
1248 name="Target Object",
1249 description="Material or object target for replacement",
1250 default="")
1252 material_target_slot: IntProperty(name="Asset Index", description='asset index in search results', default=0)
1253 model_location: FloatVectorProperty(name='Asset Location', default=(0, 0, 0))
1254 model_rotation: FloatVectorProperty(name='Asset Rotation', default=(0, 0, 0))
1256 replace: BoolProperty(name='Replace', description='replace selection with the asset', default=False)
1258 replace_resolution: BoolProperty(name='Replace resolution', description='replace resolution of the active asset',
1259 default=False)
1261 invoke_resolution: BoolProperty(name='Replace resolution popup',
1262 description='pop up to ask which resolution to download', default=False)
1264 resolution: EnumProperty(
1265 items=available_resolutions_callback,
1266 default=0,
1267 description='Replace resolution'
1270 max_resolution: IntProperty(
1271 name="Max resolution",
1272 description="",
1273 default=0)
1274 # has_res_0_5k: BoolProperty(name='512',
1275 # description='', default=False)
1277 cast_parent: StringProperty(
1278 name="Particles Target Object",
1279 description="",
1280 default="")
1282 # @classmethod
1283 # def poll(cls, context):
1284 # return bpy.context.window_manager.BlenderKitModelThumbnails is not ''
1285 def get_asset_data(self, context):
1286 # get asset data - it can come from scene, or from search results.
1287 s = bpy.context.scene
1289 if self.asset_index > -1:
1290 # either get the data from search results
1291 sr = bpy.context.window_manager['search results']
1292 asset_data = sr[
1293 self.asset_index].to_dict() # TODO CHECK ALL OCCURRENCES OF PASSING BLENDER ID PROPS TO THREADS!
1294 asset_base_id = asset_data['assetBaseId']
1295 else:
1296 # or from the scene.
1297 asset_base_id = self.asset_base_id
1299 au = s.get('assets used')
1300 if au == None:
1301 s['assets used'] = {}
1302 if asset_base_id in s.get('assets used'):
1303 # already used assets have already download link and especially file link.
1304 asset_data = s['assets used'][asset_base_id].to_dict()
1305 return asset_data
1307 def execute(self, context):
1308 sprops = utils.get_search_props()
1310 self.asset_data = self.get_asset_data(context)
1312 # print('after getting asset data')
1313 # print(self.asset_base_id)
1315 atype = self.asset_data['assetType']
1316 if bpy.context.mode != 'OBJECT' and (
1317 atype == 'model' or atype == 'material') and bpy.context.view_layer.objects.active is not None:
1318 bpy.ops.object.mode_set(mode='OBJECT')
1320 if self.resolution == 0 or self.resolution == '':
1321 resolution = sprops.resolution
1322 else:
1323 resolution = self.resolution
1325 resolution = resolutions.resolution_props_to_server[resolution]
1326 if self.replace: # cleanup first, assign later.
1327 obs = utils.get_selected_replace_adepts()
1328 # print(obs)
1329 for ob in obs:
1330 # print('replace attempt ', ob.name)
1331 if self.asset_base_id != '':
1332 # this is for a case when replace is called from a panel,
1333 # this uses active object as replacement source instead of target.
1334 if ob.get('asset_data') is not None and \
1335 (ob['asset_data']['assetBaseId'] == self.asset_base_id and ob['asset_data'][
1336 'resolution'] == resolution):
1337 # print('skipping this one')
1338 continue;
1339 parent = ob.parent
1340 if parent:
1341 parent = ob.parent.name # after this, parent is either name or None.
1343 kwargs = {
1344 'cast_parent': self.cast_parent,
1345 'target_object': ob.name,
1346 'material_target_slot': ob.active_material_index,
1347 'model_location': tuple(ob.matrix_world.translation),
1348 'model_rotation': tuple(ob.matrix_world.to_euler()),
1349 'replace': True,
1350 'replace_resolution': False,
1351 'parent': parent,
1352 'resolution': resolution
1354 # TODO - move this After download, not before, so that the replacement
1355 utils.delete_hierarchy(ob)
1356 start_download(self.asset_data, **kwargs)
1357 else:
1358 # replace resolution needs to replace all instances of the resolution in the scene
1359 # and deleting originals has to be thus done after the downlaod
1361 kwargs = {
1362 'cast_parent': self.cast_parent,
1363 'target_object': self.target_object,
1364 'material_target_slot': self.material_target_slot,
1365 'model_location': tuple(self.model_location),
1366 'model_rotation': tuple(self.model_rotation),
1367 'replace': False,
1368 'replace_resolution': self.replace_resolution,
1369 'resolution': resolution
1372 start_download(self.asset_data, **kwargs)
1373 return {'FINISHED'}
1375 def draw(self, context):
1376 layout = self.layout
1377 layout.prop(self, 'resolution', expand=True, icon_only=False)
1379 def invoke(self, context, event):
1380 print(self.asset_base_id)
1381 wm = context.window_manager
1382 # only make a pop up in case of switching resolutions
1383 if self.invoke_resolution:
1384 # show_enum_values(self, 'resolution')
1385 # print('ENUM VALUES')
1386 self.asset_data = self.get_asset_data(context)
1387 sprops = utils.get_search_props()
1388 if int(sprops.resolution) <= int(self.max_resolution):
1389 self.resolution = sprops.resolution
1390 elif int(self.max_resolution) > 0:
1391 self.resolution = self.max_resolution
1392 else:
1393 self.resolution = 'ORIGINAL'
1394 return wm.invoke_props_dialog(self)
1397 def register_download():
1398 bpy.utils.register_class(BlenderkitDownloadOperator)
1399 bpy.utils.register_class(BlenderkitKillDownloadOperator)
1400 bpy.app.handlers.load_post.append(scene_load)
1401 bpy.app.handlers.save_pre.append(scene_save)
1402 user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
1403 if user_preferences.use_timers:
1404 bpy.app.timers.register(timer_update)
1407 def unregister_download():
1408 bpy.utils.unregister_class(BlenderkitDownloadOperator)
1409 bpy.utils.unregister_class(BlenderkitKillDownloadOperator)
1410 bpy.app.handlers.load_post.remove(scene_load)
1411 bpy.app.handlers.save_pre.remove(scene_save)
1412 if bpy.app.timers.is_registered(timer_update):
1413 bpy.app.timers.unregister(timer_update)