blenderkit/upload_bg.py

   1 # ##### BEGIN GPL LICENSE BLOCK #####
   2 #
   3 #  This program is free software; you can redistribute it and/or
   4 #  modify it under the terms of the GNU General Public License
   5 #  as published by the Free Software Foundation; either version 2
   6 #  of the License, or (at your option) any later version.
   7 #
   8 #  This program is distributed in the hope that it will be useful,
   9 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 #  GNU General Public License for more details.
  12 #
  13 #  You should have received a copy of the GNU General Public License
  14 #  along with this program; if not, write to the Free Software Foundation,
  15 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16 #
  17 # ##### END GPL LICENSE BLOCK #####
  18
  19
  20 if "bpy" in locals():
  21     from importlib import reload
  22
  23     paths = reload(paths)
  24     append_link = reload(append_link)
  25     bg_blender = reload(bg_blender)
  26     utils = reload(utils)
  27     rerequests = reload(rerequests)
  28 else:
  29     from blenderkit import paths, append_link, bg_blender, utils, rerequests
  30
  31 import sys, json, os, time
  32 import requests
  33 import logging
  34
  35 import bpy
  36
  37 BLENDERKIT_EXPORT_DATA = sys.argv[-1]
  38
  39
  40 def start_logging():
  41     logging.basicConfig()
  42     logging.getLogger().setLevel(logging.DEBUG)
  43     requests_log = logging.getLogger("requests.packages.urllib3")
  44     requests_log.setLevel(logging.DEBUG)
  45     requests_log.propagate = True
  46
  47
  48 def print_gap():
  49     print('\n\n\n\n')
  50
  51
  52 class upload_in_chunks(object):
  53     def __init__(self, filename, chunksize=1 << 13, report_name='file'):
  54         self.filename = filename
  55         self.chunksize = chunksize
  56         self.totalsize = os.path.getsize(filename)
  57         self.readsofar = 0
  58         self.report_name = report_name
  59
  60     def __iter__(self):
  61         with open(self.filename, 'rb') as file:
  62             while True:
  63                 data = file.read(self.chunksize)
  64                 if not data:
  65                     sys.stderr.write("\n")
  66                     break
  67                 self.readsofar += len(data)
  68                 percent = self.readsofar * 1e2 / self.totalsize
  69                 bg_blender.progress('uploading %s' % self.report_name, percent)
  70                 # sys.stderr.write("\r{percent:3.0f}%".format(percent=percent))
  71                 yield data
  72
  73     def __len__(self):
  74         return self.totalsize
  75
  76
  77 def upload_file(upload_data, f):
  78     headers = utils.get_headers(upload_data['token'])
  79     version_id = upload_data['id']
  80     bg_blender.progress('uploading %s' % f['type'])
  81     upload_info = {
  82         'assetId': version_id,
  83         'fileType': f['type'],
  84         'fileIndex': f['index'],
  85         'originalFilename': os.path.basename(f['file_path'])
  86     }
  87     upload_create_url = paths.get_api_url() + 'uploads/'
  88     upload = rerequests.post(upload_create_url, json=upload_info, headers=headers, verify=True)
  89     upload = upload.json()
  90     #
  91     chunk_size = 1024 * 1024 * 2
  92     utils.pprint(upload)
  93     # file gets uploaded here:
  94     uploaded = False
  95     # s3 upload is now the only option
  96     for a in range(0, 5):
  97         if not uploaded:
  98             try:
  99                 upload_response = requests.put(upload['s3UploadUrl'],
 100                                                data=upload_in_chunks(f['file_path'], chunk_size, f['type']),
 101                                                stream=True, verify=True)
 102
 103                 if upload_response.status_code == 200:
 104                     uploaded = True
 105                 else:
 106                     print(upload_response.text)
 107                     bg_blender.progress(f'Upload failed, retry. {a}')
 108             except Exception as e:
 109                 print(e)
 110                 bg_blender.progress('Upload %s failed, retrying' % f['type'])
 111                 time.sleep(1)
 112
 113             # confirm single file upload to bkit server
 114             upload_done_url = paths.get_api_url() + 'uploads_s3/' + upload['id'] + '/upload-file/'
 115             upload_response = rerequests.post(upload_done_url, headers=headers, verify=True)
 116
 117     bg_blender.progress('finished uploading')
 118
 119     return uploaded
 120
 121
 122 def upload_files(upload_data, files):
 123     uploaded_all = True
 124     for f in files:
 125         uploaded = upload_file(upload_data, f)
 126         if not uploaded:
 127             uploaded_all = False
 128         bg_blender.progress('finished uploading')
 129     return uploaded_all
 130
 131
 132 if __name__ == "__main__":
 133
 134
 135     try:
 136         bg_blender.progress('preparing scene - append data')
 137         with open(BLENDERKIT_EXPORT_DATA, 'r') as s:
 138             data = json.load(s)
 139
 140         bpy.app.debug_value = data.get('debug_value', 0)
 141         export_data = data['export_data']
 142         upload_data = data['upload_data']
 143
 144         upload_set = data['upload_set']
 145         if 'MAINFILE' in upload_set:
 146             bpy.data.scenes.new('upload')
 147             for s in bpy.data.scenes:
 148                 if s.name != 'upload':
 149                     bpy.data.scenes.remove(s)
 150
 151             if export_data['type'] == 'MODEL':
 152                 obnames = export_data['models']
 153                 main_source, allobs = append_link.append_objects(file_name=data['source_filepath'],
 154                                                                  obnames=obnames,
 155                                                                  rotation=(0, 0, 0))
 156                 g = bpy.data.collections.new(upload_data['name'])
 157                 for o in allobs:
 158                     g.objects.link(o)
 159                 bpy.context.scene.collection.children.link(g)
 160             if export_data['type'] == 'SCENE':
 161                 sname = export_data['scene']
 162                 main_source = append_link.append_scene(file_name=data['source_filepath'],
 163                                                        scenename=sname)
 164                 bpy.data.scenes.remove(bpy.data.scenes['upload'])
 165                 main_source.name = sname
 166             elif export_data['type'] == 'MATERIAL':
 167                 matname = export_data['material']
 168                 main_source = append_link.append_material(file_name=data['source_filepath'], matname=matname)
 169
 170             elif export_data['type'] == 'BRUSH':
 171                 brushname = export_data['brush']
 172                 main_source = append_link.append_brush(file_name=data['source_filepath'], brushname=brushname)
 173
 174             bpy.ops.file.pack_all()
 175
 176             main_source.blenderkit.uploading = False
 177             fpath = os.path.join(data['temp_dir'], upload_data['assetBaseId'] + '.blend')
 178
 179             bpy.ops.wm.save_as_mainfile(filepath=fpath, compress=True, copy=False)
 180             os.remove(data['source_filepath'])
 181
 182         bg_blender.progress('preparing scene - open files')
 183
 184         files = []
 185         if 'THUMBNAIL' in upload_set:
 186             files.append({
 187                 "type": "thumbnail",
 188                 "index": 0,
 189                 "file_path": export_data["thumbnail_path"]
 190             })
 191         if 'MAINFILE' in upload_set:
 192             files.append({
 193                 "type": "blend",
 194                 "index": 0,
 195                 "file_path": fpath
 196             })
 197
 198         bg_blender.progress('uploading')
 199
 200         uploaded = upload_files(upload_data, files)
 201
 202         if uploaded:
 203             # mark on server as uploaded
 204             if 'MAINFILE' in upload_set:
 205                 confirm_data = {
 206                     "verificationStatus": "uploaded"
 207                 }
 208
 209                 url = paths.get_api_url() + 'assets/'
 210
 211                 headers = utils.get_headers(upload_data['token'])
 212
 213                 url += upload_data["id"] + '/'
 214
 215                 r = rerequests.patch(url, json=confirm_data, headers=headers, verify=True)  # files = files,
 216
 217             bg_blender.progress('upload finished successfully')
 218         else:
 219             bg_blender.progress('upload failed.')
 220
 221     except Exception as e:
 222         print(e)
 223         bg_blender.progress(e)
 224         sys.exit(1)