lib/vfs/mc-vfs/extfs/s3.in

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 #
   5 #  Midnight Commander compatible EXTFS for accessing Amazon Web Services S3.
   6 #  Written by Jakob Kemi <jakob.kemi@gmail.com> 2009
   7 #
   8 #  Copyright (c) 2009 Free Software Foundation, Inc.
   9 #  This program is free software; you can redistribute it and/or modify
  10 #  it under the terms of the GNU General Public License as published by
  11 #  the Free Software Foundation; either version 2 of the License, or
  12 #  (at your option) any later version.
  13 #
  14 #  This program is distributed in the hope that it will be useful,
  15 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 #  GNU General Public License for more details.
  18 #
  19 #  You should have received a copy of the GNU General Public License
  20 #  along with this program; if not, write to the Free Software
  21 #  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
  22 #
  23 #
  24 # Notes:
  25 #  This EXTFS exposes buckets as directories and keys as files
  26 #  Due to EXTFS limitations all buckets & keys have to be read initially which might
  27 #  take quite some time.
  28 #  Tested on Debian with Python 2.4-2.6 and boto 1.4c and 1.6b
  29 #    (Python 2.6 might need -W ignore::DeprecationWarning due to boto using
  30 #    deprecated module Popen2)
  31 #
  32 #
  33 # Installation:
  34 #  Make sure that boto <http://code.google.com/p/boto> (python-boto in Debian) is installed.
  35 #  Preferably pytz (package python-tz in Debian) should be installed as well.
  36 #
  37 #  Save as executable file /share/mc/extfs/s3 (or wherever your mc expects to find extfs modules)
  38 #  Add the the following to your extfs.ini (might exists as /usr/share/mc/extfs/extfs.ini):
  39 # ----- begin extfs.ini -----
  40 #  # Amazon S3
  41 #  s3:
  42 # ----- end extfs.ini   -----
  43 #
  44 #
  45 # Settings: (should be set via environment)
  46 #  Required:
  47 #    AWS_ACCESS_KEY_ID         : Amazon AWS acces key (required)
  48 #    AWS_SECRET_ACCESS_KEY     : Amazon AWS secret access key (required)
  49 #  Optional:
  50 #    MCVFS_EXTFS_S3_LOCATION   : where to create new buckets, "EU"(default) or "US"
  51 #    MCVFS_EXTFS_S3_DEBUGFILE  : write debug info to this file (no info default)
  52 #
  53 #
  54 # Usage:
  55 #  Open dialog "Quick cd" (<alt-c>) and type: #s3 <enter> (or simply type ''cd #s3'' in shell line)
  56 #
  57 #
  58 # History:
  59 #  2009-02-07 Jakob Kemi <jakob.kemi@gmail.com>
  60 #   - Updated instructions.
  61 #   - Improved error reporting.
  62 #
  63 #  2009-02-06 Jakob Kemi <jakob.kemi@gmail.com>
  64 #   - Threaded list command.
  65 #   - Handle rm of empty "subdirectories" (as seen in mc).
  66 #   - List most recent datetime and total size of keys as directory properties.
  67 #   - List modification time in local time.
  68 #
  69 #  2009-02-05 Jakob Kemi <jakob.kemi@gmail.com>
  70 #   - Initial version.
  71 #
  72
  73 import sys
  74 import os
  75 import time
  76 import re
  77 import datetime
  78
  79
  80 import boto
  81 from boto.s3.connection import S3Connection
  82 from boto.s3.key import Key
  83 from boto.exception import BotoServerError
  84
  85
  86 # Get settings from environment
  87 USER=os.getenv('USER','0')
  88 AWS_ACCESS_KEY_ID=os.getenv('AWS_ACCESS_KEY_ID')
  89 AWS_SECRET_ACCESS_KEY=os.getenv('AWS_SECRET_ACCESS_KEY')
  90 LOCATION = os.getenv('MCVFS_EXTFS_S3_LOCATION', 'EU').lower()
  91 DEBUGFILE = os.getenv('MCVFS_EXTFS_S3_DEBUGFILE')
  92
  93 if not AWS_ACCESS_KEY_ID or not AWS_SECRET_ACCESS_KEY:
  94         sys.stderr.write('Missing AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY environment variables.\n')
  95         sys.exit(1)
  96
  97 # Setup logging
  98 if DEBUGFILE:
  99         import logging
 100         logging.basicConfig(
 101                 filename=DEBUGFILE,
 102                 level=logging.DEBUG,
 103                 format='%(asctime)s %(levelname)s %(message)s')
 104         logging.getLogger('boto').setLevel(logging.WARNING)
 105 else:
 106         class Void(object):
 107                 def __getattr__(self, attr):
 108                         return self
 109                 def __call__(self, *args, **kw):
 110                         return self
 111         logging = Void()
 112
 113 logger=logging.getLogger('s3extfs')
 114
 115
 116 def threadmap(fun, iterable, maxthreads=16):
 117         """
 118         Quick and dirty threaded version of builtin method map.
 119         Propagates exception safely.
 120         """
 121         from threading import Thread
 122         import Queue
 123
 124         items = list(iterable)
 125         nitems = len(items)
 126         if nitems < 2:
 127                 return map(fun, items)
 128
 129         # Create and fill input queue
 130         input = Queue.Queue()
 131         output = Queue.Queue()
 132
 133         for i,item in enumerate(items):
 134                 input.put( (i,item) )
 135
 136         class WorkThread(Thread):
 137                 """
 138                 Takes one item from input queue (thread terminates when input queue is empty),
 139                 performs fun, puts result in output queue
 140                 """
 141                 def run(self):
 142                         while True:
 143                                 try:
 144                                         (i,item) = input.get_nowait()
 145                                         try:
 146                                                 result = fun(item)
 147                                                 output.put( (i,result) )
 148                                         except:
 149                                                 output.put( (None,sys.exc_info()) )
 150                                 except Queue.Empty:
 151                                         return
 152
 153         # Start threads
 154         for i in range( min(len(items), maxthreads) ):
 155                 t = WorkThread()
 156                 t.setDaemon(True)
 157                 t.start()
 158
 159         # Wait for all threads to finish & collate results
 160         ret = []
 161         for i in range(nitems):
 162                 try:
 163                         i,res = output.get()
 164                         if i == None:
 165                                 raise res[0],res[1],res[2]
 166                 except Queue.Empty:
 167                         break
 168                 ret.append(res)
 169
 170         return ret
 171
 172 logger.debug('started')
 173
 174 # Global S3 connection
 175 s3 = S3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
 176 if LOCATION == 'eu':
 177         logger.debug('Using location EU for new buckets')
 178         S3LOCATION = boto.s3.connection.Location.EU
 179 else:
 180         logger.debug('Using location US for new buckets')
 181         S3LOCATION = boto.s3.connection.Location.US
 182
 183 logger.debug('argv: ' + str(sys.argv))
 184
 185 try:
 186         cmd = sys.argv[1]
 187         args = sys.argv[2:]
 188 except:
 189         sys.stderr.write('This program should be called from within MC\n')
 190         sys.exit(1)
 191
 192 def handleServerError(msg):
 193         e = sys.exc_info()
 194         msg += ', reason: ' + e[1].reason
 195         logger.error(msg, exc_info=e)
 196         sys.stderr.write(msg+'\n')
 197         sys.exit(1)
 198
 199 #
 200 # Lists all S3 contents
 201 #
 202 if cmd == 'list':
 203         if len(args) > 0:
 204                 path = args[0]
 205         else:
 206                 path = ''
 207
 208         logger.info('list')
 209
 210         rs = s3.get_all_buckets()
 211
 212         # Import python timezones (pytz)
 213         try:
 214                 import pytz
 215         except:
 216                 logger.warning('Missing pytz module, timestamps will be off')
 217                 # A fallback UTC tz stub
 218                 class pytzutc(datetime.tzinfo):
 219                         def __init__(self):
 220                                 datetime.tzinfo.__init__(self)
 221                                 self.utc = self
 222                                 self.zone = 'UTC'
 223                         def utcoffset(self, dt):
 224                                 return datetime.timedelta(0)
 225                         def tzname(self, dt):
 226                                 return "UTC"
 227                         def dst(self, dt):
 228                                 return datetime.timedelta(0)
 229                 pytz = pytzutc()
 230
 231
 232         # Find timezone
 233         # (yes, timeZONE as in _geographic zone_ not EST/CEST or whatever crap we get from time.tzname)
 234         # http://regebro.wordpress.com/2008/05/10/python-and-time-zones-part-2-the-beast-returns/
 235         def getGuessedTimezone():
 236                 # 1. check TZ env. var
 237                 try:
 238                         tz = os.getenv('TZ', '')
 239                         return pytz.timezone(tz)
 240                 except:
 241                         pass
 242                 # 2. check if /etc/timezone exists (Debian at least)
 243                 try:
 244                         if os.path.isfile('/etc/timezone'):
 245                                 tz = open('/etc/timezone', 'r').readline().strip()
 246                                 return pytz.timezone(tz)
 247                 except:
 248                         pass
 249                 # 3. check if /etc/localtime is a _link_ to something useful
 250                 try:
 251                         if os.path.islink('/etc/localtime'):
 252                                 link = os.readlink('/etc/localtime')
 253                                 tz = '/'.join(p.split(os.path.sep)[-2:])
 254                                 return pytz.timezone(tz)
 255                 except:
 256                         pass
 257                 # 4. use time.tzname which will probably be wrong by an hour 50% of the time.
 258                 try:
 259                         return pytz.timezone(time.tzname[0])
 260                 except:
 261                         pass
 262                 # 5. use plain UTC ...
 263                 return pytz.utc
 264
 265         tz=getGuessedTimezone()
 266         logger.debug('Using timezone: ' + tz.zone)
 267
 268         # AWS time is on format: 2009-01-07T16:43:39.000Z
 269         # we "want" MM-DD-YYYY hh:mm (in localtime)
 270         expr = re.compile(r'^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})\.\d{3}Z$')
 271         def convDate(awsdatetime):
 272                 m = expr.match(awsdatetime)
 273                 ye,mo,da,ho,mi,se = map(int,m.groups())
 274
 275                 dt = datetime.datetime(ye,mo,da,ho,mi,se, tzinfo=pytz.utc)
 276                 return dt.astimezone(tz).strftime('%m-%d-%Y %H:%M')
 277
 278
 279         def bucketList(b):
 280                 totsz = 0
 281                 mostrecent = '1970-01-01T00:00:00.000Z'
 282                 ret = []
 283                 for k in b.list():
 284                         mostrecent = max(mostrecent, k.last_modified)
 285                         datetime = convDate(k.last_modified)
 286                         ret.append('%10s %3d %-8s %-8s %d %s %s\n' % (
 287                                 '-rw-r--r--', 1, USER, USER, k.size, datetime, b.name+'/'+k.name)
 288                         )
 289                         totsz += k.size
 290
 291                 datetime=convDate(mostrecent)
 292                 sys.stdout.write('%10s %3d %-8s %-8s %d %s %s\n' % (
 293                         'drwxr-xr-x', 1, USER, USER, totsz, datetime, b.name)
 294                 )
 295                 for line in ret:
 296                         sys.stdout.write(line)
 297
 298         threadmap(bucketList, rs)
 299
 300 #
 301 # Fetch file from S3
 302 #
 303 elif cmd == 'copyout':
 304         archivename = args[0]
 305         storedfilename = args[1]
 306         extractto = args[2]
 307
 308         bucket,key = storedfilename.split('/', 1)
 309         logger.info('copyout bucket: %s, key: %s'%(bucket, key))
 310
 311         try:
 312                 b = s3.get_bucket(bucket)
 313                 k = b.get_key(key)
 314
 315                 out = open(extractto, 'w')
 316
 317                 k.open(mode='r')
 318                 for buf in k:
 319                         out.write(buf)
 320                 k.close()
 321                 out.close()
 322         except BotoServerError:
 323                 handleServerError('Unable to fetch key "%s"'%(key))
 324
 325 #
 326 # Upload file to S3
 327 #
 328 elif cmd == 'copyin':
 329         archivename = args[0]
 330         storedfilename = args[1]
 331         sourcefile = args[2]
 332
 333         bucket,key = storedfilename.split('/', 1)
 334         logger.info('copyin bucket: %s, key: %s'%(bucket, key))
 335
 336         try:
 337                 b = s3.get_bucket(bucket)
 338                 k = b.new_key(key)
 339                 k.set_contents_from_file(fp=open(sourcefile,'r'))
 340         except BotoServerError:
 341                 handleServerError('Unable to upload key "%s"' % (key))
 342
 343 #
 344 # Remove file from S3
 345 #
 346 elif cmd == 'rm':
 347         archivename = args[0]
 348         storedfilename = args[1]
 349
 350         bucket,key = storedfilename.split('/', 1)
 351         logger.info('rm bucket: %s, key: %s'%(bucket, key))
 352
 353         try:
 354                 b = s3.get_bucket(bucket)
 355                 b.delete_key(key)
 356         except BotoServerError:
 357                 handleServerError('Unable to remove key "%s"' % (key))
 358
 359 #
 360 # Create directory
 361 #
 362 elif cmd == 'mkdir':
 363         archivename = args[0]
 364         dirname = args[1]
 365
 366         logger.info('mkdir dir: %s' %(dirname))
 367         if '/' in dirname:
 368                 logger.warning('skipping mkdir')
 369                 pass
 370         else:
 371                 bucket = dirname
 372                 try:
 373                         s3.create_bucket(bucket, location=boto.s3.connection.Location.EU)
 374                 except BotoServerError:
 375                         handleServerError('Unable to create bucket "%s"' % (bucket))
 376
 377 #
 378 # Remove directory
 379 #
 380 elif cmd == 'rmdir':
 381         archivename = args[0]
 382         dirname = args[1]
 383
 384         logger.info('rmdir dir: %s' %(dirname))
 385         if '/' in dirname:
 386                 logger.warning('skipping rmdir')
 387                 pass
 388         else:
 389                 bucket = dirname
 390                 try:
 391                         b = s3.get_bucket(bucket)
 392                         s3.delete_bucket(b)
 393                 except BotoServerError:
 394                         handleServerError('Unable to delete bucket "%s"' % (bucket))
 395
 396 #
 397 # Run from S3
 398 #
 399 elif cmd == 'run':
 400         archivename = args[0]
 401         storedfilename = args[1]
 402         arguments = args[2:]
 403
 404         bucket,key = storedfilename.split('/', 1)
 405         logger.info('run bucket: %s, key: %s'%(bucket, key))
 406
 407         os.execv(storedfilename, arguments)
 408 else:
 409         logger.error('unhandled, bye')
 410         sys.exit(1)
 411
 412 logger.debug('command handled')
 413 sys.exit(0)
 414