python/google/appengine/api/files/file.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21 """App Engine Files API."""
  22
  23 from __future__ import with_statement
  24
  25
  26 __all__ = [
  27            'ApiTemporaryUnavailableError',
  28            'BLOBSTORE_FILESYSTEM',
  29            'Error',
  30            'ExclusiveLockFailedError',
  31            'ExistenceError',
  32            'FileNotOpenedError',
  33            'FileTemporaryUnavailableError',
  34            'FILESYSTEMS',
  35            'FinalizationError',
  36            'GS_FILESYSTEM',
  37            'InvalidArgumentError',
  38            'InvalidFileNameError',
  39            'InvalidParameterError',
  40            'OperationNotSupportedError',
  41            'PermissionDeniedError',
  42            'ReadOnlyError',
  43            'SequenceKeyOutOfOrderError',
  44            'UnknownError',
  45            'UnsupportedContentTypeError',
  46            'UnsupportedOpenModeError',
  47            'WrongContentTypeError' ,
  48            'WrongOpenModeError',
  49
  50            'RAW',
  51            'READ_BLOCK_SIZE',
  52
  53            'delete',
  54            'finalize',
  55            'listdir',
  56            'open',
  57            'stat',
  58
  59            'BufferedFile',
  60            ]
  61
  62 import os
  63 import sys
  64 import StringIO
  65
  66 from google.appengine.api import apiproxy_stub_map
  67 from google.appengine.api.files import file_service_pb
  68 from google.appengine.runtime import apiproxy_errors
  69
  70
  71 BLOBSTORE_FILESYSTEM = 'blobstore'
  72 GS_FILESYSTEM = 'gs'
  73 FILESYSTEMS = (BLOBSTORE_FILESYSTEM, GS_FILESYSTEM)
  74 READ_BLOCK_SIZE = 1024 * 512
  75 _CREATION_HANDLE_PREFIX = 'writable:'
  76 _DEFAULT_BUFFER_SIZE = 512 * 1024
  77
  78
  79 class Error(Exception):
  80   """Base error class for this module."""
  81
  82
  83 class UnsupportedOpenModeError(Error):
  84   """Unsupported file open mode was specified."""
  85
  86
  87 class UnsupportedContentTypeError(Error):
  88   """Specified file content type is not supported by this api."""
  89
  90
  91 class InvalidArgumentError(Error):
  92   """Function argument has invalid value."""
  93
  94
  95 class FinalizationError(Error):
  96   """File is in wrong finalization state."""
  97
  98
  99 class ExistenceError(Error):
 100   """File is in wrong existence state."""
 101
 102
 103 class UnknownError(Error):
 104   """Unknown unexpected io error occured."""
 105
 106
 107 class SequenceKeyOutOfOrderError(Error):
 108   """Sequence key specified is out of order.
 109
 110   Attributes:
 111     last_sequence_key: last sequence key which was written to the file.
 112   """
 113
 114   def __init__(self, last_sequence_key, cause=None):
 115     Error.__init__(self, cause)
 116     self.last_sequence_key = last_sequence_key
 117
 118
 119 class InvalidFileNameError(Error):
 120   """File name is invalid."""
 121
 122
 123 class FileNotOpenedError(Error):
 124   """File was not opened."""
 125
 126
 127 class ReadOnlyError(Error):
 128   """File is read-only mode."""
 129
 130
 131 class WrongContentTypeError(Error):
 132   """File has a different content type."""
 133
 134
 135 class WrongOpenModeError(Error):
 136   """Incorrect file open mode."""
 137
 138
 139 class OperationNotSupportedError(Error):
 140   """Incorrect file open mode."""
 141
 142
 143 class PermissionDeniedError(Error):
 144   """Application doesn't have permissions to perform the operation."""
 145
 146
 147 class ApiTemporaryUnavailableError(Error):
 148   """Files API is temporary unavailable. Request should be retried soon."""
 149
 150
 151 class FileTemporaryUnavailableError(Error):
 152   """File is temporary unavailable. Request should be retried soon."""
 153
 154
 155 class InvalidParameterError(Error):
 156   """Parameter specified in Create() call is invalid."""
 157
 158
 159 class ExclusiveLockFailedError(Error):
 160   """Exclusive lock can't be obtained."""
 161
 162
 163
 164 RAW = file_service_pb.FileContentType.RAW
 165
 166
 167 def _raise_app_error(e):
 168   """Convert RPC error into api-specific exception."""
 169   if (e.application_error in
 170       [file_service_pb.FileServiceErrors.EXISTENCE_ERROR,
 171        file_service_pb.FileServiceErrors.EXISTENCE_ERROR_METADATA_NOT_FOUND,
 172        file_service_pb.FileServiceErrors.EXISTENCE_ERROR_METADATA_FOUND,
 173        file_service_pb.FileServiceErrors.EXISTENCE_ERROR_SHARDING_MISMATCH,
 174        file_service_pb.FileServiceErrors.EXISTENCE_ERROR_OBJECT_NOT_FOUND,
 175        file_service_pb.FileServiceErrors.EXISTENCE_ERROR_BUCKET_NOT_FOUND,
 176        ]):
 177     raise ExistenceError(e)
 178   elif (e.application_error ==
 179         file_service_pb.FileServiceErrors.API_TEMPORARILY_UNAVAILABLE):
 180     raise ApiTemporaryUnavailableError(e)
 181   elif (e.application_error ==
 182         file_service_pb.FileServiceErrors.FINALIZATION_ERROR):
 183     raise FinalizationError(e)
 184   elif (e.application_error ==
 185         file_service_pb.FileServiceErrors.IO_ERROR):
 186     raise UnknownError(e)
 187   elif (e.application_error ==
 188         file_service_pb.FileServiceErrors.SEQUENCE_KEY_OUT_OF_ORDER):
 189     raise SequenceKeyOutOfOrderError(e.error_detail, e)
 190   elif (e.application_error ==
 191         file_service_pb.FileServiceErrors.INVALID_FILE_NAME):
 192     raise InvalidFileNameError(e)
 193   elif (e.application_error ==
 194         file_service_pb.FileServiceErrors.FILE_NOT_OPENED):
 195     raise FileNotOpenedError(e)
 196   elif (e.application_error ==
 197         file_service_pb.FileServiceErrors.READ_ONLY):
 198     raise ReadOnlyError(e)
 199   elif (e.application_error ==
 200         file_service_pb.FileServiceErrors.WRONG_CONTENT_TYPE):
 201     raise WrongContentTypeError(e)
 202   elif (e.application_error ==
 203         file_service_pb.FileServiceErrors.WRONG_OPEN_MODE):
 204     raise WrongOpenModeError(e)
 205   elif (e.application_error ==
 206         file_service_pb.FileServiceErrors.OPERATION_NOT_SUPPORTED):
 207     raise OperationNotSupportedError(e)
 208   elif (e.application_error ==
 209         file_service_pb.FileServiceErrors.PERMISSION_DENIED):
 210     raise PermissionDeniedError(e)
 211   elif (e.application_error ==
 212         file_service_pb.FileServiceErrors.FILE_TEMPORARILY_UNAVAILABLE):
 213     raise FileTemporaryUnavailableError(e)
 214   elif (e.application_error ==
 215         file_service_pb.FileServiceErrors.INVALID_PARAMETER):
 216     raise InvalidParameterError(e)
 217   elif (e.application_error ==
 218         file_service_pb.FileServiceErrors.EXCLUSIVE_LOCK_FAILED):
 219     raise ExclusiveLockFailedError(e)
 220   raise Error(e)
 221
 222
 223 def _create_rpc(deadline):
 224   """Create RPC object for file service.
 225
 226   Args:
 227     deadling: Request deadline in seconds.
 228   """
 229   return apiproxy_stub_map.UserRPC('file', deadline)
 230
 231
 232 def _make_call(method, request, response,
 233                deadline=30):
 234   """Perform File RPC call.
 235
 236   Args:
 237     method: Service method name as string.
 238     request: Request protocol buffer.
 239     response: Response protocol buffer.
 240     deadline: Request deadline in seconds.
 241
 242   Raises:
 243     Error or it's descendant if any File API specific error has happened.
 244   """
 245
 246   rpc = _create_rpc(deadline=deadline)
 247   rpc.make_call(method, request, response)
 248   rpc.wait()
 249   try:
 250     rpc.check_success()
 251   except apiproxy_errors.ApplicationError, e:
 252     _raise_app_error(e)
 253
 254
 255 class _File(object):
 256   """File object.
 257
 258   File object must be obtained by open() function and closed by its close()
 259   method. It supports scoped closing by with operator.
 260   """
 261
 262   def __init__(self, filename, mode, content_type, exclusive_lock):
 263     """Constructor.
 264
 265     Args:
 266       filename: File's name as string.
 267       content_type: File's content type. Value from FileContentType.ContentType
 268         enum.
 269     """
 270     self._filename = filename
 271     self._closed = False
 272     self._content_type = content_type
 273     self._mode = mode
 274     self._exclusive_lock = exclusive_lock
 275     self._offset = 0
 276     self._open()
 277
 278   def close(self, finalize=False):
 279     """Close file.
 280
 281     Args:
 282       finalize: Specifies if file should be finalized upon closing.
 283     """
 284     if self._closed:
 285       return
 286     self._closed = True
 287     request = file_service_pb.CloseRequest()
 288     response = file_service_pb.CloseResponse()
 289     request.set_filename(self._filename)
 290     request.set_finalize(finalize)
 291     self._make_rpc_call_with_retry('Close', request, response)
 292
 293   def __enter__(self):
 294     return self
 295
 296   def __exit__(self, atype, value, traceback):
 297     self.close()
 298
 299   def write(self, data, sequence_key=None):
 300     """Write data to file.
 301
 302     Args:
 303       data: Data to be written to the file. For RAW files it should be a string
 304         or byte sequence.
 305       sequence_key: Sequence key to use for write. Is used for RAW files only.
 306         File API infrastructure ensures that sequence_key are monotonically
 307         increasing. If sequence key less than previous one is used, a
 308         SequenceKeyOutOfOrderError exception with last recorded sequence key
 309         will be raised. If part of already written content is lost due to
 310         infrastructure failure, last_sequence_key will point to last
 311         successfully written key.
 312
 313     Raises:
 314       SequenceKeyOutOfOrderError: Raised when passed sequence keys are not
 315         monotonically increasing.
 316       InvalidArgumentError: Raised when wrong object type is apssed in as data.
 317       Error: Error or its descendants are raised when other error has happened.
 318     """
 319     if self._content_type == RAW:
 320       request = file_service_pb.AppendRequest()
 321       response = file_service_pb.AppendResponse()
 322       request.set_filename(self._filename)
 323       request.set_data(data)
 324       if sequence_key:
 325         request.set_sequence_key(sequence_key)
 326       self._make_rpc_call_with_retry('Append', request, response)
 327     else:
 328       raise UnsupportedContentTypeError(
 329           'Unsupported content type: %s' % self._content_type)
 330
 331   def tell(self):
 332     """Return file's current position.
 333
 334     Is valid only when file is opened for read.
 335     """
 336     self._verify_read_mode()
 337     return self._offset
 338
 339   def seek(self, offset, whence=os.SEEK_SET):
 340     """Set the file's current position.
 341
 342     Args:
 343       offset: seek offset as number.
 344       whence: seek mode. Supported modes are os.SEEK_SET (absolute seek),
 345         and os.SEEK_CUR (seek relative to the current position) and os.SEEK_END
 346         (seek relative to the end, offset should be negative).
 347     """
 348     self._verify_read_mode()
 349     if whence == os.SEEK_SET:
 350       self._offset = offset
 351     elif whence == os.SEEK_CUR:
 352       self._offset += offset
 353     elif whence == os.SEEK_END:
 354       file_stat = self.stat()
 355       self._offset = file_stat.st_size + offset
 356     else:
 357       raise InvalidArgumentError('Whence mode %d is not supported', whence)
 358
 359   def read(self, size=None):
 360     """Read data from RAW file.
 361
 362     Args:
 363       size: Number of bytes to read as integer. Actual number of bytes
 364         read might be less than specified, but it's never 0 unless current
 365         offset is at the end of the file. If it is None, then file is read
 366         until the end.
 367
 368     Returns:
 369       A string with data read.
 370     """
 371     self._verify_read_mode()
 372     if self._content_type != RAW:
 373       raise UnsupportedContentTypeError(
 374           'Unsupported content type: %s' % self._content_type)
 375
 376     buf = StringIO.StringIO()
 377     original_offset = self._offset
 378
 379     try:
 380       if size is None:
 381         size = sys.maxint
 382
 383       while size > 0:
 384         request = file_service_pb.ReadRequest()
 385         response = file_service_pb.ReadResponse()
 386         request.set_filename(self._filename)
 387         request.set_pos(self._offset)
 388         request.set_max_bytes(min(READ_BLOCK_SIZE, size))
 389         self._make_rpc_call_with_retry('Read', request, response)
 390         chunk = response.data()
 391         self._offset += len(chunk)
 392         if len(chunk) == 0:
 393           break
 394         buf.write(chunk)
 395         size -= len(chunk)
 396
 397       return buf.getvalue()
 398     except:
 399       self._offset = original_offset
 400       raise
 401     finally:
 402       buf.close()
 403
 404   def _verify_read_mode(self):
 405     if self._mode not in ('r', 'rb'):
 406       raise WrongOpenModeError('File is opened for write.')
 407
 408   def _open(self):
 409     request = file_service_pb.OpenRequest()
 410     response = file_service_pb.OpenResponse()
 411
 412     request.set_filename(self._filename)
 413     request.set_exclusive_lock(self._exclusive_lock)
 414     request.set_content_type(self._content_type)
 415
 416     if self._mode in ('a', 'ab'):
 417       request.set_open_mode(file_service_pb.OpenRequest.APPEND)
 418     elif self._mode in ('r', 'rb'):
 419       request.set_open_mode(file_service_pb.OpenRequest.READ)
 420     else:
 421       raise UnsupportedOpenModeError('Unsupported open mode: %s', self._mode)
 422
 423     self._make_rpc_call_with_retry('Open', request, response)
 424
 425   def _make_rpc_call_with_retry(self, method, request, response):
 426     try:
 427       _make_call(method, request, response)
 428     except (ApiTemporaryUnavailableError, FileTemporaryUnavailableError):
 429
 430       if method == 'Open':
 431         _make_call(method, request, response)
 432         return
 433       if self._exclusive_lock:
 434
 435         raise
 436
 437       self._open()
 438       _make_call(method, request, response)
 439
 440   def stat(self):
 441     """Get status of a finalized file.
 442
 443     Returns:
 444       a _FileStat object similar to that returned by python's os.stat(path).
 445
 446     Throws:
 447       FinalizationError if file is not finalized.
 448     """
 449     self._verify_read_mode()
 450
 451     request = file_service_pb.StatRequest()
 452     response = file_service_pb.StatResponse()
 453     request.set_filename(self._filename)
 454
 455     _make_call('Stat', request, response)
 456
 457     if response.stat_size() == 0:
 458       raise ExistenceError("File %s not found." % self._filename)
 459
 460     if response.stat_size() > 1:
 461       raise ValueError(
 462           "Requested stat for one file. Got more than one response.")
 463
 464     file_stat_pb = response.stat(0)
 465     file_stat = _FileStat()
 466     file_stat.filename = file_stat_pb.filename()
 467     file_stat.finalized = file_stat_pb.finalized()
 468     file_stat.st_size = file_stat_pb.length()
 469     file_stat.st_mtime = file_stat_pb.mtime()
 470     file_stat.st_ctime = file_stat_pb.ctime()
 471
 472     return file_stat
 473
 474
 475 def open(filename,
 476          mode='r',
 477          content_type=RAW,
 478          exclusive_lock=False,
 479          buffering=0):
 480   """Open a file.
 481
 482   Args:
 483     filename: A name of the file as string.
 484     mode: File open mode. Either 'a' or 'r'.
 485     content_type: File's content type. Value from FileContentType.ContentType
 486       enum.
 487     exclusive_lock: If file should be exclusively locked. All other exclusive
 488       lock attempts will file until file is correctly closed.
 489     buffering: optional argument similar to the one in Python's open.
 490       It specifies the file's desired buffer size: 0 means unbuffered, positive
 491       value means use a buffer of that size, any negative value means the
 492       default size. Only read buffering is supported.
 493
 494   Returns:
 495     File object.
 496
 497   Raises:
 498     InvalidArgumentError: Raised when given illegal argument value or type.
 499   """
 500   if not filename:
 501     raise InvalidArgumentError('Filename is empty')
 502   if not isinstance(filename, basestring):
 503     raise InvalidArgumentError('Filename should be a string but is %s (%s)' %
 504                                (filename.__class__, filename))
 505   if content_type != RAW:
 506     raise InvalidArgumentError('Invalid content type')
 507   if not (isinstance(buffering, int) or isinstance(buffering, long)):
 508     raise InvalidArgumentError('buffering should be an int but is %s'
 509                                % buffering)
 510
 511   if mode == 'r' or mode == 'rb':
 512     if buffering > 0:
 513       return BufferedFile(filename, buffering)
 514     elif buffering < 0:
 515       return BufferedFile(filename, _DEFAULT_BUFFER_SIZE)
 516
 517   return _File(filename,
 518                mode=mode,
 519                content_type=content_type,
 520                exclusive_lock=exclusive_lock)
 521
 522
 523 def listdir(path, **kwargs):
 524   """Return a sorted list of filenames (matching a pattern) in the given path.
 525
 526   Only Google Cloud Storage paths are supported in current implementation.
 527
 528   Args:
 529     path: a Google Cloud Storage path of "/gs/bucketname" form.
 530     kwargs: other keyword arguments to be relayed to Google Cloud Storage.
 531       This can be used to select certain files with names matching a pattern.
 532       See google.appengine.api.files.gs.listdir for details.
 533
 534   Returns:
 535     a list containing filenames (matching a pattern) from the given path.
 536     Sorted by Python String.
 537   """
 538
 539   from google.appengine.api.files import gs
 540
 541   if not isinstance(path, basestring):
 542     raise InvalidArgumentError('path should be a string, but is %s(%r)' %
 543                                (path.__class__.__name__, path))
 544
 545   if path.startswith(gs._GS_PREFIX):
 546     return gs.listdir(path, kwargs)
 547   else:
 548     raise InvalidFileNameError('Unsupported path: %s' % path)
 549
 550
 551 def finalize(filename, content_type=RAW):
 552   """Finalize a file.
 553
 554   Args:
 555     filename: File name as string.
 556     content_type: File's content type. Value from FileContentType.ContentType
 557       enum.
 558   """
 559   if not filename:
 560     raise InvalidArgumentError('Filename is empty')
 561   if not isinstance(filename, basestring):
 562     raise InvalidArgumentError('Filename should be a string')
 563   if content_type != RAW:
 564     raise InvalidArgumentError('Invalid content type')
 565
 566   try:
 567     f = open(filename, 'a', exclusive_lock=True, content_type=content_type)
 568     f.close(finalize=True)
 569   except FinalizationError:
 570
 571     pass
 572
 573
 574 class _FileStat(object):
 575   """_FileStat contains file attributes.
 576
 577   Attributes:
 578     filename: the uploaded filename of the file;
 579     finalized: whether the file is finalized. This is always true by now;
 580     st_size: number of bytes of the file;
 581     st_ctime: creation time;
 582     st_mtime: modification time.
 583   """
 584   def __init__(self):
 585     self.filename = None
 586     self.finalized = True
 587     self.st_size = None
 588     self.st_ctime = None
 589     self.st_mtime = None
 590
 591
 592 def stat(filename):
 593   """Get status of a finalized file given it's full path filename.
 594
 595   Returns:
 596     a _FileStat object similar to that returned by python's os.stat(path).
 597
 598   Throws:
 599     FinalizationError if file is not finalized.
 600   """
 601   if not filename:
 602     raise InvalidArgumentError('Filename is empty')
 603   if not isinstance(filename, basestring):
 604     raise InvalidArgumentError('Filename should be a string')
 605
 606   with open(filename, 'r') as f:
 607     return f.stat()
 608
 609
 610 def _create(filesystem, content_type=RAW, filename=None, params=None):
 611   """Create a file.
 612
 613   Args:
 614     filesystem: File system to create a file at as string.
 615     content_type: File content type.
 616     filename: Requested file name as string. Some file system require this
 617       to be filled in, some require it to be None.
 618     params: {string: string} dict of file parameters. Each filesystem
 619       interprets them differently.
 620   """
 621   if not filesystem:
 622     raise InvalidArgumentError('Filesystem is empty')
 623   if not isinstance(filesystem, basestring):
 624     raise InvalidArgumentError('Filesystem should be a string')
 625   if content_type != RAW:
 626     raise InvalidArgumentError('Invalid content type')
 627
 628   request = file_service_pb.CreateRequest()
 629   response = file_service_pb.CreateResponse()
 630
 631   request.set_filesystem(filesystem)
 632   request.set_content_type(content_type)
 633
 634   if filename:
 635     if not isinstance(filename, basestring):
 636       raise InvalidArgumentError('Filename should be a string')
 637     request.set_filename(filename)
 638
 639   if params:
 640     if not isinstance(params, dict):
 641       raise InvalidArgumentError('Parameters should be a dictionary')
 642     for k,v in params.items():
 643       param = request.add_parameters()
 644       param.set_name(k)
 645       param.set_value(v)
 646
 647   _make_call('Create', request, response)
 648   return response.filename()
 649
 650
 651 def __checkIsFinalizedName(filename):
 652   """Check if a filename is finalized.
 653
 654   A filename is finalized when it has creation handle prefix, which is the same
 655   for both blobstore and gs files.
 656
 657   Args:
 658     filename: a gs or blobstore filename that starts with '/gs/' or
 659       '/blobstore/'
 660
 661   Raises:
 662     InvalidFileNameError: raised when filename is finalized.
 663   """
 664   if filename.split('/')[2].startswith(_CREATION_HANDLE_PREFIX):
 665     raise InvalidFileNameError('File %s should have finalized filename' %
 666                                filename)
 667
 668
 669 def delete(*filenames):
 670   """Permanently delete files.
 671
 672   Delete on non-finalized/non-existent files is a no-op.
 673
 674   Args:
 675     filenames: finalized file names as strings. filename should has format
 676       "/gs/bucket/filename" or "/blobstore/blobkey".
 677
 678   Raises:
 679     InvalidFileNameError: Raised when any filename is not of valid format or
 680       not a finalized name.
 681     IOError: Raised if any problem occurs contacting the backend system.
 682   """
 683
 684   from google.appengine.api.files import blobstore as files_blobstore
 685   from google.appengine.api.files import gs
 686   from google.appengine.ext import blobstore
 687
 688   blobkeys = []
 689
 690   for filename in filenames:
 691     if not isinstance(filename, basestring):
 692       raise InvalidArgumentError('Filename should be a string, but is %s(%r)' %
 693                                  (filename.__class__.__name__, filename))
 694     if filename.startswith(files_blobstore._BLOBSTORE_DIRECTORY):
 695       __checkIsFinalizedName(filename)
 696       blobkey = files_blobstore.get_blob_key(filename)
 697       if blobkey:
 698         blobkeys.append(blobkey)
 699     elif filename.startswith(gs._GS_PREFIX):
 700
 701       __checkIsFinalizedName(filename)
 702       blobkeys.append(blobstore.create_gs_key(filename))
 703     else:
 704       raise InvalidFileNameError('Filename should start with /%s or /%s' %
 705                                  (files_blobstore._BLOBSTORE_DIRECTORY,
 706                                  gs._GS_PREFIX))
 707
 708   try:
 709     blobstore.delete(blobkeys)
 710   except Exception, e:
 711     raise IOError('Blobstore failure.', e)
 712
 713
 714 def _get_capabilities():
 715   """Get files API capabilities.
 716
 717   Returns:
 718     An instance of file_service_pb.GetCapabilitiesResponse.
 719   """
 720   request = file_service_pb.GetCapabilitiesRequest()
 721   response = file_service_pb.GetCapabilitiesResponse()
 722
 723   _make_call('GetCapabilities', request, response)
 724   return response
 725
 726
 727 class BufferedFile(object):
 728   """BufferedFile is a file-like object reading underlying file in chunks."""
 729
 730   def __init__(self, filename, buffer_size=_DEFAULT_BUFFER_SIZE):
 731     """Constructor.
 732
 733     Args:
 734       filename: the name of the file to read as string.
 735       buffer_size: buffer read size to use as int.
 736     """
 737     self._filename = filename
 738     self._position = 0
 739     self._buffer = ''
 740     self._buffer_pos = 0
 741     self._buffer_size = buffer_size
 742     self._eof = False
 743
 744   def __enter__(self):
 745     return self
 746
 747   def __exit__(self, atype, value, traceback):
 748     self.close()
 749
 750   def close(self):
 751     self._buffer = ''
 752     self._eof = True
 753     self._buffer_pos = 0
 754
 755   def tell(self):
 756     """Return file's current position."""
 757     return self._position
 758
 759   def read(self, size=None):
 760     """Read data from RAW file.
 761
 762     Args:
 763       size: Number of bytes to read as integer. Actual number of bytes
 764         read is always equal to size unless end if file was reached.
 765
 766     Returns:
 767       A string with data read.
 768     """
 769     if size is None:
 770       size = sys.maxint
 771     data_list = []
 772     while True:
 773       result = self.__readBuffer(size)
 774       data_list.append(result)
 775       size -= len(result)
 776       if size == 0 or self._eof:
 777         return ''.join(data_list)
 778       self.__refillBuffer()
 779
 780   def readline(self, size=-1):
 781     """Read one line delimited by '\n' from the file.
 782
 783     A trailing newline character is kept in the string. It may be absent when a
 784     file ends with an incomplete line. If the size argument is non-negative,
 785     it specifies the maximum string size (counting the newline) to return. An
 786     empty string is returned only when EOF is encountered immediately.
 787
 788     Args:
 789       size: Maximum number of bytes to read. If not specified, readline stops
 790         only on '\n' or EOF.
 791
 792     Returns:
 793       The data read as a string.
 794     """
 795     data_list = []
 796
 797     while True:
 798
 799       if size < 0:
 800         end_pos = len(self._buffer)
 801       else:
 802         end_pos = self._buffer_pos + size
 803       newline_pos = self._buffer.find('\n', self._buffer_pos, end_pos)
 804
 805       if newline_pos != -1:
 806
 807         data_list.append(self.__readBuffer(newline_pos + 1 - self._buffer_pos))
 808         return ''.join(data_list)
 809       else:
 810         result = self.__readBuffer(size)
 811         data_list.append(result)
 812         size -= len(result)
 813         if size == 0 or self._eof:
 814           return ''.join(data_list)
 815         self.__refillBuffer()
 816
 817   def __readBuffer(self, size):
 818     """Read chars from self._buffer.
 819
 820     Args:
 821       size: number of chars to read. Read the entire buffer if negative.
 822
 823     Returns:
 824       chars read in string.
 825     """
 826     if size < 0:
 827       size = len(self._buffer) - self._buffer_pos
 828     result = self._buffer[self._buffer_pos:self._buffer_pos+size]
 829
 830     self._position += len(result)
 831
 832     self._buffer_pos += len(result)
 833     return result
 834
 835   def __refillBuffer(self):
 836     """Refill _buffer with another read from source."""
 837     with open(self._filename, 'r') as f:
 838       f.seek(self._position)
 839       data = f.read(self._buffer_size)
 840     self._eof = len(data) < self._buffer_size
 841     self._buffer = data
 842     self._buffer_pos = 0
 843
 844   def seek(self, offset, whence=os.SEEK_SET):
 845     """Set the file's current position.
 846
 847     Args:
 848       offset: seek offset as number.
 849       whence: seek mode. Supported modes are os.SEEK_SET (absolute seek),
 850         os.SEEK_CUR (seek relative to the current position), and os.SEEK_END
 851         (seek relative to the end, offset should be negative).
 852     """
 853     if whence == os.SEEK_SET:
 854       self._position = offset
 855     elif whence == os.SEEK_CUR:
 856       self._position += offset
 857     elif whence == os.SEEK_END:
 858       file_stat = stat(self._filename)
 859       self._position = file_stat.st_size + offset
 860     else:
 861       raise InvalidArgumentError('Whence mode %d is not supported', whence)
 862     self._buffer = ''
 863     self._buffer_pos = 0
 864     self._eof = False
 865
 866
 867 def _default_gs_bucket_name():
 868   """Return the default Google Storage bucket name for the application.
 869
 870   Returns:
 871     A string that is the default bucket name for the application.
 872   """
 873   request = file_service_pb.GetDefaultGsBucketNameRequest()
 874   response = file_service_pb.GetDefaultGsBucketNameResponse()
 875
 876   _make_call('GetDefaultGsBucketName', request, response)
 877
 878   return response.default_gs_bucket_name()