1 // Copyright 2010 Google Inc. All Rights Reserved.
3 package com
.google
.appengine
.api
.files
;
5 import static java
.nio
.charset
.StandardCharsets
.US_ASCII
;
7 import com
.google
.appengine
.api
.NamespaceManager
;
8 import com
.google
.appengine
.api
.blobstore
.BlobInfo
;
9 import com
.google
.appengine
.api
.blobstore
.BlobInfoFactory
;
10 import com
.google
.appengine
.api
.blobstore
.BlobKey
;
11 import com
.google
.appengine
.api
.blobstore
.BlobstoreFailureException
;
12 import com
.google
.appengine
.api
.blobstore
.BlobstoreService
;
13 import com
.google
.appengine
.api
.blobstore
.BlobstoreServiceFactory
;
14 import com
.google
.appengine
.api
.datastore
.DatastoreService
;
15 import com
.google
.appengine
.api
.datastore
.DatastoreServiceFactory
;
16 import com
.google
.appengine
.api
.datastore
.Entity
;
17 import com
.google
.appengine
.api
.datastore
.EntityNotFoundException
;
18 import com
.google
.appengine
.api
.datastore
.KeyFactory
;
19 import com
.google
.appengine
.api
.datastore
.Query
;
20 import com
.google
.appengine
.api
.files
.FileServicePb
.AppendRequest
;
21 import com
.google
.appengine
.api
.files
.FileServicePb
.AppendResponse
;
22 import com
.google
.appengine
.api
.files
.FileServicePb
.CloseRequest
;
23 import com
.google
.appengine
.api
.files
.FileServicePb
.CloseResponse
;
24 import com
.google
.appengine
.api
.files
.FileServicePb
.CreateRequest
;
25 import com
.google
.appengine
.api
.files
.FileServicePb
.CreateResponse
;
26 import com
.google
.appengine
.api
.files
.FileServicePb
.FileContentType
.ContentType
;
27 import com
.google
.appengine
.api
.files
.FileServicePb
.FileServiceErrors
;
28 import com
.google
.appengine
.api
.files
.FileServicePb
.GetDefaultGsBucketNameRequest
;
29 import com
.google
.appengine
.api
.files
.FileServicePb
.GetDefaultGsBucketNameResponse
;
30 import com
.google
.appengine
.api
.files
.FileServicePb
.OpenRequest
;
31 import com
.google
.appengine
.api
.files
.FileServicePb
.OpenRequest
.OpenMode
;
32 import com
.google
.appengine
.api
.files
.FileServicePb
.OpenResponse
;
33 import com
.google
.appengine
.api
.files
.FileServicePb
.ReadRequest
;
34 import com
.google
.appengine
.api
.files
.FileServicePb
.ReadResponse
;
35 import com
.google
.appengine
.api
.files
.FileServicePb
.StatRequest
;
36 import com
.google
.appengine
.api
.files
.FileServicePb
.StatResponse
;
37 import com
.google
.apphosting
.api
.ApiProxy
;
38 import com
.google
.common
.base
.Preconditions
;
39 import com
.google
.common
.hash
.Hashing
;
40 import com
.google
.protobuf
.ByteString
;
41 import com
.google
.protobuf
.InvalidProtocolBufferException
;
42 import com
.google
.protobuf
.Message
;
44 import java
.io
.FileNotFoundException
;
45 import java
.io
.IOException
;
46 import java
.nio
.ByteBuffer
;
47 import java
.util
.ArrayList
;
49 import java
.util
.TreeMap
;
52 * Implements {@link FileService} by using {@link ApiProxy} to make RPC calls to
53 * the App Engine File API.
57 class FileServiceImpl
implements FileService
{
59 static final String PACKAGE
= "file";
61 static final String FILESYSTEM_BLOBSTORE
= AppEngineFile
.FileSystem
.BLOBSTORE
.getName();
62 static final String PARAMETER_MIME_TYPE
= "content_type";
63 static final String PARAMETER_BLOB_INFO_UPLOADED_FILE_NAME
= "file_name";
64 static final String DEFAULT_MIME_TYPE
= "application/octet-stream";
65 static final String FILESYSTEM_GS
= AppEngineFile
.FileSystem
.GS
.getName();
66 static final String GS_FILESYSTEM_PREFIX
= "/gs/";
67 static final String GS_PARAMETER_MIME_TYPE
= "content_type";
68 static final String GS_PARAMETER_CANNED_ACL
= "acl";
69 static final String GS_PARAMETER_CONTENT_ENCODING
= "content_encoding";
70 static final String GS_PARAMETER_CONTENT_DISPOSITION
= "content_disposition";
71 static final String GS_PARAMETER_CACHE_CONTROL
= "cache_control";
72 static final int DATASTORE_MAX_PROPERTY_SIZE
= 500;
74 static final String GS_DEFAULT_MIME_TYPE
= "application/octet-stream";
76 private static final String BLOB_INFO_CREATION_HANDLE_PROPERTY
= "creation_handle";
77 static final String GS_CREATION_HANDLE_PREFIX
= "writable:";
78 static final String CREATION_HANDLE_PREFIX
= "writable:";
80 BlobstoreService blobstoreService
;
81 DatastoreService datastoreService
;
83 public FileServiceImpl() {
84 blobstoreService
= BlobstoreServiceFactory
.getBlobstoreService();
85 datastoreService
= DatastoreServiceFactory
.getDatastoreService();
92 public AppEngineFile
createNewBlobFile(String mimeType
) throws IOException
{
93 return createNewBlobFile(mimeType
, "");
100 public AppEngineFile
createNewBlobFile(String mimeType
, String blobInfoUploadedFileName
)
102 if (mimeType
== null || mimeType
.trim().isEmpty()) {
103 mimeType
= DEFAULT_MIME_TYPE
;
106 Map
<String
, String
> params
= new TreeMap
<String
, String
>();
107 params
.put(PARAMETER_MIME_TYPE
, mimeType
);
108 if (blobInfoUploadedFileName
!= null && !blobInfoUploadedFileName
.isEmpty()) {
109 params
.put(PARAMETER_BLOB_INFO_UPLOADED_FILE_NAME
, blobInfoUploadedFileName
);
111 String filePath
= create(FILESYSTEM_BLOBSTORE
, null, ContentType
.RAW
, params
);
112 AppEngineFile file
= new AppEngineFile(filePath
);
113 if (!file
.getNamePart().startsWith(CREATION_HANDLE_PREFIX
)) {
114 throw new RuntimeException("Expected creation handle: " + file
.getFullPath());
123 public AppEngineFile
createNewGSFile(final GSFileOptions options
) throws IOException
{
124 if (options
.fileName
== null || options
.fileName
.isEmpty() ||
125 !options
.fileName
.startsWith(GS_FILESYSTEM_PREFIX
)) {
126 throw new IllegalArgumentException("Invalid fileName, should be of the form: /gs/bucket/key");
128 Map
<String
, String
> params
= new TreeMap
<String
, String
>();
129 params
.put(GS_PARAMETER_MIME_TYPE
, options
.mimeType
);
130 if (options
.acl
!= null && !options
.acl
.trim().isEmpty()) {
131 params
.put(GS_PARAMETER_CANNED_ACL
, options
.acl
);
133 if (options
.cacheControl
!= null && !options
.cacheControl
.trim().isEmpty()) {
134 params
.put(GS_PARAMETER_CACHE_CONTROL
, options
.cacheControl
);
136 if (options
.contentEncoding
!= null && !options
.contentEncoding
.trim().isEmpty()) {
137 params
.put(GS_PARAMETER_CONTENT_ENCODING
, options
.contentEncoding
);
139 if (options
.contentDisposition
!= null && !options
.contentDisposition
.trim().isEmpty()) {
140 params
.put(GS_PARAMETER_CONTENT_DISPOSITION
, options
.contentDisposition
);
142 if (options
.userMetadata
!= null) {
143 for (String key
: options
.userMetadata
.keySet()) {
144 if (key
== null || key
.isEmpty()) {
145 throw new IllegalArgumentException(
146 "Empty or null key in userMetadata");
148 String value
= options
.userMetadata
.get(key
);
149 if (value
== null || value
.isEmpty()) {
150 throw new IllegalArgumentException(
151 "Empty or null value in userMetadata for key: " + key
);
153 params
.put(GSFileOptions
.GS_USER_METADATA_PREFIX
+ key
, value
);
156 AppEngineFile file
= new AppEngineFile(
157 create(FILESYSTEM_GS
, options
.fileName
, ContentType
.RAW
, params
));
158 if (!file
.getNamePart().startsWith(GS_CREATION_HANDLE_PREFIX
)) {
159 throw new RuntimeException("Expected creation handle: " + file
.getFullPath());
168 public FileWriteChannel
openWriteChannel(AppEngineFile file
, boolean lock
)
169 throws FileNotFoundException
, FinalizationException
, LockException
, IOException
{
170 FileWriteChannel channel
= new FileWriteChannelImpl(file
, lock
, this);
171 openForAppend(file
, lock
);
176 * Open the given file for append and optionally lock it.
178 * @param file the file to open
179 * @param lock should the file be locked for exclusive access?
180 * @throws FileNotFoundException if the file does not exist in the File Proxy
181 * @throws FinalizationException if the file has already been finalized. The
182 * file may have been finalized by another request.
183 * @throws LockException if the file is locked in a different App Engine
184 * request, or if {@code lock = true} and the file is opened in a
185 * different App Engine request
186 * @throws IOException if any other unexpected problem occurs
188 void openForAppend(AppEngineFile file
, boolean lock
)
189 throws FileNotFoundException
, FinalizationException
, LockException
, IOException
{
190 openForAppend(file
.getFullPath(), ContentType
.RAW
, lock
);
197 public FileReadChannel
openReadChannel(AppEngineFile file
, boolean lock
)
198 throws FileNotFoundException
, LockException
, IOException
{
199 FileReadChannel channel
= new FileReadChannelImpl(file
, this);
200 openForRead(file
, lock
);
208 public RecordReadChannel
openRecordReadChannel(AppEngineFile file
, boolean lock
)
209 throws FileNotFoundException
, LockException
, IOException
{
210 FileReadChannel fileReadChannel
= new BufferedFileReadChannelImpl(
211 openReadChannel(file
, lock
), RecordConstants
.BLOCK_SIZE
* 2);
212 RecordReadChannel channel
= new RecordReadChannelImpl(fileReadChannel
);
220 public RecordWriteChannel
openRecordWriteChannel(AppEngineFile file
, boolean lock
)
221 throws FileNotFoundException
, LockException
, IOException
{
222 RecordWriteChannel channel
= new RecordWriteChannelImpl(openWriteChannel(file
, lock
));
227 public void delete(AppEngineFile
... files
) throws IOException
{
228 Preconditions
.checkNotNull(files
, "No file given");
230 if (files
.length
== 0) {
234 ArrayList
<BlobKey
> blobKeys
= new ArrayList
<BlobKey
>();
236 for (int i
= 0; i
< files
.length
; i
++) {
237 AppEngineFile file
= files
[i
];
238 Preconditions
.checkNotNull(file
, String
.format("File at index %d is null", i
));
240 if (!file
.hasFinalizedName()) {
241 throw new UnsupportedOperationException(
242 String
.format("File %s does not have a finalized name", file
.getFullPath()));
245 if (file
.getFileSystem().equals((AppEngineFile
.FileSystem
.BLOBSTORE
))) {
246 BlobKey blobKey
= getBlobKey(file
);
247 if (blobKey
!= null) {
248 blobKeys
.add(blobKey
);
250 } else if (file
.getFileSystem().equals((AppEngineFile
.FileSystem
.GS
))) {
251 blobKeys
.add(blobstoreService
.createGsBlobKey(file
.getFullPath()));
253 throw new UnsupportedOperationException(
254 String
.format("File at index %d not supported by delete"));
258 if (!blobKeys
.isEmpty()) {
260 blobstoreService
.delete(blobKeys
.toArray(new BlobKey
[blobKeys
.size()]));
261 } catch (BlobstoreFailureException e
) {
262 throw new IOException("Blobstore failure", e
);
268 * Appends bytes from the given buffer to the end of the given file.
270 * @param file the file to which to append bytes. Must be opened for append in
271 * the current request
272 * @param buffer The buffer from which bytes are to be retrieved
273 * @param sequenceKey the sequence key. See the explanation of the {@code
274 * sequenceKey} paramater at
275 * {@link FileWriteChannel#write(ByteBuffer, String)}
276 * @throws IllegalArgumentException if {@code file} is not writable
277 * @throws KeyOrderingException if {@code sequenceKey} is not {@code null} and
278 * the backend system already has recorded a last good sequence key
279 * for this file and {@code sequenceKey} is not strictly
280 * lexicographically greater than the last good sequence key
281 * @throws IOException if the file is not opened for append in the current App
282 * Engine request or any other unexpected problem occurs
284 int append(AppEngineFile file
, ByteBuffer buffer
, String sequenceKey
) throws IOException
{
285 if (null == buffer
) {
286 throw new NullPointerException("buffer is null");
289 throw new NullPointerException("file is null");
291 ByteString data
= ByteString
.copyFrom(buffer
);
292 append(file
.getFullPath(), data
, sequenceKey
);
296 static final String BLOB_FILE_INDEX_KIND
= "__BlobFileIndex__";
302 public BlobKey
getBlobKey(AppEngineFile file
) {
304 throw new NullPointerException("file is null");
306 if (file
.getFileSystem() != AppEngineFile
.FileSystem
.BLOBSTORE
) {
307 throw new IllegalArgumentException("file is not of type BLOBSTORE");
309 BlobKey cached
= file
.getCachedBlobKey();
310 if (null != cached
) {
313 String namePart
= file
.getNamePart();
314 String creationHandle
= (namePart
.startsWith(CREATION_HANDLE_PREFIX
) ? namePart
: null);
316 if (null == creationHandle
) {
317 return new BlobKey(namePart
);
320 String origNamespace
= NamespaceManager
.get();
322 Entity blobInfoEntity
= null;
324 NamespaceManager
.set("");
326 Entity blobFileIndexEntity
= datastoreService
.get(null, KeyFactory
.createKey(
327 BLOB_FILE_INDEX_KIND
, getBlobFileIndexKeyName(creationHandle
)));
328 String blobKey
= (String
) blobFileIndexEntity
.getProperty("blob_key");
329 blobInfoEntity
= datastoreService
.get(
330 null, KeyFactory
.createKey(BlobInfoFactory
.KIND
, blobKey
));
331 } catch (EntityNotFoundException ex
) {
332 if (creationHandle
.length() < DATASTORE_MAX_PROPERTY_SIZE
) {
333 query
= new Query(BlobInfoFactory
.KIND
);
334 query
.addFilter(BLOB_INFO_CREATION_HANDLE_PROPERTY
, Query
.FilterOperator
.EQUAL
,
336 blobInfoEntity
= datastoreService
.prepare(query
).asSingleEntity();
340 NamespaceManager
.set(origNamespace
);
343 if (null == blobInfoEntity
) {
346 BlobInfo blobInfo
= new BlobInfoFactory(datastoreService
).createBlobInfo(blobInfoEntity
);
347 return blobInfo
.getBlobKey();
350 private static String
getBlobFileIndexKeyName(String creationHandle
) {
351 if (creationHandle
.length() < DATASTORE_MAX_PROPERTY_SIZE
) {
352 return creationHandle
;
355 return Hashing
.sha512().hashString(creationHandle
, US_ASCII
).toString();
362 public AppEngineFile
getBlobFile(BlobKey blobKey
) {
363 if (null == blobKey
) {
364 throw new NullPointerException("blobKey is null");
366 String namePart
= blobKey
.getKeyString();
367 AppEngineFile file
= new AppEngineFile(AppEngineFile
.FileSystem
.BLOBSTORE
, namePart
);
368 file
.setCachedBlobKey(blobKey
);
376 public FileStat
stat(AppEngineFile file
) throws IOException
{
377 Preconditions
.checkNotNull(file
, "file is null");
379 StatRequest
.Builder statRequestBuilder
= StatRequest
.newBuilder();
380 statRequestBuilder
.setFilename(file
.getFullPath());
381 StatResponse
.Builder statResponseBuilder
= StatResponse
.newBuilder();
382 openForRead(file
, false);
384 makeSyncCall("Stat", statRequestBuilder
, statResponseBuilder
);
389 if (statResponseBuilder
.getStatCount() != 1) {
390 throw new IllegalStateException(
391 "Requested stat for one file. Got zero or more than one response.");
393 FileServicePb
.FileStat fileStatPb
= statResponseBuilder
.build().getStat(0);
394 FileStat fileStat
= new FileStat();
395 fileStat
.setFilename(fileStatPb
.getFilename());
396 fileStat
.setFinalized(fileStatPb
.getFinalized());
397 fileStat
.setLength(fileStatPb
.getLength());
398 if (fileStatPb
.hasCtime()) {
399 fileStat
.setCtime(fileStatPb
.getCtime());
401 if (fileStatPb
.hasMtime()) {
402 fileStat
.setMtime(fileStatPb
.getMtime());
408 * Reads bytes from {@code file} starting from {@code startingPos} and puts
409 * the bytes into the {@code buffer}. Returns the number of bytes read. The
410 * number of bytes read will be the minumum of the number of bytes available
411 * in the file and the buffer's {@link ByteBuffer#remaining() free bytes}.
413 * @param file the file from which to read bytes. Must be opened for read in
414 * the current request
415 * @param buffer the destination buffer
416 * @return the number of bytes read
417 * @throws IOException if the file is not opened for read in the current App
418 * Engine request or any other unexpected problem occurs
420 int read(AppEngineFile file
, ByteBuffer buffer
, long startingPos
) throws IOException
{
421 if (startingPos
< 0) {
422 throw new IllegalArgumentException("startingPos is negative: " + startingPos
);
424 if (buffer
== null) {
425 throw new NullPointerException("buffer is null");
427 long remaining
= buffer
.remaining();
428 if (buffer
.remaining() < 1) {
431 ByteString byteString
= read(file
.getFullPath(), startingPos
, remaining
);
432 byteString
.copyTo(buffer
);
433 int numBytesRead
= byteString
.size();
434 if (numBytesRead
<= 0) {
441 * Change the state of the given file to closed and optionally finalize the
442 * file. After the file is finalized it may be read, and it may no longer be
445 * @param file the file to close and optionally finalize. The file must be
446 * opened in the current request.
447 * @param finalize should the file be finalized? The file may only be
448 * finalized if the current request holds the lock for the file
449 * @throws IllegalStateException if {@code finalize = true} and the current
450 * request does not hold the exclusive lock on {@code file}
451 * @throws IOException if the file is not opened in the current request, if
452 * {@code finalize = true} and the file is already finalized or if any
453 * other unexpected problem occurs
455 void close(AppEngineFile file
, boolean finalize
) throws IOException
{
457 close(file
.getFullPath(), finalize
);
458 } catch (LockException e
) {
460 throw new IllegalStateException("The current request does not hold the exclusive lock.");
467 * Opens a file for appending by making the "Open" RPC call with mode=APPEND.
469 private void openForAppend(String fileName
, ContentType contentType
, boolean lock
)
471 open(fileName
, contentType
, OpenMode
.APPEND
, lock
);
474 private void openForRead(AppEngineFile file
, boolean lock
)
475 throws FileNotFoundException
, LockException
, IOException
{
477 throw new NullPointerException("file is null");
479 openForRead(file
.getFullPath(), ContentType
.RAW
, lock
);
483 * Opens a file for reading by making the "Open" RPC call with mode=READ
485 private void openForRead(String fileName
, ContentType contentType
, boolean lock
)
487 open(fileName
, contentType
, OpenMode
.READ
, lock
);
491 * Makes the "Create" RPC call.
493 * @return created file name.
495 private String
create(
496 String fileSystem
, String fileName
, ContentType contentType
, Map
<String
, String
> parameters
)
498 CreateRequest
.Builder request
= CreateRequest
.newBuilder();
499 request
.setFilesystem(fileSystem
);
500 if (fileName
!= null && !fileName
.isEmpty()) {
501 request
.setFilename(fileName
);
503 request
.setContentType(contentType
);
504 if (parameters
!= null) {
505 for (Map
.Entry
<String
, String
> e
: parameters
.entrySet()) {
506 CreateRequest
.Parameter
.Builder parameter
= request
.addParametersBuilder();
507 parameter
.setName(e
.getKey());
508 parameter
.setValue(e
.getValue());
511 CreateResponse
.Builder response
= CreateResponse
.newBuilder();
512 makeSyncCall("Create", request
, response
);
513 return response
.build().getFilename();
517 * Makes the "Open" RPC call
519 private void open(String fileName
, ContentType contentType
, OpenMode openMode
, boolean lock
)
521 OpenRequest
.Builder openRequest
= OpenRequest
.newBuilder();
522 openRequest
.setFilename(fileName
);
523 openRequest
.setContentType(contentType
);
524 openRequest
.setOpenMode(openMode
);
525 openRequest
.setExclusiveLock(lock
);
526 OpenResponse
.Builder openResponse
= OpenResponse
.newBuilder();
527 makeSyncCall("Open", openRequest
, openResponse
);
531 * Makes the 'Append' RPC call
533 private void append(String fileName
, ByteString data
, String sequenceKey
) throws IOException
{
534 AppendRequest
.Builder appendRequest
= AppendRequest
.newBuilder();
535 appendRequest
.setFilename(fileName
);
536 appendRequest
.setData(data
);
537 if (null != sequenceKey
) {
538 appendRequest
.setSequenceKey(sequenceKey
);
540 AppendResponse
.Builder appendResponse
= AppendResponse
.newBuilder();
541 makeSyncCall("Append", appendRequest
, appendResponse
);
545 * Makes the "Read" RPC call
547 private ByteString
read(String fileName
, long pos
, long maxBytes
) throws IOException
{
548 ReadRequest
.Builder readRequest
= ReadRequest
.newBuilder();
549 readRequest
.setFilename(fileName
);
550 readRequest
.setMaxBytes(maxBytes
);
551 readRequest
.setPos(pos
);
552 ReadResponse
.Builder readResponse
= ReadResponse
.newBuilder();
553 makeSyncCall("Read", readRequest
, readResponse
);
554 return readResponse
.build().getData();
558 * Makes the "Close" RPC call
560 private void close(String fileName
, boolean finalize
) throws IOException
{
561 CloseRequest
.Builder closeRequest
= CloseRequest
.newBuilder();
562 closeRequest
.setFilename(fileName
);
563 closeRequest
.setFinalize(finalize
);
564 CloseResponse
.Builder closeResponse
= CloseResponse
.newBuilder();
565 makeSyncCall("Close", closeRequest
, closeResponse
);
569 * Makes the "GetDefaultGSBucketName" RPC call.
572 public String
getDefaultGsBucketName() throws IOException
{
573 GetDefaultGsBucketNameRequest
.Builder request
= GetDefaultGsBucketNameRequest
.newBuilder();
574 GetDefaultGsBucketNameResponse
.Builder response
= GetDefaultGsBucketNameResponse
.newBuilder();
575 makeSyncCall("GetDefaultGsBucketName", request
, response
);
576 return response
.getDefaultGsBucketName();
580 * Makes a synchronous RPC call to the app server
585 * @throws IOException
587 private void makeSyncCall(String callName
, Message
.Builder request
, Message
.Builder response
)
590 byte[] responseBytes
=
591 ApiProxy
.makeSyncCall(PACKAGE
, callName
, request
.build().toByteArray());
592 response
.mergeFrom(responseBytes
);
593 } catch (ApiProxy
.ApplicationException ex
) {
594 throw translateException(ex
, null);
595 } catch (InvalidProtocolBufferException e
) {
596 throw new RuntimeException("Internal logic error: Response PB could not be parsed.", e
);
601 * Translates from an internal to a public exception
603 private static IOException
translateException(ApiProxy
.ApplicationException ex
, String message
) {
604 int errorCode
= ex
.getApplicationError();
605 FileServiceErrors
.ErrorCode errorCodeEnum
= FileServiceErrors
.ErrorCode
.valueOf(errorCode
);
606 switch (errorCodeEnum
) {
607 case EXCLUSIVE_LOCK_FAILED
:
608 return new LockException(message
, ex
);
609 case EXISTENCE_ERROR
:
610 case EXISTENCE_ERROR_METADATA_NOT_FOUND
:
611 case EXISTENCE_ERROR_METADATA_FOUND
:
612 case EXISTENCE_ERROR_SHARDING_MISMATCH
:
613 case EXISTENCE_ERROR_BUCKET_NOT_FOUND
:
614 case EXISTENCE_ERROR_OBJECT_NOT_FOUND
:
615 return new FileNotFoundException();
616 case FINALIZATION_ERROR
:
617 return new FinalizationException(message
, ex
);
618 case SEQUENCE_KEY_OUT_OF_ORDER
:
619 return new KeyOrderingException(message
, ex
);
621 return new IOException(message
, ex
);