1 // Copyright 2010 Google Inc. All Rights Reserved.
3 package com
.google
.appengine
.api
.files
;
5 import com
.google
.appengine
.api
.NamespaceManager
;
6 import com
.google
.appengine
.api
.blobstore
.BlobInfo
;
7 import com
.google
.appengine
.api
.blobstore
.BlobInfoFactory
;
8 import com
.google
.appengine
.api
.blobstore
.BlobKey
;
9 import com
.google
.appengine
.api
.blobstore
.BlobstoreFailureException
;
10 import com
.google
.appengine
.api
.blobstore
.BlobstoreService
;
11 import com
.google
.appengine
.api
.blobstore
.BlobstoreServiceFactory
;
12 import com
.google
.appengine
.api
.datastore
.DatastoreService
;
13 import com
.google
.appengine
.api
.datastore
.DatastoreServiceFactory
;
14 import com
.google
.appengine
.api
.datastore
.Entity
;
15 import com
.google
.appengine
.api
.datastore
.EntityNotFoundException
;
16 import com
.google
.appengine
.api
.datastore
.KeyFactory
;
17 import com
.google
.appengine
.api
.datastore
.Query
;
18 import com
.google
.appengine
.api
.files
.FileServicePb
.AppendRequest
;
19 import com
.google
.appengine
.api
.files
.FileServicePb
.AppendResponse
;
20 import com
.google
.appengine
.api
.files
.FileServicePb
.CloseRequest
;
21 import com
.google
.appengine
.api
.files
.FileServicePb
.CloseResponse
;
22 import com
.google
.appengine
.api
.files
.FileServicePb
.CreateRequest
;
23 import com
.google
.appengine
.api
.files
.FileServicePb
.CreateResponse
;
24 import com
.google
.appengine
.api
.files
.FileServicePb
.FileContentType
.ContentType
;
25 import com
.google
.appengine
.api
.files
.FileServicePb
.FileServiceErrors
;
26 import com
.google
.appengine
.api
.files
.FileServicePb
.GetDefaultGsBucketNameRequest
;
27 import com
.google
.appengine
.api
.files
.FileServicePb
.GetDefaultGsBucketNameResponse
;
28 import com
.google
.appengine
.api
.files
.FileServicePb
.OpenRequest
;
29 import com
.google
.appengine
.api
.files
.FileServicePb
.OpenRequest
.OpenMode
;
30 import com
.google
.appengine
.api
.files
.FileServicePb
.OpenResponse
;
31 import com
.google
.appengine
.api
.files
.FileServicePb
.ReadRequest
;
32 import com
.google
.appengine
.api
.files
.FileServicePb
.ReadResponse
;
33 import com
.google
.appengine
.api
.files
.FileServicePb
.StatRequest
;
34 import com
.google
.appengine
.api
.files
.FileServicePb
.StatResponse
;
35 import com
.google
.apphosting
.api
.ApiProxy
;
36 import com
.google
.common
.base
.Charsets
;
37 import com
.google
.common
.base
.Preconditions
;
38 import com
.google
.common
.hash
.Hashing
;
39 import com
.google
.protobuf
.ByteString
;
40 import com
.google
.protobuf
.InvalidProtocolBufferException
;
41 import com
.google
.protobuf
.Message
;
43 import java
.io
.FileNotFoundException
;
44 import java
.io
.IOException
;
45 import java
.nio
.ByteBuffer
;
46 import java
.util
.ArrayList
;
48 import java
.util
.TreeMap
;
51 * Implements {@link FileService} by using {@link ApiProxy} to make RPC calls to
52 * the App Engine File API.
55 class FileServiceImpl
implements FileService
{
57 static final String PACKAGE
= "file";
59 static final String FILESYSTEM_BLOBSTORE
= AppEngineFile
.FileSystem
.BLOBSTORE
.getName();
60 static final String PARAMETER_MIME_TYPE
= "content_type";
61 static final String PARAMETER_BLOB_INFO_UPLOADED_FILE_NAME
= "file_name";
62 static final String DEFAULT_MIME_TYPE
= "application/octet-stream";
63 static final String FILESYSTEM_GS
= AppEngineFile
.FileSystem
.GS
.getName();
64 static final String GS_FILESYSTEM_PREFIX
= "/gs/";
65 static final String GS_PARAMETER_MIME_TYPE
= "content_type";
66 static final String GS_PARAMETER_CANNED_ACL
= "acl";
67 static final String GS_PARAMETER_CONTENT_ENCODING
= "content_encoding";
68 static final String GS_PARAMETER_CONTENT_DISPOSITION
= "content_disposition";
69 static final String GS_PARAMETER_CACHE_CONTROL
= "cache_control";
70 static final String GS_USER_METADATA_PREFIX
= "x-goog-meta-";
72 static final String GS_DEFAULT_MIME_TYPE
= "application/octet-stream";
74 private static final String BLOB_INFO_CREATION_HANDLE_PROPERTY
= "creation_handle";
75 static final String GS_CREATION_HANDLE_PREFIX
= "writable:";
76 static final String CREATION_HANDLE_PREFIX
= "writable:";
78 BlobstoreService blobstoreService
;
79 DatastoreService datastoreService
;
81 public FileServiceImpl() {
82 blobstoreService
= BlobstoreServiceFactory
.getBlobstoreService();
83 datastoreService
= DatastoreServiceFactory
.getDatastoreService();
90 public AppEngineFile
createNewBlobFile(String mimeType
) throws IOException
{
91 return createNewBlobFile(mimeType
, "");
98 public AppEngineFile
createNewBlobFile(String mimeType
, String blobInfoUploadedFileName
)
100 if (mimeType
== null || mimeType
.trim().isEmpty()) {
101 mimeType
= DEFAULT_MIME_TYPE
;
104 Map
<String
, String
> params
= new TreeMap
<String
, String
>();
105 params
.put(PARAMETER_MIME_TYPE
, mimeType
);
106 if (blobInfoUploadedFileName
!= null && !blobInfoUploadedFileName
.isEmpty()) {
107 params
.put(PARAMETER_BLOB_INFO_UPLOADED_FILE_NAME
, blobInfoUploadedFileName
);
109 String filePath
= create(FILESYSTEM_BLOBSTORE
, null, ContentType
.RAW
, params
);
110 AppEngineFile file
= new AppEngineFile(filePath
);
111 if (!file
.getNamePart().startsWith(CREATION_HANDLE_PREFIX
)) {
112 throw new RuntimeException("Expected creation handle: " + file
.getFullPath());
121 public AppEngineFile
createNewGSFile(final GSFileOptions options
) throws IOException
{
122 if (options
.fileName
== null || options
.fileName
.isEmpty() ||
123 !options
.fileName
.startsWith(GS_FILESYSTEM_PREFIX
)) {
124 throw new IllegalArgumentException("Invalid fileName, should be of the form: /gs/bucket/key");
126 Map
<String
, String
> params
= new TreeMap
<String
, String
>();
127 params
.put(GS_PARAMETER_MIME_TYPE
, options
.mimeType
);
128 if (options
.acl
!= null && !options
.acl
.trim().isEmpty()) {
129 params
.put(GS_PARAMETER_CANNED_ACL
, options
.acl
);
131 if (options
.cacheControl
!= null && !options
.cacheControl
.trim().isEmpty()) {
132 params
.put(GS_PARAMETER_CACHE_CONTROL
, options
.cacheControl
);
134 if (options
.contentEncoding
!= null && !options
.contentEncoding
.trim().isEmpty()) {
135 params
.put(GS_PARAMETER_CONTENT_ENCODING
, options
.contentEncoding
);
137 if (options
.contentDisposition
!= null && !options
.contentDisposition
.trim().isEmpty()) {
138 params
.put(GS_PARAMETER_CONTENT_DISPOSITION
, options
.contentDisposition
);
140 if (options
.userMetadata
!= null) {
141 for (String key
: options
.userMetadata
.keySet()) {
142 if (key
== null || key
.isEmpty()) {
143 throw new IllegalArgumentException(
144 "Empty or null key in userMetadata");
146 String value
= options
.userMetadata
.get(key
);
147 if (value
== null || value
.isEmpty()) {
148 throw new IllegalArgumentException(
149 "Empty or null value in userMetadata for key: " + key
);
151 params
.put(GS_USER_METADATA_PREFIX
+ key
, value
);
154 AppEngineFile file
= new AppEngineFile(
155 create(FILESYSTEM_GS
, options
.fileName
, ContentType
.RAW
, params
));
156 if (!file
.getNamePart().startsWith(GS_CREATION_HANDLE_PREFIX
)) {
157 throw new RuntimeException("Expected creation handle: " + file
.getFullPath());
166 public FileWriteChannel
openWriteChannel(AppEngineFile file
, boolean lock
)
167 throws FileNotFoundException
, FinalizationException
, LockException
, IOException
{
168 FileWriteChannel channel
= new FileWriteChannelImpl(file
, lock
, this);
169 openForAppend(file
, lock
);
174 * Open the given file for append and optionally lock it.
176 * @param file the file to open
177 * @param lock should the file be locked for exclusive access?
178 * @throws FileNotFoundException if the file does not exist in the File Proxy
179 * @throws FinalizationException if the file has already been finalized. The
180 * file may have been finalized by another request.
181 * @throws LockException if the file is locked in a different App Engine
182 * request, or if {@code lock = true} and the file is opened in a
183 * different App Engine request
184 * @throws IOException if any other unexpected problem occurs
186 void openForAppend(AppEngineFile file
, boolean lock
)
187 throws FileNotFoundException
, FinalizationException
, LockException
, IOException
{
188 openForAppend(file
.getFullPath(), ContentType
.RAW
, lock
);
195 public FileReadChannel
openReadChannel(AppEngineFile file
, boolean lock
)
196 throws FileNotFoundException
, LockException
, IOException
{
197 FileReadChannel channel
= new FileReadChannelImpl(file
, this);
198 openForRead(file
, lock
);
206 public RecordReadChannel
openRecordReadChannel(AppEngineFile file
, boolean lock
)
207 throws FileNotFoundException
, LockException
, IOException
{
208 FileReadChannel fileReadChannel
= new BufferedFileReadChannelImpl(
209 openReadChannel(file
, lock
), RecordConstants
.BLOCK_SIZE
* 2);
210 RecordReadChannel channel
= new RecordReadChannelImpl(fileReadChannel
);
218 public RecordWriteChannel
openRecordWriteChannel(AppEngineFile file
, boolean lock
)
219 throws FileNotFoundException
, LockException
, IOException
{
220 RecordWriteChannel channel
= new RecordWriteChannelImpl(openWriteChannel(file
, lock
));
225 public void delete(AppEngineFile
... files
) throws IOException
{
226 Preconditions
.checkNotNull(files
, "No file given");
228 if (files
.length
== 0) {
232 ArrayList
<BlobKey
> blobKeys
= new ArrayList
<BlobKey
>();
234 for (int i
= 0; i
< files
.length
; i
++) {
235 AppEngineFile file
= files
[i
];
236 Preconditions
.checkNotNull(file
, String
.format("File at index %d is null", i
));
238 if (!file
.hasFinalizedName()) {
239 throw new UnsupportedOperationException(
240 String
.format("File %s does not have a finalized name", file
.getFullPath()));
243 if (file
.getFileSystem().equals((AppEngineFile
.FileSystem
.BLOBSTORE
))) {
244 BlobKey blobKey
= getBlobKey(file
);
245 if (blobKey
!= null) {
246 blobKeys
.add(blobKey
);
248 } else if (file
.getFileSystem().equals((AppEngineFile
.FileSystem
.GS
))) {
249 blobKeys
.add(blobstoreService
.createGsBlobKey(file
.getFullPath()));
251 throw new UnsupportedOperationException(
252 String
.format("File at index %d not supported by delete"));
256 if (blobKeys
.size() != 0) {
258 blobstoreService
.delete(blobKeys
.toArray(new BlobKey
[blobKeys
.size()]));
259 } catch (BlobstoreFailureException e
) {
260 throw new IOException("Blobstore failure", e
);
266 * Appends bytes from the given buffer to the end of the given file.
268 * @param file the file to which to append bytes. Must be opened for append in
269 * the current request
270 * @param buffer The buffer from which bytes are to be retrieved
271 * @param sequenceKey the sequence key. See the explanation of the {@code
272 * sequenceKey} paramater at
273 * {@link FileWriteChannel#write(ByteBuffer, String)}
274 * @throws IllegalArgumentException if {@code file} is not writable
275 * @throws KeyOrderingException if {@code sequenceKey} is not {@code null} and
276 * the backend system already has recorded a last good sequence key
277 * for this file and {@code sequenceKey} is not strictly
278 * lexicographically greater than the last good sequence key
279 * @throws IOException if the file is not opened for append in the current App
280 * Engine request or any other unexpected problem occurs
282 int append(AppEngineFile file
, ByteBuffer buffer
, String sequenceKey
) throws IOException
{
283 if (null == buffer
) {
284 throw new NullPointerException("buffer is null");
287 throw new NullPointerException("file is null");
289 ByteString data
= ByteString
.copyFrom(buffer
);
290 append(file
.getFullPath(), data
, sequenceKey
);
294 static final String BLOB_FILE_INDEX_KIND
= "__BlobFileIndex__";
300 public BlobKey
getBlobKey(AppEngineFile file
) {
302 throw new NullPointerException("file is null");
304 if (file
.getFileSystem() != AppEngineFile
.FileSystem
.BLOBSTORE
) {
305 throw new IllegalArgumentException("file is not of type BLOBSTORE");
307 BlobKey cached
= file
.getCachedBlobKey();
308 if (null != cached
) {
311 String namePart
= file
.getNamePart();
312 String creationHandle
= (namePart
.startsWith(CREATION_HANDLE_PREFIX
) ? namePart
: null);
314 if (null == creationHandle
) {
315 return new BlobKey(namePart
);
318 String origNamespace
= NamespaceManager
.get();
320 Entity blobInfoEntity
;
322 NamespaceManager
.set("");
324 Entity blobFileIndexEntity
=
325 datastoreService
.get(null, KeyFactory
.createKey(BLOB_FILE_INDEX_KIND
, getBlobFileIndexKeyName(creationHandle
)));
326 String blobKey
= (String
) blobFileIndexEntity
.getProperty("blob_key");
327 blobInfoEntity
= datastoreService
.get(null, KeyFactory
.createKey(BlobInfoFactory
.KIND
, blobKey
));
328 } catch (EntityNotFoundException ex
) {
329 query
= new Query(BlobInfoFactory
.KIND
);
330 query
.addFilter(BLOB_INFO_CREATION_HANDLE_PROPERTY
, Query
.FilterOperator
.EQUAL
,
332 blobInfoEntity
= datastoreService
.prepare(query
).asSingleEntity();
335 NamespaceManager
.set(origNamespace
);
338 if (null == blobInfoEntity
) {
341 BlobInfo blobInfo
= new BlobInfoFactory(datastoreService
).createBlobInfo(blobInfoEntity
);
342 return blobInfo
.getBlobKey();
345 private static String
getBlobFileIndexKeyName(String creationHandle
) {
346 if (creationHandle
.length() < 500) {
347 return creationHandle
;
350 return Hashing
.sha512().hashString(creationHandle
, Charsets
.US_ASCII
).toString();
357 public AppEngineFile
getBlobFile(BlobKey blobKey
) {
358 if (null == blobKey
) {
359 throw new NullPointerException("blobKey is null");
361 String namePart
= blobKey
.getKeyString();
362 AppEngineFile file
= new AppEngineFile(AppEngineFile
.FileSystem
.BLOBSTORE
, namePart
);
363 file
.setCachedBlobKey(blobKey
);
371 public FileStat
stat(AppEngineFile file
) throws IOException
{
372 Preconditions
.checkNotNull(file
, "file is null");
374 StatRequest
.Builder statRequestBuilder
= StatRequest
.newBuilder();
375 statRequestBuilder
.setFilename(file
.getFullPath());
376 StatResponse
.Builder statResponseBuilder
= StatResponse
.newBuilder();
377 openForRead(file
, false);
379 makeSyncCall("Stat", statRequestBuilder
, statResponseBuilder
);
384 if (statResponseBuilder
.getStatCount() != 1) {
385 throw new IllegalStateException(
386 "Requested stat for one file. Got zero or more than one response.");
388 FileServicePb
.FileStat fileStatPb
= statResponseBuilder
.build().getStat(0);
389 FileStat fileStat
= new FileStat();
390 fileStat
.setFilename(fileStatPb
.getFilename());
391 fileStat
.setFinalized(fileStatPb
.getFinalized());
392 fileStat
.setLength(fileStatPb
.getLength());
393 if (fileStatPb
.hasCtime()) {
394 fileStat
.setCtime(fileStatPb
.getCtime());
396 if (fileStatPb
.hasMtime()) {
397 fileStat
.setMtime(fileStatPb
.getMtime());
403 * Reads bytes from {@code file} starting from {@code startingPos} and puts
404 * the bytes into the {@code buffer}. Returns the number of bytes read. The
405 * number of bytes read will be the minumum of the number of bytes available
406 * in the file and the buffer's {@link ByteBuffer#remaining() free bytes}.
408 * @param file the file from which to read bytes. Must be opened for read in
409 * the current request
410 * @param buffer the destination buffer
411 * @return the number of bytes read
412 * @throws IOException if the file is not opened for read in the current App
413 * Engine request or any other unexpected problem occurs
415 int read(AppEngineFile file
, ByteBuffer buffer
, long startingPos
) throws IOException
{
416 if (startingPos
< 0) {
417 throw new IllegalArgumentException("startingPos is negative: " + startingPos
);
419 if (buffer
== null) {
420 throw new NullPointerException("buffer is null");
422 long remaining
= buffer
.remaining();
423 if (buffer
.remaining() < 1) {
426 ByteString byteString
= read(file
.getFullPath(), startingPos
, remaining
);
427 byteString
.copyTo(buffer
);
428 int numBytesRead
= byteString
.size();
429 if (numBytesRead
<= 0) {
436 * Change the state of the given file to closed and optionally finalize the
437 * file. After the file is finalized it may be read, and it may no longer be
440 * @param file the file to close and optionally finalize. The file must be
441 * opened in the current request.
442 * @param finalize should the file be finalized? The file may only be
443 * finalized if the current request holds the lock for the file
444 * @throws IllegalStateException if {@code finalize = true} and the current
445 * request does not hold the exclusive lock on {@code file}
446 * @throws IOException if the file is not opened in the current request, if
447 * {@code finalize = true} and the file is already finalized or if any
448 * other unexpected problem occurs
450 void close(AppEngineFile file
, boolean finalize
) throws IOException
{
452 close(file
.getFullPath(), finalize
);
453 } catch (LockException e
) {
455 throw new IllegalStateException("The current request does not hold the exclusive lock.");
462 * Opens a file for appending by making the "Open" RPC call with mode=APPEND.
464 private void openForAppend(String fileName
, ContentType contentType
, boolean lock
)
466 open(fileName
, contentType
, OpenMode
.APPEND
, lock
);
469 private void openForRead(AppEngineFile file
, boolean lock
)
470 throws FileNotFoundException
, LockException
, IOException
{
472 throw new NullPointerException("file is null");
474 openForRead(file
.getFullPath(), ContentType
.RAW
, lock
);
478 * Opens a file for reading by making the "Open" RPC call with mode=READ
480 private void openForRead(String fileName
, ContentType contentType
, boolean lock
)
482 open(fileName
, contentType
, OpenMode
.READ
, lock
);
486 * Makes the "Create" RPC call.
488 * @return created file name.
490 private String
create(
491 String fileSystem
, String fileName
, ContentType contentType
, Map
<String
, String
> parameters
)
493 CreateRequest
.Builder request
= CreateRequest
.newBuilder();
494 request
.setFilesystem(fileSystem
);
495 if (fileName
!= null && !fileName
.isEmpty()) {
496 request
.setFilename(fileName
);
498 request
.setContentType(contentType
);
499 if (parameters
!= null) {
500 for (Map
.Entry
<String
, String
> e
: parameters
.entrySet()) {
501 CreateRequest
.Parameter
.Builder parameter
= request
.addParametersBuilder();
502 parameter
.setName(e
.getKey());
503 parameter
.setValue(e
.getValue());
506 CreateResponse
.Builder response
= CreateResponse
.newBuilder();
507 makeSyncCall("Create", request
, response
);
508 return response
.build().getFilename();
512 * Makes the "Open" RPC call
514 private void open(String fileName
, ContentType contentType
, OpenMode openMode
, boolean lock
)
516 OpenRequest
.Builder openRequest
= OpenRequest
.newBuilder();
517 openRequest
.setFilename(fileName
);
518 openRequest
.setContentType(contentType
);
519 openRequest
.setOpenMode(openMode
);
520 openRequest
.setExclusiveLock(lock
);
521 OpenResponse
.Builder openResponse
= OpenResponse
.newBuilder();
522 makeSyncCall("Open", openRequest
, openResponse
);
526 * Makes the 'Append' RPC call
528 private void append(String fileName
, ByteString data
, String sequenceKey
) throws IOException
{
529 AppendRequest
.Builder appendRequest
= AppendRequest
.newBuilder();
530 appendRequest
.setFilename(fileName
);
531 appendRequest
.setData(data
);
532 if (null != sequenceKey
) {
533 appendRequest
.setSequenceKey(sequenceKey
);
535 AppendResponse
.Builder appendResponse
= AppendResponse
.newBuilder();
536 makeSyncCall("Append", appendRequest
, appendResponse
);
540 * Makes the "Read" RPC call
542 private ByteString
read(String fileName
, long pos
, long maxBytes
) throws IOException
{
543 ReadRequest
.Builder readRequest
= ReadRequest
.newBuilder();
544 readRequest
.setFilename(fileName
);
545 readRequest
.setMaxBytes(maxBytes
);
546 readRequest
.setPos(pos
);
547 ReadResponse
.Builder readResponse
= ReadResponse
.newBuilder();
548 makeSyncCall("Read", readRequest
, readResponse
);
549 return readResponse
.build().getData();
553 * Makes the "Close" RPC call
555 private void close(String fileName
, boolean finalize
) throws IOException
{
556 CloseRequest
.Builder closeRequest
= CloseRequest
.newBuilder();
557 closeRequest
.setFilename(fileName
);
558 closeRequest
.setFinalize(finalize
);
559 CloseResponse
.Builder closeResponse
= CloseResponse
.newBuilder();
560 makeSyncCall("Close", closeRequest
, closeResponse
);
564 * Makes the "GetDefaultGSBucketName" RPC call.
567 public String
getDefaultGsBucketName() throws IOException
{
568 GetDefaultGsBucketNameRequest
.Builder request
= GetDefaultGsBucketNameRequest
.newBuilder();
569 GetDefaultGsBucketNameResponse
.Builder response
= GetDefaultGsBucketNameResponse
.newBuilder();
570 makeSyncCall("GetDefaultGsBucketName", request
, response
);
571 return response
.getDefaultGsBucketName();
575 * Makes a synchronous RPC call to the app server
580 * @throws IOException
582 private void makeSyncCall(String callName
, Message
.Builder request
, Message
.Builder response
)
585 byte[] responseBytes
=
586 ApiProxy
.makeSyncCall(PACKAGE
, callName
, request
.build().toByteArray());
587 response
.mergeFrom(responseBytes
);
588 } catch (ApiProxy
.ApplicationException ex
) {
589 throw translateException(ex
, null);
590 } catch (InvalidProtocolBufferException e
) {
591 throw new RuntimeException("Internal logic error: Response PB could not be parsed.", e
);
596 * Translates from an internal to a public exception
598 private static IOException
translateException(ApiProxy
.ApplicationException ex
, String message
) {
599 int errorCode
= ex
.getApplicationError();
600 FileServiceErrors
.ErrorCode errorCodeEnum
= FileServiceErrors
.ErrorCode
.valueOf(errorCode
);
601 switch (errorCodeEnum
) {
602 case EXCLUSIVE_LOCK_FAILED
:
603 return new LockException(message
, ex
);
604 case EXISTENCE_ERROR
:
605 case EXISTENCE_ERROR_METADATA_NOT_FOUND
:
606 case EXISTENCE_ERROR_METADATA_FOUND
:
607 case EXISTENCE_ERROR_SHARDING_MISMATCH
:
608 case EXISTENCE_ERROR_BUCKET_NOT_FOUND
:
609 case EXISTENCE_ERROR_OBJECT_NOT_FOUND
:
610 return new FileNotFoundException();
611 case FINALIZATION_ERROR
:
612 return new FinalizationException(message
, ex
);
613 case SEQUENCE_KEY_OUT_OF_ORDER
:
614 return new KeyOrderingException(message
, ex
);
616 return new IOException(message
, ex
);