1 // Copyright 2010 Google Inc. All Rights Reserved.
3 package com
.google
.appengine
.api
.files
;
5 import com
.google
.appengine
.api
.NamespaceManager
;
6 import com
.google
.appengine
.api
.blobstore
.BlobInfo
;
7 import com
.google
.appengine
.api
.blobstore
.BlobInfoFactory
;
8 import com
.google
.appengine
.api
.blobstore
.BlobKey
;
9 import com
.google
.appengine
.api
.datastore
.DatastoreService
;
10 import com
.google
.appengine
.api
.datastore
.DatastoreServiceFactory
;
11 import com
.google
.appengine
.api
.datastore
.Entity
;
12 import com
.google
.appengine
.api
.datastore
.EntityNotFoundException
;
13 import com
.google
.appengine
.api
.datastore
.KeyFactory
;
14 import com
.google
.appengine
.api
.datastore
.Query
;
15 import com
.google
.appengine
.api
.files
.FileServicePb
.AppendRequest
;
16 import com
.google
.appengine
.api
.files
.FileServicePb
.AppendResponse
;
17 import com
.google
.appengine
.api
.files
.FileServicePb
.CloseRequest
;
18 import com
.google
.appengine
.api
.files
.FileServicePb
.CloseResponse
;
19 import com
.google
.appengine
.api
.files
.FileServicePb
.CreateRequest
;
20 import com
.google
.appengine
.api
.files
.FileServicePb
.CreateResponse
;
21 import com
.google
.appengine
.api
.files
.FileServicePb
.DeleteRequest
;
22 import com
.google
.appengine
.api
.files
.FileServicePb
.DeleteResponse
;
23 import com
.google
.appengine
.api
.files
.FileServicePb
.StatRequest
;
24 import com
.google
.appengine
.api
.files
.FileServicePb
.StatResponse
;
25 import com
.google
.appengine
.api
.files
.FileServicePb
.FileContentType
.ContentType
;
26 import com
.google
.appengine
.api
.files
.FileServicePb
.FileServiceErrors
;
27 import com
.google
.appengine
.api
.files
.FileServicePb
.GetDefaultGsBucketNameRequest
;
28 import com
.google
.appengine
.api
.files
.FileServicePb
.GetDefaultGsBucketNameResponse
;
29 import com
.google
.appengine
.api
.files
.FileServicePb
.OpenRequest
;
30 import com
.google
.appengine
.api
.files
.FileServicePb
.OpenRequest
.OpenMode
;
31 import com
.google
.appengine
.api
.files
.FileServicePb
.OpenResponse
;
32 import com
.google
.appengine
.api
.files
.FileServicePb
.ReadRequest
;
33 import com
.google
.appengine
.api
.files
.FileServicePb
.ReadResponse
;
34 import com
.google
.apphosting
.api
.ApiProxy
;
35 import com
.google
.common
.base
.Preconditions
;
36 import com
.google
.protobuf
.ByteString
;
37 import com
.google
.protobuf
.InvalidProtocolBufferException
;
38 import com
.google
.protobuf
.Message
;
40 import java
.io
.FileNotFoundException
;
41 import java
.io
.IOException
;
42 import java
.nio
.ByteBuffer
;
44 import java
.util
.TreeMap
;
47 * Implements {@link FileService} by using {@link ApiProxy} to make RPC calls to
48 * the App Engine File API.
51 class FileServiceImpl
implements FileService
{
53 static final String PACKAGE
= "file";
55 static final String FILESYSTEM_BLOBSTORE
= AppEngineFile
.FileSystem
.BLOBSTORE
.getName();
56 static final String PARAMETER_MIME_TYPE
= "content_type";
57 static final String PARAMETER_BLOB_INFO_UPLOADED_FILE_NAME
= "file_name";
58 static final String DEFAULT_MIME_TYPE
= "application/octet-stream";
59 static final String FILESYSTEM_GS
= AppEngineFile
.FileSystem
.GS
.getName();
60 static final String GS_FILESYSTEM_PREFIX
= "/gs/";
61 static final String GS_PARAMETER_MIME_TYPE
= "content_type";
62 static final String GS_PARAMETER_CANNED_ACL
= "acl";
63 static final String GS_PARAMETER_CONTENT_ENCODING
= "content_encoding";
64 static final String GS_PARAMETER_CONTENT_DISPOSITION
= "content_disposition";
65 static final String GS_PARAMETER_CACHE_CONTROL
= "cache_control";
66 static final String GS_USER_METADATA_PREFIX
= "x-goog-meta-";
68 static final String GS_DEFAULT_MIME_TYPE
= "application/octet-stream";
70 private static final String BLOB_INFO_CREATION_HANDLE_PROPERTY
= "creation_handle";
71 static final String GS_CREATION_HANDLE_PREFIX
= "writable:";
72 static final String CREATION_HANDLE_PREFIX
= "writable:";
78 public AppEngineFile
createNewBlobFile(String mimeType
) throws IOException
{
79 return createNewBlobFile(mimeType
, "");
86 public AppEngineFile
createNewBlobFile(String mimeType
, String blobInfoUploadedFileName
)
88 if (mimeType
== null || mimeType
.trim().isEmpty()) {
89 mimeType
= DEFAULT_MIME_TYPE
;
92 Map
<String
, String
> params
= new TreeMap
<String
, String
>();
93 params
.put(PARAMETER_MIME_TYPE
, mimeType
);
94 if (blobInfoUploadedFileName
!= null && !blobInfoUploadedFileName
.isEmpty()) {
95 params
.put(PARAMETER_BLOB_INFO_UPLOADED_FILE_NAME
, blobInfoUploadedFileName
);
97 String filePath
= create(FILESYSTEM_BLOBSTORE
, null, ContentType
.RAW
, params
);
98 AppEngineFile file
= new AppEngineFile(filePath
);
99 if (!file
.getNamePart().startsWith(CREATION_HANDLE_PREFIX
)) {
100 throw new RuntimeException("Expected creation handle: " + file
.getFullPath());
109 public AppEngineFile
createNewGSFile(final GSFileOptions options
) throws IOException
{
110 if (options
.fileName
== null || options
.fileName
.isEmpty() ||
111 !options
.fileName
.startsWith(GS_FILESYSTEM_PREFIX
)) {
112 throw new IllegalArgumentException("Invalid fileName, should be of the form: /gs/bucket/key");
114 Map
<String
, String
> params
= new TreeMap
<String
, String
>();
115 params
.put(GS_PARAMETER_MIME_TYPE
, options
.mimeType
);
116 if (options
.acl
!= null && !options
.acl
.trim().isEmpty()) {
117 params
.put(GS_PARAMETER_CANNED_ACL
, options
.acl
);
119 if (options
.cacheControl
!= null && !options
.cacheControl
.trim().isEmpty()) {
120 params
.put(GS_PARAMETER_CACHE_CONTROL
, options
.cacheControl
);
122 if (options
.contentEncoding
!= null && !options
.contentEncoding
.trim().isEmpty()) {
123 params
.put(GS_PARAMETER_CONTENT_ENCODING
, options
.contentEncoding
);
125 if (options
.contentDisposition
!= null && !options
.contentDisposition
.trim().isEmpty()) {
126 params
.put(GS_PARAMETER_CONTENT_DISPOSITION
, options
.contentDisposition
);
128 if (options
.userMetadata
!= null) {
129 for (String key
: options
.userMetadata
.keySet()) {
130 if (key
== null || key
.isEmpty()) {
131 throw new IllegalArgumentException(
132 "Empty or null key in userMetadata");
134 String value
= options
.userMetadata
.get(key
);
135 if (value
== null || value
.isEmpty()) {
136 throw new IllegalArgumentException(
137 "Empty or null value in userMetadata for key: " + key
);
139 params
.put(GS_USER_METADATA_PREFIX
+ key
, value
);
142 AppEngineFile file
= new AppEngineFile(
143 create(FILESYSTEM_GS
, options
.fileName
, ContentType
.RAW
, params
));
144 if (!file
.getNamePart().startsWith(GS_CREATION_HANDLE_PREFIX
)) {
145 throw new RuntimeException("Expected creation handle: " + file
.getFullPath());
154 public FileWriteChannel
openWriteChannel(AppEngineFile file
, boolean lock
)
155 throws FileNotFoundException
, FinalizationException
, LockException
, IOException
{
156 FileWriteChannel channel
= new FileWriteChannelImpl(file
, lock
, this);
157 openForAppend(file
, lock
);
162 * Open the given file for append and optionally lock it.
164 * @param file the file to open
165 * @param lock should the file be locked for exclusive access?
166 * @throws FileNotFoundException if the file does not exist in the File Proxy
167 * @throws FinalizationException if the file has already been finalized. The
168 * file may have been finalized by another request.
169 * @throws LockException if the file is locked in a different App Engine
170 * request, or if {@code lock = true} and the file is opened in a
171 * different App Engine request
172 * @throws IOException if any other unexpected problem occurs
174 void openForAppend(AppEngineFile file
, boolean lock
)
175 throws FileNotFoundException
, FinalizationException
, LockException
, IOException
{
176 openForAppend(file
.getFullPath(), ContentType
.RAW
, lock
);
183 public FileReadChannel
openReadChannel(AppEngineFile file
, boolean lock
)
184 throws FileNotFoundException
, LockException
, IOException
{
185 FileReadChannel channel
= new FileReadChannelImpl(file
, this);
186 openForRead(file
, lock
);
194 public RecordReadChannel
openRecordReadChannel(AppEngineFile file
, boolean lock
)
195 throws FileNotFoundException
, LockException
, IOException
{
196 RecordReadChannel channel
= new RecordReadChannelImpl(openReadChannel(file
, lock
));
204 public RecordWriteChannel
openRecordWriteChannel(AppEngineFile file
, boolean lock
)
205 throws FileNotFoundException
, LockException
, IOException
{
206 RecordWriteChannel channel
= new RecordWriteChannelImpl(openWriteChannel(file
, lock
));
210 public void delete(AppEngineFile file
) throws IOException
{
211 delete(file
.getFullPath());
215 * Appends bytes from the given buffer to the end of the given file.
217 * @param file the file to which to append bytes. Must be opened for append in
218 * the current request
219 * @param buffer The buffer from which bytes are to be retrieved
220 * @param sequenceKey the sequence key. See the explanation of the {@code
221 * sequenceKey} paramater at
222 * {@link FileWriteChannel#write(ByteBuffer, String)}
223 * @throws IllegalArgumentException if {@code file} is not writable
224 * @throws KeyOrderingException if {@code sequenceKey} is not {@code null} and
225 * the backend system already has recorded a last good sequence key
226 * for this file and {@code sequenceKey} is not strictly
227 * lexicographically greater than the last good sequence key
228 * @throws IOException if the file is not opened for append in the current App
229 * Engine request or any other unexpected problem occurs
231 int append(AppEngineFile file
, ByteBuffer buffer
, String sequenceKey
) throws IOException
{
232 if (null == buffer
) {
233 throw new NullPointerException("buffer is null");
236 throw new NullPointerException("file is null");
238 ByteString data
= ByteString
.copyFrom(buffer
);
239 append(file
.getFullPath(), data
, sequenceKey
);
243 private static final String BLOB_FILE_INDEX_KIND
= "__BlobFileIndex__";
245 private static final String BLOB_KEY_PROPERTY_NAME
= "blob_key";
251 public BlobKey
getBlobKey(AppEngineFile file
) {
253 throw new NullPointerException("file is null");
255 if (file
.getFileSystem() != AppEngineFile
.FileSystem
.BLOBSTORE
) {
256 throw new IllegalArgumentException("file is not of type BLOBSTORE");
258 BlobKey cached
= file
.getCachedBlobKey();
259 if (null != cached
) {
262 String namePart
= file
.getNamePart();
263 String creationHandle
= (namePart
.startsWith(CREATION_HANDLE_PREFIX
) ? namePart
: null);
265 if (null == creationHandle
) {
266 return new BlobKey(namePart
);
269 DatastoreService datastore
= DatastoreServiceFactory
.getDatastoreService();
270 String origNamespace
= NamespaceManager
.get();
272 Entity blobInfoEntity
;
274 NamespaceManager
.set("");
276 Entity blobFileIndexEntity
=
277 datastore
.get(null, KeyFactory
.createKey(BLOB_FILE_INDEX_KIND
, creationHandle
));
278 String blobKey
= (String
) blobFileIndexEntity
.getProperty("blob_key");
279 blobInfoEntity
= datastore
.get(null, KeyFactory
.createKey(BlobInfoFactory
.KIND
, blobKey
));
280 } catch (EntityNotFoundException ex
) {
281 query
= new Query(BlobInfoFactory
.KIND
);
282 query
.addFilter(BLOB_INFO_CREATION_HANDLE_PROPERTY
, Query
.FilterOperator
.EQUAL
,
284 blobInfoEntity
= datastore
.prepare(query
).asSingleEntity();
287 NamespaceManager
.set(origNamespace
);
290 if (null == blobInfoEntity
) {
293 BlobInfo blobInfo
= new BlobInfoFactory().createBlobInfo(blobInfoEntity
);
294 return blobInfo
.getBlobKey();
301 public AppEngineFile
getBlobFile(BlobKey blobKey
) {
302 if (null == blobKey
) {
303 throw new NullPointerException("blobKey is null");
305 String namePart
= blobKey
.getKeyString();
306 AppEngineFile file
= new AppEngineFile(AppEngineFile
.FileSystem
.BLOBSTORE
, namePart
);
307 file
.setCachedBlobKey(blobKey
);
315 public FileStat
stat(AppEngineFile file
) throws IOException
{
316 Preconditions
.checkNotNull(file
, "file is null");
318 StatRequest
.Builder statRequestBuilder
= StatRequest
.newBuilder();
319 statRequestBuilder
.setFilename(file
.getFullPath());
320 StatResponse
.Builder statResponseBuilder
= StatResponse
.newBuilder();
321 openForRead(file
, false);
323 makeSyncCall("Stat", statRequestBuilder
, statResponseBuilder
);
328 if (statResponseBuilder
.getStatCount() != 1) {
329 throw new IllegalStateException(
330 "Requested stat for one file. Got zero or more than one response.");
332 FileServicePb
.FileStat fileStatPb
= statResponseBuilder
.build().getStat(0);
333 FileStat fileStat
= new FileStat();
334 fileStat
.setFilename(fileStatPb
.getFilename());
335 fileStat
.setFinalized(fileStatPb
.getFinalized());
336 fileStat
.setLength(fileStatPb
.getLength());
337 if (fileStatPb
.hasCtime()) {
338 fileStat
.setCtime(fileStatPb
.getCtime());
340 if (fileStatPb
.hasMtime()) {
341 fileStat
.setMtime(fileStatPb
.getMtime());
347 * Reads bytes from {@code file} starting from {@code startingPos} and puts
348 * the bytes into the {@code buffer}. Returns the number of bytes read. The
349 * number of bytes read will be the minumum of the number of bytes available
350 * in the file and the buffer's {@link ByteBuffer#remaining() free bytes}.
352 * @param file the file from which to read bytes. Must be opened for read in
353 * the current request
354 * @param buffer the destination buffer
355 * @return the number of bytes read
356 * @throws IOException if the file is not opened for read in the current App
357 * Engine request or any other unexpected problem occurs
359 int read(AppEngineFile file
, ByteBuffer buffer
, long startingPos
) throws IOException
{
360 if (startingPos
< 0) {
361 throw new IllegalArgumentException("startingPos is negative: " + startingPos
);
363 if (buffer
== null) {
364 throw new NullPointerException("buffer is null");
366 long remaining
= buffer
.remaining();
367 if (buffer
.remaining() < 1) {
370 ByteString byteString
= read(file
.getFullPath(), startingPos
, remaining
);
371 byteString
.copyTo(buffer
);
372 int numBytesRead
= byteString
.size();
373 if (numBytesRead
<= 0) {
380 * Change the state of the given file to closed and optionally finalize the
381 * file. After the file is finalized it may be read, and it may no longer be
384 * @param file the file to close and optionally finalize. The file must be
385 * opened in the current request.
386 * @param finalize should the file be finalized? The file may only be
387 * finalized if the current request holds the lock for the file
388 * @throws IllegalStateException if {@code finalize = true} and the current
389 * request does not hold the exclusive lock on {@code file}
390 * @throws IOException if the file is not opened in the current request, if
391 * {@code finalize = true} and the file is already finalized or if any
392 * other unexpected problem occurs
394 void close(AppEngineFile file
, boolean finalize
) throws IOException
{
396 close(file
.getFullPath(), finalize
);
397 } catch (LockException e
) {
399 throw new IllegalStateException("The current request does not hold the exclusive lock.");
406 * Opens a file for appending by making the "Open" RPC call with mode=APPEND.
408 private void openForAppend(String fileName
, ContentType contentType
, boolean lock
)
410 open(fileName
, contentType
, OpenMode
.APPEND
, lock
);
413 private void openForRead(AppEngineFile file
, boolean lock
)
414 throws FileNotFoundException
, LockException
, IOException
{
416 throw new NullPointerException("file is null");
418 openForRead(file
.getFullPath(), ContentType
.RAW
, lock
);
422 * Opens a file for reading by making the "Open" RPC call with mode=READ
424 private void openForRead(String fileName
, ContentType contentType
, boolean lock
)
426 open(fileName
, contentType
, OpenMode
.READ
, lock
);
430 * Makes the "Create" RPC call.
432 * @return created file name.
434 private String
create(
435 String fileSystem
, String fileName
, ContentType contentType
, Map
<String
, String
> parameters
)
437 CreateRequest
.Builder request
= CreateRequest
.newBuilder();
438 request
.setFilesystem(fileSystem
);
439 if (fileName
!= null && !fileName
.isEmpty()) {
440 request
.setFilename(fileName
);
442 request
.setContentType(contentType
);
443 if (parameters
!= null) {
444 for (Map
.Entry
<String
, String
> e
: parameters
.entrySet()) {
445 CreateRequest
.Parameter
.Builder parameter
= request
.addParametersBuilder();
446 parameter
.setName(e
.getKey());
447 parameter
.setValue(e
.getValue());
450 CreateResponse
.Builder response
= CreateResponse
.newBuilder();
451 makeSyncCall("Create", request
, response
);
452 return response
.build().getFilename();
456 * Makes the "Open" RPC call
458 private void open(String fileName
, ContentType contentType
, OpenMode openMode
, boolean lock
)
460 OpenRequest
.Builder openRequest
= OpenRequest
.newBuilder();
461 openRequest
.setFilename(fileName
);
462 openRequest
.setContentType(contentType
);
463 openRequest
.setOpenMode(openMode
);
464 openRequest
.setExclusiveLock(lock
);
465 OpenResponse
.Builder openResponse
= OpenResponse
.newBuilder();
466 makeSyncCall("Open", openRequest
, openResponse
);
470 * Makes the 'Append' RPC call
472 private void append(String fileName
, ByteString data
, String sequenceKey
) throws IOException
{
473 AppendRequest
.Builder appendRequest
= AppendRequest
.newBuilder();
474 appendRequest
.setFilename(fileName
);
475 appendRequest
.setData(data
);
476 if (null != sequenceKey
) {
477 appendRequest
.setSequenceKey(sequenceKey
);
479 AppendResponse
.Builder appendResponse
= AppendResponse
.newBuilder();
480 makeSyncCall("Append", appendRequest
, appendResponse
);
484 * Makes the "Read" RPC call
486 private ByteString
read(String fileName
, long pos
, long maxBytes
) throws IOException
{
487 ReadRequest
.Builder readRequest
= ReadRequest
.newBuilder();
488 readRequest
.setFilename(fileName
);
489 readRequest
.setMaxBytes(maxBytes
);
490 readRequest
.setPos(pos
);
491 ReadResponse
.Builder readResponse
= ReadResponse
.newBuilder();
492 makeSyncCall("Read", readRequest
, readResponse
);
493 return readResponse
.build().getData();
497 * Makes the "Close" RPC call
499 private void close(String fileName
, boolean finalize
) throws IOException
{
500 CloseRequest
.Builder closeRequest
= CloseRequest
.newBuilder();
501 closeRequest
.setFilename(fileName
);
502 closeRequest
.setFinalize(finalize
);
503 CloseResponse
.Builder closeResponse
= CloseResponse
.newBuilder();
504 makeSyncCall("Close", closeRequest
, closeResponse
);
508 * Makes the "Delete" RPC call
510 private void delete(String fileName
) throws IOException
{
511 DeleteRequest
.Builder deleteRequest
= DeleteRequest
.newBuilder();
512 deleteRequest
.setFilename(fileName
);
513 DeleteResponse
.Builder deleteResponse
= DeleteResponse
.newBuilder();
514 makeSyncCall("Delete", deleteRequest
, deleteResponse
);
518 * Makes the "GetDefaultGSBucketName" RPC call.
521 public String
getDefaultGsBucketName() throws IOException
{
522 GetDefaultGsBucketNameRequest
.Builder request
= GetDefaultGsBucketNameRequest
.newBuilder();
523 GetDefaultGsBucketNameResponse
.Builder response
= GetDefaultGsBucketNameResponse
.newBuilder();
524 makeSyncCall("GetDefaultGsBucketName", request
, response
);
525 return response
.getDefaultGsBucketName();
529 * Makes a synchronous RPC call to the app server
534 * @throws IOException
536 private void makeSyncCall(String callName
, Message
.Builder request
, Message
.Builder response
)
539 byte[] responseBytes
=
540 ApiProxy
.makeSyncCall(PACKAGE
, callName
, request
.build().toByteArray());
541 response
.mergeFrom(responseBytes
);
542 } catch (ApiProxy
.ApplicationException ex
) {
543 throw translateException(ex
, null);
544 } catch (InvalidProtocolBufferException e
) {
545 throw new RuntimeException("Internal logic error: Response PB could not be parsed.", e
);
550 * Translates from an internal to a public exception
552 private static IOException
translateException(ApiProxy
.ApplicationException ex
, String message
) {
553 int errorCode
= ex
.getApplicationError();
554 FileServiceErrors
.ErrorCode errorCodeEnum
= FileServiceErrors
.ErrorCode
.valueOf(errorCode
);
555 switch (errorCodeEnum
) {
556 case EXCLUSIVE_LOCK_FAILED
:
557 return new LockException(message
, ex
);
558 case EXISTENCE_ERROR
:
559 case EXISTENCE_ERROR_METADATA_NOT_FOUND
:
560 case EXISTENCE_ERROR_METADATA_FOUND
:
561 case EXISTENCE_ERROR_SHARDING_MISMATCH
:
562 case EXISTENCE_ERROR_BUCKET_NOT_FOUND
:
563 case EXISTENCE_ERROR_OBJECT_NOT_FOUND
:
564 return new FileNotFoundException();
565 case FINALIZATION_ERROR
:
566 return new FinalizationException(message
, ex
);
567 case SEQUENCE_KEY_OUT_OF_ORDER
:
568 return new KeyOrderingException(message
, ex
);
570 return new IOException(message
, ex
);