App Engine Java SDK version 1.7.0
[gae.git] / java / src / main / com / google / appengine / api / files / FileServiceImpl.java
blob80cf19393683879a34f94c77a3b0256867b00db0
1 // Copyright 2010 Google Inc. All Rights Reserved.
3 package com.google.appengine.api.files;
5 import com.google.appengine.api.NamespaceManager;
6 import com.google.appengine.api.blobstore.BlobInfo;
7 import com.google.appengine.api.blobstore.BlobInfoFactory;
8 import com.google.appengine.api.blobstore.BlobKey;
9 import com.google.appengine.api.datastore.DatastoreService;
10 import com.google.appengine.api.datastore.DatastoreServiceFactory;
11 import com.google.appengine.api.datastore.Entity;
12 import com.google.appengine.api.datastore.EntityNotFoundException;
13 import com.google.appengine.api.datastore.KeyFactory;
14 import com.google.appengine.api.datastore.Query;
15 import com.google.appengine.api.files.FileServicePb.AppendRequest;
16 import com.google.appengine.api.files.FileServicePb.AppendResponse;
17 import com.google.appengine.api.files.FileServicePb.CloseRequest;
18 import com.google.appengine.api.files.FileServicePb.CloseResponse;
19 import com.google.appengine.api.files.FileServicePb.CreateRequest;
20 import com.google.appengine.api.files.FileServicePb.CreateResponse;
21 import com.google.appengine.api.files.FileServicePb.DeleteRequest;
22 import com.google.appengine.api.files.FileServicePb.DeleteResponse;
23 import com.google.appengine.api.files.FileServicePb.StatRequest;
24 import com.google.appengine.api.files.FileServicePb.StatResponse;
25 import com.google.appengine.api.files.FileServicePb.FileContentType.ContentType;
26 import com.google.appengine.api.files.FileServicePb.FileServiceErrors;
27 import com.google.appengine.api.files.FileServicePb.GetDefaultGsBucketNameRequest;
28 import com.google.appengine.api.files.FileServicePb.GetDefaultGsBucketNameResponse;
29 import com.google.appengine.api.files.FileServicePb.OpenRequest;
30 import com.google.appengine.api.files.FileServicePb.OpenRequest.OpenMode;
31 import com.google.appengine.api.files.FileServicePb.OpenResponse;
32 import com.google.appengine.api.files.FileServicePb.ReadRequest;
33 import com.google.appengine.api.files.FileServicePb.ReadResponse;
34 import com.google.apphosting.api.ApiProxy;
35 import com.google.common.base.Preconditions;
36 import com.google.protobuf.ByteString;
37 import com.google.protobuf.InvalidProtocolBufferException;
38 import com.google.protobuf.Message;
40 import java.io.FileNotFoundException;
41 import java.io.IOException;
42 import java.nio.ByteBuffer;
43 import java.util.Map;
44 import java.util.TreeMap;
46 /**
47 * Implements {@link FileService} by using {@link ApiProxy} to make RPC calls to
48 * the App Engine File API.
51 class FileServiceImpl implements FileService {
53 static final String PACKAGE = "file";
55 static final String FILESYSTEM_BLOBSTORE = AppEngineFile.FileSystem.BLOBSTORE.getName();
56 static final String PARAMETER_MIME_TYPE = "content_type";
57 static final String PARAMETER_BLOB_INFO_UPLOADED_FILE_NAME = "file_name";
58 static final String DEFAULT_MIME_TYPE = "application/octet-stream";
59 static final String FILESYSTEM_GS = AppEngineFile.FileSystem.GS.getName();
60 static final String GS_FILESYSTEM_PREFIX = "/gs/";
61 static final String GS_PARAMETER_MIME_TYPE = "content_type";
62 static final String GS_PARAMETER_CANNED_ACL = "acl";
63 static final String GS_PARAMETER_CONTENT_ENCODING = "content_encoding";
64 static final String GS_PARAMETER_CONTENT_DISPOSITION = "content_disposition";
65 static final String GS_PARAMETER_CACHE_CONTROL = "cache_control";
66 static final String GS_USER_METADATA_PREFIX = "x-goog-meta-";
68 static final String GS_DEFAULT_MIME_TYPE = "application/octet-stream";
70 private static final String BLOB_INFO_CREATION_HANDLE_PROPERTY = "creation_handle";
71 static final String GS_CREATION_HANDLE_PREFIX = "writable:";
72 static final String CREATION_HANDLE_PREFIX = "writable:";
74 /**
75 * {@inheritDoc}
77 @Override
78 public AppEngineFile createNewBlobFile(String mimeType) throws IOException {
79 return createNewBlobFile(mimeType, "");
82 /**
83 * {@inheritDoc}
85 @Override
86 public AppEngineFile createNewBlobFile(String mimeType, String blobInfoUploadedFileName)
87 throws IOException {
88 if (mimeType == null || mimeType.trim().isEmpty()) {
89 mimeType = DEFAULT_MIME_TYPE;
92 Map<String, String> params = new TreeMap<String, String>();
93 params.put(PARAMETER_MIME_TYPE, mimeType);
94 if (blobInfoUploadedFileName != null && !blobInfoUploadedFileName.isEmpty()) {
95 params.put(PARAMETER_BLOB_INFO_UPLOADED_FILE_NAME, blobInfoUploadedFileName);
97 String filePath = create(FILESYSTEM_BLOBSTORE, null, ContentType.RAW, params);
98 AppEngineFile file = new AppEngineFile(filePath);
99 if (!file.getNamePart().startsWith(CREATION_HANDLE_PREFIX)) {
100 throw new RuntimeException("Expected creation handle: " + file.getFullPath());
102 return file;
106 * {@inheritDoc}
108 @Override
109 public AppEngineFile createNewGSFile(final GSFileOptions options) throws IOException {
110 if (options.fileName == null || options.fileName.isEmpty() ||
111 !options.fileName.startsWith(GS_FILESYSTEM_PREFIX)) {
112 throw new IllegalArgumentException("Invalid fileName, should be of the form: /gs/bucket/key");
114 Map<String, String> params = new TreeMap<String, String>();
115 params.put(GS_PARAMETER_MIME_TYPE, options.mimeType);
116 if (options.acl != null && !options.acl.trim().isEmpty()) {
117 params.put(GS_PARAMETER_CANNED_ACL, options.acl);
119 if (options.cacheControl != null && !options.cacheControl.trim().isEmpty()) {
120 params.put(GS_PARAMETER_CACHE_CONTROL, options.cacheControl);
122 if (options.contentEncoding != null && !options.contentEncoding.trim().isEmpty()) {
123 params.put(GS_PARAMETER_CONTENT_ENCODING, options.contentEncoding);
125 if (options.contentDisposition != null && !options.contentDisposition.trim().isEmpty()) {
126 params.put(GS_PARAMETER_CONTENT_DISPOSITION, options.contentDisposition);
128 if (options.userMetadata != null) {
129 for (String key : options.userMetadata.keySet()) {
130 if (key == null || key.isEmpty()) {
131 throw new IllegalArgumentException(
132 "Empty or null key in userMetadata");
134 String value = options.userMetadata.get(key);
135 if (value == null || value.isEmpty()) {
136 throw new IllegalArgumentException(
137 "Empty or null value in userMetadata for key: " + key);
139 params.put(GS_USER_METADATA_PREFIX + key, value);
142 AppEngineFile file = new AppEngineFile(
143 create(FILESYSTEM_GS, options.fileName, ContentType.RAW, params));
144 if (!file.getNamePart().startsWith(GS_CREATION_HANDLE_PREFIX)) {
145 throw new RuntimeException("Expected creation handle: " + file.getFullPath());
147 return file;
151 * {@inheritDoc}
153 @Override
154 public FileWriteChannel openWriteChannel(AppEngineFile file, boolean lock)
155 throws FileNotFoundException, FinalizationException, LockException, IOException {
156 FileWriteChannel channel = new FileWriteChannelImpl(file, lock, this);
157 openForAppend(file, lock);
158 return channel;
162 * Open the given file for append and optionally lock it.
164 * @param file the file to open
165 * @param lock should the file be locked for exclusive access?
166 * @throws FileNotFoundException if the file does not exist in the File Proxy
167 * @throws FinalizationException if the file has already been finalized. The
168 * file may have been finalized by another request.
169 * @throws LockException if the file is locked in a different App Engine
170 * request, or if {@code lock = true} and the file is opened in a
171 * different App Engine request
172 * @throws IOException if any other unexpected problem occurs
174 void openForAppend(AppEngineFile file, boolean lock)
175 throws FileNotFoundException, FinalizationException, LockException, IOException {
176 openForAppend(file.getFullPath(), ContentType.RAW, lock);
180 * {@inheritDoc}
182 @Override
183 public FileReadChannel openReadChannel(AppEngineFile file, boolean lock)
184 throws FileNotFoundException, LockException, IOException {
185 FileReadChannel channel = new FileReadChannelImpl(file, this);
186 openForRead(file, lock);
187 return channel;
191 * {@inheritDoc}
193 @Override
194 public RecordReadChannel openRecordReadChannel(AppEngineFile file, boolean lock)
195 throws FileNotFoundException, LockException, IOException {
196 RecordReadChannel channel = new RecordReadChannelImpl(openReadChannel(file, lock));
197 return channel;
201 * {@inheritDoc}
203 @Override
204 public RecordWriteChannel openRecordWriteChannel(AppEngineFile file, boolean lock)
205 throws FileNotFoundException, LockException, IOException {
206 RecordWriteChannel channel = new RecordWriteChannelImpl(openWriteChannel(file, lock));
207 return channel;
210 public void delete(AppEngineFile file) throws IOException {
211 delete(file.getFullPath());
215 * Appends bytes from the given buffer to the end of the given file.
217 * @param file the file to which to append bytes. Must be opened for append in
218 * the current request
219 * @param buffer The buffer from which bytes are to be retrieved
220 * @param sequenceKey the sequence key. See the explanation of the {@code
221 * sequenceKey} paramater at
222 * {@link FileWriteChannel#write(ByteBuffer, String)}
223 * @throws IllegalArgumentException if {@code file} is not writable
224 * @throws KeyOrderingException if {@code sequenceKey} is not {@code null} and
225 * the backend system already has recorded a last good sequence key
226 * for this file and {@code sequenceKey} is not strictly
227 * lexicographically greater than the last good sequence key
228 * @throws IOException if the file is not opened for append in the current App
229 * Engine request or any other unexpected problem occurs
231 int append(AppEngineFile file, ByteBuffer buffer, String sequenceKey) throws IOException {
232 if (null == buffer) {
233 throw new NullPointerException("buffer is null");
235 if (null == file) {
236 throw new NullPointerException("file is null");
238 ByteString data = ByteString.copyFrom(buffer);
239 append(file.getFullPath(), data, sequenceKey);
240 return data.size();
243 private static final String BLOB_FILE_INDEX_KIND = "__BlobFileIndex__";
245 private static final String BLOB_KEY_PROPERTY_NAME = "blob_key";
248 * {@inheritDoc}
250 @Override
251 public BlobKey getBlobKey(AppEngineFile file) {
252 if (null == file) {
253 throw new NullPointerException("file is null");
255 if (file.getFileSystem() != AppEngineFile.FileSystem.BLOBSTORE) {
256 throw new IllegalArgumentException("file is not of type BLOBSTORE");
258 BlobKey cached = file.getCachedBlobKey();
259 if (null != cached) {
260 return cached;
262 String namePart = file.getNamePart();
263 String creationHandle = (namePart.startsWith(CREATION_HANDLE_PREFIX) ? namePart : null);
265 if (null == creationHandle) {
266 return new BlobKey(namePart);
269 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
270 String origNamespace = NamespaceManager.get();
271 Query query;
272 Entity blobInfoEntity;
273 try {
274 NamespaceManager.set("");
275 try {
276 Entity blobFileIndexEntity =
277 datastore.get(null, KeyFactory.createKey(BLOB_FILE_INDEX_KIND, creationHandle));
278 String blobKey = (String) blobFileIndexEntity.getProperty("blob_key");
279 blobInfoEntity = datastore.get(null, KeyFactory.createKey(BlobInfoFactory.KIND, blobKey));
280 } catch (EntityNotFoundException ex) {
281 query = new Query(BlobInfoFactory.KIND);
282 query.addFilter(BLOB_INFO_CREATION_HANDLE_PROPERTY, Query.FilterOperator.EQUAL,
283 creationHandle);
284 blobInfoEntity = datastore.prepare(query).asSingleEntity();
286 } finally {
287 NamespaceManager.set(origNamespace);
290 if (null == blobInfoEntity) {
291 return null;
293 BlobInfo blobInfo = new BlobInfoFactory().createBlobInfo(blobInfoEntity);
294 return blobInfo.getBlobKey();
298 * {@inheritDoc}
300 @Override
301 public AppEngineFile getBlobFile(BlobKey blobKey) {
302 if (null == blobKey) {
303 throw new NullPointerException("blobKey is null");
305 String namePart = blobKey.getKeyString();
306 AppEngineFile file = new AppEngineFile(AppEngineFile.FileSystem.BLOBSTORE, namePart);
307 file.setCachedBlobKey(blobKey);
308 return file;
312 * {@inheritDoc}
314 @Override
315 public FileStat stat(AppEngineFile file) throws IOException {
316 Preconditions.checkNotNull(file, "file is null");
318 StatRequest.Builder statRequestBuilder = StatRequest.newBuilder();
319 statRequestBuilder.setFilename(file.getFullPath());
320 StatResponse.Builder statResponseBuilder = StatResponse.newBuilder();
321 openForRead(file, false);
322 try {
323 makeSyncCall("Stat", statRequestBuilder, statResponseBuilder);
324 } finally {
325 close(file, false);
328 if (statResponseBuilder.getStatCount() != 1) {
329 throw new IllegalStateException(
330 "Requested stat for one file. Got zero or more than one response.");
332 FileServicePb.FileStat fileStatPb = statResponseBuilder.build().getStat(0);
333 FileStat fileStat = new FileStat();
334 fileStat.setFilename(fileStatPb.getFilename());
335 fileStat.setFinalized(fileStatPb.getFinalized());
336 fileStat.setLength(fileStatPb.getLength());
337 if (fileStatPb.hasCtime()) {
338 fileStat.setCtime(fileStatPb.getCtime());
340 if (fileStatPb.hasMtime()) {
341 fileStat.setMtime(fileStatPb.getMtime());
343 return fileStat;
347 * Reads bytes from {@code file} starting from {@code startingPos} and puts
348 * the bytes into the {@code buffer}. Returns the number of bytes read. The
349 * number of bytes read will be the minumum of the number of bytes available
350 * in the file and the buffer's {@link ByteBuffer#remaining() free bytes}.
352 * @param file the file from which to read bytes. Must be opened for read in
353 * the current request
354 * @param buffer the destination buffer
355 * @return the number of bytes read
356 * @throws IOException if the file is not opened for read in the current App
357 * Engine request or any other unexpected problem occurs
359 int read(AppEngineFile file, ByteBuffer buffer, long startingPos) throws IOException {
360 if (startingPos < 0) {
361 throw new IllegalArgumentException("startingPos is negative: " + startingPos);
363 if (buffer == null) {
364 throw new NullPointerException("buffer is null");
366 long remaining = buffer.remaining();
367 if (buffer.remaining() < 1) {
368 return 0;
370 ByteString byteString = read(file.getFullPath(), startingPos, remaining);
371 byteString.copyTo(buffer);
372 int numBytesRead = byteString.size();
373 if (numBytesRead <= 0) {
374 numBytesRead = -1;
376 return numBytesRead;
380 * Change the state of the given file to closed and optionally finalize the
381 * file. After the file is finalized it may be read, and it may no longer be
382 * written.
384 * @param file the file to close and optionally finalize. The file must be
385 * opened in the current request.
386 * @param finalize should the file be finalized? The file may only be
387 * finalized if the current request holds the lock for the file
388 * @throws IllegalStateException if {@code finalize = true} and the current
389 * request does not hold the exclusive lock on {@code file}
390 * @throws IOException if the file is not opened in the current request, if
391 * {@code finalize = true} and the file is already finalized or if any
392 * other unexpected problem occurs
394 void close(AppEngineFile file, boolean finalize) throws IOException {
395 try {
396 close(file.getFullPath(), finalize);
397 } catch (LockException e) {
398 if (finalize) {
399 throw new IllegalStateException("The current request does not hold the exclusive lock.");
401 throw e;
406 * Opens a file for appending by making the "Open" RPC call with mode=APPEND.
408 private void openForAppend(String fileName, ContentType contentType, boolean lock)
409 throws IOException {
410 open(fileName, contentType, OpenMode.APPEND, lock);
413 private void openForRead(AppEngineFile file, boolean lock)
414 throws FileNotFoundException, LockException, IOException {
415 if (null == file) {
416 throw new NullPointerException("file is null");
418 openForRead(file.getFullPath(), ContentType.RAW, lock);
422 * Opens a file for reading by making the "Open" RPC call with mode=READ
424 private void openForRead(String fileName, ContentType contentType, boolean lock)
425 throws IOException {
426 open(fileName, contentType, OpenMode.READ, lock);
430 * Makes the "Create" RPC call.
432 * @return created file name.
434 private String create(
435 String fileSystem, String fileName, ContentType contentType, Map<String, String> parameters)
436 throws IOException {
437 CreateRequest.Builder request = CreateRequest.newBuilder();
438 request.setFilesystem(fileSystem);
439 if (fileName != null && !fileName.isEmpty()) {
440 request.setFilename(fileName);
442 request.setContentType(contentType);
443 if (parameters != null) {
444 for (Map.Entry<String, String> e : parameters.entrySet()) {
445 CreateRequest.Parameter.Builder parameter = request.addParametersBuilder();
446 parameter.setName(e.getKey());
447 parameter.setValue(e.getValue());
450 CreateResponse.Builder response = CreateResponse.newBuilder();
451 makeSyncCall("Create", request, response);
452 return response.build().getFilename();
456 * Makes the "Open" RPC call
458 private void open(String fileName, ContentType contentType, OpenMode openMode, boolean lock)
459 throws IOException {
460 OpenRequest.Builder openRequest = OpenRequest.newBuilder();
461 openRequest.setFilename(fileName);
462 openRequest.setContentType(contentType);
463 openRequest.setOpenMode(openMode);
464 openRequest.setExclusiveLock(lock);
465 OpenResponse.Builder openResponse = OpenResponse.newBuilder();
466 makeSyncCall("Open", openRequest, openResponse);
470 * Makes the 'Append' RPC call
472 private void append(String fileName, ByteString data, String sequenceKey) throws IOException {
473 AppendRequest.Builder appendRequest = AppendRequest.newBuilder();
474 appendRequest.setFilename(fileName);
475 appendRequest.setData(data);
476 if (null != sequenceKey) {
477 appendRequest.setSequenceKey(sequenceKey);
479 AppendResponse.Builder appendResponse = AppendResponse.newBuilder();
480 makeSyncCall("Append", appendRequest, appendResponse);
484 * Makes the "Read" RPC call
486 private ByteString read(String fileName, long pos, long maxBytes) throws IOException {
487 ReadRequest.Builder readRequest = ReadRequest.newBuilder();
488 readRequest.setFilename(fileName);
489 readRequest.setMaxBytes(maxBytes);
490 readRequest.setPos(pos);
491 ReadResponse.Builder readResponse = ReadResponse.newBuilder();
492 makeSyncCall("Read", readRequest, readResponse);
493 return readResponse.build().getData();
497 * Makes the "Close" RPC call
499 private void close(String fileName, boolean finalize) throws IOException {
500 CloseRequest.Builder closeRequest = CloseRequest.newBuilder();
501 closeRequest.setFilename(fileName);
502 closeRequest.setFinalize(finalize);
503 CloseResponse.Builder closeResponse = CloseResponse.newBuilder();
504 makeSyncCall("Close", closeRequest, closeResponse);
508 * Makes the "Delete" RPC call
510 private void delete(String fileName) throws IOException {
511 DeleteRequest.Builder deleteRequest = DeleteRequest.newBuilder();
512 deleteRequest.setFilename(fileName);
513 DeleteResponse.Builder deleteResponse = DeleteResponse.newBuilder();
514 makeSyncCall("Delete", deleteRequest, deleteResponse);
518 * Makes the "GetDefaultGSBucketName" RPC call.
520 @Override
521 public String getDefaultGsBucketName() throws IOException {
522 GetDefaultGsBucketNameRequest.Builder request = GetDefaultGsBucketNameRequest.newBuilder();
523 GetDefaultGsBucketNameResponse.Builder response = GetDefaultGsBucketNameResponse.newBuilder();
524 makeSyncCall("GetDefaultGsBucketName", request, response);
525 return response.getDefaultGsBucketName();
529 * Makes a synchronous RPC call to the app server
531 * @param callName
532 * @param request
533 * @param response
534 * @throws IOException
536 private void makeSyncCall(String callName, Message.Builder request, Message.Builder response)
537 throws IOException {
538 try {
539 byte[] responseBytes =
540 ApiProxy.makeSyncCall(PACKAGE, callName, request.build().toByteArray());
541 response.mergeFrom(responseBytes);
542 } catch (ApiProxy.ApplicationException ex) {
543 throw translateException(ex, null);
544 } catch (InvalidProtocolBufferException e) {
545 throw new RuntimeException("Internal logic error: Response PB could not be parsed.", e);
550 * Translates from an internal to a public exception
552 private static IOException translateException(ApiProxy.ApplicationException ex, String message) {
553 int errorCode = ex.getApplicationError();
554 FileServiceErrors.ErrorCode errorCodeEnum = FileServiceErrors.ErrorCode.valueOf(errorCode);
555 switch (errorCodeEnum) {
556 case EXCLUSIVE_LOCK_FAILED:
557 return new LockException(message, ex);
558 case EXISTENCE_ERROR:
559 case EXISTENCE_ERROR_METADATA_NOT_FOUND:
560 case EXISTENCE_ERROR_METADATA_FOUND:
561 case EXISTENCE_ERROR_SHARDING_MISMATCH:
562 case EXISTENCE_ERROR_BUCKET_NOT_FOUND:
563 case EXISTENCE_ERROR_OBJECT_NOT_FOUND:
564 return new FileNotFoundException();
565 case FINALIZATION_ERROR:
566 return new FinalizationException(message, ex);
567 case SEQUENCE_KEY_OUT_OF_ORDER:
568 return new KeyOrderingException(message, ex);
569 default:
570 return new IOException(message, ex);