Revision created by MOE tool push_codebase.
[gae.git] / java / src / main / com / google / appengine / api / files / FileServiceImpl.java
blob4530398fb7329443e2ee3a89349c11d74890b0bd
1 // Copyright 2010 Google Inc. All Rights Reserved.
3 package com.google.appengine.api.files;
5 import com.google.appengine.api.NamespaceManager;
6 import com.google.appengine.api.blobstore.BlobInfo;
7 import com.google.appengine.api.blobstore.BlobInfoFactory;
8 import com.google.appengine.api.blobstore.BlobKey;
9 import com.google.appengine.api.blobstore.BlobstoreFailureException;
10 import com.google.appengine.api.blobstore.BlobstoreService;
11 import com.google.appengine.api.blobstore.BlobstoreServiceFactory;
12 import com.google.appengine.api.datastore.DatastoreService;
13 import com.google.appengine.api.datastore.DatastoreServiceFactory;
14 import com.google.appengine.api.datastore.Entity;
15 import com.google.appengine.api.datastore.EntityNotFoundException;
16 import com.google.appengine.api.datastore.KeyFactory;
17 import com.google.appengine.api.datastore.Query;
18 import com.google.appengine.api.files.FileServicePb.AppendRequest;
19 import com.google.appengine.api.files.FileServicePb.AppendResponse;
20 import com.google.appengine.api.files.FileServicePb.CloseRequest;
21 import com.google.appengine.api.files.FileServicePb.CloseResponse;
22 import com.google.appengine.api.files.FileServicePb.CreateRequest;
23 import com.google.appengine.api.files.FileServicePb.CreateResponse;
24 import com.google.appengine.api.files.FileServicePb.FileContentType.ContentType;
25 import com.google.appengine.api.files.FileServicePb.FileServiceErrors;
26 import com.google.appengine.api.files.FileServicePb.GetDefaultGsBucketNameRequest;
27 import com.google.appengine.api.files.FileServicePb.GetDefaultGsBucketNameResponse;
28 import com.google.appengine.api.files.FileServicePb.OpenRequest;
29 import com.google.appengine.api.files.FileServicePb.OpenRequest.OpenMode;
30 import com.google.appengine.api.files.FileServicePb.OpenResponse;
31 import com.google.appengine.api.files.FileServicePb.ReadRequest;
32 import com.google.appengine.api.files.FileServicePb.ReadResponse;
33 import com.google.appengine.api.files.FileServicePb.StatRequest;
34 import com.google.appengine.api.files.FileServicePb.StatResponse;
35 import com.google.apphosting.api.ApiProxy;
36 import com.google.common.base.Charsets;
37 import com.google.common.base.Preconditions;
38 import com.google.common.hash.Hashing;
39 import com.google.protobuf.ByteString;
40 import com.google.protobuf.InvalidProtocolBufferException;
41 import com.google.protobuf.Message;
43 import java.io.FileNotFoundException;
44 import java.io.IOException;
45 import java.nio.ByteBuffer;
46 import java.util.ArrayList;
47 import java.util.Map;
48 import java.util.TreeMap;
50 /**
51 * Implements {@link FileService} by using {@link ApiProxy} to make RPC calls to
52 * the App Engine File API.
55 class FileServiceImpl implements FileService {
57 static final String PACKAGE = "file";
59 static final String FILESYSTEM_BLOBSTORE = AppEngineFile.FileSystem.BLOBSTORE.getName();
60 static final String PARAMETER_MIME_TYPE = "content_type";
61 static final String PARAMETER_BLOB_INFO_UPLOADED_FILE_NAME = "file_name";
62 static final String DEFAULT_MIME_TYPE = "application/octet-stream";
63 static final String FILESYSTEM_GS = AppEngineFile.FileSystem.GS.getName();
64 static final String GS_FILESYSTEM_PREFIX = "/gs/";
65 static final String GS_PARAMETER_MIME_TYPE = "content_type";
66 static final String GS_PARAMETER_CANNED_ACL = "acl";
67 static final String GS_PARAMETER_CONTENT_ENCODING = "content_encoding";
68 static final String GS_PARAMETER_CONTENT_DISPOSITION = "content_disposition";
69 static final String GS_PARAMETER_CACHE_CONTROL = "cache_control";
70 static final String GS_USER_METADATA_PREFIX = "x-goog-meta-";
72 static final String GS_DEFAULT_MIME_TYPE = "application/octet-stream";
74 private static final String BLOB_INFO_CREATION_HANDLE_PROPERTY = "creation_handle";
75 static final String GS_CREATION_HANDLE_PREFIX = "writable:";
76 static final String CREATION_HANDLE_PREFIX = "writable:";
78 BlobstoreService blobstoreService;
79 DatastoreService datastoreService;
81 public FileServiceImpl() {
82 blobstoreService = BlobstoreServiceFactory.getBlobstoreService();
83 datastoreService = DatastoreServiceFactory.getDatastoreService();
86 /**
87 * {@inheritDoc}
89 @Override
90 public AppEngineFile createNewBlobFile(String mimeType) throws IOException {
91 return createNewBlobFile(mimeType, "");
94 /**
95 * {@inheritDoc}
97 @Override
98 public AppEngineFile createNewBlobFile(String mimeType, String blobInfoUploadedFileName)
99 throws IOException {
100 if (mimeType == null || mimeType.trim().isEmpty()) {
101 mimeType = DEFAULT_MIME_TYPE;
104 Map<String, String> params = new TreeMap<String, String>();
105 params.put(PARAMETER_MIME_TYPE, mimeType);
106 if (blobInfoUploadedFileName != null && !blobInfoUploadedFileName.isEmpty()) {
107 params.put(PARAMETER_BLOB_INFO_UPLOADED_FILE_NAME, blobInfoUploadedFileName);
109 String filePath = create(FILESYSTEM_BLOBSTORE, null, ContentType.RAW, params);
110 AppEngineFile file = new AppEngineFile(filePath);
111 if (!file.getNamePart().startsWith(CREATION_HANDLE_PREFIX)) {
112 throw new RuntimeException("Expected creation handle: " + file.getFullPath());
114 return file;
118 * {@inheritDoc}
120 @Override
121 public AppEngineFile createNewGSFile(final GSFileOptions options) throws IOException {
122 if (options.fileName == null || options.fileName.isEmpty() ||
123 !options.fileName.startsWith(GS_FILESYSTEM_PREFIX)) {
124 throw new IllegalArgumentException("Invalid fileName, should be of the form: /gs/bucket/key");
126 Map<String, String> params = new TreeMap<String, String>();
127 params.put(GS_PARAMETER_MIME_TYPE, options.mimeType);
128 if (options.acl != null && !options.acl.trim().isEmpty()) {
129 params.put(GS_PARAMETER_CANNED_ACL, options.acl);
131 if (options.cacheControl != null && !options.cacheControl.trim().isEmpty()) {
132 params.put(GS_PARAMETER_CACHE_CONTROL, options.cacheControl);
134 if (options.contentEncoding != null && !options.contentEncoding.trim().isEmpty()) {
135 params.put(GS_PARAMETER_CONTENT_ENCODING, options.contentEncoding);
137 if (options.contentDisposition != null && !options.contentDisposition.trim().isEmpty()) {
138 params.put(GS_PARAMETER_CONTENT_DISPOSITION, options.contentDisposition);
140 if (options.userMetadata != null) {
141 for (String key : options.userMetadata.keySet()) {
142 if (key == null || key.isEmpty()) {
143 throw new IllegalArgumentException(
144 "Empty or null key in userMetadata");
146 String value = options.userMetadata.get(key);
147 if (value == null || value.isEmpty()) {
148 throw new IllegalArgumentException(
149 "Empty or null value in userMetadata for key: " + key);
151 params.put(GS_USER_METADATA_PREFIX + key, value);
154 AppEngineFile file = new AppEngineFile(
155 create(FILESYSTEM_GS, options.fileName, ContentType.RAW, params));
156 if (!file.getNamePart().startsWith(GS_CREATION_HANDLE_PREFIX)) {
157 throw new RuntimeException("Expected creation handle: " + file.getFullPath());
159 return file;
163 * {@inheritDoc}
165 @Override
166 public FileWriteChannel openWriteChannel(AppEngineFile file, boolean lock)
167 throws FileNotFoundException, FinalizationException, LockException, IOException {
168 FileWriteChannel channel = new FileWriteChannelImpl(file, lock, this);
169 openForAppend(file, lock);
170 return channel;
174 * Open the given file for append and optionally lock it.
176 * @param file the file to open
177 * @param lock should the file be locked for exclusive access?
178 * @throws FileNotFoundException if the file does not exist in the File Proxy
179 * @throws FinalizationException if the file has already been finalized. The
180 * file may have been finalized by another request.
181 * @throws LockException if the file is locked in a different App Engine
182 * request, or if {@code lock = true} and the file is opened in a
183 * different App Engine request
184 * @throws IOException if any other unexpected problem occurs
186 void openForAppend(AppEngineFile file, boolean lock)
187 throws FileNotFoundException, FinalizationException, LockException, IOException {
188 openForAppend(file.getFullPath(), ContentType.RAW, lock);
192 * {@inheritDoc}
194 @Override
195 public FileReadChannel openReadChannel(AppEngineFile file, boolean lock)
196 throws FileNotFoundException, LockException, IOException {
197 FileReadChannel channel = new FileReadChannelImpl(file, this);
198 openForRead(file, lock);
199 return channel;
203 * {@inheritDoc}
205 @Override
206 public RecordReadChannel openRecordReadChannel(AppEngineFile file, boolean lock)
207 throws FileNotFoundException, LockException, IOException {
208 FileReadChannel fileReadChannel = new BufferedFileReadChannelImpl(
209 openReadChannel(file, lock), RecordConstants.BLOCK_SIZE * 2);
210 RecordReadChannel channel = new RecordReadChannelImpl(fileReadChannel);
211 return channel;
215 * {@inheritDoc}
217 @Override
218 public RecordWriteChannel openRecordWriteChannel(AppEngineFile file, boolean lock)
219 throws FileNotFoundException, LockException, IOException {
220 RecordWriteChannel channel = new RecordWriteChannelImpl(openWriteChannel(file, lock));
221 return channel;
224 @Override
225 public void delete(AppEngineFile... files) throws IOException {
226 Preconditions.checkNotNull(files, "No file given");
228 if (files.length == 0) {
229 return;
232 ArrayList<BlobKey> blobKeys = new ArrayList<BlobKey>();
234 for (int i = 0; i < files.length; i++) {
235 AppEngineFile file = files[i];
236 Preconditions.checkNotNull(file, String.format("File at index %d is null", i));
238 if (!file.hasFinalizedName()) {
239 throw new UnsupportedOperationException(
240 String.format("File %s does not have a finalized name", file.getFullPath()));
243 if (file.getFileSystem().equals((AppEngineFile.FileSystem.BLOBSTORE))) {
244 BlobKey blobKey = getBlobKey(file);
245 if (blobKey != null) {
246 blobKeys.add(blobKey);
248 } else if (file.getFileSystem().equals((AppEngineFile.FileSystem.GS))) {
249 blobKeys.add(blobstoreService.createGsBlobKey(file.getFullPath()));
250 } else {
251 throw new UnsupportedOperationException(
252 String.format("File at index %d not supported by delete"));
256 if (blobKeys.size() != 0) {
257 try {
258 blobstoreService.delete(blobKeys.toArray(new BlobKey[blobKeys.size()]));
259 } catch (BlobstoreFailureException e) {
260 throw new IOException("Blobstore failure", e);
266 * Appends bytes from the given buffer to the end of the given file.
268 * @param file the file to which to append bytes. Must be opened for append in
269 * the current request
270 * @param buffer The buffer from which bytes are to be retrieved
271 * @param sequenceKey the sequence key. See the explanation of the {@code
272 * sequenceKey} paramater at
273 * {@link FileWriteChannel#write(ByteBuffer, String)}
274 * @throws IllegalArgumentException if {@code file} is not writable
275 * @throws KeyOrderingException if {@code sequenceKey} is not {@code null} and
276 * the backend system already has recorded a last good sequence key
277 * for this file and {@code sequenceKey} is not strictly
278 * lexicographically greater than the last good sequence key
279 * @throws IOException if the file is not opened for append in the current App
280 * Engine request or any other unexpected problem occurs
282 int append(AppEngineFile file, ByteBuffer buffer, String sequenceKey) throws IOException {
283 if (null == buffer) {
284 throw new NullPointerException("buffer is null");
286 if (null == file) {
287 throw new NullPointerException("file is null");
289 ByteString data = ByteString.copyFrom(buffer);
290 append(file.getFullPath(), data, sequenceKey);
291 return data.size();
294 static final String BLOB_FILE_INDEX_KIND = "__BlobFileIndex__";
297 * {@inheritDoc}
299 @Override
300 public BlobKey getBlobKey(AppEngineFile file) {
301 if (null == file) {
302 throw new NullPointerException("file is null");
304 if (file.getFileSystem() != AppEngineFile.FileSystem.BLOBSTORE) {
305 throw new IllegalArgumentException("file is not of type BLOBSTORE");
307 BlobKey cached = file.getCachedBlobKey();
308 if (null != cached) {
309 return cached;
311 String namePart = file.getNamePart();
312 String creationHandle = (namePart.startsWith(CREATION_HANDLE_PREFIX) ? namePart : null);
314 if (null == creationHandle) {
315 return new BlobKey(namePart);
318 String origNamespace = NamespaceManager.get();
319 Query query;
320 Entity blobInfoEntity;
321 try {
322 NamespaceManager.set("");
323 try {
324 Entity blobFileIndexEntity =
325 datastoreService.get(null, KeyFactory.createKey(BLOB_FILE_INDEX_KIND, getBlobFileIndexKeyName(creationHandle)));
326 String blobKey = (String) blobFileIndexEntity.getProperty("blob_key");
327 blobInfoEntity = datastoreService.get(null, KeyFactory.createKey(BlobInfoFactory.KIND, blobKey));
328 } catch (EntityNotFoundException ex) {
329 query = new Query(BlobInfoFactory.KIND);
330 query.addFilter(BLOB_INFO_CREATION_HANDLE_PROPERTY, Query.FilterOperator.EQUAL,
331 creationHandle);
332 blobInfoEntity = datastoreService.prepare(query).asSingleEntity();
334 } finally {
335 NamespaceManager.set(origNamespace);
338 if (null == blobInfoEntity) {
339 return null;
341 BlobInfo blobInfo = new BlobInfoFactory(datastoreService).createBlobInfo(blobInfoEntity);
342 return blobInfo.getBlobKey();
345 private static String getBlobFileIndexKeyName(String creationHandle) {
346 if (creationHandle.length() < 500) {
347 return creationHandle;
350 return Hashing.sha512().hashString(creationHandle, Charsets.US_ASCII).toString();
354 * {@inheritDoc}
356 @Override
357 public AppEngineFile getBlobFile(BlobKey blobKey) {
358 if (null == blobKey) {
359 throw new NullPointerException("blobKey is null");
361 String namePart = blobKey.getKeyString();
362 AppEngineFile file = new AppEngineFile(AppEngineFile.FileSystem.BLOBSTORE, namePart);
363 file.setCachedBlobKey(blobKey);
364 return file;
368 * {@inheritDoc}
370 @Override
371 public FileStat stat(AppEngineFile file) throws IOException {
372 Preconditions.checkNotNull(file, "file is null");
374 StatRequest.Builder statRequestBuilder = StatRequest.newBuilder();
375 statRequestBuilder.setFilename(file.getFullPath());
376 StatResponse.Builder statResponseBuilder = StatResponse.newBuilder();
377 openForRead(file, false);
378 try {
379 makeSyncCall("Stat", statRequestBuilder, statResponseBuilder);
380 } finally {
381 close(file, false);
384 if (statResponseBuilder.getStatCount() != 1) {
385 throw new IllegalStateException(
386 "Requested stat for one file. Got zero or more than one response.");
388 FileServicePb.FileStat fileStatPb = statResponseBuilder.build().getStat(0);
389 FileStat fileStat = new FileStat();
390 fileStat.setFilename(fileStatPb.getFilename());
391 fileStat.setFinalized(fileStatPb.getFinalized());
392 fileStat.setLength(fileStatPb.getLength());
393 if (fileStatPb.hasCtime()) {
394 fileStat.setCtime(fileStatPb.getCtime());
396 if (fileStatPb.hasMtime()) {
397 fileStat.setMtime(fileStatPb.getMtime());
399 return fileStat;
403 * Reads bytes from {@code file} starting from {@code startingPos} and puts
404 * the bytes into the {@code buffer}. Returns the number of bytes read. The
405 * number of bytes read will be the minumum of the number of bytes available
406 * in the file and the buffer's {@link ByteBuffer#remaining() free bytes}.
408 * @param file the file from which to read bytes. Must be opened for read in
409 * the current request
410 * @param buffer the destination buffer
411 * @return the number of bytes read
412 * @throws IOException if the file is not opened for read in the current App
413 * Engine request or any other unexpected problem occurs
415 int read(AppEngineFile file, ByteBuffer buffer, long startingPos) throws IOException {
416 if (startingPos < 0) {
417 throw new IllegalArgumentException("startingPos is negative: " + startingPos);
419 if (buffer == null) {
420 throw new NullPointerException("buffer is null");
422 long remaining = buffer.remaining();
423 if (buffer.remaining() < 1) {
424 return 0;
426 ByteString byteString = read(file.getFullPath(), startingPos, remaining);
427 byteString.copyTo(buffer);
428 int numBytesRead = byteString.size();
429 if (numBytesRead <= 0) {
430 numBytesRead = -1;
432 return numBytesRead;
436 * Change the state of the given file to closed and optionally finalize the
437 * file. After the file is finalized it may be read, and it may no longer be
438 * written.
440 * @param file the file to close and optionally finalize. The file must be
441 * opened in the current request.
442 * @param finalize should the file be finalized? The file may only be
443 * finalized if the current request holds the lock for the file
444 * @throws IllegalStateException if {@code finalize = true} and the current
445 * request does not hold the exclusive lock on {@code file}
446 * @throws IOException if the file is not opened in the current request, if
447 * {@code finalize = true} and the file is already finalized or if any
448 * other unexpected problem occurs
450 void close(AppEngineFile file, boolean finalize) throws IOException {
451 try {
452 close(file.getFullPath(), finalize);
453 } catch (LockException e) {
454 if (finalize) {
455 throw new IllegalStateException("The current request does not hold the exclusive lock.");
457 throw e;
462 * Opens a file for appending by making the "Open" RPC call with mode=APPEND.
464 private void openForAppend(String fileName, ContentType contentType, boolean lock)
465 throws IOException {
466 open(fileName, contentType, OpenMode.APPEND, lock);
469 private void openForRead(AppEngineFile file, boolean lock)
470 throws FileNotFoundException, LockException, IOException {
471 if (null == file) {
472 throw new NullPointerException("file is null");
474 openForRead(file.getFullPath(), ContentType.RAW, lock);
478 * Opens a file for reading by making the "Open" RPC call with mode=READ
480 private void openForRead(String fileName, ContentType contentType, boolean lock)
481 throws IOException {
482 open(fileName, contentType, OpenMode.READ, lock);
486 * Makes the "Create" RPC call.
488 * @return created file name.
490 private String create(
491 String fileSystem, String fileName, ContentType contentType, Map<String, String> parameters)
492 throws IOException {
493 CreateRequest.Builder request = CreateRequest.newBuilder();
494 request.setFilesystem(fileSystem);
495 if (fileName != null && !fileName.isEmpty()) {
496 request.setFilename(fileName);
498 request.setContentType(contentType);
499 if (parameters != null) {
500 for (Map.Entry<String, String> e : parameters.entrySet()) {
501 CreateRequest.Parameter.Builder parameter = request.addParametersBuilder();
502 parameter.setName(e.getKey());
503 parameter.setValue(e.getValue());
506 CreateResponse.Builder response = CreateResponse.newBuilder();
507 makeSyncCall("Create", request, response);
508 return response.build().getFilename();
512 * Makes the "Open" RPC call
514 private void open(String fileName, ContentType contentType, OpenMode openMode, boolean lock)
515 throws IOException {
516 OpenRequest.Builder openRequest = OpenRequest.newBuilder();
517 openRequest.setFilename(fileName);
518 openRequest.setContentType(contentType);
519 openRequest.setOpenMode(openMode);
520 openRequest.setExclusiveLock(lock);
521 OpenResponse.Builder openResponse = OpenResponse.newBuilder();
522 makeSyncCall("Open", openRequest, openResponse);
526 * Makes the 'Append' RPC call
528 private void append(String fileName, ByteString data, String sequenceKey) throws IOException {
529 AppendRequest.Builder appendRequest = AppendRequest.newBuilder();
530 appendRequest.setFilename(fileName);
531 appendRequest.setData(data);
532 if (null != sequenceKey) {
533 appendRequest.setSequenceKey(sequenceKey);
535 AppendResponse.Builder appendResponse = AppendResponse.newBuilder();
536 makeSyncCall("Append", appendRequest, appendResponse);
540 * Makes the "Read" RPC call
542 private ByteString read(String fileName, long pos, long maxBytes) throws IOException {
543 ReadRequest.Builder readRequest = ReadRequest.newBuilder();
544 readRequest.setFilename(fileName);
545 readRequest.setMaxBytes(maxBytes);
546 readRequest.setPos(pos);
547 ReadResponse.Builder readResponse = ReadResponse.newBuilder();
548 makeSyncCall("Read", readRequest, readResponse);
549 return readResponse.build().getData();
553 * Makes the "Close" RPC call
555 private void close(String fileName, boolean finalize) throws IOException {
556 CloseRequest.Builder closeRequest = CloseRequest.newBuilder();
557 closeRequest.setFilename(fileName);
558 closeRequest.setFinalize(finalize);
559 CloseResponse.Builder closeResponse = CloseResponse.newBuilder();
560 makeSyncCall("Close", closeRequest, closeResponse);
564 * Makes the "GetDefaultGSBucketName" RPC call.
566 @Override
567 public String getDefaultGsBucketName() throws IOException {
568 GetDefaultGsBucketNameRequest.Builder request = GetDefaultGsBucketNameRequest.newBuilder();
569 GetDefaultGsBucketNameResponse.Builder response = GetDefaultGsBucketNameResponse.newBuilder();
570 makeSyncCall("GetDefaultGsBucketName", request, response);
571 return response.getDefaultGsBucketName();
575 * Makes a synchronous RPC call to the app server
577 * @param callName
578 * @param request
579 * @param response
580 * @throws IOException
582 private void makeSyncCall(String callName, Message.Builder request, Message.Builder response)
583 throws IOException {
584 try {
585 byte[] responseBytes =
586 ApiProxy.makeSyncCall(PACKAGE, callName, request.build().toByteArray());
587 response.mergeFrom(responseBytes);
588 } catch (ApiProxy.ApplicationException ex) {
589 throw translateException(ex, null);
590 } catch (InvalidProtocolBufferException e) {
591 throw new RuntimeException("Internal logic error: Response PB could not be parsed.", e);
596 * Translates from an internal to a public exception
598 private static IOException translateException(ApiProxy.ApplicationException ex, String message) {
599 int errorCode = ex.getApplicationError();
600 FileServiceErrors.ErrorCode errorCodeEnum = FileServiceErrors.ErrorCode.valueOf(errorCode);
601 switch (errorCodeEnum) {
602 case EXCLUSIVE_LOCK_FAILED:
603 return new LockException(message, ex);
604 case EXISTENCE_ERROR:
605 case EXISTENCE_ERROR_METADATA_NOT_FOUND:
606 case EXISTENCE_ERROR_METADATA_FOUND:
607 case EXISTENCE_ERROR_SHARDING_MISMATCH:
608 case EXISTENCE_ERROR_BUCKET_NOT_FOUND:
609 case EXISTENCE_ERROR_OBJECT_NOT_FOUND:
610 return new FileNotFoundException();
611 case FINALIZATION_ERROR:
612 return new FinalizationException(message, ex);
613 case SEQUENCE_KEY_OUT_OF_ORDER:
614 return new KeyOrderingException(message, ex);
615 default:
616 return new IOException(message, ex);