1.9.30 sync.
[gae.git] / java / src / main / com / google / appengine / api / files / FileServiceImpl.java
blobc9991c26995bf8318d01164f79d330b79f83c583
1 // Copyright 2010 Google Inc. All Rights Reserved.
3 package com.google.appengine.api.files;
5 import static java.nio.charset.StandardCharsets.US_ASCII;
7 import com.google.appengine.api.NamespaceManager;
8 import com.google.appengine.api.blobstore.BlobInfo;
9 import com.google.appengine.api.blobstore.BlobInfoFactory;
10 import com.google.appengine.api.blobstore.BlobKey;
11 import com.google.appengine.api.blobstore.BlobstoreFailureException;
12 import com.google.appengine.api.blobstore.BlobstoreService;
13 import com.google.appengine.api.blobstore.BlobstoreServiceFactory;
14 import com.google.appengine.api.datastore.DatastoreService;
15 import com.google.appengine.api.datastore.DatastoreServiceFactory;
16 import com.google.appengine.api.datastore.Entity;
17 import com.google.appengine.api.datastore.EntityNotFoundException;
18 import com.google.appengine.api.datastore.KeyFactory;
19 import com.google.appengine.api.datastore.Query;
20 import com.google.appengine.api.files.FileServicePb.AppendRequest;
21 import com.google.appengine.api.files.FileServicePb.AppendResponse;
22 import com.google.appengine.api.files.FileServicePb.CloseRequest;
23 import com.google.appengine.api.files.FileServicePb.CloseResponse;
24 import com.google.appengine.api.files.FileServicePb.CreateRequest;
25 import com.google.appengine.api.files.FileServicePb.CreateResponse;
26 import com.google.appengine.api.files.FileServicePb.FileContentType.ContentType;
27 import com.google.appengine.api.files.FileServicePb.FileServiceErrors;
28 import com.google.appengine.api.files.FileServicePb.GetDefaultGsBucketNameRequest;
29 import com.google.appengine.api.files.FileServicePb.GetDefaultGsBucketNameResponse;
30 import com.google.appengine.api.files.FileServicePb.OpenRequest;
31 import com.google.appengine.api.files.FileServicePb.OpenRequest.OpenMode;
32 import com.google.appengine.api.files.FileServicePb.OpenResponse;
33 import com.google.appengine.api.files.FileServicePb.ReadRequest;
34 import com.google.appengine.api.files.FileServicePb.ReadResponse;
35 import com.google.appengine.api.files.FileServicePb.StatRequest;
36 import com.google.appengine.api.files.FileServicePb.StatResponse;
37 import com.google.apphosting.api.ApiProxy;
38 import com.google.common.base.Preconditions;
39 import com.google.common.hash.Hashing;
40 import com.google.protobuf.ByteString;
41 import com.google.protobuf.InvalidProtocolBufferException;
42 import com.google.protobuf.Message;
44 import java.io.FileNotFoundException;
45 import java.io.IOException;
46 import java.nio.ByteBuffer;
47 import java.util.ArrayList;
48 import java.util.Map;
49 import java.util.TreeMap;
51 /**
52 * Implements {@link FileService} by using {@link ApiProxy} to make RPC calls to
53 * the App Engine File API.
56 @Deprecated
57 class FileServiceImpl implements FileService {
59 static final String PACKAGE = "file";
61 static final String FILESYSTEM_BLOBSTORE = AppEngineFile.FileSystem.BLOBSTORE.getName();
62 static final String PARAMETER_MIME_TYPE = "content_type";
63 static final String PARAMETER_BLOB_INFO_UPLOADED_FILE_NAME = "file_name";
64 static final String DEFAULT_MIME_TYPE = "application/octet-stream";
65 static final String FILESYSTEM_GS = AppEngineFile.FileSystem.GS.getName();
66 static final String GS_FILESYSTEM_PREFIX = "/gs/";
67 static final String GS_PARAMETER_MIME_TYPE = "content_type";
68 static final String GS_PARAMETER_CANNED_ACL = "acl";
69 static final String GS_PARAMETER_CONTENT_ENCODING = "content_encoding";
70 static final String GS_PARAMETER_CONTENT_DISPOSITION = "content_disposition";
71 static final String GS_PARAMETER_CACHE_CONTROL = "cache_control";
72 static final int DATASTORE_MAX_PROPERTY_SIZE = 500;
74 static final String GS_DEFAULT_MIME_TYPE = "application/octet-stream";
76 private static final String BLOB_INFO_CREATION_HANDLE_PROPERTY = "creation_handle";
77 static final String GS_CREATION_HANDLE_PREFIX = "writable:";
78 static final String CREATION_HANDLE_PREFIX = "writable:";
80 BlobstoreService blobstoreService;
81 DatastoreService datastoreService;
83 public FileServiceImpl() {
84 blobstoreService = BlobstoreServiceFactory.getBlobstoreService();
85 datastoreService = DatastoreServiceFactory.getDatastoreService();
88 /**
89 * {@inheritDoc}
91 @Override
92 public AppEngineFile createNewBlobFile(String mimeType) throws IOException {
93 return createNewBlobFile(mimeType, "");
96 /**
97 * {@inheritDoc}
99 @Override
100 public AppEngineFile createNewBlobFile(String mimeType, String blobInfoUploadedFileName)
101 throws IOException {
102 if (mimeType == null || mimeType.trim().isEmpty()) {
103 mimeType = DEFAULT_MIME_TYPE;
106 Map<String, String> params = new TreeMap<String, String>();
107 params.put(PARAMETER_MIME_TYPE, mimeType);
108 if (blobInfoUploadedFileName != null && !blobInfoUploadedFileName.isEmpty()) {
109 params.put(PARAMETER_BLOB_INFO_UPLOADED_FILE_NAME, blobInfoUploadedFileName);
111 String filePath = create(FILESYSTEM_BLOBSTORE, null, ContentType.RAW, params);
112 AppEngineFile file = new AppEngineFile(filePath);
113 if (!file.getNamePart().startsWith(CREATION_HANDLE_PREFIX)) {
114 throw new RuntimeException("Expected creation handle: " + file.getFullPath());
116 return file;
120 * {@inheritDoc}
122 @Override
123 public AppEngineFile createNewGSFile(final GSFileOptions options) throws IOException {
124 if (options.fileName == null || options.fileName.isEmpty() ||
125 !options.fileName.startsWith(GS_FILESYSTEM_PREFIX)) {
126 throw new IllegalArgumentException("Invalid fileName, should be of the form: /gs/bucket/key");
128 Map<String, String> params = new TreeMap<String, String>();
129 params.put(GS_PARAMETER_MIME_TYPE, options.mimeType);
130 if (options.acl != null && !options.acl.trim().isEmpty()) {
131 params.put(GS_PARAMETER_CANNED_ACL, options.acl);
133 if (options.cacheControl != null && !options.cacheControl.trim().isEmpty()) {
134 params.put(GS_PARAMETER_CACHE_CONTROL, options.cacheControl);
136 if (options.contentEncoding != null && !options.contentEncoding.trim().isEmpty()) {
137 params.put(GS_PARAMETER_CONTENT_ENCODING, options.contentEncoding);
139 if (options.contentDisposition != null && !options.contentDisposition.trim().isEmpty()) {
140 params.put(GS_PARAMETER_CONTENT_DISPOSITION, options.contentDisposition);
142 if (options.userMetadata != null) {
143 for (String key : options.userMetadata.keySet()) {
144 if (key == null || key.isEmpty()) {
145 throw new IllegalArgumentException(
146 "Empty or null key in userMetadata");
148 String value = options.userMetadata.get(key);
149 if (value == null || value.isEmpty()) {
150 throw new IllegalArgumentException(
151 "Empty or null value in userMetadata for key: " + key);
153 params.put(GSFileOptions.GS_USER_METADATA_PREFIX + key, value);
156 AppEngineFile file = new AppEngineFile(
157 create(FILESYSTEM_GS, options.fileName, ContentType.RAW, params));
158 if (!file.getNamePart().startsWith(GS_CREATION_HANDLE_PREFIX)) {
159 throw new RuntimeException("Expected creation handle: " + file.getFullPath());
161 return file;
165 * {@inheritDoc}
167 @Override
168 public FileWriteChannel openWriteChannel(AppEngineFile file, boolean lock)
169 throws FileNotFoundException, FinalizationException, LockException, IOException {
170 FileWriteChannel channel = new FileWriteChannelImpl(file, lock, this);
171 openForAppend(file, lock);
172 return channel;
176 * Open the given file for append and optionally lock it.
178 * @param file the file to open
179 * @param lock should the file be locked for exclusive access?
180 * @throws FileNotFoundException if the file does not exist in the File Proxy
181 * @throws FinalizationException if the file has already been finalized. The
182 * file may have been finalized by another request.
183 * @throws LockException if the file is locked in a different App Engine
184 * request, or if {@code lock = true} and the file is opened in a
185 * different App Engine request
186 * @throws IOException if any other unexpected problem occurs
188 void openForAppend(AppEngineFile file, boolean lock)
189 throws FileNotFoundException, FinalizationException, LockException, IOException {
190 openForAppend(file.getFullPath(), ContentType.RAW, lock);
194 * {@inheritDoc}
196 @Override
197 public FileReadChannel openReadChannel(AppEngineFile file, boolean lock)
198 throws FileNotFoundException, LockException, IOException {
199 FileReadChannel channel = new FileReadChannelImpl(file, this);
200 openForRead(file, lock);
201 return channel;
205 * {@inheritDoc}
207 @Override
208 public RecordReadChannel openRecordReadChannel(AppEngineFile file, boolean lock)
209 throws FileNotFoundException, LockException, IOException {
210 FileReadChannel fileReadChannel = new BufferedFileReadChannelImpl(
211 openReadChannel(file, lock), RecordConstants.BLOCK_SIZE * 2);
212 RecordReadChannel channel = new RecordReadChannelImpl(fileReadChannel);
213 return channel;
217 * {@inheritDoc}
219 @Override
220 public RecordWriteChannel openRecordWriteChannel(AppEngineFile file, boolean lock)
221 throws FileNotFoundException, LockException, IOException {
222 RecordWriteChannel channel = new RecordWriteChannelImpl(openWriteChannel(file, lock));
223 return channel;
226 @Override
227 public void delete(AppEngineFile... files) throws IOException {
228 Preconditions.checkNotNull(files, "No file given");
230 if (files.length == 0) {
231 return;
234 ArrayList<BlobKey> blobKeys = new ArrayList<BlobKey>();
236 for (int i = 0; i < files.length; i++) {
237 AppEngineFile file = files[i];
238 Preconditions.checkNotNull(file, String.format("File at index %d is null", i));
240 if (!file.hasFinalizedName()) {
241 throw new UnsupportedOperationException(
242 String.format("File %s does not have a finalized name", file.getFullPath()));
245 if (file.getFileSystem().equals((AppEngineFile.FileSystem.BLOBSTORE))) {
246 BlobKey blobKey = getBlobKey(file);
247 if (blobKey != null) {
248 blobKeys.add(blobKey);
250 } else if (file.getFileSystem().equals((AppEngineFile.FileSystem.GS))) {
251 blobKeys.add(blobstoreService.createGsBlobKey(file.getFullPath()));
252 } else {
253 throw new UnsupportedOperationException(
254 String.format("File at index %d not supported by delete"));
258 if (!blobKeys.isEmpty()) {
259 try {
260 blobstoreService.delete(blobKeys.toArray(new BlobKey[blobKeys.size()]));
261 } catch (BlobstoreFailureException e) {
262 throw new IOException("Blobstore failure", e);
268 * Appends bytes from the given buffer to the end of the given file.
270 * @param file the file to which to append bytes. Must be opened for append in
271 * the current request
272 * @param buffer The buffer from which bytes are to be retrieved
273 * @param sequenceKey the sequence key. See the explanation of the {@code
274 * sequenceKey} paramater at
275 * {@link FileWriteChannel#write(ByteBuffer, String)}
276 * @throws IllegalArgumentException if {@code file} is not writable
277 * @throws KeyOrderingException if {@code sequenceKey} is not {@code null} and
278 * the backend system already has recorded a last good sequence key
279 * for this file and {@code sequenceKey} is not strictly
280 * lexicographically greater than the last good sequence key
281 * @throws IOException if the file is not opened for append in the current App
282 * Engine request or any other unexpected problem occurs
284 int append(AppEngineFile file, ByteBuffer buffer, String sequenceKey) throws IOException {
285 if (null == buffer) {
286 throw new NullPointerException("buffer is null");
288 if (null == file) {
289 throw new NullPointerException("file is null");
291 ByteString data = ByteString.copyFrom(buffer);
292 append(file.getFullPath(), data, sequenceKey);
293 return data.size();
296 static final String BLOB_FILE_INDEX_KIND = "__BlobFileIndex__";
299 * {@inheritDoc}
301 @Override
302 public BlobKey getBlobKey(AppEngineFile file) {
303 if (null == file) {
304 throw new NullPointerException("file is null");
306 if (file.getFileSystem() != AppEngineFile.FileSystem.BLOBSTORE) {
307 throw new IllegalArgumentException("file is not of type BLOBSTORE");
309 BlobKey cached = file.getCachedBlobKey();
310 if (null != cached) {
311 return cached;
313 String namePart = file.getNamePart();
314 String creationHandle = (namePart.startsWith(CREATION_HANDLE_PREFIX) ? namePart : null);
316 if (null == creationHandle) {
317 return new BlobKey(namePart);
320 String origNamespace = NamespaceManager.get();
321 Query query;
322 Entity blobInfoEntity = null;
323 try {
324 NamespaceManager.set("");
325 try {
326 Entity blobFileIndexEntity = datastoreService.get(null, KeyFactory.createKey(
327 BLOB_FILE_INDEX_KIND, getBlobFileIndexKeyName(creationHandle)));
328 String blobKey = (String) blobFileIndexEntity.getProperty("blob_key");
329 blobInfoEntity = datastoreService.get(
330 null, KeyFactory.createKey(BlobInfoFactory.KIND, blobKey));
331 } catch (EntityNotFoundException ex) {
332 if (creationHandle.length() < DATASTORE_MAX_PROPERTY_SIZE) {
333 query = new Query(BlobInfoFactory.KIND);
334 query.addFilter(BLOB_INFO_CREATION_HANDLE_PROPERTY, Query.FilterOperator.EQUAL,
335 creationHandle);
336 blobInfoEntity = datastoreService.prepare(query).asSingleEntity();
339 } finally {
340 NamespaceManager.set(origNamespace);
343 if (null == blobInfoEntity) {
344 return null;
346 BlobInfo blobInfo = new BlobInfoFactory(datastoreService).createBlobInfo(blobInfoEntity);
347 return blobInfo.getBlobKey();
350 private static String getBlobFileIndexKeyName(String creationHandle) {
351 if (creationHandle.length() < DATASTORE_MAX_PROPERTY_SIZE) {
352 return creationHandle;
355 return Hashing.sha512().hashString(creationHandle, US_ASCII).toString();
359 * {@inheritDoc}
361 @Override
362 public AppEngineFile getBlobFile(BlobKey blobKey) {
363 if (null == blobKey) {
364 throw new NullPointerException("blobKey is null");
366 String namePart = blobKey.getKeyString();
367 AppEngineFile file = new AppEngineFile(AppEngineFile.FileSystem.BLOBSTORE, namePart);
368 file.setCachedBlobKey(blobKey);
369 return file;
373 * {@inheritDoc}
375 @Override
376 public FileStat stat(AppEngineFile file) throws IOException {
377 Preconditions.checkNotNull(file, "file is null");
379 StatRequest.Builder statRequestBuilder = StatRequest.newBuilder();
380 statRequestBuilder.setFilename(file.getFullPath());
381 StatResponse.Builder statResponseBuilder = StatResponse.newBuilder();
382 openForRead(file, false);
383 try {
384 makeSyncCall("Stat", statRequestBuilder, statResponseBuilder);
385 } finally {
386 close(file, false);
389 if (statResponseBuilder.getStatCount() != 1) {
390 throw new IllegalStateException(
391 "Requested stat for one file. Got zero or more than one response.");
393 FileServicePb.FileStat fileStatPb = statResponseBuilder.build().getStat(0);
394 FileStat fileStat = new FileStat();
395 fileStat.setFilename(fileStatPb.getFilename());
396 fileStat.setFinalized(fileStatPb.getFinalized());
397 fileStat.setLength(fileStatPb.getLength());
398 if (fileStatPb.hasCtime()) {
399 fileStat.setCtime(fileStatPb.getCtime());
401 if (fileStatPb.hasMtime()) {
402 fileStat.setMtime(fileStatPb.getMtime());
404 return fileStat;
408 * Reads bytes from {@code file} starting from {@code startingPos} and puts
409 * the bytes into the {@code buffer}. Returns the number of bytes read. The
410 * number of bytes read will be the minumum of the number of bytes available
411 * in the file and the buffer's {@link ByteBuffer#remaining() free bytes}.
413 * @param file the file from which to read bytes. Must be opened for read in
414 * the current request
415 * @param buffer the destination buffer
416 * @return the number of bytes read
417 * @throws IOException if the file is not opened for read in the current App
418 * Engine request or any other unexpected problem occurs
420 int read(AppEngineFile file, ByteBuffer buffer, long startingPos) throws IOException {
421 if (startingPos < 0) {
422 throw new IllegalArgumentException("startingPos is negative: " + startingPos);
424 if (buffer == null) {
425 throw new NullPointerException("buffer is null");
427 long remaining = buffer.remaining();
428 if (buffer.remaining() < 1) {
429 return 0;
431 ByteString byteString = read(file.getFullPath(), startingPos, remaining);
432 byteString.copyTo(buffer);
433 int numBytesRead = byteString.size();
434 if (numBytesRead <= 0) {
435 numBytesRead = -1;
437 return numBytesRead;
441 * Change the state of the given file to closed and optionally finalize the
442 * file. After the file is finalized it may be read, and it may no longer be
443 * written.
445 * @param file the file to close and optionally finalize. The file must be
446 * opened in the current request.
447 * @param finalize should the file be finalized? The file may only be
448 * finalized if the current request holds the lock for the file
449 * @throws IllegalStateException if {@code finalize = true} and the current
450 * request does not hold the exclusive lock on {@code file}
451 * @throws IOException if the file is not opened in the current request, if
452 * {@code finalize = true} and the file is already finalized or if any
453 * other unexpected problem occurs
455 void close(AppEngineFile file, boolean finalize) throws IOException {
456 try {
457 close(file.getFullPath(), finalize);
458 } catch (LockException e) {
459 if (finalize) {
460 throw new IllegalStateException("The current request does not hold the exclusive lock.");
462 throw e;
467 * Opens a file for appending by making the "Open" RPC call with mode=APPEND.
469 private void openForAppend(String fileName, ContentType contentType, boolean lock)
470 throws IOException {
471 open(fileName, contentType, OpenMode.APPEND, lock);
474 private void openForRead(AppEngineFile file, boolean lock)
475 throws FileNotFoundException, LockException, IOException {
476 if (null == file) {
477 throw new NullPointerException("file is null");
479 openForRead(file.getFullPath(), ContentType.RAW, lock);
483 * Opens a file for reading by making the "Open" RPC call with mode=READ
485 private void openForRead(String fileName, ContentType contentType, boolean lock)
486 throws IOException {
487 open(fileName, contentType, OpenMode.READ, lock);
491 * Makes the "Create" RPC call.
493 * @return created file name.
495 private String create(
496 String fileSystem, String fileName, ContentType contentType, Map<String, String> parameters)
497 throws IOException {
498 CreateRequest.Builder request = CreateRequest.newBuilder();
499 request.setFilesystem(fileSystem);
500 if (fileName != null && !fileName.isEmpty()) {
501 request.setFilename(fileName);
503 request.setContentType(contentType);
504 if (parameters != null) {
505 for (Map.Entry<String, String> e : parameters.entrySet()) {
506 CreateRequest.Parameter.Builder parameter = request.addParametersBuilder();
507 parameter.setName(e.getKey());
508 parameter.setValue(e.getValue());
511 CreateResponse.Builder response = CreateResponse.newBuilder();
512 makeSyncCall("Create", request, response);
513 return response.build().getFilename();
517 * Makes the "Open" RPC call
519 private void open(String fileName, ContentType contentType, OpenMode openMode, boolean lock)
520 throws IOException {
521 OpenRequest.Builder openRequest = OpenRequest.newBuilder();
522 openRequest.setFilename(fileName);
523 openRequest.setContentType(contentType);
524 openRequest.setOpenMode(openMode);
525 openRequest.setExclusiveLock(lock);
526 OpenResponse.Builder openResponse = OpenResponse.newBuilder();
527 makeSyncCall("Open", openRequest, openResponse);
531 * Makes the 'Append' RPC call
533 private void append(String fileName, ByteString data, String sequenceKey) throws IOException {
534 AppendRequest.Builder appendRequest = AppendRequest.newBuilder();
535 appendRequest.setFilename(fileName);
536 appendRequest.setData(data);
537 if (null != sequenceKey) {
538 appendRequest.setSequenceKey(sequenceKey);
540 AppendResponse.Builder appendResponse = AppendResponse.newBuilder();
541 makeSyncCall("Append", appendRequest, appendResponse);
545 * Makes the "Read" RPC call
547 private ByteString read(String fileName, long pos, long maxBytes) throws IOException {
548 ReadRequest.Builder readRequest = ReadRequest.newBuilder();
549 readRequest.setFilename(fileName);
550 readRequest.setMaxBytes(maxBytes);
551 readRequest.setPos(pos);
552 ReadResponse.Builder readResponse = ReadResponse.newBuilder();
553 makeSyncCall("Read", readRequest, readResponse);
554 return readResponse.build().getData();
558 * Makes the "Close" RPC call
560 private void close(String fileName, boolean finalize) throws IOException {
561 CloseRequest.Builder closeRequest = CloseRequest.newBuilder();
562 closeRequest.setFilename(fileName);
563 closeRequest.setFinalize(finalize);
564 CloseResponse.Builder closeResponse = CloseResponse.newBuilder();
565 makeSyncCall("Close", closeRequest, closeResponse);
569 * Makes the "GetDefaultGSBucketName" RPC call.
571 @Override
572 public String getDefaultGsBucketName() throws IOException {
573 GetDefaultGsBucketNameRequest.Builder request = GetDefaultGsBucketNameRequest.newBuilder();
574 GetDefaultGsBucketNameResponse.Builder response = GetDefaultGsBucketNameResponse.newBuilder();
575 makeSyncCall("GetDefaultGsBucketName", request, response);
576 return response.getDefaultGsBucketName();
580 * Makes a synchronous RPC call to the app server
582 * @param callName
583 * @param request
584 * @param response
585 * @throws IOException
587 private void makeSyncCall(String callName, Message.Builder request, Message.Builder response)
588 throws IOException {
589 try {
590 byte[] responseBytes =
591 ApiProxy.makeSyncCall(PACKAGE, callName, request.build().toByteArray());
592 response.mergeFrom(responseBytes);
593 } catch (ApiProxy.ApplicationException ex) {
594 throw translateException(ex, null);
595 } catch (InvalidProtocolBufferException e) {
596 throw new RuntimeException("Internal logic error: Response PB could not be parsed.", e);
601 * Translates from an internal to a public exception
603 private static IOException translateException(ApiProxy.ApplicationException ex, String message) {
604 int errorCode = ex.getApplicationError();
605 FileServiceErrors.ErrorCode errorCodeEnum = FileServiceErrors.ErrorCode.valueOf(errorCode);
606 switch (errorCodeEnum) {
607 case EXCLUSIVE_LOCK_FAILED:
608 return new LockException(message, ex);
609 case EXISTENCE_ERROR:
610 case EXISTENCE_ERROR_METADATA_NOT_FOUND:
611 case EXISTENCE_ERROR_METADATA_FOUND:
612 case EXISTENCE_ERROR_SHARDING_MISMATCH:
613 case EXISTENCE_ERROR_BUCKET_NOT_FOUND:
614 case EXISTENCE_ERROR_OBJECT_NOT_FOUND:
615 return new FileNotFoundException();
616 case FINALIZATION_ERROR:
617 return new FinalizationException(message, ex);
618 case SEQUENCE_KEY_OUT_OF_ORDER:
619 return new KeyOrderingException(message, ex);
620 default:
621 return new IOException(message, ex);