java/src/main/com/google/appengine/api/search/IndexImpl.java

   1 // Copyright 2010 Google Inc. All Rights Reserved.
   2 package com.google.appengine.api.search;
   3
   4 import static com.google.appengine.api.search.FutureHelper.quietGet;
   5
   6 import com.google.appengine.api.search.SearchServicePb.SearchServiceError.ErrorCode;
   7 import com.google.appengine.api.search.checkers.DocumentChecker;
   8 import com.google.appengine.api.search.checkers.Preconditions;
   9 import com.google.appengine.api.search.checkers.SearchApiLimits;
  10 import com.google.appengine.api.utils.FutureWrapper;
  11 import com.google.apphosting.api.search.DocumentPb;
  12 import com.google.common.collect.Iterables;
  13
  14 import java.util.ArrayList;
  15 import java.util.Arrays;
  16 import java.util.Collections;
  17 import java.util.HashMap;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.concurrent.Future;
  21
  22 /**
  23  * The default implementation of {@link Index}. This class uses a
  24  * {@link SearchApiHelper} to forward all requests to an appserver.
  25  *
  26  */
  27 class IndexImpl implements Index {
  28
  29   private final SearchApiHelper apiHelper;
  30   private final IndexSpec spec;
  31   private final Schema schema;
  32   private final SearchServiceConfig config;
  33   private final Long storageUsage;
  34   private final Long storageLimit;
  35
  36   /**
  37    * Creates new index specification.
  38    *
  39    * @param apiHelper the helper used to forward all calls
  40    * @param config a {@link SearchServiceConfig} instance that describes the
  41    * index implementation.
  42    * @param indexSpec the index specification
  43    */
  44   IndexImpl(SearchApiHelper apiHelper, SearchServiceConfig config, IndexSpec indexSpec) {
  45     this(apiHelper, config, indexSpec, null, null, null);
  46   }
  47
  48   /**
  49    * Creates new index specification.
  50    *
  51    * @param apiHelper the helper used to forward all calls
  52    * @param config a {@link SearchServiceConfig} instance that describes the
  53    * index implementation.
  54    * @param indexSpec the index specification
  55    * @param schema the {@link Schema} defining the names and types of fields
  56    * supported
  57    */
  58   IndexImpl(SearchApiHelper apiHelper, SearchServiceConfig config, IndexSpec indexSpec,
  59       Schema schema, Long amount, Long limit) {
  60     this.apiHelper = Preconditions.checkNotNull(apiHelper, "Internal error");
  61     Preconditions.checkNotNull(config.getNamespace(), "Internal error");
  62     this.spec = Preconditions.checkNotNull(indexSpec, "Internal error");
  63     this.schema = schema;
  64     this.config = config;
  65     this.storageUsage = amount;
  66     this.storageLimit = limit;
  67   }
  68
  69   @Override
  70   public String getName() {
  71     return spec.getName();
  72   }
  73
  74   @Override
  75   public String getNamespace() {
  76     return config.getNamespace();
  77   }
  78
  79   @Override
  80   public Schema getSchema() {
  81     return schema;
  82   }
  83
  84   private RuntimeException noStorageInfo() {
  85     return new UnsupportedOperationException("Storage information is not available");
  86   }
  87
  88   @Override
  89   public long getStorageUsage() {
  90     if (storageUsage == null) {
  91       throw noStorageInfo();
  92     }
  93     return storageUsage;
  94   }
  95
  96   @Override
  97   public long getStorageLimit() {
  98     if (storageLimit == null) {
  99       throw noStorageInfo();
 100     }
 101     return storageLimit;
 102   }
 103
 104   @Override
 105   public int hashCode() {
 106     final int prime = 31;
 107     int result = 1;
 108     result = prime * result + spec.hashCode();
 109     result = prime * result + ((schema == null) ? 0 : schema.hashCode());
 110     result = prime * result + ((storageUsage == null) ? 0 : storageUsage.hashCode());
 111     result = prime * result + ((storageLimit == null) ? 0 : storageLimit.hashCode());
 112     return result;
 113   }
 114
 115   @Override
 116   public boolean equals(Object obj) {
 117     if (this == obj) {
 118       return true;
 119     }
 120     if (obj == null) {
 121       return false;
 122     }
 123     if (getClass() != obj.getClass()) {
 124       return false;
 125     }
 126     IndexImpl other = (IndexImpl) obj;
 127     return Util.equalObjects(spec, other.spec) &&
 128         Util.equalObjects(schema, other.schema) &&
 129         Util.equalObjects(storageUsage, other.storageUsage) &&
 130         Util.equalObjects(storageLimit, other.storageLimit);
 131   }
 132
 133   @Override
 134   public String toString() {
 135     String storageInfo =
 136         (storageUsage == null || storageLimit == null) ?
 137         "(no storage data)" :
 138         String.format(" (%d/%d)", storageUsage.longValue(), storageLimit.longValue());
 139     return String.format("IndexImpl{namespace: %s, %s, %s%s}", config.getNamespace(), spec,
 140         (schema == null ? "(null schema)" : schema), storageInfo);
 141   }
 142
 143   @Deprecated
 144   @Override
 145   public Future<Void> deleteSchemaAsync() {
 146     SearchServicePb.DeleteSchemaParams.Builder builder =
 147         SearchServicePb.DeleteSchemaParams.newBuilder().addIndexSpec(
 148             spec.copyToProtocolBuffer(config.getNamespace()));
 149
 150     Future<SearchServicePb.DeleteSchemaResponse.Builder> future =
 151         apiHelper.makeAsyncDeleteSchemaCall(builder, config.getDeadline());
 152     return new FutureWrapper<SearchServicePb.DeleteSchemaResponse.Builder,
 153            Void>(future) {
 154       @Override
 155       protected Throwable convertException(Throwable cause) {
 156         OperationResult result = OperationResult.convertToOperationResult(cause);
 157         return (result == null) ? cause : new DeleteException(result);
 158       }
 159
 160       @Override
 161       protected Void wrap(SearchServicePb.DeleteSchemaResponse.Builder key)
 162           throws Exception {
 163         SearchServicePb.DeleteSchemaResponse response = key.build();
 164         ArrayList<OperationResult> results = new ArrayList<OperationResult>(
 165             response.getStatusCount());
 166         for (SearchServicePb.RequestStatus status : response.getStatusList()) {
 167           results.add(new OperationResult(status));
 168         }
 169         if (response.getStatusList().size() != 1) {
 170           throw new DeleteException(
 171               new OperationResult(
 172                   StatusCode.INTERNAL_ERROR,
 173                   String.format("Expected 1 removed schema, but got %d",
 174                                 response.getStatusList().size())),
 175               results);
 176         }
 177         for (OperationResult result : results) {
 178           if (result.getCode() != StatusCode.OK) {
 179             throw new DeleteException(result, results);
 180           }
 181         }
 182         return null;
 183       }
 184     };
 185   }
 186
 187   @Override
 188   public Future<Void> deleteAsync(String... documentIds) {
 189     return deleteAsync(Arrays.asList(documentIds));
 190   }
 191
 192   @Override
 193   public Future<Void> deleteAsync(final Iterable<String> documentIds) {
 194     Preconditions.checkArgument(documentIds != null,
 195         "Delete documents given null collection of document ids");
 196     SearchServicePb.DeleteDocumentParams.Builder builder =
 197         SearchServicePb.DeleteDocumentParams.newBuilder().setIndexSpec(
 198             spec.copyToProtocolBuffer(config.getNamespace()));
 199     int size = 0;
 200     for (String documentId : documentIds) {
 201       size++;
 202       builder.addDocId(DocumentChecker.checkDocumentId(documentId));
 203     }
 204     if (size > SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST) {
 205       throw new IllegalArgumentException(
 206           String.format("number of doc ids, %s, exceeds maximum %s", size,
 207                         SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST));
 208     }
 209     final int documentIdsSize = size;
 210     Future<SearchServicePb.DeleteDocumentResponse.Builder> future =
 211         apiHelper.makeAsyncDeleteDocumentCall(builder, config.getDeadline());
 212     return new FutureWrapper<SearchServicePb.DeleteDocumentResponse.Builder,
 213            Void>(future) {
 214       @Override
 215       protected Throwable convertException(Throwable cause) {
 216         OperationResult result = OperationResult.convertToOperationResult(cause);
 217         return (result == null) ? cause : new DeleteException(result);
 218       }
 219
 220       @Override
 221       protected Void wrap(SearchServicePb.DeleteDocumentResponse.Builder key)
 222           throws Exception {
 223         SearchServicePb.DeleteDocumentResponse response = key.build();
 224         ArrayList<OperationResult> results = new ArrayList<OperationResult>(
 225             response.getStatusCount());
 226         for (SearchServicePb.RequestStatus status : response.getStatusList()) {
 227           results.add(new OperationResult(status));
 228         }
 229         if (documentIdsSize != response.getStatusList().size()) {
 230           throw new DeleteException(
 231               new OperationResult(
 232                   StatusCode.INTERNAL_ERROR,
 233                   String.format("Expected %d removed documents, but got %d", documentIdsSize,
 234                                 response.getStatusList().size())),
 235               results);
 236         }
 237         for (OperationResult result : results) {
 238           if (result.getCode() != StatusCode.OK) {
 239             throw new DeleteException(result, results);
 240           }
 241         }
 242         return null;
 243       }
 244     };
 245   }
 246
 247   @Override
 248   public Future<PutResponse> putAsync(Document... documents) {
 249     return putAsync(Arrays.asList(documents));
 250   }
 251
 252   @Override
 253   public Future<PutResponse> putAsync(Document.Builder... builders) {
 254     List<Document> documents = new ArrayList<Document>();
 255     for (int i = 0; i < builders.length; i++) {
 256       documents.add(builders[i].build());
 257     }
 258     return putAsync(documents);
 259   }
 260
 261   @Override
 262   public Future<PutResponse> putAsync(final Iterable<Document> documents) {
 263     Preconditions.checkNotNull(documents, "document list cannot be null");
 264     if (Iterables.isEmpty(documents)) {
 265       return new FutureHelper.FakeFuture<PutResponse>(
 266           new PutResponse(Collections.<OperationResult>emptyList(),
 267                                      Collections.<String>emptyList()));
 268     }
 269     SearchServicePb.IndexDocumentParams.Builder builder =
 270         SearchServicePb.IndexDocumentParams.newBuilder()
 271             .setIndexSpec(spec.copyToProtocolBuffer(config.getNamespace()));
 272     Map<String, Document> docMap = new HashMap<String, Document>();
 273     int size = 0;
 274     for (Document document : documents) {
 275       Document other = null;
 276       if (document.getId() != null) {
 277         other = docMap.put(document.getId(), document);
 278       }
 279       if (other != null) {
 280         if (!document.isIdenticalTo(other)) {
 281           throw new IllegalArgumentException(
 282               String.format(
 283                   "Put request with documents with the same ID \"%s\" but different content",
 284                   document.getId()));
 285         }
 286       }
 287       if (other == null) {
 288         size++;
 289         builder.addDocument(Preconditions.checkNotNull(document, "document cannot be null")
 290             .copyToProtocolBuffer());
 291       }
 292     }
 293     if (size > SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST) {
 294       throw new IllegalArgumentException(
 295           String.format("number of documents, %s, exceeds maximum %s", size,
 296                         SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST));
 297     }
 298     final int documentsSize = size;
 299     Future<SearchServicePb.IndexDocumentResponse.Builder> future =
 300         apiHelper.makeAsyncIndexDocumentCall(builder, config.getDeadline());
 301     return new FutureWrapper<SearchServicePb.IndexDocumentResponse.Builder,
 302            PutResponse>(future) {
 303       @Override
 304       protected Throwable convertException(Throwable cause) {
 305         OperationResult result = OperationResult.convertToOperationResult(cause);
 306         return (result == null) ? cause : new PutException(result);
 307       }
 308
 309       @Override
 310       protected PutResponse wrap(SearchServicePb.IndexDocumentResponse.Builder key)
 311           throws Exception {
 312         SearchServicePb.IndexDocumentResponse response = key.build();
 313         List<OperationResult> results = newOperationResultList(response);
 314         if (documentsSize != response.getStatusList().size()) {
 315           throw new PutException(
 316               new OperationResult(
 317                   StatusCode.INTERNAL_ERROR,
 318                   String.format("Expected %d indexed documents, but got %d", documentsSize,
 319                       response.getStatusList().size())), results, response.getDocIdList());
 320         }
 321         for (OperationResult result : results) {
 322           if (result.getCode() != StatusCode.OK) {
 323             throw new PutException(result, results, response.getDocIdList());
 324           }
 325         }
 326         return new PutResponse(results, response.getDocIdList());
 327       }
 328
 329       /**
 330        * Constructs a list of OperationResult from an index document response.
 331        *
 332        * @param response the index document response to extract operation
 333        * results from
 334        * @return a list of OperationResult
 335        */
 336       private List<OperationResult> newOperationResultList(
 337           SearchServicePb.IndexDocumentResponse response) {
 338         ArrayList<OperationResult> results = new ArrayList<OperationResult>(
 339             response.getStatusCount());
 340         for (SearchServicePb.RequestStatus status : response.getStatusList()) {
 341           results.add(new OperationResult(status));
 342         }
 343         return results;
 344       }
 345     };
 346   }
 347
 348   private Future<Results<ScoredDocument>> executeSearchForResults(
 349       SearchServicePb.SearchParams.Builder params) {
 350     Future<SearchServicePb.SearchResponse.Builder> future =
 351         apiHelper.makeAsyncSearchCall(params, config.getDeadline());
 352     return new FutureWrapper<SearchServicePb.SearchResponse.Builder,
 353            Results<ScoredDocument>>(future) {
 354       @Override
 355       protected Throwable convertException(Throwable cause) {
 356         OperationResult result = OperationResult.convertToOperationResult(cause);
 357         return (result == null) ? cause : new SearchException(result);
 358       }
 359
 360       @Override
 361       protected Results<ScoredDocument> wrap(SearchServicePb.SearchResponse.Builder key)
 362       throws Exception {
 363         SearchServicePb.SearchResponse response = key.build();
 364         SearchServicePb.RequestStatus status = response.getStatus();
 365         if (status.getCode() != SearchServicePb.SearchServiceError.ErrorCode.OK) {
 366           throw new SearchException(new OperationResult(status));
 367         }
 368         List<ScoredDocument> scoredDocs = new ArrayList<ScoredDocument>();
 369         for (SearchServicePb.SearchResult result : response.getResultList()) {
 370           List<Field> expressions = new ArrayList<Field>();
 371           for (DocumentPb.Field expression : result.getExpressionList()) {
 372             expressions.add(Field.newBuilder(expression).build());
 373           }
 374           ScoredDocument.Builder scoredDocBuilder = ScoredDocument.newBuilder(result.getDocument());
 375           for (Double score : result.getScoreList()) {
 376             scoredDocBuilder.addScore(score);
 377           }
 378           for (Field expression : expressions) {
 379             scoredDocBuilder.addExpression(expression);
 380           }
 381           if (result.hasCursor()) {
 382             scoredDocBuilder.setCursor(
 383                 Cursor.newBuilder().build("true:" + result.getCursor()));
 384           }
 385           scoredDocs.add(scoredDocBuilder.build());
 386         }
 387         Results<ScoredDocument> scoredResults = new Results<ScoredDocument>(
 388             new OperationResult(status),
 389             scoredDocs, response.getMatchedCount(), response.getResultCount(),
 390             (response.hasCursor() ?
 391                 Cursor.newBuilder().build("false:" + response.getCursor()) : null));
 392
 393         return scoredResults;
 394       }
 395     };
 396   }
 397
 398   @Override
 399   public Future<Results<ScoredDocument>> searchAsync(String query) {
 400     return searchAsync(Query.newBuilder().build(
 401         Preconditions.checkNotNull(query, "query cannot be null")));
 402   }
 403
 404   @Override
 405   public Future<Results<ScoredDocument>> searchAsync(Query query) {
 406     Preconditions.checkNotNull(query, "query cannot be null");
 407     return executeSearchForResults(
 408         query.copyToProtocolBuffer().setIndexSpec(spec.copyToProtocolBuffer(
 409             config.getNamespace())));
 410   }
 411
 412   @Override
 413   public Future<GetResponse<Document>> getRangeAsync(GetRequest.Builder builder) {
 414     return getRangeAsync(builder.build());
 415   }
 416
 417   @Override
 418   public Future<GetResponse<Document>> getRangeAsync(GetRequest request) {
 419     Preconditions.checkNotNull(request, "list documents request cannot be null");
 420
 421     SearchServicePb.ListDocumentsParams.Builder params =
 422         request.copyToProtocolBuffer().setIndexSpec(spec.copyToProtocolBuffer(
 423         config.getNamespace()));
 424
 425     Future<SearchServicePb.ListDocumentsResponse.Builder> future =
 426         apiHelper.makeAsyncListDocumentsCall(params, config.getDeadline());
 427     return new FutureWrapper<SearchServicePb.ListDocumentsResponse.Builder,
 428         GetResponse<Document>>(future) {
 429       @Override
 430       protected Throwable convertException(Throwable cause) {
 431         OperationResult result = OperationResult.convertToOperationResult(cause);
 432         return (result == null) ? cause : new GetException(result);
 433       }
 434
 435       @Override
 436       protected GetResponse<Document> wrap(
 437           SearchServicePb.ListDocumentsResponse.Builder key) throws Exception {
 438         SearchServicePb.ListDocumentsResponse response = key.build();
 439         SearchServicePb.RequestStatus status = response.getStatus();
 440
 441         if (status.getCode() != ErrorCode.OK) {
 442           throw new GetException(new OperationResult(status));
 443         }
 444
 445         List<Document> results = new ArrayList<Document>();
 446         for (DocumentPb.Document document : response.getDocumentList()) {
 447           results.add(Document.newBuilder(document).build());
 448         }
 449         return new GetResponse<Document>(results);
 450       }
 451     };
 452   }
 453
 454   @Override
 455   public Document get(String documentId) {
 456     Preconditions.checkNotNull(documentId, "documentId must not be null");
 457     GetResponse<Document> response =
 458         getRange(GetRequest.newBuilder().setStartId(documentId).setLimit(1));
 459     for (Document document : response) {
 460       if (documentId.equals(document.getId())) {
 461         return document;
 462       }
 463     }
 464     return null;
 465   }
 466
 467   @Deprecated
 468   @Override
 469   public void deleteSchema() {
 470     quietGet(deleteSchemaAsync());
 471   }
 472
 473   @Override
 474   public void delete(String... documentIds) {
 475     quietGet(deleteAsync(documentIds));
 476   }
 477
 478   @Override
 479   public void delete(Iterable<String> documentIds) {
 480     quietGet(deleteAsync(documentIds));
 481   }
 482
 483   @Override
 484   public PutResponse put(Document... documents) {
 485     return quietGet(putAsync(documents));
 486   }
 487
 488   @Override
 489   public PutResponse put(Document.Builder... builders) {
 490     return quietGet(putAsync(builders));
 491   }
 492
 493   @Override
 494   public PutResponse put(Iterable<Document> documents) {
 495     return quietGet(putAsync(documents));
 496   }
 497
 498   @Override
 499   public Results<ScoredDocument> search(String query) {
 500     return quietGet(searchAsync(query));
 501   }
 502
 503   @Override
 504   public Results<ScoredDocument> search(Query query) {
 505     return quietGet(searchAsync(query));
 506   }
 507
 508   @Override
 509   public GetResponse<Document> getRange(GetRequest request) {
 510     return quietGet(getRangeAsync(request));
 511   }
 512
 513   @Override
 514   public GetResponse<Document> getRange(GetRequest.Builder builder) {
 515     return getRange(builder.build());
 516   }
 517 }