java/src/main/com/google/appengine/api/search/IndexImpl.java

   1 package com.google.appengine.api.search;
   2
   3 import static com.google.appengine.api.search.FutureHelper.quietGet;
   4
   5 import com.google.appengine.api.search.SearchServicePb.SearchServiceError.ErrorCode;
   6 import com.google.appengine.api.search.checkers.DocumentChecker;
   7 import com.google.appengine.api.search.checkers.Preconditions;
   8 import com.google.appengine.api.search.checkers.SearchApiLimits;
   9 import com.google.appengine.api.utils.FutureWrapper;
  10 import com.google.apphosting.api.search.DocumentPb;
  11 import com.google.common.collect.Iterables;
  12
  13 import java.util.ArrayList;
  14 import java.util.Arrays;
  15 import java.util.Collections;
  16 import java.util.HashMap;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.concurrent.Future;
  20
  21 /**
  22  * The default implementation of {@link Index}. This class uses a
  23  * {@link SearchApiHelper} to forward all requests to an appserver.
  24  */
  25 class IndexImpl implements Index {
  26
  27   private final SearchApiHelper apiHelper;
  28   private final IndexSpec spec;
  29   private final Schema schema;
  30   private final SearchServiceConfig config;
  31   private final Long storageUsage;
  32   private final Long storageLimit;
  33
  34   /**
  35    * Creates new index specification.
  36    *
  37    * @param apiHelper the helper used to forward all calls
  38    * @param config a {@link SearchServiceConfig} instance that describes the
  39    * index implementation.
  40    * @param indexSpec the index specification
  41    */
  42   IndexImpl(SearchApiHelper apiHelper, SearchServiceConfig config, IndexSpec indexSpec) {
  43     this(apiHelper, config, indexSpec, null, null, null);
  44   }
  45
  46   /**
  47    * Creates new index specification.
  48    *
  49    * @param apiHelper the helper used to forward all calls
  50    * @param config a {@link SearchServiceConfig} instance that describes the
  51    * index implementation.
  52    * @param indexSpec the index specification
  53    * @param schema the {@link Schema} defining the names and types of fields
  54    * supported
  55    */
  56   IndexImpl(SearchApiHelper apiHelper, SearchServiceConfig config, IndexSpec indexSpec,
  57       Schema schema, Long amount, Long limit) {
  58     this.apiHelper = Preconditions.checkNotNull(apiHelper, "Internal error");
  59     Preconditions.checkNotNull(config.getNamespace(), "Internal error");
  60     this.spec = Preconditions.checkNotNull(indexSpec, "Internal error");
  61     this.schema = schema;
  62     this.config = config;
  63     this.storageUsage = amount;
  64     this.storageLimit = limit;
  65   }
  66
  67   @Override
  68   public String getName() {
  69     return spec.getName();
  70   }
  71
  72   @Override
  73   public String getNamespace() {
  74     return config.getNamespace();
  75   }
  76
  77   @Override
  78   public Schema getSchema() {
  79     return schema;
  80   }
  81
  82   private RuntimeException noStorageInfo() {
  83     return new UnsupportedOperationException("Storage information is not available");
  84   }
  85
  86   @Override
  87   public long getStorageUsage() {
  88     if (storageUsage == null) {
  89       throw noStorageInfo();
  90     }
  91     return storageUsage;
  92   }
  93
  94   @Override
  95   public long getStorageLimit() {
  96     if (storageLimit == null) {
  97       throw noStorageInfo();
  98     }
  99     return storageLimit;
 100   }
 101
 102   @Override
 103   public int hashCode() {
 104     final int prime = 31;
 105     int result = 1;
 106     result = prime * result + spec.hashCode();
 107     result = prime * result + ((schema == null) ? 0 : schema.hashCode());
 108     result = prime * result + ((storageUsage == null) ? 0 : storageUsage.hashCode());
 109     result = prime * result + ((storageLimit == null) ? 0 : storageLimit.hashCode());
 110     return result;
 111   }
 112
 113   @Override
 114   public boolean equals(Object obj) {
 115     if (this == obj) {
 116       return true;
 117     }
 118     if (obj == null) {
 119       return false;
 120     }
 121     if (getClass() != obj.getClass()) {
 122       return false;
 123     }
 124     IndexImpl other = (IndexImpl) obj;
 125     return Util.equalObjects(spec, other.spec)
 126         && Util.equalObjects(schema, other.schema)
 127         && Util.equalObjects(storageUsage, other.storageUsage)
 128         && Util.equalObjects(storageLimit, other.storageLimit);
 129   }
 130
 131   @Override
 132   public String toString() {
 133     String storageInfo =
 134         (storageUsage == null || storageLimit == null)
 135         ? "(no storage data)"
 136         : String.format(" (%d/%d)", storageUsage.longValue(), storageLimit.longValue());
 137     return String.format("IndexImpl{namespace: %s, %s, %s%s}", config.getNamespace(), spec,
 138         (schema == null ? "(null schema)" : schema), storageInfo);
 139   }
 140
 141   @Deprecated
 142   @Override
 143   public Future<Void> deleteSchemaAsync() {
 144     SearchServicePb.DeleteSchemaParams.Builder builder =
 145         SearchServicePb.DeleteSchemaParams.newBuilder().addIndexSpec(
 146             spec.copyToProtocolBuffer(config.getNamespace()));
 147
 148     Future<SearchServicePb.DeleteSchemaResponse.Builder> future =
 149         apiHelper.makeAsyncDeleteSchemaCall(builder, config.getDeadline());
 150     return new FutureWrapper<SearchServicePb.DeleteSchemaResponse.Builder,
 151            Void>(future) {
 152       @Override
 153       protected Throwable convertException(Throwable cause) {
 154         OperationResult result = OperationResult.convertToOperationResult(cause);
 155         return (result == null) ? cause : new DeleteException(result);
 156       }
 157
 158       @Override
 159       protected Void wrap(SearchServicePb.DeleteSchemaResponse.Builder key)
 160           throws Exception {
 161         SearchServicePb.DeleteSchemaResponse response = key.build();
 162         ArrayList<OperationResult> results = new ArrayList<>(
 163             response.getStatusCount());
 164         for (SearchServicePb.RequestStatus status : response.getStatusList()) {
 165           results.add(new OperationResult(status));
 166         }
 167         if (response.getStatusList().size() != 1) {
 168           throw new DeleteException(
 169               new OperationResult(
 170                   StatusCode.INTERNAL_ERROR,
 171                   String.format("Expected 1 removed schema, but got %d",
 172                                 response.getStatusList().size())),
 173               results);
 174         }
 175         for (OperationResult result : results) {
 176           if (result.getCode() != StatusCode.OK) {
 177             throw new DeleteException(result, results);
 178           }
 179         }
 180         return null;
 181       }
 182     };
 183   }
 184
 185   @Override
 186   public Future<Void> deleteAsync(String... documentIds) {
 187     return deleteAsync(Arrays.asList(documentIds));
 188   }
 189
 190   @Override
 191   public Future<Void> deleteAsync(final Iterable<String> documentIds) {
 192     Preconditions.checkArgument(documentIds != null,
 193         "Delete documents given null collection of document ids");
 194     SearchServicePb.DeleteDocumentParams.Builder builder =
 195         SearchServicePb.DeleteDocumentParams.newBuilder().setIndexSpec(
 196             spec.copyToProtocolBuffer(config.getNamespace()));
 197     int size = 0;
 198     for (String documentId : documentIds) {
 199       size++;
 200       builder.addDocId(DocumentChecker.checkDocumentId(documentId));
 201     }
 202     if (size > SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST) {
 203       throw new IllegalArgumentException(
 204           String.format("number of doc ids, %s, exceeds maximum %s", size,
 205                         SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST));
 206     }
 207     final int documentIdsSize = size;
 208     Future<SearchServicePb.DeleteDocumentResponse.Builder> future =
 209         apiHelper.makeAsyncDeleteDocumentCall(builder, config.getDeadline());
 210     return new FutureWrapper<SearchServicePb.DeleteDocumentResponse.Builder,
 211            Void>(future) {
 212       @Override
 213       protected Throwable convertException(Throwable cause) {
 214         OperationResult result = OperationResult.convertToOperationResult(cause);
 215         return (result == null) ? cause : new DeleteException(result);
 216       }
 217
 218       @Override
 219       protected Void wrap(SearchServicePb.DeleteDocumentResponse.Builder key)
 220           throws Exception {
 221         SearchServicePb.DeleteDocumentResponse response = key.build();
 222         ArrayList<OperationResult> results = new ArrayList<>(
 223             response.getStatusCount());
 224         for (SearchServicePb.RequestStatus status : response.getStatusList()) {
 225           results.add(new OperationResult(status));
 226         }
 227         if (documentIdsSize != response.getStatusList().size()) {
 228           throw new DeleteException(
 229               new OperationResult(
 230                   StatusCode.INTERNAL_ERROR,
 231                   String.format("Expected %d removed documents, but got %d", documentIdsSize,
 232                                 response.getStatusList().size())),
 233               results);
 234         }
 235         for (OperationResult result : results) {
 236           if (result.getCode() != StatusCode.OK) {
 237             throw new DeleteException(result, results);
 238           }
 239         }
 240         return null;
 241       }
 242     };
 243   }
 244
 245   @Override
 246   public Future<PutResponse> putAsync(Document... documents) {
 247     return putAsync(Arrays.asList(documents));
 248   }
 249
 250   @Override
 251   public Future<PutResponse> putAsync(Document.Builder... builders) {
 252     List<Document> documents = new ArrayList<>();
 253     for (int i = 0; i < builders.length; i++) {
 254       documents.add(builders[i].build());
 255     }
 256     return putAsync(documents);
 257   }
 258
 259   @Override
 260   public Future<PutResponse> putAsync(final Iterable<Document> documents) {
 261     Preconditions.checkNotNull(documents, "document list cannot be null");
 262     if (Iterables.isEmpty(documents)) {
 263       return new FutureHelper.FakeFuture<>(
 264           new PutResponse(Collections.<OperationResult>emptyList(),
 265                                      Collections.<String>emptyList()));
 266     }
 267     SearchServicePb.IndexDocumentParams.Builder builder =
 268         SearchServicePb.IndexDocumentParams.newBuilder()
 269             .setIndexSpec(spec.copyToProtocolBuffer(config.getNamespace()));
 270     Map<String, Document> docMap = new HashMap<>();
 271     int size = 0;
 272     for (Document document : documents) {
 273       Document other = null;
 274       if (document.getId() != null) {
 275         other = docMap.put(document.getId(), document);
 276       }
 277       if (other != null) {
 278         if (!document.isIdenticalTo(other)) {
 279           throw new IllegalArgumentException(
 280               String.format(
 281                   "Put request with documents with the same ID \"%s\" but different content",
 282                   document.getId()));
 283         }
 284       }
 285       if (other == null) {
 286         size++;
 287         builder.addDocument(Preconditions.checkNotNull(document, "document cannot be null")
 288             .copyToProtocolBuffer());
 289       }
 290     }
 291     if (size > SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST) {
 292       throw new IllegalArgumentException(
 293           String.format("number of documents, %s, exceeds maximum %s", size,
 294                         SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST));
 295     }
 296     final int documentsSize = size;
 297     Future<SearchServicePb.IndexDocumentResponse.Builder> future =
 298         apiHelper.makeAsyncIndexDocumentCall(builder, config.getDeadline());
 299     return new FutureWrapper<SearchServicePb.IndexDocumentResponse.Builder,
 300            PutResponse>(future) {
 301       @Override
 302       protected Throwable convertException(Throwable cause) {
 303         OperationResult result = OperationResult.convertToOperationResult(cause);
 304         return (result == null) ? cause : new PutException(result);
 305       }
 306
 307       @Override
 308       protected PutResponse wrap(SearchServicePb.IndexDocumentResponse.Builder key)
 309           throws Exception {
 310         SearchServicePb.IndexDocumentResponse response = key.build();
 311         List<OperationResult> results = newOperationResultList(response);
 312         if (documentsSize != response.getStatusList().size()) {
 313           throw new PutException(
 314               new OperationResult(
 315                   StatusCode.INTERNAL_ERROR,
 316                   String.format("Expected %d indexed documents, but got %d", documentsSize,
 317                       response.getStatusList().size())), results, response.getDocIdList());
 318         }
 319         for (OperationResult result : results) {
 320           if (result.getCode() != StatusCode.OK) {
 321             throw new PutException(result, results, response.getDocIdList());
 322           }
 323         }
 324         return new PutResponse(results, response.getDocIdList());
 325       }
 326
 327       /**
 328        * Constructs a list of OperationResult from an index document response.
 329        *
 330        * @param response the index document response to extract operation
 331        * results from
 332        * @return a list of OperationResult
 333        */
 334       private List<OperationResult> newOperationResultList(
 335           SearchServicePb.IndexDocumentResponse response) {
 336         ArrayList<OperationResult> results = new ArrayList<>(
 337             response.getStatusCount());
 338         for (SearchServicePb.RequestStatus status : response.getStatusList()) {
 339           results.add(new OperationResult(status));
 340         }
 341         return results;
 342       }
 343     };
 344   }
 345
 346   private Future<Results<ScoredDocument>> executeSearchForResults(
 347       SearchServicePb.SearchParams.Builder params) {
 348     Future<SearchServicePb.SearchResponse.Builder> future =
 349         apiHelper.makeAsyncSearchCall(params, config.getDeadline());
 350     return new FutureWrapper<SearchServicePb.SearchResponse.Builder,
 351            Results<ScoredDocument>>(future) {
 352       @Override
 353       protected Throwable convertException(Throwable cause) {
 354         OperationResult result = OperationResult.convertToOperationResult(cause);
 355         return (result == null) ? cause : new SearchException(result);
 356       }
 357
 358       @Override
 359       protected Results<ScoredDocument> wrap(SearchServicePb.SearchResponse.Builder key)
 360       throws Exception {
 361         SearchServicePb.SearchResponse response = key.build();
 362         SearchServicePb.RequestStatus status = response.getStatus();
 363         if (status.getCode() != SearchServicePb.SearchServiceError.ErrorCode.OK) {
 364           throw new SearchException(new OperationResult(status));
 365         }
 366         List<ScoredDocument> scoredDocs = new ArrayList<>();
 367         for (SearchServicePb.SearchResult result : response.getResultList()) {
 368           List<Field> expressions = new ArrayList<>();
 369           for (DocumentPb.Field expression : result.getExpressionList()) {
 370             expressions.add(Field.newBuilder(expression).build());
 371           }
 372           ScoredDocument.Builder scoredDocBuilder = ScoredDocument.newBuilder(result.getDocument());
 373           for (Double score : result.getScoreList()) {
 374             scoredDocBuilder.addScore(score);
 375           }
 376           for (Field expression : expressions) {
 377             scoredDocBuilder.addExpression(expression);
 378           }
 379           if (result.hasCursor()) {
 380             scoredDocBuilder.setCursor(
 381                 Cursor.newBuilder().build("true:" + result.getCursor()));
 382           }
 383           scoredDocs.add(scoredDocBuilder.build());
 384         }
 385         List<FacetResult> facetResults = new ArrayList<>();
 386         for (SearchServicePb.FacetResult result : response.getFacetResultList()) {
 387           facetResults.add(FacetResult.newBuilder(result).build());
 388         }
 389         Results<ScoredDocument> scoredResults = new Results<>(
 390             new OperationResult(status),
 391             scoredDocs, response.getMatchedCount(), response.getResultCount(),
 392             (response.hasCursor() ? Cursor.newBuilder().build("false:" + response.getCursor())
 393                 : null), facetResults);
 394         return scoredResults;
 395       }
 396     };
 397   }
 398
 399   @Override
 400   public Future<Results<ScoredDocument>> searchAsync(String query) {
 401     return searchAsync(Query.newBuilder().build(
 402         Preconditions.checkNotNull(query, "query cannot be null")));
 403   }
 404
 405   @Override
 406   public Future<Results<ScoredDocument>> searchAsync(Query query) {
 407     Preconditions.checkNotNull(query, "query cannot be null");
 408     return executeSearchForResults(
 409         query.copyToProtocolBuffer().setIndexSpec(spec.copyToProtocolBuffer(
 410             config.getNamespace())));
 411   }
 412
 413   @Override
 414   public Future<GetResponse<Document>> getRangeAsync(GetRequest.Builder builder) {
 415     return getRangeAsync(builder.build());
 416   }
 417
 418   @Override
 419   public Future<GetResponse<Document>> getRangeAsync(GetRequest request) {
 420     Preconditions.checkNotNull(request, "list documents request cannot be null");
 421
 422     SearchServicePb.ListDocumentsParams.Builder params =
 423         request.copyToProtocolBuffer().setIndexSpec(spec.copyToProtocolBuffer(
 424         config.getNamespace()));
 425
 426     Future<SearchServicePb.ListDocumentsResponse.Builder> future =
 427         apiHelper.makeAsyncListDocumentsCall(params, config.getDeadline());
 428     return new FutureWrapper<SearchServicePb.ListDocumentsResponse.Builder,
 429         GetResponse<Document>>(future) {
 430       @Override
 431       protected Throwable convertException(Throwable cause) {
 432         OperationResult result = OperationResult.convertToOperationResult(cause);
 433         return (result == null) ? cause : new GetException(result);
 434       }
 435
 436       @Override
 437       protected GetResponse<Document> wrap(
 438           SearchServicePb.ListDocumentsResponse.Builder key) throws Exception {
 439         SearchServicePb.ListDocumentsResponse response = key.build();
 440         SearchServicePb.RequestStatus status = response.getStatus();
 441
 442         if (status.getCode() != ErrorCode.OK) {
 443           throw new GetException(new OperationResult(status));
 444         }
 445
 446         List<Document> results = new ArrayList<>();
 447         for (DocumentPb.Document document : response.getDocumentList()) {
 448           results.add(Document.newBuilder(document).build());
 449         }
 450         return new GetResponse<>(results);
 451       }
 452     };
 453   }
 454
 455   @Override
 456   public Document get(String documentId) {
 457     Preconditions.checkNotNull(documentId, "documentId must not be null");
 458     GetResponse<Document> response =
 459         getRange(GetRequest.newBuilder().setStartId(documentId).setLimit(1));
 460     for (Document document : response) {
 461       if (documentId.equals(document.getId())) {
 462         return document;
 463       }
 464     }
 465     return null;
 466   }
 467
 468   @Deprecated
 469   @Override
 470   public void deleteSchema() {
 471     quietGet(deleteSchemaAsync());
 472   }
 473
 474   @Override
 475   public void delete(String... documentIds) {
 476     quietGet(deleteAsync(documentIds));
 477   }
 478
 479   @Override
 480   public void delete(Iterable<String> documentIds) {
 481     quietGet(deleteAsync(documentIds));
 482   }
 483
 484   @Override
 485   public PutResponse put(Document... documents) {
 486     return quietGet(putAsync(documents));
 487   }
 488
 489   @Override
 490   public PutResponse put(Document.Builder... builders) {
 491     return quietGet(putAsync(builders));
 492   }
 493
 494   @Override
 495   public PutResponse put(Iterable<Document> documents) {
 496     return quietGet(putAsync(documents));
 497   }
 498
 499   @Override
 500   public Results<ScoredDocument> search(String query) {
 501     return quietGet(searchAsync(query));
 502   }
 503
 504   @Override
 505   public Results<ScoredDocument> search(Query query) {
 506     return quietGet(searchAsync(query));
 507   }
 508
 509   @Override
 510   public GetResponse<Document> getRange(GetRequest request) {
 511     return quietGet(getRangeAsync(request));
 512   }
 513
 514   @Override
 515   public GetResponse<Document> getRange(GetRequest.Builder builder) {
 516     return getRange(builder.build());
 517   }
 518 }