java/src/main/com/google/appengine/api/search/IndexImpl.java

   1 // Copyright 2010 Google Inc. All Rights Reserved.
   2 package com.google.appengine.api.search;
   3
   4 import static com.google.appengine.api.search.FutureHelper.quietGet;
   5
   6 import com.google.appengine.api.search.SearchServicePb.SearchServiceError.ErrorCode;
   7 import com.google.appengine.api.search.checkers.DocumentChecker;
   8 import com.google.appengine.api.search.checkers.Preconditions;
   9 import com.google.appengine.api.search.checkers.SearchApiLimits;
  10 import com.google.appengine.api.utils.FutureWrapper;
  11 import com.google.apphosting.api.search.DocumentPb;
  12
  13 import java.util.ArrayList;
  14 import java.util.Arrays;
  15 import java.util.Collections;
  16 import java.util.HashMap;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.concurrent.Future;
  20
  21 /**
  22  * The default implementation of {@link Index}. This class uses a
  23  * {@link SearchApiHelper} to forward all requests to an appserver.
  24  *
  25  */
  26 class IndexImpl implements Index {
  27
  28   private final SearchApiHelper apiHelper;
  29   private final IndexSpec spec;
  30   private final String namespace;
  31   private final Schema schema;
  32
  33   /**
  34    * Creates new index specification.
  35    *
  36    * @param apiHelper the helper used to forward all calls
  37    * @param indexSpec the index specification
  38    */
  39   IndexImpl(SearchApiHelper apiHelper, String namespace, IndexSpec indexSpec) {
  40     this(apiHelper, namespace, indexSpec, null);
  41   }
  42
  43   /**
  44    * Creates new index specification.
  45    *
  46    * @param apiHelper the helper used to forward all calls
  47    * @param indexSpec the index specification
  48    * @param schema the {@link Schema} defining the names and types of fields
  49    * supported
  50    */
  51   IndexImpl(SearchApiHelper apiHelper, String namespace, IndexSpec indexSpec, Schema schema) {
  52     this.apiHelper = Preconditions.checkNotNull(apiHelper, "Internal error");
  53     this.namespace = Preconditions.checkNotNull(namespace, "Internal error");
  54     this.spec = Preconditions.checkNotNull(indexSpec, "Internal error");
  55     this.schema = schema;
  56   }
  57
  58   @Override
  59   public String getName() {
  60     return spec.getName();
  61   }
  62
  63   @Override
  64   public String getNamespace() {
  65     return namespace;
  66   }
  67
  68   @Override
  69   public Consistency getConsistency() {
  70     return spec.getConsistency();
  71   }
  72
  73   @Override
  74   public Schema getSchema() {
  75     return schema;
  76   }
  77
  78   @Override
  79   public int hashCode() {
  80     final int prime = 31;
  81     int result = 1;
  82     result = prime * result + spec.hashCode();
  83     result = prime * result + ((schema == null) ? 0 : schema.hashCode());
  84     return result;
  85   }
  86
  87   @Override
  88   public boolean equals(Object obj) {
  89     if (this == obj) {
  90       return true;
  91     }
  92     if (obj == null) {
  93       return false;
  94     }
  95     if (getClass() != obj.getClass()) {
  96       return false;
  97     }
  98     IndexImpl other = (IndexImpl) obj;
  99     return Util.equalObjects(spec, other.spec) &&
 100         Util.equalObjects(schema, other.schema);
 101   }
 102
 103   @Override
 104   public String toString() {
 105     return String.format("IndexImpl{namespace: %s, %s, %s}", namespace, spec,
 106                          (schema == null ? "(null schema)" : schema));
 107   }
 108
 109   @Override
 110   public Future<Void> deleteSchemaAsync() {
 111     SearchServicePb.DeleteSchemaParams.Builder builder =
 112         SearchServicePb.DeleteSchemaParams.newBuilder().addIndexSpec(
 113             spec.copyToProtocolBuffer(namespace));
 114     SearchServicePb.DeleteSchemaRequest request =
 115         SearchServicePb.DeleteSchemaRequest.newBuilder().setParams(builder).build();
 116     SearchServicePb.DeleteSchemaResponse.Builder responseBuilder =
 117         SearchServicePb.DeleteSchemaResponse.newBuilder();
 118
 119     Future<SearchServicePb.DeleteSchemaResponse.Builder> future =
 120         apiHelper.makeAsyncCall("DeleteSchema", request, responseBuilder);
 121     return new FutureWrapper<SearchServicePb.DeleteSchemaResponse.Builder,
 122            Void>(future) {
 123       @Override
 124       protected Throwable convertException(Throwable cause) {
 125         OperationResult result = OperationResult.convertToOperationResult(cause);
 126         return (result == null) ? cause : new DeleteException(result);
 127       }
 128
 129       @Override
 130       protected Void wrap(SearchServicePb.DeleteSchemaResponse.Builder key)
 131           throws Exception {
 132         SearchServicePb.DeleteSchemaResponse response = key.build();
 133         ArrayList<OperationResult> results = new ArrayList<OperationResult>(
 134             response.getStatusCount());
 135         for (SearchServicePb.RequestStatus status : response.getStatusList()) {
 136           results.add(new OperationResult(status));
 137         }
 138         if (response.getStatusList().size() != 1) {
 139           throw new DeleteException(
 140               new OperationResult(
 141                   StatusCode.INTERNAL_ERROR,
 142                   String.format("Expected 1 removed schema, but got %d",
 143                                 response.getStatusList().size())),
 144               results);
 145         }
 146         for (OperationResult result : results) {
 147           if (result.getCode() != StatusCode.OK) {
 148             throw new DeleteException(result, results);
 149           }
 150         }
 151         return null;
 152       }
 153     };
 154   }
 155
 156   @Override
 157   public Future<Void> deleteAsync(String... documentIds) {
 158     return deleteAsync(Arrays.asList(documentIds));
 159   }
 160
 161   @Override
 162   public Future<Void> deleteAsync(final Iterable<String> documentIds) {
 163     Preconditions.checkArgument(documentIds != null,
 164         "Delete documents given null collection of document ids");
 165     SearchServicePb.DeleteDocumentParams.Builder builder =
 166         SearchServicePb.DeleteDocumentParams.newBuilder().setIndexSpec(
 167             spec.copyToProtocolBuffer(namespace));
 168     int size = 0;
 169     for (String documentId : documentIds) {
 170       size++;
 171       builder.addDocId(DocumentChecker.checkDocumentId(documentId));
 172     }
 173     if (size > SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST) {
 174       throw new IllegalArgumentException(
 175           String.format("number of doc ids, %s, exceeds maximum %s", size,
 176                         SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST));
 177     }
 178     final int documentIdsSize = size;
 179     SearchServicePb.DeleteDocumentRequest request =
 180         SearchServicePb.DeleteDocumentRequest.newBuilder().setParams(builder).build();
 181     SearchServicePb.DeleteDocumentResponse.Builder responseBuilder =
 182         SearchServicePb.DeleteDocumentResponse.newBuilder();
 183
 184     Future<SearchServicePb.DeleteDocumentResponse.Builder> future =
 185         apiHelper.makeAsyncCall("DeleteDocument", request, responseBuilder);
 186     return new FutureWrapper<SearchServicePb.DeleteDocumentResponse.Builder,
 187            Void>(future) {
 188       @Override
 189       protected Throwable convertException(Throwable cause) {
 190         OperationResult result = OperationResult.convertToOperationResult(cause);
 191         return (result == null) ? cause : new DeleteException(result);
 192       }
 193
 194       @Override
 195       protected Void wrap(SearchServicePb.DeleteDocumentResponse.Builder key)
 196           throws Exception {
 197         SearchServicePb.DeleteDocumentResponse response = key.build();
 198         ArrayList<OperationResult> results = new ArrayList<OperationResult>(
 199             response.getStatusCount());
 200         for (SearchServicePb.RequestStatus status : response.getStatusList()) {
 201           results.add(new OperationResult(status));
 202         }
 203         if (documentIdsSize != response.getStatusList().size()) {
 204           throw new DeleteException(
 205               new OperationResult(
 206                   StatusCode.INTERNAL_ERROR,
 207                   String.format("Expected %d removed documents, but got %d", documentIdsSize,
 208                                 response.getStatusList().size())),
 209               results);
 210         }
 211         for (OperationResult result : results) {
 212           if (result.getCode() != StatusCode.OK) {
 213             throw new DeleteException(result, results);
 214           }
 215         }
 216         return null;
 217       }
 218     };
 219   }
 220
 221   @Override
 222   public Future<PutResponse> putAsync(Document... documents) {
 223     return putAsync(Arrays.asList(documents));
 224   }
 225
 226   @Override
 227   public Future<PutResponse> putAsync(Document.Builder... builders) {
 228     List<Document> documents = new ArrayList<Document>();
 229     for (int i = 0; i < builders.length; i++) {
 230       documents.add(builders[i].build());
 231     }
 232     return putAsync(documents);
 233   }
 234
 235   @Override
 236   public Future<PutResponse> putAsync(final Iterable<Document> documents) {
 237     Preconditions.checkNotNull(documents, "document list cannot be null");
 238     if (!documents.iterator().hasNext()) {
 239       return new FutureHelper.FakeFuture<PutResponse>(
 240           new PutResponse(Collections.<OperationResult>emptyList(),
 241                                      Collections.<String>emptyList()));
 242     }
 243     SearchServicePb.IndexDocumentParams.Builder builder =
 244         SearchServicePb.IndexDocumentParams.newBuilder()
 245             .setIndexSpec(spec.copyToProtocolBuffer(namespace));
 246     Map<String, Document> docMap = new HashMap<String, Document>();
 247     int size = 0;
 248     for (Document document : documents) {
 249       Document other = null;
 250       if (document.getId() != null) {
 251         other = docMap.put(document.getId(), document);
 252       }
 253       if (other != null) {
 254         if (!document.isIdenticalTo(other)) {
 255           throw new IllegalArgumentException(
 256               String.format(
 257                   "Put request with documents with the same ID \"%s\" but differnt content",
 258                   document.getId()));
 259         }
 260       }
 261       if (other == null) {
 262         size++;
 263         builder.addDocument(Preconditions.checkNotNull(document, "document cannot be null")
 264             .copyToProtocolBuffer());
 265       }
 266     }
 267     if (size > SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST) {
 268       throw new IllegalArgumentException(
 269           String.format("number of documents, %s, exceeds maximum %s", size,
 270                         SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST));
 271     }
 272     final int documentsSize = size;
 273     SearchServicePb.IndexDocumentRequest request =
 274         SearchServicePb.IndexDocumentRequest.newBuilder().setParams(builder).build();
 275     SearchServicePb.IndexDocumentResponse.Builder responseBuilder =
 276         SearchServicePb.IndexDocumentResponse.newBuilder();
 277     Future<SearchServicePb.IndexDocumentResponse.Builder> future =
 278         apiHelper.makeAsyncCall("IndexDocument", request, responseBuilder);
 279     return new FutureWrapper<SearchServicePb.IndexDocumentResponse.Builder,
 280            PutResponse>(future) {
 281       @Override
 282       protected Throwable convertException(Throwable cause) {
 283         OperationResult result = OperationResult.convertToOperationResult(cause);
 284         return (result == null) ? cause : new PutException(result);
 285       }
 286
 287       @Override
 288       protected PutResponse wrap(SearchServicePb.IndexDocumentResponse.Builder key)
 289           throws Exception {
 290         SearchServicePb.IndexDocumentResponse response = key.build();
 291         List<OperationResult> results = newOperationResultList(response);
 292         if (documentsSize != response.getStatusList().size()) {
 293           throw new PutException(
 294               new OperationResult(
 295                   StatusCode.INTERNAL_ERROR,
 296                   String.format("Expected %d indexed documents, but got %d", documentsSize,
 297                       response.getStatusList().size())), results, response.getDocIdList());
 298         }
 299         for (OperationResult result : results) {
 300           if (result.getCode() != StatusCode.OK) {
 301             throw new PutException(result, results, response.getDocIdList());
 302           }
 303         }
 304         return new PutResponse(results, response.getDocIdList());
 305       }
 306
 307       /**
 308        * Constructs a list of OperationResult from an index document response.
 309        *
 310        * @param response the index document response to extract operation
 311        * results from
 312        * @return a list of OperationResult
 313        */
 314       private List<OperationResult> newOperationResultList(
 315           SearchServicePb.IndexDocumentResponse response) {
 316         ArrayList<OperationResult> results = new ArrayList<OperationResult>(
 317             response.getStatusCount());
 318         for (SearchServicePb.RequestStatus status : response.getStatusList()) {
 319           results.add(new OperationResult(status));
 320         }
 321         return results;
 322       }
 323     };
 324   }
 325
 326   private Future<Results<ScoredDocument>> executeSearchForResults(
 327       SearchServicePb.SearchParams.Builder params) {
 328     SearchServicePb.SearchResponse.Builder responseBuilder =
 329         SearchServicePb.SearchResponse.newBuilder();
 330     SearchServicePb.SearchRequest request = SearchServicePb.SearchRequest.newBuilder()
 331         .setParams(params).build();
 332
 333     Future<SearchServicePb.SearchResponse.Builder> future =
 334         apiHelper.makeAsyncCall("Search", request, responseBuilder);
 335     return new FutureWrapper<SearchServicePb.SearchResponse.Builder,
 336            Results<ScoredDocument>>(future) {
 337       @Override
 338       protected Throwable convertException(Throwable cause) {
 339         OperationResult result = OperationResult.convertToOperationResult(cause);
 340         return (result == null) ? cause : new SearchException(result);
 341       }
 342
 343       @Override
 344       protected Results<ScoredDocument> wrap(SearchServicePb.SearchResponse.Builder key)
 345       throws Exception {
 346         SearchServicePb.SearchResponse response = key.build();
 347         SearchServicePb.RequestStatus status = response.getStatus();
 348         if (status.getCode() != SearchServicePb.SearchServiceError.ErrorCode.OK) {
 349           throw new SearchException(new OperationResult(status));
 350         }
 351         List<ScoredDocument> scoredDocs = new ArrayList<ScoredDocument>();
 352         for (SearchServicePb.SearchResult result : response.getResultList()) {
 353           List<Field> expressions = new ArrayList<Field>();
 354           for (DocumentPb.Field expression : result.getExpressionList()) {
 355             expressions.add(Field.newBuilder(expression).build());
 356           }
 357           ScoredDocument.Builder scoredDocBuilder = ScoredDocument.newBuilder(result.getDocument());
 358           for (Double score : result.getScoreList()) {
 359             scoredDocBuilder.addScore(score);
 360           }
 361           for (Field expression : expressions) {
 362             scoredDocBuilder.addExpression(expression);
 363           }
 364           if (result.hasCursor()) {
 365             scoredDocBuilder.setCursor(
 366                 Cursor.newBuilder().build("true:" + result.getCursor()));
 367           }
 368           scoredDocs.add(scoredDocBuilder.build());
 369         }
 370         Results<ScoredDocument> scoredResults = new Results<ScoredDocument>(
 371             new OperationResult(status),
 372             scoredDocs, response.getMatchedCount(), response.getResultCount(),
 373             (response.hasCursor() ?
 374                 Cursor.newBuilder().build("false:" + response.getCursor()) : null));
 375
 376         return scoredResults;
 377       }
 378     };
 379   }
 380
 381   @Override
 382   public Future<Results<ScoredDocument>> searchAsync(String query) {
 383     return searchAsync(Query.newBuilder().build(
 384         Preconditions.checkNotNull(query, "query cannot be null")));
 385   }
 386
 387   @Override
 388   public Future<Results<ScoredDocument>> searchAsync(Query query) {
 389     Preconditions.checkNotNull(query, "query cannot be null");
 390     return executeSearchForResults(
 391         query.copyToProtocolBuffer().setIndexSpec(spec.copyToProtocolBuffer(namespace)));
 392   }
 393
 394   @Override
 395   public Future<GetResponse<Document>> getRangeAsync(GetRequest.Builder builder) {
 396     return getRangeAsync(builder.build());
 397   }
 398
 399   @Override
 400   public Future<GetResponse<Document>> getRangeAsync(GetRequest request) {
 401     Preconditions.checkNotNull(request, "list documents request cannot be null");
 402
 403     SearchServicePb.ListDocumentsParams.Builder params =
 404         request.copyToProtocolBuffer().setIndexSpec(spec.copyToProtocolBuffer(namespace));
 405     SearchServicePb.ListDocumentsResponse.Builder responseBuilder =
 406         SearchServicePb.ListDocumentsResponse.newBuilder();
 407     SearchServicePb.ListDocumentsRequest requestPb = SearchServicePb.ListDocumentsRequest
 408         .newBuilder().setParams(params).build();
 409
 410     Future<SearchServicePb.ListDocumentsResponse.Builder> future =
 411         apiHelper.makeAsyncCall("ListDocuments", requestPb, responseBuilder);
 412     return new FutureWrapper<SearchServicePb.ListDocumentsResponse.Builder,
 413         GetResponse<Document>>(future) {
 414       @Override
 415       protected Throwable convertException(Throwable cause) {
 416         OperationResult result = OperationResult.convertToOperationResult(cause);
 417         return (result == null) ? cause : new GetException(result);
 418       }
 419
 420       @Override
 421       protected GetResponse<Document> wrap(
 422           SearchServicePb.ListDocumentsResponse.Builder key) throws Exception {
 423         SearchServicePb.ListDocumentsResponse response = key.build();
 424         SearchServicePb.RequestStatus status = response.getStatus();
 425
 426         if (status.getCode() != ErrorCode.OK) {
 427           throw new GetException(new OperationResult(status));
 428         }
 429
 430         List<Document> results = new ArrayList<Document>();
 431         for (DocumentPb.Document document : response.getDocumentList()) {
 432           results.add(Document.newBuilder(document).build());
 433         }
 434         return new GetResponse<Document>(results);
 435       }
 436     };
 437   }
 438
 439   @Override
 440   public Document get(String documentId) {
 441     Preconditions.checkNotNull(documentId, "documentId must not be null");
 442     GetResponse<Document> response =
 443         getRange(GetRequest.newBuilder().setStartId(documentId).setLimit(1));
 444     for (Document document : response) {
 445       if (documentId.equals(document.getId())) {
 446         return document;
 447       }
 448     }
 449     return null;
 450   }
 451
 452   @Override
 453   public void deleteSchema() {
 454     quietGet(deleteSchemaAsync());
 455   }
 456
 457   @Override
 458   public void delete(String... documentIds) {
 459     quietGet(deleteAsync(documentIds));
 460   }
 461
 462   @Override
 463   public void delete(Iterable<String> documentIds) {
 464     quietGet(deleteAsync(documentIds));
 465   }
 466
 467   @Override
 468   public PutResponse put(Document... documents) {
 469     return quietGet(putAsync(documents));
 470   }
 471
 472   @Override
 473   public PutResponse put(Document.Builder... builders) {
 474     return quietGet(putAsync(builders));
 475   }
 476
 477   @Override
 478   public PutResponse put(Iterable<Document> documents) {
 479     return quietGet(putAsync(documents));
 480   }
 481
 482   @Override
 483   public Results<ScoredDocument> search(String query) {
 484     return quietGet(searchAsync(query));
 485   }
 486
 487   @Override
 488   public Results<ScoredDocument> search(Query query) {
 489     return quietGet(searchAsync(query));
 490   }
 491
 492   @Override
 493   public GetResponse<Document> getRange(GetRequest request) {
 494     return quietGet(getRangeAsync(request));
 495   }
 496
 497   @Override
 498   public GetResponse<Document> getRange(GetRequest.Builder builder) {
 499     return getRange(builder.build());
 500   }
 501 }