1 // Copyright 2010 Google Inc. All Rights Reserved.
2 package com
.google
.appengine
.api
.search
;
4 import static com
.google
.appengine
.api
.search
.FutureHelper
.quietGet
;
6 import com
.google
.appengine
.api
.search
.SearchServicePb
.SearchServiceError
.ErrorCode
;
7 import com
.google
.appengine
.api
.search
.checkers
.DocumentChecker
;
8 import com
.google
.appengine
.api
.search
.checkers
.Preconditions
;
9 import com
.google
.appengine
.api
.search
.checkers
.SearchApiLimits
;
10 import com
.google
.appengine
.api
.utils
.FutureWrapper
;
11 import com
.google
.apphosting
.api
.search
.DocumentPb
;
13 import java
.util
.ArrayList
;
14 import java
.util
.Arrays
;
15 import java
.util
.Collections
;
16 import java
.util
.HashMap
;
17 import java
.util
.List
;
19 import java
.util
.concurrent
.Future
;
22 * The default implementation of {@link Index}. This class uses a
23 * {@link SearchApiHelper} to forward all requests to an appserver.
26 class IndexImpl
implements Index
{
28 private final SearchApiHelper apiHelper
;
29 private final IndexSpec spec
;
30 private final String namespace
;
31 private final Schema schema
;
34 * Creates new index specification.
36 * @param apiHelper the helper used to forward all calls
37 * @param indexSpec the index specification
39 IndexImpl(SearchApiHelper apiHelper
, String namespace
, IndexSpec indexSpec
) {
40 this(apiHelper
, namespace
, indexSpec
, null);
44 * Creates new index specification.
46 * @param apiHelper the helper used to forward all calls
47 * @param indexSpec the index specification
48 * @param schema the {@link Schema} defining the names and types of fields
51 IndexImpl(SearchApiHelper apiHelper
, String namespace
, IndexSpec indexSpec
, Schema schema
) {
52 this.apiHelper
= Preconditions
.checkNotNull(apiHelper
, "Internal error");
53 this.namespace
= Preconditions
.checkNotNull(namespace
, "Internal error");
54 this.spec
= Preconditions
.checkNotNull(indexSpec
, "Internal error");
59 public String
getName() {
60 return spec
.getName();
64 public String
getNamespace() {
69 public Consistency
getConsistency() {
70 return spec
.getConsistency();
74 public Schema
getSchema() {
79 public int hashCode() {
82 result
= prime
* result
+ spec
.hashCode();
83 result
= prime
* result
+ ((schema
== null) ?
0 : schema
.hashCode());
88 public boolean equals(Object obj
) {
95 if (getClass() != obj
.getClass()) {
98 IndexImpl other
= (IndexImpl
) obj
;
99 return Util
.equalObjects(spec
, other
.spec
) &&
100 Util
.equalObjects(schema
, other
.schema
);
104 public String
toString() {
105 return String
.format("IndexImpl{namespace: %s, %s, %s}", namespace
, spec
,
106 (schema
== null ?
"(null schema)" : schema
));
110 public Future
<Void
> deleteSchemaAsync() {
111 SearchServicePb
.DeleteSchemaParams
.Builder builder
=
112 SearchServicePb
.DeleteSchemaParams
.newBuilder().addIndexSpec(
113 spec
.copyToProtocolBuffer(namespace
));
114 SearchServicePb
.DeleteSchemaRequest request
=
115 SearchServicePb
.DeleteSchemaRequest
.newBuilder().setParams(builder
).build();
116 SearchServicePb
.DeleteSchemaResponse
.Builder responseBuilder
=
117 SearchServicePb
.DeleteSchemaResponse
.newBuilder();
119 Future
<SearchServicePb
.DeleteSchemaResponse
.Builder
> future
=
120 apiHelper
.makeAsyncCall("DeleteSchema", request
, responseBuilder
);
121 return new FutureWrapper
<SearchServicePb
.DeleteSchemaResponse
.Builder
,
124 protected Throwable
convertException(Throwable cause
) {
125 OperationResult result
= OperationResult
.convertToOperationResult(cause
);
126 return (result
== null) ? cause
: new DeleteException(result
);
130 protected Void
wrap(SearchServicePb
.DeleteSchemaResponse
.Builder key
)
132 SearchServicePb
.DeleteSchemaResponse response
= key
.build();
133 ArrayList
<OperationResult
> results
= new ArrayList
<OperationResult
>(
134 response
.getStatusCount());
135 for (SearchServicePb
.RequestStatus status
: response
.getStatusList()) {
136 results
.add(new OperationResult(status
));
138 if (response
.getStatusList().size() != 1) {
139 throw new DeleteException(
141 StatusCode
.INTERNAL_ERROR
,
142 String
.format("Expected 1 removed schema, but got %d",
143 response
.getStatusList().size())),
146 for (OperationResult result
: results
) {
147 if (result
.getCode() != StatusCode
.OK
) {
148 throw new DeleteException(result
, results
);
157 public Future
<Void
> deleteAsync(String
... documentIds
) {
158 return deleteAsync(Arrays
.asList(documentIds
));
162 public Future
<Void
> deleteAsync(final Iterable
<String
> documentIds
) {
163 Preconditions
.checkArgument(documentIds
!= null,
164 "Delete documents given null collection of document ids");
165 SearchServicePb
.DeleteDocumentParams
.Builder builder
=
166 SearchServicePb
.DeleteDocumentParams
.newBuilder().setIndexSpec(
167 spec
.copyToProtocolBuffer(namespace
));
169 for (String documentId
: documentIds
) {
171 builder
.addDocId(DocumentChecker
.checkDocumentId(documentId
));
173 if (size
> SearchApiLimits
.PUT_MAXIMUM_DOCS_PER_REQUEST
) {
174 throw new IllegalArgumentException(
175 String
.format("number of doc ids, %s, exceeds maximum %s", size
,
176 SearchApiLimits
.PUT_MAXIMUM_DOCS_PER_REQUEST
));
178 final int documentIdsSize
= size
;
179 SearchServicePb
.DeleteDocumentRequest request
=
180 SearchServicePb
.DeleteDocumentRequest
.newBuilder().setParams(builder
).build();
181 SearchServicePb
.DeleteDocumentResponse
.Builder responseBuilder
=
182 SearchServicePb
.DeleteDocumentResponse
.newBuilder();
184 Future
<SearchServicePb
.DeleteDocumentResponse
.Builder
> future
=
185 apiHelper
.makeAsyncCall("DeleteDocument", request
, responseBuilder
);
186 return new FutureWrapper
<SearchServicePb
.DeleteDocumentResponse
.Builder
,
189 protected Throwable
convertException(Throwable cause
) {
190 OperationResult result
= OperationResult
.convertToOperationResult(cause
);
191 return (result
== null) ? cause
: new DeleteException(result
);
195 protected Void
wrap(SearchServicePb
.DeleteDocumentResponse
.Builder key
)
197 SearchServicePb
.DeleteDocumentResponse response
= key
.build();
198 ArrayList
<OperationResult
> results
= new ArrayList
<OperationResult
>(
199 response
.getStatusCount());
200 for (SearchServicePb
.RequestStatus status
: response
.getStatusList()) {
201 results
.add(new OperationResult(status
));
203 if (documentIdsSize
!= response
.getStatusList().size()) {
204 throw new DeleteException(
206 StatusCode
.INTERNAL_ERROR
,
207 String
.format("Expected %d removed documents, but got %d", documentIdsSize
,
208 response
.getStatusList().size())),
211 for (OperationResult result
: results
) {
212 if (result
.getCode() != StatusCode
.OK
) {
213 throw new DeleteException(result
, results
);
222 public Future
<PutResponse
> putAsync(Document
... documents
) {
223 return putAsync(Arrays
.asList(documents
));
227 public Future
<PutResponse
> putAsync(Document
.Builder
... builders
) {
228 List
<Document
> documents
= new ArrayList
<Document
>();
229 for (int i
= 0; i
< builders
.length
; i
++) {
230 documents
.add(builders
[i
].build());
232 return putAsync(documents
);
236 public Future
<PutResponse
> putAsync(final Iterable
<Document
> documents
) {
237 Preconditions
.checkNotNull(documents
, "document list cannot be null");
238 if (!documents
.iterator().hasNext()) {
239 return new FutureHelper
.FakeFuture
<PutResponse
>(
240 new PutResponse(Collections
.<OperationResult
>emptyList(),
241 Collections
.<String
>emptyList()));
243 SearchServicePb
.IndexDocumentParams
.Builder builder
=
244 SearchServicePb
.IndexDocumentParams
.newBuilder()
245 .setIndexSpec(spec
.copyToProtocolBuffer(namespace
));
246 Map
<String
, Document
> docMap
= new HashMap
<String
, Document
>();
248 for (Document document
: documents
) {
249 Document other
= null;
250 if (document
.getId() != null) {
251 other
= docMap
.put(document
.getId(), document
);
254 if (!document
.isIdenticalTo(other
)) {
255 throw new IllegalArgumentException(
257 "Put request with documents with the same ID \"%s\" but differnt content",
263 builder
.addDocument(Preconditions
.checkNotNull(document
, "document cannot be null")
264 .copyToProtocolBuffer());
267 if (size
> SearchApiLimits
.PUT_MAXIMUM_DOCS_PER_REQUEST
) {
268 throw new IllegalArgumentException(
269 String
.format("number of documents, %s, exceeds maximum %s", size
,
270 SearchApiLimits
.PUT_MAXIMUM_DOCS_PER_REQUEST
));
272 final int documentsSize
= size
;
273 SearchServicePb
.IndexDocumentRequest request
=
274 SearchServicePb
.IndexDocumentRequest
.newBuilder().setParams(builder
).build();
275 SearchServicePb
.IndexDocumentResponse
.Builder responseBuilder
=
276 SearchServicePb
.IndexDocumentResponse
.newBuilder();
277 Future
<SearchServicePb
.IndexDocumentResponse
.Builder
> future
=
278 apiHelper
.makeAsyncCall("IndexDocument", request
, responseBuilder
);
279 return new FutureWrapper
<SearchServicePb
.IndexDocumentResponse
.Builder
,
280 PutResponse
>(future
) {
282 protected Throwable
convertException(Throwable cause
) {
283 OperationResult result
= OperationResult
.convertToOperationResult(cause
);
284 return (result
== null) ? cause
: new PutException(result
);
288 protected PutResponse
wrap(SearchServicePb
.IndexDocumentResponse
.Builder key
)
290 SearchServicePb
.IndexDocumentResponse response
= key
.build();
291 List
<OperationResult
> results
= newOperationResultList(response
);
292 if (documentsSize
!= response
.getStatusList().size()) {
293 throw new PutException(
295 StatusCode
.INTERNAL_ERROR
,
296 String
.format("Expected %d indexed documents, but got %d", documentsSize
,
297 response
.getStatusList().size())), results
, response
.getDocIdList());
299 for (OperationResult result
: results
) {
300 if (result
.getCode() != StatusCode
.OK
) {
301 throw new PutException(result
, results
, response
.getDocIdList());
304 return new PutResponse(results
, response
.getDocIdList());
308 * Constructs a list of OperationResult from an index document response.
310 * @param response the index document response to extract operation
312 * @return a list of OperationResult
314 private List
<OperationResult
> newOperationResultList(
315 SearchServicePb
.IndexDocumentResponse response
) {
316 ArrayList
<OperationResult
> results
= new ArrayList
<OperationResult
>(
317 response
.getStatusCount());
318 for (SearchServicePb
.RequestStatus status
: response
.getStatusList()) {
319 results
.add(new OperationResult(status
));
326 private Future
<Results
<ScoredDocument
>> executeSearchForResults(
327 SearchServicePb
.SearchParams
.Builder params
) {
328 SearchServicePb
.SearchResponse
.Builder responseBuilder
=
329 SearchServicePb
.SearchResponse
.newBuilder();
330 SearchServicePb
.SearchRequest request
= SearchServicePb
.SearchRequest
.newBuilder()
331 .setParams(params
).build();
333 Future
<SearchServicePb
.SearchResponse
.Builder
> future
=
334 apiHelper
.makeAsyncCall("Search", request
, responseBuilder
);
335 return new FutureWrapper
<SearchServicePb
.SearchResponse
.Builder
,
336 Results
<ScoredDocument
>>(future
) {
338 protected Throwable
convertException(Throwable cause
) {
339 OperationResult result
= OperationResult
.convertToOperationResult(cause
);
340 return (result
== null) ? cause
: new SearchException(result
);
344 protected Results
<ScoredDocument
> wrap(SearchServicePb
.SearchResponse
.Builder key
)
346 SearchServicePb
.SearchResponse response
= key
.build();
347 SearchServicePb
.RequestStatus status
= response
.getStatus();
348 if (status
.getCode() != SearchServicePb
.SearchServiceError
.ErrorCode
.OK
) {
349 throw new SearchException(new OperationResult(status
));
351 List
<ScoredDocument
> scoredDocs
= new ArrayList
<ScoredDocument
>();
352 for (SearchServicePb
.SearchResult result
: response
.getResultList()) {
353 List
<Field
> expressions
= new ArrayList
<Field
>();
354 for (DocumentPb
.Field expression
: result
.getExpressionList()) {
355 expressions
.add(Field
.newBuilder(expression
).build());
357 ScoredDocument
.Builder scoredDocBuilder
= ScoredDocument
.newBuilder(result
.getDocument());
358 for (Double score
: result
.getScoreList()) {
359 scoredDocBuilder
.addScore(score
);
361 for (Field expression
: expressions
) {
362 scoredDocBuilder
.addExpression(expression
);
364 if (result
.hasCursor()) {
365 scoredDocBuilder
.setCursor(
366 Cursor
.newBuilder().build("true:" + result
.getCursor()));
368 scoredDocs
.add(scoredDocBuilder
.build());
370 Results
<ScoredDocument
> scoredResults
= new Results
<ScoredDocument
>(
371 new OperationResult(status
),
372 scoredDocs
, response
.getMatchedCount(), response
.getResultCount(),
373 (response
.hasCursor() ?
374 Cursor
.newBuilder().build("false:" + response
.getCursor()) : null));
376 return scoredResults
;
382 public Future
<Results
<ScoredDocument
>> searchAsync(String query
) {
383 return searchAsync(Query
.newBuilder().build(
384 Preconditions
.checkNotNull(query
, "query cannot be null")));
388 public Future
<Results
<ScoredDocument
>> searchAsync(Query query
) {
389 Preconditions
.checkNotNull(query
, "query cannot be null");
390 return executeSearchForResults(
391 query
.copyToProtocolBuffer().setIndexSpec(spec
.copyToProtocolBuffer(namespace
)));
395 public Future
<GetResponse
<Document
>> getRangeAsync(GetRequest
.Builder builder
) {
396 return getRangeAsync(builder
.build());
400 public Future
<GetResponse
<Document
>> getRangeAsync(GetRequest request
) {
401 Preconditions
.checkNotNull(request
, "list documents request cannot be null");
403 SearchServicePb
.ListDocumentsParams
.Builder params
=
404 request
.copyToProtocolBuffer().setIndexSpec(spec
.copyToProtocolBuffer(namespace
));
405 SearchServicePb
.ListDocumentsResponse
.Builder responseBuilder
=
406 SearchServicePb
.ListDocumentsResponse
.newBuilder();
407 SearchServicePb
.ListDocumentsRequest requestPb
= SearchServicePb
.ListDocumentsRequest
408 .newBuilder().setParams(params
).build();
410 Future
<SearchServicePb
.ListDocumentsResponse
.Builder
> future
=
411 apiHelper
.makeAsyncCall("ListDocuments", requestPb
, responseBuilder
);
412 return new FutureWrapper
<SearchServicePb
.ListDocumentsResponse
.Builder
,
413 GetResponse
<Document
>>(future
) {
415 protected Throwable
convertException(Throwable cause
) {
416 OperationResult result
= OperationResult
.convertToOperationResult(cause
);
417 return (result
== null) ? cause
: new GetException(result
);
421 protected GetResponse
<Document
> wrap(
422 SearchServicePb
.ListDocumentsResponse
.Builder key
) throws Exception
{
423 SearchServicePb
.ListDocumentsResponse response
= key
.build();
424 SearchServicePb
.RequestStatus status
= response
.getStatus();
426 if (status
.getCode() != ErrorCode
.OK
) {
427 throw new GetException(new OperationResult(status
));
430 List
<Document
> results
= new ArrayList
<Document
>();
431 for (DocumentPb
.Document document
: response
.getDocumentList()) {
432 results
.add(Document
.newBuilder(document
).build());
434 return new GetResponse
<Document
>(results
);
440 public Document
get(String documentId
) {
441 Preconditions
.checkNotNull(documentId
, "documentId must not be null");
442 GetResponse
<Document
> response
=
443 getRange(GetRequest
.newBuilder().setStartId(documentId
).setLimit(1));
444 for (Document document
: response
) {
445 if (documentId
.equals(document
.getId())) {
453 public void deleteSchema() {
454 quietGet(deleteSchemaAsync());
458 public void delete(String
... documentIds
) {
459 quietGet(deleteAsync(documentIds
));
463 public void delete(Iterable
<String
> documentIds
) {
464 quietGet(deleteAsync(documentIds
));
468 public PutResponse
put(Document
... documents
) {
469 return quietGet(putAsync(documents
));
473 public PutResponse
put(Document
.Builder
... builders
) {
474 return quietGet(putAsync(builders
));
478 public PutResponse
put(Iterable
<Document
> documents
) {
479 return quietGet(putAsync(documents
));
483 public Results
<ScoredDocument
> search(String query
) {
484 return quietGet(searchAsync(query
));
488 public Results
<ScoredDocument
> search(Query query
) {
489 return quietGet(searchAsync(query
));
493 public GetResponse
<Document
> getRange(GetRequest request
) {
494 return quietGet(getRangeAsync(request
));
498 public GetResponse
<Document
> getRange(GetRequest
.Builder builder
) {
499 return getRange(builder
.build());