1 // Copyright 2010 Google Inc. All Rights Reserved.
2 package com
.google
.appengine
.api
.search
;
4 import static com
.google
.appengine
.api
.search
.FutureHelper
.quietGet
;
6 import com
.google
.appengine
.api
.search
.SearchServicePb
.SearchServiceError
.ErrorCode
;
7 import com
.google
.appengine
.api
.search
.checkers
.DocumentChecker
;
8 import com
.google
.appengine
.api
.search
.checkers
.Preconditions
;
9 import com
.google
.appengine
.api
.search
.checkers
.SearchApiLimits
;
10 import com
.google
.appengine
.api
.utils
.FutureWrapper
;
11 import com
.google
.apphosting
.api
.search
.DocumentPb
;
12 import com
.google
.common
.collect
.Iterables
;
14 import java
.util
.ArrayList
;
15 import java
.util
.Arrays
;
16 import java
.util
.Collections
;
17 import java
.util
.HashMap
;
18 import java
.util
.List
;
20 import java
.util
.concurrent
.Future
;
23 * The default implementation of {@link Index}. This class uses a
24 * {@link SearchApiHelper} to forward all requests to an appserver.
27 class IndexImpl
implements Index
{
29 private final SearchApiHelper apiHelper
;
30 private final IndexSpec spec
;
31 private final Schema schema
;
32 private final SearchServiceConfig config
;
33 private final Long storageUsage
;
34 private final Long storageLimit
;
37 * Creates new index specification.
39 * @param apiHelper the helper used to forward all calls
40 * @param config a {@link SearchServiceConfig} instance that describes the
41 * index implementation.
42 * @param indexSpec the index specification
44 IndexImpl(SearchApiHelper apiHelper
, SearchServiceConfig config
, IndexSpec indexSpec
) {
45 this(apiHelper
, config
, indexSpec
, null, null, null);
49 * Creates new index specification.
51 * @param apiHelper the helper used to forward all calls
52 * @param config a {@link SearchServiceConfig} instance that describes the
53 * index implementation.
54 * @param indexSpec the index specification
55 * @param schema the {@link Schema} defining the names and types of fields
58 IndexImpl(SearchApiHelper apiHelper
, SearchServiceConfig config
, IndexSpec indexSpec
,
59 Schema schema
, Long amount
, Long limit
) {
60 this.apiHelper
= Preconditions
.checkNotNull(apiHelper
, "Internal error");
61 Preconditions
.checkNotNull(config
.getNamespace(), "Internal error");
62 this.spec
= Preconditions
.checkNotNull(indexSpec
, "Internal error");
65 this.storageUsage
= amount
;
66 this.storageLimit
= limit
;
70 public String
getName() {
71 return spec
.getName();
75 public String
getNamespace() {
76 return config
.getNamespace();
80 public Schema
getSchema() {
84 private RuntimeException
noStorageInfo() {
85 return new UnsupportedOperationException("Storage information is not available");
89 public long getStorageUsage() {
90 if (storageUsage
== null) {
91 throw noStorageInfo();
97 public long getStorageLimit() {
98 if (storageLimit
== null) {
99 throw noStorageInfo();
105 public int hashCode() {
106 final int prime
= 31;
108 result
= prime
* result
+ spec
.hashCode();
109 result
= prime
* result
+ ((schema
== null) ?
0 : schema
.hashCode());
110 result
= prime
* result
+ ((storageUsage
== null) ?
0 : storageUsage
.hashCode());
111 result
= prime
* result
+ ((storageLimit
== null) ?
0 : storageLimit
.hashCode());
116 public boolean equals(Object obj
) {
123 if (getClass() != obj
.getClass()) {
126 IndexImpl other
= (IndexImpl
) obj
;
127 return Util
.equalObjects(spec
, other
.spec
) &&
128 Util
.equalObjects(schema
, other
.schema
) &&
129 Util
.equalObjects(storageUsage
, other
.storageUsage
) &&
130 Util
.equalObjects(storageLimit
, other
.storageLimit
);
134 public String
toString() {
136 (storageUsage
== null || storageLimit
== null) ?
137 "(no storage data)" :
138 String
.format(" (%d/%d)", storageUsage
.longValue(), storageLimit
.longValue());
139 return String
.format("IndexImpl{namespace: %s, %s, %s%s}", config
.getNamespace(), spec
,
140 (schema
== null ?
"(null schema)" : schema
), storageInfo
);
145 public Future
<Void
> deleteSchemaAsync() {
146 SearchServicePb
.DeleteSchemaParams
.Builder builder
=
147 SearchServicePb
.DeleteSchemaParams
.newBuilder().addIndexSpec(
148 spec
.copyToProtocolBuffer(config
.getNamespace()));
150 Future
<SearchServicePb
.DeleteSchemaResponse
.Builder
> future
=
151 apiHelper
.makeAsyncDeleteSchemaCall(builder
, config
.getDeadline());
152 return new FutureWrapper
<SearchServicePb
.DeleteSchemaResponse
.Builder
,
155 protected Throwable
convertException(Throwable cause
) {
156 OperationResult result
= OperationResult
.convertToOperationResult(cause
);
157 return (result
== null) ? cause
: new DeleteException(result
);
161 protected Void
wrap(SearchServicePb
.DeleteSchemaResponse
.Builder key
)
163 SearchServicePb
.DeleteSchemaResponse response
= key
.build();
164 ArrayList
<OperationResult
> results
= new ArrayList
<OperationResult
>(
165 response
.getStatusCount());
166 for (SearchServicePb
.RequestStatus status
: response
.getStatusList()) {
167 results
.add(new OperationResult(status
));
169 if (response
.getStatusList().size() != 1) {
170 throw new DeleteException(
172 StatusCode
.INTERNAL_ERROR
,
173 String
.format("Expected 1 removed schema, but got %d",
174 response
.getStatusList().size())),
177 for (OperationResult result
: results
) {
178 if (result
.getCode() != StatusCode
.OK
) {
179 throw new DeleteException(result
, results
);
188 public Future
<Void
> deleteAsync(String
... documentIds
) {
189 return deleteAsync(Arrays
.asList(documentIds
));
193 public Future
<Void
> deleteAsync(final Iterable
<String
> documentIds
) {
194 Preconditions
.checkArgument(documentIds
!= null,
195 "Delete documents given null collection of document ids");
196 SearchServicePb
.DeleteDocumentParams
.Builder builder
=
197 SearchServicePb
.DeleteDocumentParams
.newBuilder().setIndexSpec(
198 spec
.copyToProtocolBuffer(config
.getNamespace()));
200 for (String documentId
: documentIds
) {
202 builder
.addDocId(DocumentChecker
.checkDocumentId(documentId
));
204 if (size
> SearchApiLimits
.PUT_MAXIMUM_DOCS_PER_REQUEST
) {
205 throw new IllegalArgumentException(
206 String
.format("number of doc ids, %s, exceeds maximum %s", size
,
207 SearchApiLimits
.PUT_MAXIMUM_DOCS_PER_REQUEST
));
209 final int documentIdsSize
= size
;
210 Future
<SearchServicePb
.DeleteDocumentResponse
.Builder
> future
=
211 apiHelper
.makeAsyncDeleteDocumentCall(builder
, config
.getDeadline());
212 return new FutureWrapper
<SearchServicePb
.DeleteDocumentResponse
.Builder
,
215 protected Throwable
convertException(Throwable cause
) {
216 OperationResult result
= OperationResult
.convertToOperationResult(cause
);
217 return (result
== null) ? cause
: new DeleteException(result
);
221 protected Void
wrap(SearchServicePb
.DeleteDocumentResponse
.Builder key
)
223 SearchServicePb
.DeleteDocumentResponse response
= key
.build();
224 ArrayList
<OperationResult
> results
= new ArrayList
<OperationResult
>(
225 response
.getStatusCount());
226 for (SearchServicePb
.RequestStatus status
: response
.getStatusList()) {
227 results
.add(new OperationResult(status
));
229 if (documentIdsSize
!= response
.getStatusList().size()) {
230 throw new DeleteException(
232 StatusCode
.INTERNAL_ERROR
,
233 String
.format("Expected %d removed documents, but got %d", documentIdsSize
,
234 response
.getStatusList().size())),
237 for (OperationResult result
: results
) {
238 if (result
.getCode() != StatusCode
.OK
) {
239 throw new DeleteException(result
, results
);
248 public Future
<PutResponse
> putAsync(Document
... documents
) {
249 return putAsync(Arrays
.asList(documents
));
253 public Future
<PutResponse
> putAsync(Document
.Builder
... builders
) {
254 List
<Document
> documents
= new ArrayList
<Document
>();
255 for (int i
= 0; i
< builders
.length
; i
++) {
256 documents
.add(builders
[i
].build());
258 return putAsync(documents
);
262 public Future
<PutResponse
> putAsync(final Iterable
<Document
> documents
) {
263 Preconditions
.checkNotNull(documents
, "document list cannot be null");
264 if (Iterables
.isEmpty(documents
)) {
265 return new FutureHelper
.FakeFuture
<PutResponse
>(
266 new PutResponse(Collections
.<OperationResult
>emptyList(),
267 Collections
.<String
>emptyList()));
269 SearchServicePb
.IndexDocumentParams
.Builder builder
=
270 SearchServicePb
.IndexDocumentParams
.newBuilder()
271 .setIndexSpec(spec
.copyToProtocolBuffer(config
.getNamespace()));
272 Map
<String
, Document
> docMap
= new HashMap
<String
, Document
>();
274 for (Document document
: documents
) {
275 Document other
= null;
276 if (document
.getId() != null) {
277 other
= docMap
.put(document
.getId(), document
);
280 if (!document
.isIdenticalTo(other
)) {
281 throw new IllegalArgumentException(
283 "Put request with documents with the same ID \"%s\" but different content",
289 builder
.addDocument(Preconditions
.checkNotNull(document
, "document cannot be null")
290 .copyToProtocolBuffer());
293 if (size
> SearchApiLimits
.PUT_MAXIMUM_DOCS_PER_REQUEST
) {
294 throw new IllegalArgumentException(
295 String
.format("number of documents, %s, exceeds maximum %s", size
,
296 SearchApiLimits
.PUT_MAXIMUM_DOCS_PER_REQUEST
));
298 final int documentsSize
= size
;
299 Future
<SearchServicePb
.IndexDocumentResponse
.Builder
> future
=
300 apiHelper
.makeAsyncIndexDocumentCall(builder
, config
.getDeadline());
301 return new FutureWrapper
<SearchServicePb
.IndexDocumentResponse
.Builder
,
302 PutResponse
>(future
) {
304 protected Throwable
convertException(Throwable cause
) {
305 OperationResult result
= OperationResult
.convertToOperationResult(cause
);
306 return (result
== null) ? cause
: new PutException(result
);
310 protected PutResponse
wrap(SearchServicePb
.IndexDocumentResponse
.Builder key
)
312 SearchServicePb
.IndexDocumentResponse response
= key
.build();
313 List
<OperationResult
> results
= newOperationResultList(response
);
314 if (documentsSize
!= response
.getStatusList().size()) {
315 throw new PutException(
317 StatusCode
.INTERNAL_ERROR
,
318 String
.format("Expected %d indexed documents, but got %d", documentsSize
,
319 response
.getStatusList().size())), results
, response
.getDocIdList());
321 for (OperationResult result
: results
) {
322 if (result
.getCode() != StatusCode
.OK
) {
323 throw new PutException(result
, results
, response
.getDocIdList());
326 return new PutResponse(results
, response
.getDocIdList());
330 * Constructs a list of OperationResult from an index document response.
332 * @param response the index document response to extract operation
334 * @return a list of OperationResult
336 private List
<OperationResult
> newOperationResultList(
337 SearchServicePb
.IndexDocumentResponse response
) {
338 ArrayList
<OperationResult
> results
= new ArrayList
<OperationResult
>(
339 response
.getStatusCount());
340 for (SearchServicePb
.RequestStatus status
: response
.getStatusList()) {
341 results
.add(new OperationResult(status
));
348 private Future
<Results
<ScoredDocument
>> executeSearchForResults(
349 SearchServicePb
.SearchParams
.Builder params
) {
350 Future
<SearchServicePb
.SearchResponse
.Builder
> future
=
351 apiHelper
.makeAsyncSearchCall(params
, config
.getDeadline());
352 return new FutureWrapper
<SearchServicePb
.SearchResponse
.Builder
,
353 Results
<ScoredDocument
>>(future
) {
355 protected Throwable
convertException(Throwable cause
) {
356 OperationResult result
= OperationResult
.convertToOperationResult(cause
);
357 return (result
== null) ? cause
: new SearchException(result
);
361 protected Results
<ScoredDocument
> wrap(SearchServicePb
.SearchResponse
.Builder key
)
363 SearchServicePb
.SearchResponse response
= key
.build();
364 SearchServicePb
.RequestStatus status
= response
.getStatus();
365 if (status
.getCode() != SearchServicePb
.SearchServiceError
.ErrorCode
.OK
) {
366 throw new SearchException(new OperationResult(status
));
368 List
<ScoredDocument
> scoredDocs
= new ArrayList
<ScoredDocument
>();
369 for (SearchServicePb
.SearchResult result
: response
.getResultList()) {
370 List
<Field
> expressions
= new ArrayList
<Field
>();
371 for (DocumentPb
.Field expression
: result
.getExpressionList()) {
372 expressions
.add(Field
.newBuilder(expression
).build());
374 ScoredDocument
.Builder scoredDocBuilder
= ScoredDocument
.newBuilder(result
.getDocument());
375 for (Double score
: result
.getScoreList()) {
376 scoredDocBuilder
.addScore(score
);
378 for (Field expression
: expressions
) {
379 scoredDocBuilder
.addExpression(expression
);
381 if (result
.hasCursor()) {
382 scoredDocBuilder
.setCursor(
383 Cursor
.newBuilder().build("true:" + result
.getCursor()));
385 scoredDocs
.add(scoredDocBuilder
.build());
387 Results
<ScoredDocument
> scoredResults
= new Results
<ScoredDocument
>(
388 new OperationResult(status
),
389 scoredDocs
, response
.getMatchedCount(), response
.getResultCount(),
390 (response
.hasCursor() ?
391 Cursor
.newBuilder().build("false:" + response
.getCursor()) : null));
393 return scoredResults
;
399 public Future
<Results
<ScoredDocument
>> searchAsync(String query
) {
400 return searchAsync(Query
.newBuilder().build(
401 Preconditions
.checkNotNull(query
, "query cannot be null")));
405 public Future
<Results
<ScoredDocument
>> searchAsync(Query query
) {
406 Preconditions
.checkNotNull(query
, "query cannot be null");
407 return executeSearchForResults(
408 query
.copyToProtocolBuffer().setIndexSpec(spec
.copyToProtocolBuffer(
409 config
.getNamespace())));
413 public Future
<GetResponse
<Document
>> getRangeAsync(GetRequest
.Builder builder
) {
414 return getRangeAsync(builder
.build());
418 public Future
<GetResponse
<Document
>> getRangeAsync(GetRequest request
) {
419 Preconditions
.checkNotNull(request
, "list documents request cannot be null");
421 SearchServicePb
.ListDocumentsParams
.Builder params
=
422 request
.copyToProtocolBuffer().setIndexSpec(spec
.copyToProtocolBuffer(
423 config
.getNamespace()));
425 Future
<SearchServicePb
.ListDocumentsResponse
.Builder
> future
=
426 apiHelper
.makeAsyncListDocumentsCall(params
, config
.getDeadline());
427 return new FutureWrapper
<SearchServicePb
.ListDocumentsResponse
.Builder
,
428 GetResponse
<Document
>>(future
) {
430 protected Throwable
convertException(Throwable cause
) {
431 OperationResult result
= OperationResult
.convertToOperationResult(cause
);
432 return (result
== null) ? cause
: new GetException(result
);
436 protected GetResponse
<Document
> wrap(
437 SearchServicePb
.ListDocumentsResponse
.Builder key
) throws Exception
{
438 SearchServicePb
.ListDocumentsResponse response
= key
.build();
439 SearchServicePb
.RequestStatus status
= response
.getStatus();
441 if (status
.getCode() != ErrorCode
.OK
) {
442 throw new GetException(new OperationResult(status
));
445 List
<Document
> results
= new ArrayList
<Document
>();
446 for (DocumentPb
.Document document
: response
.getDocumentList()) {
447 results
.add(Document
.newBuilder(document
).build());
449 return new GetResponse
<Document
>(results
);
455 public Document
get(String documentId
) {
456 Preconditions
.checkNotNull(documentId
, "documentId must not be null");
457 GetResponse
<Document
> response
=
458 getRange(GetRequest
.newBuilder().setStartId(documentId
).setLimit(1));
459 for (Document document
: response
) {
460 if (documentId
.equals(document
.getId())) {
469 public void deleteSchema() {
470 quietGet(deleteSchemaAsync());
474 public void delete(String
... documentIds
) {
475 quietGet(deleteAsync(documentIds
));
479 public void delete(Iterable
<String
> documentIds
) {
480 quietGet(deleteAsync(documentIds
));
484 public PutResponse
put(Document
... documents
) {
485 return quietGet(putAsync(documents
));
489 public PutResponse
put(Document
.Builder
... builders
) {
490 return quietGet(putAsync(builders
));
494 public PutResponse
put(Iterable
<Document
> documents
) {
495 return quietGet(putAsync(documents
));
499 public Results
<ScoredDocument
> search(String query
) {
500 return quietGet(searchAsync(query
));
504 public Results
<ScoredDocument
> search(Query query
) {
505 return quietGet(searchAsync(query
));
509 public GetResponse
<Document
> getRange(GetRequest request
) {
510 return quietGet(getRangeAsync(request
));
514 public GetResponse
<Document
> getRange(GetRequest
.Builder builder
) {
515 return getRange(builder
.build());