From 8f369fc00f43559684858aff7b6faa8bfb0397da Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 28 Jan 2022 14:35:31 +0000 Subject: [PATCH 01/51] 267 lucene 8_6_0 DoublePoint rename --- src/main/java/org/icatproject/core/entity/Parameter.java | 2 +- src/main/java/org/icatproject/core/manager/LuceneApi.java | 4 ++-- src/test/java/org/icatproject/core/manager/TestLucene.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/icatproject/core/entity/Parameter.java b/src/main/java/org/icatproject/core/entity/Parameter.java index e043da96..1d4cc89c 100644 --- a/src/main/java/org/icatproject/core/entity/Parameter.java +++ b/src/main/java/org/icatproject/core/entity/Parameter.java @@ -168,7 +168,7 @@ public void getDoc(JsonGenerator gen) { if (stringValue != null) { LuceneApi.encodeStringField(gen, "stringValue", stringValue); } else if (numericValue != null) { - LuceneApi.encodeDoubleField(gen, "numericValue", numericValue); + LuceneApi.encodeDoublePoint(gen, "numericValue", numericValue); } else if (dateTimeValue != null) { LuceneApi.encodeStringField(gen, "dateTimeValue", dateTimeValue); } diff --git a/src/main/java/org/icatproject/core/manager/LuceneApi.java b/src/main/java/org/icatproject/core/manager/LuceneApi.java index 5f1b0d0d..867424b8 100644 --- a/src/main/java/org/icatproject/core/manager/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/LuceneApi.java @@ -62,8 +62,8 @@ public static void encodeStringField(JsonGenerator gen, String name, Date value) gen.writeStartObject().write("type", "StringField").write("name", name).write("value", timeString).writeEnd(); } - public static void encodeDoubleField(JsonGenerator gen, String name, Double value) { - gen.writeStartObject().write("type", "DoubleField").write("name", name).write("value", value) + public static void encodeDoublePoint(JsonGenerator gen, String name, Double value) { + gen.writeStartObject().write("type", "DoublePoint").write("name", name).write("value", value) .write("store", true).writeEnd(); } diff --git a/src/test/java/org/icatproject/core/manager/TestLucene.java b/src/test/java/org/icatproject/core/manager/TestLucene.java index a0242526..bd4a2cf9 100644 --- a/src/test/java/org/icatproject/core/manager/TestLucene.java +++ b/src/test/java/org/icatproject/core/manager/TestLucene.java @@ -306,7 +306,7 @@ private void fillParms(JsonGenerator gen, int i, String rel) { gen.writeStartArray(); LuceneApi.encodeStringField(gen, "name", "N" + name); LuceneApi.encodeStringField(gen, "units", units); - LuceneApi.encodeDoubleField(gen, "numericValue", new Double(j * j)); + LuceneApi.encodeDoublePoint(gen, "numericValue", new Double(j * j)); LuceneApi.encodeSortedDocValuesField(gen, rel, new Long(i)); gen.writeEnd(); System.out.println(rel + " " + i + " '" + "N" + name + "' '" + units + "' " + new Double(j * j)); From 3770bbf642594fa7eecae7572f1013bc4c55eade Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 11 Feb 2022 23:09:06 +0000 Subject: [PATCH 02/51] Add getReadableIds and use in filterReadAccess #277 --- .../core/manager/EntityBeanManager.java | 50 +++++------- .../icatproject/core/manager/GateKeeper.java | 78 +++++++++++++++++++ 2 files changed, 98 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 4654d681..b46a2407 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -148,6 +148,7 @@ public enum PersistMode { private boolean luceneActive; private int maxEntities; + private int maxIdsInQuery; private long exportCacheSize; private Set rootUserNames; @@ -786,23 +787,23 @@ private void filterReadAccess(List results, List allIds = new ArrayList<>(); + allResults.forEach(r -> allIds.add(r.getEntityBaseBeanId())); + List allowedIds = gateKeeper.getReadableIds(userId, allIds, klass, manager); for (ScoredEntityBaseBean sr : allResults) { - long entityId = sr.getEntityBaseBeanId(); - EntityBaseBean beanManaged = manager.find(klass, entityId); - if (beanManaged != null) { - try { - gateKeeper.performAuthorisation(userId, beanManaged, AccessType.READ, manager); - results.add(new ScoredEntityBaseBean(entityId, sr.getScore())); - if (results.size() > maxEntities) { - throw new IcatException(IcatExceptionType.VALIDATION, - "attempt to return more than " + maxEntities + " entities"); - } - if (results.size() == maxCount) { - break; - } - } catch (IcatException e) { - // Nothing to do + try { + if (allowedIds.contains(sr.getEntityBaseBeanId())) { + results.add(sr); } + if (results.size() > maxEntities) { + throw new IcatException(IcatExceptionType.VALIDATION, + "attempt to return more than " + maxEntities + " entities"); + } + if (results.size() == maxCount) { + break; + } + } catch (IcatException e) { + // Nothing to do } } } @@ -1154,6 +1155,7 @@ void init() { notificationRequests = propertyHandler.getNotificationRequests(); luceneActive = lucene.isActive(); maxEntities = propertyHandler.getMaxEntities(); + maxIdsInQuery = propertyHandler.getMaxIdsInQuery(); exportCacheSize = propertyHandler.getImportCacheSize(); rootUserNames = propertyHandler.getRootUserNames(); key = propertyHandler.getKey(); @@ -1398,11 +1400,7 @@ public List luceneDatafiles(String userName, String user, LuceneSearchResult last = null; Long uid = null; List allResults = Collections.emptyList(); - /* - * As results may be rejected and maxCount may be 1 ensure that we - * don't make a huge number of calls to Lucene - */ - int blockSize = Math.max(1000, maxCount); + int blockSize = maxIdsInQuery; do { if (last == null) { @@ -1441,11 +1439,7 @@ public List luceneDatasets(String userName, String user, S LuceneSearchResult last = null; Long uid = null; List allResults = Collections.emptyList(); - /* - * As results may be rejected and maxCount may be 1 ensure that we - * don't make a huge number of calls to Lucene - */ - int blockSize = Math.max(1000, maxCount); + int blockSize = maxIdsInQuery; do { if (last == null) { @@ -1492,11 +1486,7 @@ public List luceneInvestigations(String userName, String u LuceneSearchResult last = null; Long uid = null; List allResults = Collections.emptyList(); - /* - * As results may be rejected and maxCount may be 1 ensure that we - * don't make a huge number of calls to Lucene - */ - int blockSize = Math.max(1000, maxCount); + int blockSize = maxIdsInQuery; do { if (last == null) { diff --git a/src/main/java/org/icatproject/core/manager/GateKeeper.java b/src/main/java/org/icatproject/core/manager/GateKeeper.java index d01470f8..f4cfe2eb 100644 --- a/src/main/java/org/icatproject/core/manager/GateKeeper.java +++ b/src/main/java/org/icatproject/core/manager/GateKeeper.java @@ -245,6 +245,84 @@ public List getReadable(String userId, List bean return results; } + public List getReadableIds(String userId, List ids, + Class objectClass, EntityManager manager) { + if (ids.size() == 0) { + return ids; + } + + String simpleName = objectClass.getSimpleName(); + if (rootUserNames.contains(userId)) { + logger.info("\"Root\" user " + userId + " is allowed READ to " + simpleName); + return ids; + } + + TypedQuery query = manager.createNamedQuery(Rule.INCLUDE_QUERY, String.class) + .setParameter("member", userId).setParameter("bean", simpleName); + + List restrictions = query.getResultList(); + logger.debug("Got " + restrictions.size() + " authz queries for READ by " + userId + " to a " + + objectClass.getSimpleName()); + + for (String restriction : restrictions) { + logger.debug("Query: " + restriction); + if (restriction == null) { + logger.info("Null restriction => READ permitted to " + simpleName); + return ids; + } + } + + /* + * IDs are processed in batches to avoid Oracle error: ORA-01795: + * maximum number of expressions in a list is 1000 + */ + + List idLists = new ArrayList<>(); + StringBuilder sb = null; + + int i = 0; + for (Long id : ids) { + if (i == 0) { + sb = new StringBuilder(); + sb.append(id); + i = 1; + } else { + sb.append("," + id); + i++; + } + if (i == maxIdsInQuery) { + i = 0; + idLists.add(sb.toString()); + sb = null; + } + } + if (sb != null) { + idLists.add(sb.toString()); + } + + logger.debug("Check readability of " + ids.size() + " beans has been divided into " + idLists.size() + + " queries."); + + Set readableIds = new HashSet<>(); + for (String idList : idLists) { + for (String qString : restrictions) { + TypedQuery q = manager.createQuery(qString.replace(":pkids", idList), Long.class); + if (qString.contains(":user")) { + q.setParameter("user", userId); + } + readableIds.addAll(q.getResultList()); + } + } + + List results = new ArrayList<>(); + for (Long id : ids) { + if (readableIds.contains(id)) { + results.add(id); + } + } + return results; + } + public Set getRootUserNames() { return rootUserNames; } From 44a9d80d6d4d69f8a53e8f8c730eff03ed3805a3 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Wed, 9 Mar 2022 13:45:36 +0000 Subject: [PATCH 03/51] Implement Elasticsearch Java client functions #267 --- pom.xml | 19 +- .../org/icatproject/core/entity/Datafile.java | 18 +- .../core/entity/DatafileParameter.java | 8 +- .../org/icatproject/core/entity/Dataset.java | 21 +- .../core/entity/DatasetParameter.java | 8 +- .../core/entity/EntityBaseBean.java | 13 +- .../core/entity/Investigation.java | 45 +- .../core/entity/InvestigationParameter.java | 8 +- .../core/entity/InvestigationUser.java | 10 +- .../icatproject/core/entity/Parameter.java | 14 +- .../org/icatproject/core/entity/Sample.java | 12 +- .../core/manager/ElasticsearchApi.java | 495 ++++++++++++++++ .../core/manager/ElasticsearchDocument.java | 140 +++++ .../core/manager/EntityBeanManager.java | 190 ++----- .../core/manager/EntityInfoHandler.java | 18 +- .../core/manager/FacetDimension.java | 27 + .../icatproject/core/manager/FacetLabel.java | 26 + .../icatproject/core/manager/LuceneApi.java | 415 ++++++-------- .../core/manager/PropertyHandler.java | 104 ++-- .../icatproject/core/manager/SearchApi.java | 172 ++++++ ...{LuceneManager.java => SearchManager.java} | 186 +++--- ...eneSearchResult.java => SearchResult.java} | 8 +- .../org/icatproject/exposed/ICATRest.java | 102 ++-- src/main/resources/logback.xml | 2 +- src/main/resources/run.properties | 13 +- .../core/manager/TestElasticsearchApi.java | 535 ++++++++++++++++++ .../core/manager/TestEntityInfo.java | 6 +- .../icatproject/core/manager/TestLucene.java | 181 +++--- .../org/icatproject/integration/WSession.java | 4 +- src/test/scripts/prepare_test.py | 14 +- 30 files changed, 2043 insertions(+), 771 deletions(-) create mode 100644 src/main/java/org/icatproject/core/manager/ElasticsearchApi.java create mode 100644 src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java create mode 100644 src/main/java/org/icatproject/core/manager/FacetDimension.java create mode 100644 src/main/java/org/icatproject/core/manager/FacetLabel.java create mode 100644 src/main/java/org/icatproject/core/manager/SearchApi.java rename src/main/java/org/icatproject/core/manager/{LuceneManager.java => SearchManager.java} (70%) rename src/main/java/org/icatproject/core/manager/{LuceneSearchResult.java => SearchResult.java} (71%) create mode 100644 src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java diff --git a/pom.xml b/pom.xml index 07b036f4..b513f3c7 100644 --- a/pom.xml +++ b/pom.xml @@ -124,7 +124,7 @@ org.apache.httpcomponents httpclient - 4.3.6 + 4.5.13 @@ -133,6 +133,17 @@ 4.3.4 + + co.elastic.clients + elasticsearch-java + 7.16.3 + + + com.fasterxml.jackson.core + jackson-databind + 2.12.3 + + @@ -225,6 +236,7 @@ ${javax.net.ssl.trustStore} ${luceneUrl} + ${searchUrls} false @@ -243,6 +255,7 @@ ${javax.net.ssl.trustStore} ${serverUrl} ${luceneUrl} + ${searchUrls} @@ -322,6 +335,7 @@ ${containerHome} ${serverUrl} ${luceneUrl} + ${searchUrls} @@ -399,6 +413,3 @@ - - - diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index 77bec24e..9f5b061f 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -20,7 +20,7 @@ import javax.persistence.UniqueConstraint; import javax.xml.bind.annotation.XmlRootElement; -import org.icatproject.core.manager.LuceneApi; +import org.icatproject.core.manager.SearchApi; @Comment("A data file") @SuppressWarnings("serial") @@ -194,7 +194,7 @@ public void setSourceDatafiles(List sourceDatafiles) { } @Override - public void getDoc(JsonGenerator gen) { + public void getDoc(JsonGenerator gen, SearchApi searchApi) { StringBuilder sb = new StringBuilder(name); if (description != null) { sb.append(" " + description); @@ -205,15 +205,17 @@ public void getDoc(JsonGenerator gen) { if (datafileFormat != null) { sb.append(" " + datafileFormat.getName()); } - LuceneApi.encodeTextfield(gen, "text", sb.toString()); + searchApi.encodeTextField(gen, "text", sb.toString()); if (datafileModTime != null) { - LuceneApi.encodeStringField(gen, "date", datafileModTime); + searchApi.encodeStringField(gen, "date", datafileModTime); } else if (datafileCreateTime != null) { - LuceneApi.encodeStringField(gen, "date", datafileCreateTime); + searchApi.encodeStringField(gen, "date", datafileCreateTime); } else { - LuceneApi.encodeStringField(gen, "date", modTime); + searchApi.encodeStringField(gen, "date", modTime); } - LuceneApi.encodeStoredId(gen, id); - LuceneApi.encodeStringField(gen, "dataset", dataset.id); + searchApi.encodeStoredId(gen, id); + searchApi.encodeStringField(gen, "dataset", dataset.id); + + // TODO User and Parameter support for Elasticsearch } } diff --git a/src/main/java/org/icatproject/core/entity/DatafileParameter.java b/src/main/java/org/icatproject/core/entity/DatafileParameter.java index c781e4df..466adce2 100644 --- a/src/main/java/org/icatproject/core/entity/DatafileParameter.java +++ b/src/main/java/org/icatproject/core/entity/DatafileParameter.java @@ -14,7 +14,7 @@ import org.icatproject.core.IcatException; import org.icatproject.core.manager.EntityBeanManager.PersistMode; import org.icatproject.core.manager.GateKeeper; -import org.icatproject.core.manager.LuceneApi; +import org.icatproject.core.manager.SearchApi; @Comment("A parameter associated with a data file") @SuppressWarnings("serial") @@ -54,9 +54,9 @@ public void setDatafile(Datafile datafile) { } @Override - public void getDoc(JsonGenerator gen) { - super.getDoc(gen); - LuceneApi.encodeSortedDocValuesField(gen, "datafile", datafile.id); + public void getDoc(JsonGenerator gen, SearchApi searchApi) { + super.getDoc(gen, searchApi); + searchApi.encodeSortedDocValuesField(gen, "datafile", datafile.id); } } diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index 7f40c71a..27a324d8 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -19,7 +19,7 @@ import javax.persistence.UniqueConstraint; import javax.xml.bind.annotation.XmlRootElement; -import org.icatproject.core.manager.LuceneApi; +import org.icatproject.core.manager.SearchApi; @Comment("A collection of data files and part of an investigation") @SuppressWarnings("serial") @@ -182,7 +182,7 @@ public void setType(DatasetType type) { } @Override - public void getDoc(JsonGenerator gen) { + public void getDoc(JsonGenerator gen, SearchApi searchApi) { StringBuilder sb = new StringBuilder(name + " " + type.getName() + " " + type.getName()); if (description != null) { @@ -200,25 +200,26 @@ public void getDoc(JsonGenerator gen) { } } - LuceneApi.encodeTextfield(gen, "text", sb.toString()); + searchApi.encodeTextField(gen, "text", sb.toString()); if (startDate != null) { - LuceneApi.encodeStringField(gen, "startDate", startDate); + searchApi.encodeStringField(gen, "startDate", startDate); } else { - LuceneApi.encodeStringField(gen, "startDate", createTime); + searchApi.encodeStringField(gen, "startDate", createTime); } if (endDate != null) { - LuceneApi.encodeStringField(gen, "endDate", endDate); + searchApi.encodeStringField(gen, "endDate", endDate); } else { - LuceneApi.encodeStringField(gen, "endDate", modTime); + searchApi.encodeStringField(gen, "endDate", modTime); } - LuceneApi.encodeStoredId(gen, id); + searchApi.encodeStoredId(gen, id); - LuceneApi.encodeSortedDocValuesField(gen, "id", id); + searchApi.encodeSortedDocValuesField(gen, "id", id); - LuceneApi.encodeStringField(gen, "investigation", investigation.id); + searchApi.encodeStringField(gen, "investigation", investigation.id); + // TODO User, Parameter and Sample support for Elasticsearch } } diff --git a/src/main/java/org/icatproject/core/entity/DatasetParameter.java b/src/main/java/org/icatproject/core/entity/DatasetParameter.java index 656bf8af..ac77c02b 100644 --- a/src/main/java/org/icatproject/core/entity/DatasetParameter.java +++ b/src/main/java/org/icatproject/core/entity/DatasetParameter.java @@ -14,7 +14,7 @@ import org.icatproject.core.IcatException; import org.icatproject.core.manager.EntityBeanManager.PersistMode; import org.icatproject.core.manager.GateKeeper; -import org.icatproject.core.manager.LuceneApi; +import org.icatproject.core.manager.SearchApi; @Comment("A parameter associated with a data set") @SuppressWarnings("serial") @@ -54,8 +54,8 @@ public void setDataset(Dataset dataset) { } @Override - public void getDoc(JsonGenerator gen) { - super.getDoc(gen); - LuceneApi.encodeSortedDocValuesField(gen, "dataset", dataset.id); + public void getDoc(JsonGenerator gen, SearchApi searchApi) { + super.getDoc(gen, searchApi); + searchApi.encodeSortedDocValuesField(gen, "dataset", dataset.id); } } \ No newline at end of file diff --git a/src/main/java/org/icatproject/core/entity/EntityBaseBean.java b/src/main/java/org/icatproject/core/entity/EntityBaseBean.java index afcc54e0..06e917d3 100644 --- a/src/main/java/org/icatproject/core/entity/EntityBaseBean.java +++ b/src/main/java/org/icatproject/core/entity/EntityBaseBean.java @@ -31,7 +31,8 @@ import org.icatproject.core.manager.EntityInfoHandler; import org.icatproject.core.manager.EntityInfoHandler.Relationship; import org.icatproject.core.manager.GateKeeper; -import org.icatproject.core.manager.LuceneManager; +import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.SearchManager; import org.icatproject.core.parser.IncludeClause.Step; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,8 +82,8 @@ void addToClone(EntityBaseBean clone) { // This is only used by the older create and createMany calls and not by the // new Restful write call - public void addToLucene(LuceneManager lucene) throws IcatException { - lucene.addDocument(this); + public void addToSearch(SearchManager searchManager) throws IcatException { + searchManager.addDocument(this); Class klass = this.getClass(); Set rs = eiHandler.getRelatedEntities(klass); Map getters = eiHandler.getGetters(klass); @@ -94,7 +95,7 @@ public void addToLucene(LuceneManager lucene) throws IcatException { List collection = (List) m.invoke(this); if (!collection.isEmpty()) { for (EntityBaseBean bean : collection) { - bean.addToLucene(lucene); + bean.addToSearch(searchManager); } } } catch (Exception e) { @@ -434,8 +435,8 @@ public String toString() { return this.getClass().getSimpleName() + ":" + id; } - /* This should be overridden by classes wishing to index things in lucene */ - public void getDoc(JsonGenerator gen) { + /* This should be overridden by classes wishing to index things in a search engine */ + public void getDoc(JsonGenerator gen, SearchApi searchApi) { } } diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index 0ecbba2d..a9d48483 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -18,7 +18,7 @@ import javax.persistence.TemporalType; import javax.persistence.UniqueConstraint; -import org.icatproject.core.manager.LuceneApi; +import org.icatproject.core.manager.SearchApi; @Comment("An investigation or experiment") @SuppressWarnings("serial") @@ -258,7 +258,7 @@ public void setVisitId(String visitId) { } @Override - public void getDoc(JsonGenerator gen) { + public void getDoc(JsonGenerator gen, SearchApi searchApi) { StringBuilder sb = new StringBuilder(visitId + " " + name + " " + facility.getName() + " " + type.getName()); if (summary != null) { sb.append(" " + summary); @@ -269,22 +269,49 @@ public void getDoc(JsonGenerator gen) { if (title != null) { sb.append(" " + title); } - LuceneApi.encodeTextfield(gen, "text", sb.toString()); + searchApi.encodeTextField(gen, "text", sb.toString()); if (startDate != null) { - LuceneApi.encodeStringField(gen, "startDate", startDate); + searchApi.encodeStringField(gen, "startDate", startDate); } else { - LuceneApi.encodeStringField(gen, "startDate", createTime); + searchApi.encodeStringField(gen, "startDate", createTime); } if (endDate != null) { - LuceneApi.encodeStringField(gen, "endDate", endDate); + searchApi.encodeStringField(gen, "endDate", endDate); } else { - LuceneApi.encodeStringField(gen, "endDate", modTime); + searchApi.encodeStringField(gen, "endDate", modTime); } - LuceneApi.encodeSortedDocValuesField(gen, "id", id); + investigationUsers.forEach((investigationUser) -> { + searchApi.encodeStringField(gen, "userName", investigationUser.getUser().getName()); + searchApi.encodeTextField(gen, "userFullName", investigationUser.getUser().getFullName()); + }); + + + samples.forEach((sample) -> { + searchApi.encodeSortedSetDocValuesFacetField(gen, "sampleName", sample.getName()); + searchApi.encodeTextField(gen, "sampleText", sample.getDocText()); + }); + + for (InvestigationParameter parameter : parameters) { + ParameterType type = parameter.type; + String parameterName = type.getName(); + String parameterUnits = type.getUnits(); + searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterName", parameterName); + searchApi.encodeStringField(gen, "parameterUnits", parameterUnits); + // TODO make all value types facetable... + if (type.getValueType() == ParameterValueType.STRING) { + searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterStringValue", parameter.getStringValue()); + } else if (type.getValueType() == ParameterValueType.DATE_AND_TIME) { + searchApi.encodeStringField(gen, "parameterDateValue", parameter.getDateTimeValue()); + } else if (type.getValueType() == ParameterValueType.NUMERIC) { + searchApi.encodeDoublePoint(gen, "parameterNumericValue", parameter.getNumericValue()); + } + } + + searchApi.encodeSortedDocValuesField(gen, "id", id); - LuceneApi.encodeStoredId(gen, id); + searchApi.encodeStoredId(gen, id); } } diff --git a/src/main/java/org/icatproject/core/entity/InvestigationParameter.java b/src/main/java/org/icatproject/core/entity/InvestigationParameter.java index 9f5fe2e4..f3e632ba 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationParameter.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationParameter.java @@ -14,7 +14,7 @@ import org.icatproject.core.IcatException; import org.icatproject.core.manager.EntityBeanManager.PersistMode; import org.icatproject.core.manager.GateKeeper; -import org.icatproject.core.manager.LuceneApi; +import org.icatproject.core.manager.SearchApi; @Comment("A parameter associated with an investigation") @SuppressWarnings("serial") @@ -55,8 +55,8 @@ public void setInvestigation(Investigation investigation) { } @Override - public void getDoc(JsonGenerator gen) { - super.getDoc(gen); - LuceneApi.encodeSortedDocValuesField(gen, "investigation", investigation.id); + public void getDoc(JsonGenerator gen, SearchApi searchApi) { + super.getDoc(gen, searchApi); + searchApi.encodeSortedDocValuesField(gen, "investigation", investigation.id); } } \ No newline at end of file diff --git a/src/main/java/org/icatproject/core/entity/InvestigationUser.java b/src/main/java/org/icatproject/core/entity/InvestigationUser.java index 8d07901b..dd45c92d 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationUser.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationUser.java @@ -10,7 +10,7 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; -import org.icatproject.core.manager.LuceneApi; +import org.icatproject.core.manager.SearchApi; @Comment("Many to many relationship between investigation and user. It is expected that this will show the association of " + "individual users with an investigation which might be derived from the proposal. It may also be used as the " @@ -38,12 +38,12 @@ public InvestigationUser() { } @Override - public void getDoc(JsonGenerator gen) { + public void getDoc(JsonGenerator gen, SearchApi searchApi) { if (user.getFullName() != null) { - LuceneApi.encodeTextfield(gen, "text", user.getFullName()); + searchApi.encodeTextField(gen, "text", user.getFullName()); } - LuceneApi.encodeStringField(gen, "name", user.getName()); - LuceneApi.encodeSortedDocValuesField(gen, "investigation", investigation.id); + searchApi.encodeStringField(gen, "name", user.getName()); + searchApi.encodeSortedDocValuesField(gen, "investigation", investigation.id); } public String getRole() { diff --git a/src/main/java/org/icatproject/core/entity/Parameter.java b/src/main/java/org/icatproject/core/entity/Parameter.java index 1d4cc89c..067ea572 100644 --- a/src/main/java/org/icatproject/core/entity/Parameter.java +++ b/src/main/java/org/icatproject/core/entity/Parameter.java @@ -17,7 +17,7 @@ import org.icatproject.core.IcatException; import org.icatproject.core.manager.EntityBeanManager.PersistMode; import org.icatproject.core.manager.GateKeeper; -import org.icatproject.core.manager.LuceneApi; +import org.icatproject.core.manager.SearchApi; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -162,15 +162,15 @@ public void postMergeFixup(EntityManager manager, GateKeeper gateKeeper) throws } @Override - public void getDoc(JsonGenerator gen) { - LuceneApi.encodeStringField(gen, "name", type.getName()); - LuceneApi.encodeStringField(gen, "units", type.getUnits()); + public void getDoc(JsonGenerator gen, SearchApi searchApi) { + searchApi.encodeStringField(gen, "name", type.getName()); + searchApi.encodeStringField(gen, "units", type.getUnits()); if (stringValue != null) { - LuceneApi.encodeStringField(gen, "stringValue", stringValue); + searchApi.encodeStringField(gen, "stringValue", stringValue); } else if (numericValue != null) { - LuceneApi.encodeDoublePoint(gen, "numericValue", numericValue); + searchApi.encodeDoublePoint(gen, "numericValue", numericValue); } else if (dateTimeValue != null) { - LuceneApi.encodeStringField(gen, "dateTimeValue", dateTimeValue); + searchApi.encodeStringField(gen, "dateTimeValue", dateTimeValue); } } diff --git a/src/main/java/org/icatproject/core/entity/Sample.java b/src/main/java/org/icatproject/core/entity/Sample.java index 9a36843e..170f7b1b 100644 --- a/src/main/java/org/icatproject/core/entity/Sample.java +++ b/src/main/java/org/icatproject/core/entity/Sample.java @@ -15,7 +15,7 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; -import org.icatproject.core.manager.LuceneApi; +import org.icatproject.core.manager.SearchApi; @Comment("A sample to be used in an investigation") @SuppressWarnings("serial") @@ -96,12 +96,16 @@ public void setType(SampleType type) { } @Override - public void getDoc(JsonGenerator gen) { + public void getDoc(JsonGenerator gen, SearchApi searchApi) { + searchApi.encodeTextField(gen, "text", getDocText()); + searchApi.encodeSortedDocValuesField(gen, "investigation", investigation.id); + } + + public String getDocText() { StringBuilder sb = new StringBuilder(name); if (type != null) { sb.append(" " + type.getName()); } - LuceneApi.encodeTextfield(gen, "text", sb.toString()); - LuceneApi.encodeSortedDocValuesField(gen, "investigation", investigation.id); + return sb.toString(); } } diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java new file mode 100644 index 00000000..991f9172 --- /dev/null +++ b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java @@ -0,0 +1,495 @@ +package org.icatproject.core.manager; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.StringReader; +import java.net.URL; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; +import java.util.concurrent.ExecutorService; + +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonNumber; +import javax.json.JsonObject; +import javax.json.JsonReader; +import javax.json.JsonValue; +import javax.json.JsonValue.ValueType; +import javax.json.stream.JsonGenerator; +import javax.persistence.EntityManager; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; +import org.icatproject.core.IcatException; +import org.icatproject.core.IcatException.IcatExceptionType; +import org.icatproject.core.entity.EntityBaseBean; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch._types.aggregations.StringTermsBucket; +import co.elastic.clients.elasticsearch._types.mapping.DynamicMapping; +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.Operator; +import co.elastic.clients.elasticsearch.core.BulkResponse; +import co.elastic.clients.elasticsearch.core.OpenPointInTimeResponse; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; +import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; +import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.indices.CreateIndexResponse; +import co.elastic.clients.json.JsonData; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.ElasticsearchTransport; +import co.elastic.clients.transport.rest_client.RestClientTransport; + +public class ElasticsearchApi extends SearchApi { + + private static ElasticsearchClient client; + private Map pitMap = new HashMap<>(); + + public ElasticsearchApi(List servers) { + List hosts = new ArrayList(); + for (URL server : servers) { + hosts.add(new HttpHost(server.getHost(), server.getPort(), server.getProtocol())); + } + RestClient restClient = RestClient.builder(hosts.toArray(new HttpHost[1])).build(); + ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper()); + new JacksonJsonpMapper(); + client = new ElasticsearchClient(transport); + try { + initMappings(); + } catch (Exception e) { + logger.warn("ElasticsearchApi init failed when setting explicit mappings"); + } + } + + private void initMappings() throws IcatException { + CreateIndexResponse response; + try { + response = client.indices().create(c -> c.index("datafile").mappings(m -> m + .dynamic(DynamicMapping.False) + .properties("id", p -> p.long_(l -> l)) + .properties("text", p -> p.text(t -> t)) + .properties("date", p -> p.date(d -> d)) + .properties("userName", p -> p.text(t -> t)) + .properties("userFullName", p -> p.text(t -> t)) + .properties("parameterName", p -> p.text(t -> t)) + .properties("parameterUnits", p -> p.text(t -> t)) + .properties("parameterStringValue", p -> p.text(t -> t)) + .properties("parameterDateValue", p -> p.date(d -> d)) + .properties("parameterNumericValue", p -> p.double_(d -> d)))); + response.acknowledged(); + response = client.indices().create(c -> c.index("dataset").mappings(m -> m + .dynamic(DynamicMapping.False) + .properties("id", p -> p.long_(l -> l)) + .properties("text", p -> p.text(t -> t)) + .properties("startDate", p -> p.date(d -> d)) + .properties("endDate", p -> p.date(d -> d)) + .properties("userName", p -> p.text(t -> t)) + .properties("userFullName", p -> p.text(t -> t)) + .properties("parameterName", p -> p.text(t -> t)) + .properties("parameterUnits", p -> p.text(t -> t)) + .properties("parameterStringValue", p -> p.text(t -> t)) + .properties("parameterDateValue", p -> p.date(d -> d)) + .properties("parameterNumericValue", p -> p.double_(d -> d)))); + response.acknowledged(); + response = client.indices().create(c -> c.index("investigation").mappings(m -> m + .dynamic(DynamicMapping.False) + .properties("id", p -> p.long_(l -> l)) + .properties("text", p -> p.text(t -> t)) + .properties("startDate", p -> p.date(d -> d)) + .properties("endDate", p -> p.date(d -> d)) + .properties("userName", p -> p.text(t -> t)) + .properties("userFullName", p -> p.text(t -> t)) + .properties("sampleName", p -> p.text(t -> t)) + .properties("sampleText", p -> p.text(t -> t)) + .properties("parameterName", p -> p.text(t -> t)) + .properties("parameterUnits", p -> p.text(t -> t)) + .properties("parameterStringValue", p -> p.text(t -> t)) + .properties("parameterDateValue", p -> p.date(d -> d)) + .properties("parameterNumericValue", p -> p.double_(d -> d)))); + response.acknowledged(); + // TODO consider both dynamic field names and nested fields + } catch (ElasticsearchException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + @Override + public void addNow(String entityName, List ids, EntityManager manager, + Class klass, ExecutorService getBeanDocExecutor) throws IcatException { + // getBeanDocExecutor is not used for the Elasticsearch implementation + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator gen = Json.createGenerator(baos); + gen.writeStartArray(); + for (Long id : ids) { + EntityBaseBean bean = (EntityBaseBean) manager.find(klass, id); + if (bean != null) { + gen.writeStartArray(); + gen.write(entityName); // Index + gen.writeNull(); // Search engine ID is null as we are adding + bean.getDoc(gen, this); // Fields + gen.writeEnd(); + } + } + gen.writeEnd(); + modify(baos.toString()); + } + + @Override + public void clear() throws IcatException { + try { + client.indices().delete(c -> c.index("_all")); + initMappings(); + } catch (ElasticsearchException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + // TODO Ideally want to write these as k:v pairs in an object, not objects in a + // list, but this is to be consistent with Lucene + + public void encodeSortedDocValuesField(JsonGenerator gen, String name, Long value) { + gen.writeStartObject().write(name, value).writeEnd(); + } + + public void encodeStoredId(JsonGenerator gen, Long id) { + gen.writeStartObject().write("id", Long.toString(id)).writeEnd(); + } + + public void encodeStringField(JsonGenerator gen, String name, Date value) { + String timeString; + synchronized (df) { + timeString = df.format(value); + } + gen.writeStartObject().write(name, timeString).writeEnd(); + } + + public void encodeDoublePoint(JsonGenerator gen, String name, Double value) { + gen.writeStartObject().write(name, value).writeEnd(); + } + + public void encodeSortedSetDocValuesFacetField(JsonGenerator gen, String name, String value) { + gen.writeStartObject().write(name, value).writeEnd(); + + } + + public void encodeStringField(JsonGenerator gen, String name, Long value) { + gen.writeStartObject().write(name, value).writeEnd(); + } + + public void encodeStringField(JsonGenerator gen, String name, String value) { + gen.writeStartObject().write(name, value).writeEnd(); + } + + public void encodeTextField(JsonGenerator gen, String name, String value) { + if (value != null) { + gen.writeStartObject().write(name, value).writeEnd(); + } + } + + @Override + public void freeSearcher(String uid) + throws IcatException { + try { + pitMap.remove(uid); + client.closePointInTime(p -> p.id(uid)); + } catch (ElasticsearchException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + @Override + public void commit() throws IcatException { + try { + client.indices().refresh(); + } catch (ElasticsearchException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + @Override + public List facetSearch(JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException { + // TODO this should be generalised + try { + String index = "_all"; + Set fields = facetQuery.keySet(); + BoolQuery.Builder builder = new BoolQuery.Builder(); + for (String field : fields) { + // Only expecting a target and text field as part of the current facet + // implementation + if (field.equals("target")) { + index = facetQuery.getString("target").toLowerCase(); + } else if (field.equals("text")) { + String text = facetQuery.getString("text"); + builder.must(m -> m.queryString(q -> q.defaultField("text").query(text))); + } + } + String indexFinal = index; + SearchResponse response = client.search(s -> s + .index(indexFinal) + .size(maxResults) + .query(q -> q.bool(builder.build())) + .aggregations("samples", a -> a.terms(t -> t.field("sampleName").size(maxLabels))) + .aggregations("parameters", a -> a.terms(t -> t.field("parameterName").size(maxLabels))), + Object.class); + + List sampleBuckets = response.aggregations().get("samples").sterms().buckets().array(); + List parameterBuckets = response.aggregations().get("parameters").sterms().buckets() + .array(); + List facetDimensions = new ArrayList<>(); + FacetDimension sampleDimension = new FacetDimension("sampleName"); + FacetDimension parameterDimension = new FacetDimension("parameterName"); + for (StringTermsBucket sampleBucket : sampleBuckets) { + sampleDimension.getFacets().add(new FacetLabel(sampleBucket.key(), sampleBucket.docCount())); + } + for (StringTermsBucket parameterBucket : parameterBuckets) { + parameterDimension.getFacets().add(new FacetLabel(parameterBucket.key(), parameterBucket.docCount())); + } + facetDimensions.add(sampleDimension); + facetDimensions.add(parameterDimension); + return facetDimensions; + } catch (ElasticsearchException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + @Override + public SearchResult getResults(JsonObject query, int maxResults) + throws IcatException { + try { + String index; + if (query.keySet().contains("target")) { + index = query.getString("target").toLowerCase(); + } else { + index = query.getString("_all"); + } + OpenPointInTimeResponse pitResponse = client.openPointInTime(p -> p + .index(index) + .keepAlive(t -> t.time("1m"))); + String pit = pitResponse.id(); + pitMap.put(pit, 0); + return getResults(pit, query, maxResults); + } catch (ElasticsearchException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + @Override + public SearchResult getResults(String uid, JsonObject query, int maxResults) + throws IcatException { + try { + Set fields = query.keySet(); + BoolQuery.Builder builder = new BoolQuery.Builder(); + for (String field : fields) { + if (field.equals("text")) { + String text = query.getString("text"); + builder.must(m -> m.queryString(q -> q.defaultField("text").query(text))); + } else if (field.equals("lower")) { + Long time = decodeTime(query.getString("lower")); + builder.must(m -> m + .bool(b -> b + .should(s -> s + .range(r -> r + .field("date") + .gte(JsonData.of(time)))) + .should(s -> s + .range(r -> r + .field("startDate") + .gte(JsonData.of(time)))))); + } else if (field.equals("upper")) { + Long time = decodeTime(query.getString("upper")); + builder.must(m -> m + .bool(b -> b + .should(s -> s + .range(r -> r + .field("date") + .lte(JsonData.of(time)))) + .should(s -> s + .range(r -> r + .field("endDate") + .lte(JsonData.of(time)))))); + } else if (field.equals("user")) { + String user = query.getString("user"); + builder.filter(f -> f.term(t -> t.field("userName").value(v -> v.stringValue(user)))); + } else if (field.equals("userFullName")) { + String userFullName = query.getString("userFullName"); + builder.filter(f -> f.queryString(q -> q.defaultField("userFullName").query(userFullName))); + } else if (field.equals("samples")) { + for (JsonValue sampleValue : query.getJsonArray("samples")) { + builder.filter( + f -> f.queryString(q -> q.defaultField("sampleText").query(sampleValue.toString()))); + } + } else if (field.equals("parameters")) { + for (JsonValue parameterValue : query.getJsonArray("parameters")) { + // TODO there are more things to support and consider here... e.g. parameters + // with a numeric range not a numeric value + BoolQuery.Builder parameterBuilder = new BoolQuery.Builder(); + JsonObject parameterObject = (JsonObject) parameterValue; + String name = parameterObject.getString("name", null); + String units = parameterObject.getString("units", null); + String stringValue = parameterObject.getString("stringValue", null); + Long lowerDate = decodeTime(parameterObject.getString("lowerDateValue", null)); + Long upperDate = decodeTime(parameterObject.getString("upperDateValue", null)); + JsonNumber lowerNumeric = parameterObject.getJsonNumber("lowerNumericValue"); + JsonNumber upperNumeric = parameterObject.getJsonNumber("upperNumericValue"); + if (name != null) { + parameterBuilder.must(m -> m.match(a -> a.field("parameterName").operator(Operator.And) + .query(q -> q.stringValue(name)))); + } + if (units != null) { + parameterBuilder.must(m -> m.match(a -> a.field("parameterUnits").operator(Operator.And) + .query(q -> q.stringValue(units)))); + } + if (stringValue != null) { + parameterBuilder.must(m -> m.match(a -> a.field("parameterStringValue") + .operator(Operator.And).query(q -> q.stringValue(stringValue)))); + } else if (lowerDate != null && upperDate != null) { + parameterBuilder.must(m -> m.range(r -> r.field("parameterDateValue") + .gte(JsonData.of(lowerDate)).lte(JsonData.of(upperDate)))); + } else if (lowerNumeric != null && upperNumeric != null) { + parameterBuilder.must(m -> m.range( + r -> r.field("parameterNumericValue").gte(JsonData.of(lowerNumeric.doubleValue())) + .lte(JsonData.of(upperNumeric.doubleValue())))); + } + builder.filter(f -> f.bool(b -> parameterBuilder)); + } + // TODO consider support for other fields (would require dynamic fields) + } + } + Integer from = pitMap.get(uid); + SearchResponse response = client.search(s -> s + .size(maxResults) + .pit(p -> p.id(uid).keepAlive(t -> t.time("1m"))) + .query(q -> q.bool(builder.build())) + .docvalueFields(d -> d.field("id")) + .from(from) + .sort(o -> o.score(c -> c.order(SortOrder.Desc))), ElasticsearchDocument.class); + SearchResult result = new SearchResult(); + result.setUid(uid); + pitMap.put(uid, from + maxResults); + List entities = result.getResults(); + for (Hit hit : response.hits().hits()) { + entities.add(new ScoredEntityBaseBean(hit.source().getId(), hit.score().floatValue())); + } + return result; + } catch (ElasticsearchException | IOException | ParseException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + // TODO does this need to be separated for different entity types? + private ElasticsearchDocument buildDocument(JsonArray jsonArray) throws IcatException { + ElasticsearchDocument document = new ElasticsearchDocument(); + try { + for (JsonValue fieldValue : jsonArray) { + JsonObject fieldObject = (JsonObject) fieldValue; + for (Entry fieldEntry : fieldObject.entrySet()) { + if (fieldEntry.getKey().equals("id")) { + if (fieldEntry.getValue().getValueType().equals(ValueType.STRING)) { + document.setId(Long.parseLong(fieldObject.getString("id"))); + } else if (fieldEntry.getValue().getValueType().equals(ValueType.NUMBER)) { + document.setId((long) fieldObject.getInt("id")); + } + } else if (fieldEntry.getKey().equals("text")) { + document.setText(fieldObject.getString("text")); + } else if (fieldEntry.getKey().equals("date")) { + document.setDate(dec(fieldObject.getString("date"))); + } else if (fieldEntry.getKey().equals("startDate")) { + document.setStartDate(dec(fieldObject.getString("startDate"))); + } else if (fieldEntry.getKey().equals("endDate")) { + document.setEndDate(dec(fieldObject.getString("endDate"))); + } else if (fieldEntry.getKey().equals("user.name")) { + document.getUserName().add(fieldObject.getString("user.name")); + } else if (fieldEntry.getKey().equals("user.fullName")) { + document.getUserFullName().add(fieldObject.getString("user.fullName")); + } else if (fieldEntry.getKey().equals("sample.name")) { + document.getSampleName().add(fieldObject.getString("sample.name")); + } else if (fieldEntry.getKey().equals("sample.text")) { + document.getSampleText().add(fieldObject.getString("sample.text")); + } else if (fieldEntry.getKey().equals("parameter.name")) { + document.getParameterName().add(fieldObject.getString("parameter.name")); + } else if (fieldEntry.getKey().equals("parameter.units")) { + document.getParameterUnits().add(fieldObject.getString("parameter.units")); + } else if (fieldEntry.getKey().equals("parameter.stringValue")) { + document.getParameterStringValue().add(fieldObject.getString("parameter.stringValue")); + } else if (fieldEntry.getKey().equals("parameter.dateValue")) { + document.getParameterDateValue().add(dec(fieldObject.getString("parameter.dateValue"))); + } else if (fieldEntry.getKey().equals("parameter.numericValue")) { + document.getParameterNumericValue() + .add(fieldObject.getJsonNumber("parameter.numericValue").doubleValue()); + } + } + } + return document; + } catch (ParseException e) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, e.getClass() + " " + e.getMessage()); + } + } + + @Override + public void modify(String json) throws IcatException { + // Format should be [[, , ], ...] + JsonReader jsonReader = Json.createReader(new StringReader(json)); + JsonArray outerArray = jsonReader.readArray(); + List operations = new ArrayList<>(); + for (JsonArray innerArray : outerArray.getValuesAs(JsonArray.class)) { + // Index should always be present + if (innerArray.isNull(0)) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Cannot modify a document without the target index"); + } + String index = innerArray.getString(0).toLowerCase(); + + if (innerArray.isNull(2)) { + // If the representation is null, delete the document with provided id + if (innerArray.isNull(1)) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Cannot modify document when both the id and object representing its fields are null"); + } + String id = String.valueOf(innerArray.getInt(1)); + operations.add(new BulkOperation.Builder().delete(c -> c.index(index).id(id)).build()); + } else { + // Both creating and updates are handled by the index operation + ElasticsearchDocument document = buildDocument(innerArray.getJsonArray(2)); + String id; + if (innerArray.isNull(1)) { + // If we weren't given an id, try and get one from the document + // Avoid using a generated id, as this prevents us updating the document later + Long documentId = document.getId(); + if (documentId == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Cannot index a document without an id"); + } + id = String.valueOf(documentId); + } else { + id = String.valueOf(innerArray.getInt(1)); + } + operations + .add(new BulkOperation.Builder().index(c -> c.index(index).id(id).document(document)).build()); + } + } + try { + BulkResponse response = client.bulk(c -> c.operations(operations)); + if (response.errors()) { + // Throw an Exception for the first error we had in the list of operations + for (BulkResponseItem responseItem : response.items()) { + if (responseItem.error().reason() != "") { + throw new IcatException(IcatExceptionType.INTERNAL, responseItem.error().reason()); + } + } + } + ; + } catch (ElasticsearchException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + +} diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java b/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java new file mode 100644 index 00000000..af7acf64 --- /dev/null +++ b/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java @@ -0,0 +1,140 @@ +package org.icatproject.core.manager; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** + * This class is required in order to map to and from JSON for Elasticsearch + * client functions + */ +public class ElasticsearchDocument { + + private Long id; + private String text; + private Date date; + private Date startDate; + private Date endDate; + private List userName = new ArrayList<>(); + private List userFullName = new ArrayList<>(); + private List sampleName = new ArrayList<>(); + private List sampleText = new ArrayList<>(); + private List parameterName = new ArrayList<>(); + private List parameterUnits = new ArrayList<>(); + private List parameterStringValue = new ArrayList<>(); + private List parameterDateValue = new ArrayList<>(); + private List parameterNumericValue = new ArrayList<>(); + + public Long getId() { + return id; + } + + public List getParameterNumericValue() { + return parameterNumericValue; + } + + public void setParameterNumericValue(List parameterNumericValue) { + this.parameterNumericValue = parameterNumericValue; + } + + public List getParameterDateValue() { + return parameterDateValue; + } + + public void setParameterDateValue(List parameterDateValue) { + this.parameterDateValue = parameterDateValue; + } + + public List getParameterStringValue() { + return parameterStringValue; + } + + public void setParameterStringValue(List parameterStringValue) { + this.parameterStringValue = parameterStringValue; + } + + public List getParameterUnits() { + return parameterUnits; + } + + public void setParameterUnits(List parameterUnits) { + this.parameterUnits = parameterUnits; + } + + public List getParameterName() { + return parameterName; + } + + public void setParameterName(List parameterName) { + this.parameterName = parameterName; + } + + public List getSampleText() { + return sampleText; + } + + public void setSampleText(List sampleText) { + this.sampleText = sampleText; + } + + public List getSampleName() { + return sampleName; + } + + public void setSampleName(List sampleName) { + this.sampleName = sampleName; + } + + public List getUserFullName() { + return userFullName; + } + + public void setUserFullName(List userFullName) { + this.userFullName = userFullName; + } + + public List getUserName() { + return userName; + } + + public void setUserName(List userName) { + this.userName = userName; + } + + public Date getEndDate() { + return endDate; + } + + public void setEndDate(Date endDate) { + this.endDate = endDate; + } + + public Date getStartDate() { + return startDate; + } + + public void setStartDate(Date startDate) { + this.startDate = startDate; + } + + public Date getDate() { + return date; + } + + public void setDate(Date date) { + this.date = date; + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public void setId(Long id) { + this.id = id; + } + +} diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 4654d681..5f67d006 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -61,9 +61,7 @@ import org.icatproject.core.IcatException; import org.icatproject.core.IcatException.IcatExceptionType; import org.icatproject.core.entity.Datafile; -import org.icatproject.core.entity.Dataset; import org.icatproject.core.entity.EntityBaseBean; -import org.icatproject.core.entity.Investigation; import org.icatproject.core.entity.ParameterValueType; import org.icatproject.core.entity.Session; import org.icatproject.core.manager.EntityInfoHandler.Relationship; @@ -135,7 +133,7 @@ public enum PersistMode { Transmitter transmitter; @EJB - LuceneManager lucene; + SearchManager searchManager; private boolean log; @@ -145,7 +143,7 @@ public enum PersistMode { private Map notificationRequests; - private boolean luceneActive; + private boolean searchActive; private int maxEntities; @@ -252,8 +250,8 @@ public CreateResponse create(String userId, EntityBaseBean bean, EntityManager m long beanId = bean.getId(); - if (luceneActive) { - bean.addToLucene(lucene); + if (searchActive) { + bean.addToSearch(searchManager); } userTransaction.commit(); if (logRequests.contains(CallType.WRITE)) { @@ -383,9 +381,9 @@ public List createMany(String userId, List beans transmitter.processMessage("createMany", ip, baos.toString(), startMillis); } - if (luceneActive) { + if (searchActive) { for (EntityBaseBean bean : beans) { - bean.addToLucene(lucene); + bean.addToSearch(searchManager); } } @@ -502,9 +500,9 @@ public void delete(String userId, List beans, EntityManager mana userTransaction.commit(); - if (luceneActive) { + if (searchActive) { for (EntityBaseBean bean : allBeansToDelete) { - lucene.deleteDocument(bean); + searchManager.deleteDocument(bean); } } @@ -785,7 +783,7 @@ private void filterReadAccess(List results, List klass) throws IcatException { - logger.debug("Got " + allResults.size() + " results from Lucene"); + logger.debug("Got " + allResults.size() + " results from search engine"); for (ScoredEntityBaseBean sr : allResults) { long entityId = sr.getEntityBaseBeanId(); EntityBaseBean beanManaged = manager.find(klass, entityId); @@ -1152,7 +1150,7 @@ void init() { logRequests = propertyHandler.getLogSet(); log = !logRequests.isEmpty(); notificationRequests = propertyHandler.getNotificationRequests(); - luceneActive = lucene.isActive(); + searchActive = searchManager.isActive(); maxEntities = propertyHandler.getMaxEntities(); exportCacheSize = propertyHandler.getImportCacheSize(); rootUserNames = propertyHandler.getRootUserNames(); @@ -1378,87 +1376,64 @@ public EntityBaseBean lookup(EntityBaseBean bean, EntityManager manager) throws return results.get(0); } - public void luceneClear() throws IcatException { - if (luceneActive) { - lucene.clear(); + public void searchClear() throws IcatException { + if (searchActive) { + searchManager.clear(); } } - public void luceneCommit() throws IcatException { - if (luceneActive) { - lucene.commit(); + public void searchCommit() throws IcatException { + if (searchActive) { + searchManager.commit(); } } - public List luceneDatafiles(String userName, String user, String text, Date lower, Date upper, - List parms, int maxCount, EntityManager manager, String ip) throws IcatException { + public List freeTextSearch(String userName, JsonObject jo, int maxCount, + EntityManager manager, String ip, Class klass) throws IcatException { long startMillis = log ? System.currentTimeMillis() : 0; List results = new ArrayList<>(); - if (luceneActive) { - LuceneSearchResult last = null; - Long uid = null; + if (searchActive) { + SearchResult last = null; + String uid = null; List allResults = Collections.emptyList(); /* * As results may be rejected and maxCount may be 1 ensure that we - * don't make a huge number of calls to Lucene + * don't make a huge number of calls to search engine */ int blockSize = Math.max(1000, maxCount); do { if (last == null) { - last = lucene.datafiles(user, text, lower, upper, parms, blockSize); + last = searchManager.freeTextSearch(jo, blockSize); uid = last.getUid(); } else { - last = lucene.datafilesAfter(uid, blockSize); + last = searchManager.freeTextSearch(uid, jo, blockSize); } allResults = last.getResults(); - filterReadAccess(results, allResults, maxCount, userName, manager, Datafile.class); + filterReadAccess(results, allResults, maxCount, userName, manager, klass); } while (results.size() != maxCount && allResults.size() == blockSize); - /* failing lucene retrieval calls clean up before throwing */ - lucene.freeSearcher(uid); - } - - if (logRequests.contains("R")) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) { - gen.write("userName", userName); - if (results.size() > 0) { - gen.write("entityId", results.get(0).getEntityBaseBeanId()); + /* failing retrieval calls clean up before throwing */ + searchManager.freeSearcher(uid); + + // TODO move this to somewhere we can manually call it + if (results.size() > 0) { + /* Get facets for the filtered list of IDs */ + String facetText = ""; + for (ScoredEntityBaseBean result: results) { + facetText += " id:" + result.getEntityBaseBeanId(); + } + JsonObject facetQuery = Json.createObjectBuilder() + .add("target", jo.getString("target")) + .add("text", facetText.substring(1)) + .build(); + List facets = searchManager.facetSearch(facetQuery, blockSize, 100); // TODO remove hardcode + for (FacetDimension dimension: facets) { + logger.debug("Facet dimension: {}", dimension.getDimension()); + for (FacetLabel facet: dimension.getFacets()) { + logger.debug("{}: {}", facet.getLabel(), facet.getValue()); + } } - gen.writeEnd(); } - transmitter.processMessage("luceneDatafiles", ip, baos.toString(), startMillis); - } - logger.debug("Returning {} results", results.size()); - return results; - } - - public List luceneDatasets(String userName, String user, String text, Date lower, Date upper, - List parms, int maxCount, EntityManager manager, String ip) throws IcatException { - long startMillis = log ? System.currentTimeMillis() : 0; - List results = new ArrayList<>(); - if (luceneActive) { - LuceneSearchResult last = null; - Long uid = null; - List allResults = Collections.emptyList(); - /* - * As results may be rejected and maxCount may be 1 ensure that we - * don't make a huge number of calls to Lucene - */ - int blockSize = Math.max(1000, maxCount); - - do { - if (last == null) { - last = lucene.datasets(user, text, lower, upper, parms, blockSize); - uid = last.getUid(); - } else { - last = lucene.datasetsAfter(uid, blockSize); - } - allResults = last.getResults(); - filterReadAccess(results, allResults, maxCount, userName, manager, Dataset.class); - } while (results.size() != maxCount && allResults.size() == blockSize); - /* failing lucene retrieval calls clean up before throwing */ - lucene.freeSearcher(uid); } if (logRequests.contains("R")) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -1469,71 +1444,28 @@ public List luceneDatasets(String userName, String user, S } gen.writeEnd(); } - transmitter.processMessage("luceneDatasets", ip, baos.toString(), startMillis); + transmitter.processMessage("freeTextSearch", ip, baos.toString(), startMillis); } logger.debug("Returning {} results", results.size()); return results; } - public List luceneGetPopulating() { - if (luceneActive) { - return lucene.getPopulating(); + public List searchGetPopulating() { + if (searchActive) { + return searchManager.getPopulating(); } else { return Collections.emptyList(); } } - public List luceneInvestigations(String userName, String user, String text, Date lower, - Date upper, List parms, List samples, String userFullName, int maxCount, - EntityManager manager, String ip) throws IcatException { - long startMillis = log ? System.currentTimeMillis() : 0; - List results = new ArrayList<>(); - if (luceneActive) { - LuceneSearchResult last = null; - Long uid = null; - List allResults = Collections.emptyList(); - /* - * As results may be rejected and maxCount may be 1 ensure that we - * don't make a huge number of calls to Lucene - */ - int blockSize = Math.max(1000, maxCount); - - do { - if (last == null) { - last = lucene.investigations(user, text, lower, upper, parms, samples, userFullName, blockSize); - uid = last.getUid(); - } else { - last = lucene.investigationsAfter(uid, blockSize); - } - allResults = last.getResults(); - filterReadAccess(results, allResults, maxCount, userName, manager, Investigation.class); - } while (results.size() != maxCount && allResults.size() == blockSize); - /* failing lucene retrieval calls clean up before throwing */ - lucene.freeSearcher(uid); - } - if (logRequests.contains("R")) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) { - gen.write("userName", userName); - if (results.size() > 0) { - gen.write("entityId", results.get(0).getEntityBaseBeanId()); - } - gen.writeEnd(); - } - transmitter.processMessage("luceneInvestigations", ip, baos.toString(), startMillis); - } - logger.debug("Returning {} results", results.size()); - return results; - } - - public void lucenePopulate(String entityName, long minid, EntityManager manager) throws IcatException { - if (luceneActive) { + public void searchPopulate(String entityName, long minid, EntityManager manager) throws IcatException { + if (searchActive) { try { Class.forName(Constants.ENTITY_PREFIX + entityName); } catch (ClassNotFoundException e) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, e.getMessage()); } - lucene.populate(entityName, minid); + searchManager.populate(entityName, minid); } } @@ -1902,8 +1834,8 @@ public NotificationMessage update(String userId, EntityBaseBean bean, EntityMana } transmitter.processMessage("update", ip, baos.toString(), startMillis); } - if (luceneActive) { - lucene.updateDocument(beanManaged); + if (searchActive) { + searchManager.updateDocument(beanManaged); } return notification; } catch (IcatException e) { @@ -1975,7 +1907,7 @@ public List write(String userId, String json, EntityManager manager, UserT userTransaction.commit(); /* - * Nothing should be able to go wrong now so log, update lucene + * Nothing should be able to go wrong now so log, update * and send notification messages */ if (logRequests.contains(CallType.WRITE)) { @@ -2001,12 +1933,12 @@ public List write(String userId, String json, EntityManager manager, UserT } } - if (luceneActive) { + if (searchActive) { for (EntityBaseBean eb : creates) { - lucene.addDocument(eb); + searchManager.addDocument(eb); } for (EntityBaseBean eb : updates) { - lucene.updateDocument(eb); + searchManager.updateDocument(eb); } } @@ -2339,7 +2271,7 @@ public long cloneEntity(String userId, String beanName, long id, String keys, En } /* - * Nothing should be able to go wrong now so log, update lucene and send + * Nothing should be able to go wrong now so log, update and send * notification messages */ if (logRequests.contains(CallType.WRITE)) { @@ -2354,9 +2286,9 @@ public long cloneEntity(String userId, String beanName, long id, String keys, En transmitter.processMessage("write", ip, baos.toString(), startMillis); } - if (luceneActive) { + if (searchActive) { for (EntityBaseBean c : clonedTo.values()) { - lucene.addDocument(c); + searchManager.addDocument(c); } } diff --git a/src/main/java/org/icatproject/core/manager/EntityInfoHandler.java b/src/main/java/org/icatproject/core/manager/EntityInfoHandler.java index 39850946..2e030251 100644 --- a/src/main/java/org/icatproject/core/manager/EntityInfoHandler.java +++ b/src/main/java/org/icatproject/core/manager/EntityInfoHandler.java @@ -103,7 +103,7 @@ private class PrivateEntityInfo { public Map gettersFromName; public Map relationshipsByName; public Set relInKey; - private boolean hasLuceneDoc; + private boolean hasSearchDoc; public PrivateEntityInfo(Set rels, List notNullableFields, Map getters, Map gettersFromName, Map stringFields, Map setters, @@ -111,7 +111,7 @@ public PrivateEntityInfo(Set rels, List notNullableFields, Map fieldComments, Set ones, Set attributes, Constructor constructor, Map fieldByName, String exportHeader, String exportNull, List fields, String exportHeaderAll, - Map relationshipsByName, Set relInKey, boolean hasLuceneDoc) { + Map relationshipsByName, Set relInKey, boolean hasSearchDoc) { this.relatedEntities = rels; this.notNullableFields = notNullableFields; this.getters = getters; @@ -132,7 +132,7 @@ public PrivateEntityInfo(Set rels, List notNullableFields, this.exportHeaderAll = exportHeaderAll; this.relationshipsByName = relationshipsByName; this.relInKey = relInKey; - this.hasLuceneDoc = hasLuceneDoc; + this.hasSearchDoc = hasSearchDoc; } } @@ -591,17 +591,17 @@ private PrivateEntityInfo buildEi(Class objectClass) t } } - boolean hasLuceneDoc = true; + boolean hasSearchDoc = true; try { - objectClass.getDeclaredMethod("getDoc", JsonGenerator.class); + objectClass.getDeclaredMethod("getDoc", JsonGenerator.class, SearchApi.class); } catch (NoSuchMethodException e) { - hasLuceneDoc = false; + hasSearchDoc = false; } return new PrivateEntityInfo(rels, notNullableFields, getters, gettersFromName, stringFields, setters, updaters, constraintFields, commentString, comments, ones, attributes, constructor, fieldsByName, exportHeader.toString(), exportNull.toString(), fields, exportHeaderAll.toString(), relationshipsByName, - relInKey, hasLuceneDoc); + relInKey, hasSearchDoc); } /** @@ -945,7 +945,7 @@ public Map getStringFields(Class objec } /** Return true if getDoc() method exists else false */ - public boolean hasLuceneDoc(Class objectClass) throws IcatException { + public boolean hasSearchDoc(Class objectClass) throws IcatException { PrivateEntityInfo ei = null; synchronized (this.map) { ei = this.map.get(objectClass); @@ -953,7 +953,7 @@ public boolean hasLuceneDoc(Class objectClass) throws ei = this.buildEi(objectClass); this.map.put(objectClass, ei); } - return ei.hasLuceneDoc; + return ei.hasSearchDoc; } } diff --git a/src/main/java/org/icatproject/core/manager/FacetDimension.java b/src/main/java/org/icatproject/core/manager/FacetDimension.java new file mode 100644 index 00000000..0b6e0b75 --- /dev/null +++ b/src/main/java/org/icatproject/core/manager/FacetDimension.java @@ -0,0 +1,27 @@ +package org.icatproject.core.manager; + +import java.util.ArrayList; +import java.util.List; + +/** + * Holds information for a single facetable dimension, or field. + * Each dimension will have a list of FacetLabels. + */ +public class FacetDimension { + + private String dimension; + private List facets = new ArrayList<>(); + + public FacetDimension(String dimension) { + this.dimension = dimension; + } + + public List getFacets() { + return facets; + } + + public String getDimension() { + return dimension; + } + +} diff --git a/src/main/java/org/icatproject/core/manager/FacetLabel.java b/src/main/java/org/icatproject/core/manager/FacetLabel.java new file mode 100644 index 00000000..60b8e389 --- /dev/null +++ b/src/main/java/org/icatproject/core/manager/FacetLabel.java @@ -0,0 +1,26 @@ +package org.icatproject.core.manager; + +/** + * Holds information for a single label value pair. + * The value is the number of times the label is present in a particular facet + * dimension. + */ +public class FacetLabel { + + private String label; + private Long value; + + public FacetLabel(String label, Long value) { + this.label = label; + this.value = value; + } + + public String getLabel() { + return label; + } + + public Long getValue() { + return value; + } + +} diff --git a/src/main/java/org/icatproject/core/manager/LuceneApi.java b/src/main/java/org/icatproject/core/manager/LuceneApi.java index 867424b8..d82c454e 100644 --- a/src/main/java/org/icatproject/core/manager/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/LuceneApi.java @@ -1,18 +1,23 @@ package org.icatproject.core.manager; -import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; import java.net.URI; import java.net.URISyntaxException; -import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; -import java.util.TimeZone; +import java.util.Map; +import java.util.concurrent.ExecutorService; import javax.json.Json; +import javax.json.JsonObject; import javax.json.stream.JsonGenerator; import javax.json.stream.JsonParser; import javax.json.stream.JsonParser.Event; +import javax.persistence.EntityManager; import javax.ws.rs.core.MediaType; import org.apache.http.client.methods.CloseableHttpResponse; @@ -20,41 +25,59 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.icatproject.core.IcatException; import org.icatproject.core.IcatException.IcatExceptionType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.icatproject.core.entity.EntityBaseBean; -public class LuceneApi { +public class LuceneApi extends SearchApi { private enum ParserState { - None, Results + None, Results, Dimensions, Labels } static String basePath = "/icat.lucene"; - final static Logger logger = LoggerFactory.getLogger(LuceneApi.class); - public static void encodeSortedDocValuesField(JsonGenerator gen, String name, Long value) { + /* + * Serves as a record of target entities where search is supported, and the + * relevant path to the search endpoint + */ + private static Map targetPaths = new HashMap<>(); + static { + targetPaths.put("Investigation", "investigations"); + targetPaths.put("Dataset", "datasets"); + targetPaths.put("Datafile", "datafiles"); + } + + private String getTargetPath(JsonObject query) throws IcatException { + if (!query.containsKey("target")) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "'target' must be present in query for LuceneApi, but it was " + query.toString()); + } + String path = targetPaths.get(query.getString("target")); + if (path == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "'target' must be one of " + targetPaths.keySet() + ", but it was " + query.toString()); + } + return path; + } + + // TODO this method of encoding an entity as an array of 3 key objects that represent single field each + // is something that should be streamlined, but would require changes to icat.lucene + + public void encodeSortedDocValuesField(JsonGenerator gen, String name, Long value) { gen.writeStartObject().write("type", "SortedDocValuesField").write("name", name).write("value", value) .writeEnd(); } - public static void encodeStoredId(JsonGenerator gen, Long id) { + public void encodeStoredId(JsonGenerator gen, Long id) { gen.writeStartObject().write("type", "StringField").write("name", "id").write("value", Long.toString(id)) .write("store", true).writeEnd(); } - private static SimpleDateFormat df; - - static { - df = new SimpleDateFormat("yyyyMMddHHmm"); - TimeZone tz = TimeZone.getTimeZone("GMT"); - df.setTimeZone(tz); - } - - public static void encodeStringField(JsonGenerator gen, String name, Date value) { + public void encodeStringField(JsonGenerator gen, String name, Date value) { String timeString; synchronized (df) { timeString = df.format(value); @@ -62,22 +85,28 @@ public static void encodeStringField(JsonGenerator gen, String name, Date value) gen.writeStartObject().write("type", "StringField").write("name", name).write("value", timeString).writeEnd(); } - public static void encodeDoublePoint(JsonGenerator gen, String name, Double value) { + public void encodeDoublePoint(JsonGenerator gen, String name, Double value) { gen.writeStartObject().write("type", "DoublePoint").write("name", name).write("value", value) .write("store", true).writeEnd(); } - public static void encodeStringField(JsonGenerator gen, String name, Long value) { + public void encodeSortedSetDocValuesFacetField(JsonGenerator gen, String name, String value) { + gen.writeStartObject().write("type", "SortedSetDocValuesFacetField").write("name", name).write("value", value) + .writeEnd(); + + } + + public void encodeStringField(JsonGenerator gen, String name, Long value) { gen.writeStartObject().write("type", "StringField").write("name", name).write("value", Long.toString(value)) .writeEnd(); } - public static void encodeStringField(JsonGenerator gen, String name, String value) { + public void encodeStringField(JsonGenerator gen, String name, String value) { gen.writeStartObject().write("type", "StringField").write("name", name).write("value", value).writeEnd(); } - public static void encodeTextfield(JsonGenerator gen, String name, String value) { + public void encodeTextField(JsonGenerator gen, String name, String value) { if (value != null) { gen.writeStartObject().write("type", "TextField").write("name", name).write("value", value).writeEnd(); } @@ -89,6 +118,44 @@ public LuceneApi(URI server) { this.server = server; } + @Override + public void addNow(String entityName, List ids, EntityManager manager, + Class klass, ExecutorService getBeanDocExecutor) throws Exception { + URI uri = new URIBuilder(server).setPath(basePath + "/addNow/" + entityName) + .build(); + + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + HttpPost httpPost = new HttpPost(uri); + PipedOutputStream beanDocs = new PipedOutputStream(); + httpPost.setEntity(new InputStreamEntity(new PipedInputStream(beanDocs))); + getBeanDocExecutor.submit(() -> { + try (JsonGenerator gen = Json.createGenerator(beanDocs)) { + gen.writeStartArray(); + for (Long id : ids) { + EntityBaseBean bean = (EntityBaseBean) manager.find(klass, id); + if (bean != null) { + gen.writeStartArray(); + bean.getDoc(gen, this); + gen.writeEnd(); + } + } + gen.writeEnd(); + return null; + } catch (Exception e) { + logger.error("About to throw internal exception because of", e); + throw new IcatException(IcatExceptionType.INTERNAL, e.getMessage()); + } finally { + manager.close(); + } + }); + + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } + } + + @Override public void clear() throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URI uri = new URIBuilder(server).setPath(basePath + "/clear").build(); @@ -102,6 +169,7 @@ public void clear() throws IcatException { } + @Override public void commit() throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URI uri = new URIBuilder(server).setPath(basePath + "/commit").build(); @@ -115,171 +183,117 @@ public void commit() throws IcatException { } } - public LuceneSearchResult datafiles(long uid, int maxResults) throws IcatException { + @Override + public List facetSearch(JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/datafiles/" + uid) - .setParameter("maxResults", Integer.toString(maxResults)).build(); - return getLsr(uri, httpclient); + String indexPath = getTargetPath(facetQuery); + URI uri = new URIBuilder(server).setPath(basePath + "/" + indexPath + "/facet") + .setParameter("maxResults", Integer.toString(maxResults)) + .setParameter("maxLabels", Integer.toString(maxLabels)).build(); + logger.trace("Making call {}", uri); + return getFacets(uri, httpclient, facetQuery.toString()); } catch (IOException | URISyntaxException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } } - public LuceneSearchResult datafiles(String user, String text, Date lower, Date upper, List parms, - int maxResults) throws IcatException { - - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/datafiles") - .setParameter("maxResults", Integer.toString(maxResults)).build(); + @Override + public void freeSearcher(String uid) throws IcatException { + try { + URI uri = new URIBuilder(server).setPath(basePath + "/freeSearcher/" + uid).build(); logger.trace("Making call {}", uri); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - if (user != null) { - gen.write("user", user); - } - if (text != null) { - gen.write("text", text); - } - if (lower != null) { - gen.write("lower", enc(lower)); - } - if (upper != null) { - gen.write("upper", enc(upper)); + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + HttpDelete httpDelete = new HttpDelete(uri); + try (CloseableHttpResponse response = httpclient.execute(httpDelete)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); } - if (parms != null && !parms.isEmpty()) { - gen.writeStartArray("params"); - for (ParameterPOJO parm : parms) { - gen.writeStartObject(); - if (parm.name != null) { - gen.write("name", parm.name); - } - if (parm.units != null) { - gen.write("units", parm.units); - } - if (parm.stringValue != null) { - gen.write("stringValue", parm.stringValue); - } - if (parm.lowerDateValue != null) { - gen.write("lowerDateValue", enc(parm.lowerDateValue)); - } - if (parm.upperDateValue != null) { - gen.write("upperDateValue", enc(parm.upperDateValue)); - } - if (parm.lowerNumericValue != null) { - gen.write("lowerNumericValue", parm.lowerNumericValue); - } - if (parm.upperNumericValue != null) { - gen.write("upperNumericValue", parm.upperNumericValue); + } + } catch (URISyntaxException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + private List getFacets(URI uri, CloseableHttpClient httpclient, String facetQueryString) + throws IcatException { + logger.debug(facetQueryString); + try { + StringEntity input = new StringEntity(facetQueryString); + input.setContentType(MediaType.APPLICATION_JSON); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(input); + + List facetDimensions = new ArrayList<>(); + ParserState state = ParserState.None; + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + try (JsonParser p = Json.createParser(response.getEntity().getContent())) { + String key = null; + while (p.hasNext()) { + // Get next event in the stream + Event e = p.next(); + if (e.equals(Event.KEY_NAME)) { + // The key name will indicate the content to expect next, and is expected to be + // one of + // "dimensions", a specific dimension, or a label within that dimension. + key = p.getString(); + } else if (e == Event.START_OBJECT) { + if (state == ParserState.None && key != null && key.equals("dimensions")) { + state = ParserState.Dimensions; + } else if (state == ParserState.Dimensions) { + facetDimensions.add(new FacetDimension(key)); + state = ParserState.Labels; + } + } else if (e == (Event.END_OBJECT)) { + if (state == ParserState.Labels) { + // We may have multiple dimensions, so change state so we can read the next one + state = ParserState.Dimensions; + } else if (state == ParserState.Dimensions) { + state = ParserState.None; + } + } else if (state == ParserState.Labels) { + FacetDimension currentFacets = facetDimensions.get(facetDimensions.size() - 1); + currentFacets.getFacets().add(new FacetLabel(key, p.getLong())); } - gen.writeEnd(); // object } - gen.writeEnd(); // array } - gen.writeEnd(); // object } - return getLsr(uri, httpclient, baos); - } catch (IOException | URISyntaxException e) { + return facetDimensions; + } catch (IOException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } - }; - - private String enc(Date dateValue) { - synchronized (df) { - return df.format(dateValue); - } } - public LuceneSearchResult datasets(Long uid, int maxResults) throws IcatException { + @Override + public SearchResult getResults(JsonObject query, int maxResults) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/datasets/" + uid) + String indexPath = getTargetPath(query); + URI uri = new URIBuilder(server).setPath(basePath + "/" + indexPath) .setParameter("maxResults", Integer.toString(maxResults)).build(); - return getLsr(uri, httpclient); + logger.trace("Making call {}", uri); + return getResults(uri, httpclient, query.toString()); } catch (IOException | URISyntaxException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } } - public LuceneSearchResult datasets(String user, String text, Date lower, Date upper, List parms, - int maxResults) throws IcatException { + @Override + public SearchResult getResults(String uid, JsonObject query, int maxResults) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/datasets") + String indexPath = getTargetPath(query); + URI uri = new URIBuilder(server).setPath(basePath + "/" + indexPath + "/" + uid) .setParameter("maxResults", Integer.toString(maxResults)).build(); - logger.trace("Making call {}", uri); + return getResults(uri, httpclient); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - if (user != null) { - gen.write("user", user); - } - if (text != null) { - gen.write("text", text); - } - if (lower != null) { - gen.write("lower", enc(lower)); - } - if (upper != null) { - gen.write("upper", enc(upper)); - } - if (parms != null && !parms.isEmpty()) { - gen.writeStartArray("params"); - for (ParameterPOJO parm : parms) { - gen.writeStartObject(); - if (parm.name != null) { - gen.write("name", parm.name); - } - if (parm.units != null) { - gen.write("units", parm.units); - } - if (parm.stringValue != null) { - gen.write("stringValue", parm.stringValue); - } - if (parm.lowerDateValue != null) { - gen.write("lowerDateValue", enc(parm.lowerDateValue)); - } - if (parm.upperDateValue != null) { - gen.write("upperDateValue", enc(parm.upperDateValue)); - } - if (parm.lowerNumericValue != null) { - gen.write("lowerNumericValue", parm.lowerNumericValue); - } - if (parm.upperNumericValue != null) { - gen.write("upperNumericValue", parm.upperNumericValue); - } - gen.writeEnd(); // object - } - gen.writeEnd(); // array - } - gen.writeEnd(); // object - } - return getLsr(uri, httpclient, baos); } catch (IOException | URISyntaxException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } } - public void freeSearcher(Long uid) throws IcatException { - try { - URI uri = new URIBuilder(server).setPath(basePath + "/freeSearcher/" + uid).build(); - logger.trace("Making call {}", uri); - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - HttpDelete httpDelete = new HttpDelete(uri); - try (CloseableHttpResponse response = httpclient.execute(httpDelete)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } - } catch (URISyntaxException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - private LuceneSearchResult getLsr(URI uri, CloseableHttpClient httpclient) throws IcatException { + private SearchResult getResults(URI uri, CloseableHttpClient httpclient) throws IcatException { HttpGet httpGet = new HttpGet(uri); - LuceneSearchResult lsr = new LuceneSearchResult(); + SearchResult lsr = new SearchResult(); List results = lsr.getResults(); ParserState state = ParserState.None; try (CloseableHttpResponse response = httpclient.execute(httpGet)) { @@ -311,16 +325,16 @@ private LuceneSearchResult getLsr(URI uri, CloseableHttpClient httpclient) throw return lsr; } - private LuceneSearchResult getLsr(URI uri, CloseableHttpClient httpclient, ByteArrayOutputStream baos) + private SearchResult getResults(URI uri, CloseableHttpClient httpclient, String queryString) throws IcatException { - logger.debug(baos.toString()); + logger.debug(queryString); try { - StringEntity input = new StringEntity(baos.toString()); + StringEntity input = new StringEntity(queryString); input.setContentType(MediaType.APPLICATION_JSON); HttpPost httpPost = new HttpPost(uri); httpPost.setEntity(input); - LuceneSearchResult lsr = new LuceneSearchResult(); + SearchResult lsr = new SearchResult(); List results = lsr.getResults(); ParserState state = ParserState.None; try (CloseableHttpResponse response = httpclient.execute(httpPost)) { @@ -341,7 +355,7 @@ private LuceneSearchResult getLsr(URI uri, CloseableHttpClient httpclient, ByteA } } else { // Not in results yet if (e == (Event.VALUE_NUMBER) && key.equals("uid")) { - lsr.setUid(p.getLong()); + lsr.setUid(String.valueOf(p.getLong())); } else if (e == Event.START_ARRAY && key.equals("results")) { state = ParserState.Results; } @@ -357,86 +371,7 @@ private LuceneSearchResult getLsr(URI uri, CloseableHttpClient httpclient, ByteA } } - public LuceneSearchResult investigations(Long uid, int maxResults) throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/investigations/" + uid) - .setParameter("maxResults", Integer.toString(maxResults)).build(); - return getLsr(uri, httpclient); - - } catch (IOException | URISyntaxException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - public LuceneSearchResult investigations(String user, String text, Date lower, Date upper, - List parms, List samples, String userFullName, int maxResults) throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/investigations") - .setParameter("maxResults", Integer.toString(maxResults)).build(); - logger.trace("Making call {}", uri); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - if (user != null) { - gen.write("user", user); - } - if (text != null) { - gen.write("text", text); - } - if (lower != null) { - gen.write("lower", enc(lower)); - } - if (upper != null) { - gen.write("upper", enc(upper)); - } - if (parms != null && !parms.isEmpty()) { - gen.writeStartArray("params"); - for (ParameterPOJO parm : parms) { - gen.writeStartObject(); - if (parm.name != null) { - gen.write("name", parm.name); - } - if (parm.units != null) { - gen.write("units", parm.units); - } - if (parm.stringValue != null) { - gen.write("stringValue", parm.stringValue); - } - if (parm.lowerDateValue != null) { - gen.write("lowerDateValue", enc(parm.lowerDateValue)); - } - if (parm.upperDateValue != null) { - gen.write("upperDateValue", enc(parm.upperDateValue)); - } - if (parm.lowerNumericValue != null) { - gen.write("lowerNumericValue", parm.lowerNumericValue); - } - if (parm.upperNumericValue != null) { - gen.write("upperNumericValue", parm.upperNumericValue); - } - gen.writeEnd(); // object - } - gen.writeEnd(); // array - } - if (samples != null && !samples.isEmpty()) { - gen.writeStartArray("samples"); - for (String sample : samples) { - gen.write(sample); - } - gen.writeEnd(); // array - } - if (userFullName != null) { - gen.write("userFullName", userFullName); - } - gen.writeEnd(); // object - } - return getLsr(uri, httpclient, baos); - } catch (IOException | URISyntaxException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - + @Override public void lock(String entityName) throws IcatException { try { URI uri = new URIBuilder(server).setPath(basePath + "/lock/" + entityName).build(); @@ -452,6 +387,7 @@ public void lock(String entityName) throws IcatException { } } + @Override public void unlock(String entityName) throws IcatException { try { URI uri = new URIBuilder(server).setPath(basePath + "/unlock/" + entityName).build(); @@ -467,6 +403,7 @@ public void unlock(String entityName) throws IcatException { } } + @Override public void modify(String json) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URI uri = new URIBuilder(server).setPath(basePath + "/modify").build(); diff --git a/src/main/java/org/icatproject/core/manager/PropertyHandler.java b/src/main/java/org/icatproject/core/manager/PropertyHandler.java index f316bb64..81f4a14e 100644 --- a/src/main/java/org/icatproject/core/manager/PropertyHandler.java +++ b/src/main/java/org/icatproject/core/manager/PropertyHandler.java @@ -225,6 +225,8 @@ public enum CallType { READ, WRITE, SESSION, INFO } + public enum SearchEngine {LUCENE, ELASTICSEARCH} + public class ExtendedAuthenticator { private Authenticator authenticator; @@ -275,7 +277,7 @@ public Set getRootUserNames() { /** - * Configure which entities will be indexed by lucene on ingest + * Configure which entities will be indexed on ingest */ private Set entitiesToIndex = new HashSet(); @@ -300,12 +302,13 @@ public int getLifetimeMinutes() { private ContainerType containerType; private String jmsTopicConnectionFactory; private String digestKey; - private URL luceneUrl; - private int lucenePopulateBlockSize; - private Path luceneDirectory; - private long luceneBacklogHandlerIntervalMillis; + private SearchEngine searchEngine; + private List searchUrls = new ArrayList<>(); + private int searchPopulateBlockSize; + private Path searchDirectory; + private long searchBacklogHandlerIntervalMillis; private Map cluster = new HashMap<>(); - private long luceneEnqueuedRequestIntervalMillis; + private long searchEnqueuedRequestIntervalMillis; @PostConstruct private void init() { @@ -379,13 +382,13 @@ private void init() { formattedProps.add("rootUserNames " + names); /* entitiesToIndex */ - key = "lucene.entitiesToIndex"; + key = "search.entitiesToIndex"; if (props.has(key)) { String indexableEntities = props.getString(key); for (String indexableEntity : indexableEntities.split("\\s+")) { entitiesToIndex.add(indexableEntity); } - logger.info("lucene.entitiesToIndex: {}", entitiesToIndex.toString()); + logger.info("search.entitiesToIndex: {}", entitiesToIndex.toString()); } else { /* If the property is not specified, we default to all the entities which * currently override the EntityBaseBean.getDoc() method. This should @@ -393,9 +396,9 @@ private void init() { */ entitiesToIndex.addAll(Arrays.asList("Datafile", "Dataset", "Investigation", "InvestigationUser", "DatafileParameter", "DatasetParameter", "InvestigationParameter", "Sample")); - logger.info("lucene.entitiesToIndex not set. Defaulting to: {}", entitiesToIndex.toString()); + logger.info("search.entitiesToIndex not set. Defaulting to: {}", entitiesToIndex.toString()); } - formattedProps.add("lucene.entitiesToIndex " + entitiesToIndex.toString()); + formattedProps.add("search.entitiesToIndex " + entitiesToIndex.toString()); /* notification.list */ key = "notification.list"; @@ -455,29 +458,54 @@ private void init() { logger.info("'log.list' entry not present so no JMS call logging will be performed"); } - /* Lucene Host */ - if (props.has("lucene.url")) { - luceneUrl = props.getURL("lucene.url"); - formattedProps.add("lucene.url" + " " + luceneUrl); + /* Search Host */ + if (props.has("search.engine")) { + try { + searchEngine = SearchEngine.valueOf(props.getString("search.engine").toUpperCase()); + } catch (IllegalArgumentException e) { + String msg = "Value " + props.getString("search.engine") + " of search.engine must be chosen from " + + Arrays.asList(SearchEngine.values()); + throw new IllegalStateException(msg); + } - lucenePopulateBlockSize = props.getPositiveInt("lucene.populateBlockSize"); - formattedProps.add("lucene.populateBlockSize" + " " + lucenePopulateBlockSize); + for (String urlString : props.getString("search.urls").split("\\s+")) { + try { + searchUrls.add(new URL(urlString)); + } catch (MalformedURLException e) { + abend("Url in search.urls " + urlString + " is not a valid URL"); + } + } - luceneDirectory = props.getPath("lucene.directory"); - if (!luceneDirectory.toFile().isDirectory()) { - String msg = luceneDirectory + " is not a directory"; + if (searchEngine == SearchEngine.LUCENE && searchUrls.size() != 1) { + String msg = "Exactly one value for search.urls must be provided when using " + searchEngine; + throw new IllegalStateException(msg); + } else if (searchUrls.size() == 0) { + String msg = "At least one value for search.urls must be provided"; + throw new IllegalStateException(msg); + } + formattedProps.add("search.urls" + " " + searchUrls.toString()); + logger.info("Using {} as search engine with url(s) {}", searchEngine, searchUrls); + + searchPopulateBlockSize = props.getPositiveInt("search.populateBlockSize"); + formattedProps.add("search.populateBlockSize" + " " + searchPopulateBlockSize); + + searchDirectory = props.getPath("search.directory"); + if (!searchDirectory.toFile().isDirectory()) { + String msg = searchDirectory + " is not a directory"; logger.error(fatal, msg); throw new IllegalStateException(msg); } - formattedProps.add("lucene.directory" + " " + luceneDirectory); + formattedProps.add("search.directory" + " " + searchDirectory); - luceneBacklogHandlerIntervalMillis = props.getPositiveLong("lucene.backlogHandlerIntervalSeconds"); - formattedProps.add("lucene.backlogHandlerIntervalSeconds" + " " + luceneBacklogHandlerIntervalMillis); - luceneBacklogHandlerIntervalMillis *= 1000; + searchBacklogHandlerIntervalMillis = props.getPositiveLong("search.backlogHandlerIntervalSeconds"); + formattedProps.add("search.backlogHandlerIntervalSeconds" + " " + searchBacklogHandlerIntervalMillis); + searchBacklogHandlerIntervalMillis *= 1000; - luceneEnqueuedRequestIntervalMillis = props.getPositiveLong("lucene.enqueuedRequestIntervalSeconds"); - formattedProps.add("lucene.enqueuedRequestIntervalSeconds" + " " + luceneEnqueuedRequestIntervalMillis); - luceneEnqueuedRequestIntervalMillis *= 1000; + searchEnqueuedRequestIntervalMillis = props.getPositiveLong("search.enqueuedRequestIntervalSeconds"); + formattedProps.add("search.enqueuedRequestIntervalSeconds" + " " + searchEnqueuedRequestIntervalMillis); + searchEnqueuedRequestIntervalMillis *= 1000; + } else { + logger.info("'search.engine' entry not present so no free text search available"); } /* @@ -604,24 +632,28 @@ public String getKey() { return digestKey; } - public URL getLuceneUrl() { - return luceneUrl; + public SearchEngine getSearchEngine() { + return searchEngine; + } + + public List getSearchUrls() { + return searchUrls; } - public int getLucenePopulateBlockSize() { - return lucenePopulateBlockSize; + public int getSearchPopulateBlockSize() { + return searchPopulateBlockSize; } - public long getLuceneBacklogHandlerIntervalMillis() { - return luceneBacklogHandlerIntervalMillis; + public long getSearchBacklogHandlerIntervalMillis() { + return searchBacklogHandlerIntervalMillis; } - public long getLuceneEnqueuedRequestIntervalMillis() { - return luceneEnqueuedRequestIntervalMillis; + public long getSearchEnqueuedRequestIntervalMillis() { + return searchEnqueuedRequestIntervalMillis; } - public Path getLuceneDirectory() { - return luceneDirectory; + public Path getSearchDirectory() { + return searchDirectory; } } diff --git a/src/main/java/org/icatproject/core/manager/SearchApi.java b/src/main/java/org/icatproject/core/manager/SearchApi.java new file mode 100644 index 00000000..fbe41d73 --- /dev/null +++ b/src/main/java/org/icatproject/core/manager/SearchApi.java @@ -0,0 +1,172 @@ +package org.icatproject.core.manager; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; +import java.util.concurrent.ExecutorService; + +import javax.json.Json; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import javax.json.stream.JsonGenerator; +import javax.persistence.EntityManager; + +import org.icatproject.core.IcatException; +import org.icatproject.core.entity.EntityBaseBean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// TODO see what functionality can live here, and possibly convert from abstract to a fully generic API +public abstract class SearchApi { + + abstract void addNow(String entityName, List ids, EntityManager manager, + Class klass, ExecutorService getBeanDocExecutor) throws Exception; + + abstract void clear() throws IcatException; + + final Logger logger = LoggerFactory.getLogger(this.getClass()); + protected static SimpleDateFormat df; + + static { + df = new SimpleDateFormat("yyyyMMddHHmm"); + TimeZone tz = TimeZone.getTimeZone("GMT"); + df.setTimeZone(tz); + } + + protected static Date dec(String value) throws java.text.ParseException { + if (value == null) { + return null; + } else { + synchronized (df) { + return df.parse(value); + } + } + } + + protected static Long decodeTime(String value) throws java.text.ParseException { + if (value == null) { + return null; + } else { + synchronized (df) { + return df.parse(value).getTime(); + } + } + } + + protected static String enc(Date dateValue) { + synchronized (df) { + return df.format(dateValue); + } + } + + public abstract void encodeSortedDocValuesField(JsonGenerator gen, String name, Long value); + + public abstract void encodeSortedSetDocValuesFacetField(JsonGenerator gen, String name, String value); + + public abstract void encodeStoredId(JsonGenerator gen, Long id); + + public abstract void encodeStringField(JsonGenerator gen, String name, Date value); + + public abstract void encodeDoublePoint(JsonGenerator gen, String name, Double value); + + public abstract void encodeStringField(JsonGenerator gen, String name, Long value); + + public abstract void encodeStringField(JsonGenerator gen, String name, String value); + + public abstract void encodeTextField(JsonGenerator gen, String name, String value); + + public abstract void freeSearcher(String uid) throws IcatException; + + public abstract void commit() throws IcatException; + + public abstract List facetSearch(JsonObject facetQuery, int maxResults, int maxLabels) + throws IcatException; + + public abstract SearchResult getResults(JsonObject query, int maxResults) throws IcatException; + + public abstract SearchResult getResults(String uid, JsonObject query, int maxResults) throws IcatException; + + public void lock(String entityName) throws IcatException { + logger.info("Manually locking index not supported, no request sent"); + } + + public void unlock(String entityName) throws IcatException { + logger.info("Manually unlocking index not supported, no request sent"); + } + + public abstract void modify(String json) throws IcatException; + + /** + * Legacy function for building a Query from individual arguments + * @param target + * @param user + * @param text + * @param lower + * @param upper + * @param parameters + * @param samples + * @param userFullName + * @return + */ + public static JsonObject buildQuery(String target, String user, String text, Date lower, Date upper, + List parameters, List samples, String userFullName) { + JsonObjectBuilder builder = Json.createObjectBuilder(); + if (target != null) { + builder.add("target", target); + } + if (user != null) { + builder.add("user", user); + } + if (text != null) { + builder.add("text", text); + } + if (lower != null) { + builder.add("lower", LuceneApi.enc(lower)); + } + if (upper != null) { + builder.add("upper", LuceneApi.enc(upper)); + } + if (parameters != null && !parameters.isEmpty()) { + JsonArrayBuilder parametersBuilder = Json.createArrayBuilder(); + for (ParameterPOJO parameter : parameters) { + JsonObjectBuilder parameterBuilder = Json.createObjectBuilder(); + if (parameter.name != null) { + parameterBuilder.add("name", parameter.name); + } + if (parameter.units != null) { + parameterBuilder.add("units", parameter.units); + } + if (parameter.stringValue != null) { + parameterBuilder.add("stringValue", parameter.stringValue); + } + if (parameter.lowerDateValue != null) { + parameterBuilder.add("lowerDateValue", LuceneApi.enc(parameter.lowerDateValue)); + } + if (parameter.upperDateValue != null) { + parameterBuilder.add("upperDateValue", LuceneApi.enc(parameter.upperDateValue)); + } + if (parameter.lowerNumericValue != null) { + parameterBuilder.add("lowerNumericValue", parameter.lowerNumericValue); + } + if (parameter.upperNumericValue != null) { + parameterBuilder.add("upperNumericValue", parameter.upperNumericValue); + } + parametersBuilder.add(parameterBuilder); + } + builder.add("parameters", parametersBuilder); + } + if (samples != null && !samples.isEmpty()) { + JsonArrayBuilder samplesBuilder = Json.createArrayBuilder(); + for (String sample : samples) { + samplesBuilder.add(sample); + } + builder.add("samples", samplesBuilder); + } + if (userFullName != null) { + builder.add("userFullName", userFullName); + } + return builder.build(); + } +} diff --git a/src/main/java/org/icatproject/core/manager/LuceneManager.java b/src/main/java/org/icatproject/core/manager/SearchManager.java similarity index 70% rename from src/main/java/org/icatproject/core/manager/LuceneManager.java rename to src/main/java/org/icatproject/core/manager/SearchManager.java index 7c799882..cdd3f0f2 100644 --- a/src/main/java/org/icatproject/core/manager/LuceneManager.java +++ b/src/main/java/org/icatproject/core/manager/SearchManager.java @@ -6,17 +6,13 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; import java.net.URI; import java.net.URL; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Date; import java.util.Set; import java.util.List; import java.util.Map.Entry; -import java.util.Set; import java.util.SortedSet; import java.util.Timer; import java.util.TimerTask; @@ -35,21 +31,17 @@ import javax.ejb.Singleton; import javax.ejb.Startup; import javax.json.Json; +import javax.json.JsonObject; import javax.json.stream.JsonGenerator; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; import javax.persistence.PersistenceUnit; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; import org.icatproject.core.Constants; import org.icatproject.core.IcatException; import org.icatproject.core.IcatException.IcatExceptionType; import org.icatproject.core.entity.EntityBaseBean; +import org.icatproject.core.manager.PropertyHandler.SearchEngine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.Marker; @@ -57,9 +49,9 @@ @Startup @Singleton -public class LuceneManager { +public class SearchManager { - public class EnqueuedLuceneRequestHandler extends TimerTask { + public class EnqueuedSearchRequestHandler extends TimerTask { @Override public void run() { @@ -83,7 +75,7 @@ public void run() { sb.append(']'); try { - luceneApi.modify(sb.toString()); + searchApi.modify(sb.toString()); } catch (IcatException e) { // Record failures in a flat file to be examined // periodically @@ -130,45 +122,14 @@ public IndexSome(String entityName, List ids, EntityManagerFactory entityM @Override public Long call() throws Exception { - if (eiHandler.hasLuceneDoc(klass)) { - - URI uri = new URIBuilder(luceneApi.server).setPath(LuceneApi.basePath + "/addNow/" + entityName) - .build(); - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - HttpPost httpPost = new HttpPost(uri); - PipedOutputStream beanDocs = new PipedOutputStream(); - httpPost.setEntity(new InputStreamEntity(new PipedInputStream(beanDocs))); - getBeanDocExecutor.submit(() -> { - try (JsonGenerator gen = Json.createGenerator(beanDocs)) { - gen.writeStartArray(); - for (Long id : ids) { - EntityBaseBean bean = (EntityBaseBean) manager.find(klass, id); - if (bean != null) { - gen.writeStartArray(); - bean.getDoc(gen); - gen.writeEnd(); - } - } - gen.writeEnd(); - return null; - } catch (Exception e) { - logger.error("About to throw internal exception because of", e); - throw new IcatException(IcatExceptionType.INTERNAL, e.getMessage()); - } finally { - manager.close(); - } - }); - - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } + if (eiHandler.hasSearchDoc(klass)) { + searchApi.addNow(entityName, ids, manager, klass, getBeanDocExecutor); } return start; } } - private class PendingLuceneRequestHandler extends TimerTask { + private class PendingSearchRequestHandler extends TimerTask { @Override public void run() { @@ -178,14 +139,14 @@ public void run() { try (BufferedReader reader = new BufferedReader(new FileReader(backlogHandlerFile))) { String line; while ((line = reader.readLine()) != null) { - luceneApi.modify(line); + searchApi.modify(line); } backlogHandlerFile.delete(); - logger.info("Pending lucene records now all inserted"); + logger.info("Pending search records now all inserted"); } catch (IOException e) { logger.error("Problems reading from {} : {}", backlogHandlerFile, e.getMessage()); } catch (IcatException e) { - logger.error("Failed to put previously failed entries into lucene " + e.getMessage()); + logger.error("Failed to put previously failed entries into search engine " + e.getMessage()); } catch (Throwable e) { logger.error("Something unexpected happened " + e.getClass() + " " + e.getMessage()); } @@ -219,11 +180,11 @@ public void run() { populatingClassEntry = populateMap.firstEntry(); if (populatingClassEntry != null) { - luceneApi.lock(populatingClassEntry.getKey()); + searchApi.lock(populatingClassEntry.getKey()); Long start = populatingClassEntry.getValue(); - logger.info("Lucene Populating " + populatingClassEntry); + logger.info("Search engine populating " + populatingClassEntry); CompletionService threads = new ExecutorCompletionService<>(populateExecutor); SortedSet tasks = new ConcurrentSkipListSet<>(); @@ -286,7 +247,7 @@ public void run() { /* * Unlock and commit the changes */ - luceneApi.unlock(populatingClassEntry.getKey()); + searchApi.unlock(populatingClassEntry.getKey()); populateMap.remove(populatingClassEntry.getKey()); } } @@ -301,7 +262,7 @@ public void run() { private static EntityInfoHandler eiHandler = EntityInfoHandler.getInstance(); - final static Logger logger = LoggerFactory.getLogger(LuceneManager.class); + final static Logger logger = LoggerFactory.getLogger(SearchManager.class); final static Marker fatal = MarkerFactory.getMarker("FATAL"); @@ -329,7 +290,7 @@ public void run() { private int maxThreads; - private LuceneApi luceneApi; + private SearchApi searchApi; private boolean active; @@ -345,13 +306,17 @@ public void run() { private File queueFile; + private SearchEngine searchEngine; + + private List urls; + public void addDocument(EntityBaseBean bean) throws IcatException { String entityName = bean.getClass().getSimpleName(); - if (eiHandler.hasLuceneDoc(bean.getClass()) && entitiesToIndex.contains(entityName)) { + if (eiHandler.hasSearchDoc(bean.getClass()) && entitiesToIndex.contains(entityName)) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartArray(); - bean.getDoc(gen); + bean.getDoc(gen, searchApi); gen.writeEnd(); } enqueue(entityName, baos.toString(), null); @@ -389,7 +354,7 @@ public void enqueue(String entityName, String json, Long id) throws IcatExceptio } public void clear() throws IcatException { - logger.info("Lucene clear called"); + logger.info("Search engine clear called"); popState = PopState.STOPPING; while (populateThread != null && populateThread.getState() != Thread.State.TERMINATED) { try { @@ -398,34 +363,16 @@ public void clear() throws IcatException { // Do nothing } } - logger.debug("Lucene population terminated"); + logger.debug("Search engine population terminated"); } public void commit() throws IcatException { pushPendingCalls(); - luceneApi.commit(); - } - - public LuceneSearchResult datafiles(String user, String text, Date lower, Date upper, List parms, - int blockSize) throws IcatException { - return luceneApi.datafiles(user, text, lower, upper, parms, blockSize); - } - - public LuceneSearchResult datafilesAfter(long uid, int blockSize) throws IcatException { - return luceneApi.datafiles(uid, blockSize); - } - - public LuceneSearchResult datasets(String user, String text, Date lower, Date upper, List parms, - int blockSize) throws IcatException { - return luceneApi.datasets(user, text, lower, upper, parms, blockSize); - } - - public LuceneSearchResult datasetsAfter(Long uid, int blockSize) throws IcatException { - return luceneApi.datasets(uid, blockSize); + searchApi.commit(); } public void deleteDocument(EntityBaseBean bean) throws IcatException { - if (eiHandler.hasLuceneDoc(bean.getClass())) { + if (eiHandler.hasSearchDoc(bean.getClass())) { String entityName = bean.getClass().getSimpleName(); Long id = bean.getId(); enqueue(entityName, null, id); @@ -433,7 +380,7 @@ public void deleteDocument(EntityBaseBean bean) throws IcatException { } private void pushPendingCalls() { - timer.schedule(new EnqueuedLuceneRequestHandler(), 0L); + timer.schedule(new EnqueuedSearchRequestHandler(), 0L); while (queueFile.length() != 0) { try { Thread.sleep(1000); @@ -445,7 +392,7 @@ private void pushPendingCalls() { @PreDestroy private void exit() { - logger.info("Closing down LuceneManager"); + logger.info("Closing down SearchManager"); if (active) { try { populateExecutor.shutdown(); @@ -453,15 +400,20 @@ private void exit() { pushPendingCalls(); timer.cancel(); timer = null; - logger.info("Closed down LuceneManager"); + logger.info("Closed down SearchManager"); } catch (Exception e) { - logger.error(fatal, "Problem closing down LuceneManager", e); + logger.error(fatal, "Problem closing down SearchManager", e); } } } - public void freeSearcher(Long uid) throws IcatException { - luceneApi.freeSearcher(uid); + public List facetSearch(JsonObject facetQuery, int maxResults, int maxLabels) + throws IcatException { + return searchApi.facetSearch(facetQuery, maxResults, maxLabels); + } + + public void freeSearcher(String uid) throws IcatException { + searchApi.freeSearcher(uid); } public List getPopulating() { @@ -472,46 +424,54 @@ public List getPopulating() { return result; } + public SearchResult freeTextSearch(JsonObject jo, int blockSize) throws IcatException { + return searchApi.getResults(jo, blockSize); + } + + public SearchResult freeTextSearch(String uid, JsonObject jo, int blockSize) throws IcatException { + return searchApi.getResults(uid, jo, blockSize); + } + @PostConstruct private void init() { - logger.info("Initialising LuceneManager"); - URL url = propertyHandler.getLuceneUrl(); - active = url != null; + searchEngine = propertyHandler.getSearchEngine(); + logger.info("Initialising SearchManager for engine {}", searchEngine); + urls = propertyHandler.getSearchUrls(); + active = urls != null && urls.size() > 0; if (active) { try { - luceneApi = new LuceneApi(new URI(propertyHandler.getLuceneUrl().toString())); - populateBlockSize = propertyHandler.getLucenePopulateBlockSize(); - Path luceneDirectory = propertyHandler.getLuceneDirectory(); - backlogHandlerFile = luceneDirectory.resolve("backLog").toFile(); - queueFile = luceneDirectory.resolve("queue").toFile(); + if (searchEngine == SearchEngine.LUCENE) { + searchApi = new LuceneApi(propertyHandler.getSearchUrls().get(0).toURI()); + } else if (searchEngine == SearchEngine.ELASTICSEARCH) { + searchApi = new ElasticsearchApi(propertyHandler.getSearchUrls()); + // TODO implement opensearch + // } else if (searchEngine == SearchEngine.OPENSEARCH) { + // throw new IllegalStateException("OPENSEARCH NYI"); + } + + populateBlockSize = propertyHandler.getSearchPopulateBlockSize(); + Path searchDirectory = propertyHandler.getSearchDirectory(); + backlogHandlerFile = searchDirectory.resolve("backLog").toFile(); + queueFile = searchDirectory.resolve("queue").toFile(); maxThreads = Runtime.getRuntime().availableProcessors(); populateExecutor = Executors.newWorkStealingPool(maxThreads); getBeanDocExecutor = Executors.newCachedThreadPool(); timer = new Timer(); - timer.schedule(new PendingLuceneRequestHandler(), 0L, - propertyHandler.getLuceneBacklogHandlerIntervalMillis()); - timer.schedule(new EnqueuedLuceneRequestHandler(), 0L, - propertyHandler.getLuceneEnqueuedRequestIntervalMillis()); + timer.schedule(new PendingSearchRequestHandler(), 0L, + propertyHandler.getSearchBacklogHandlerIntervalMillis()); + timer.schedule(new EnqueuedSearchRequestHandler(), 0L, + propertyHandler.getSearchEnqueuedRequestIntervalMillis()); entitiesToIndex = propertyHandler.getEntitiesToIndex(); - logger.info("Initialised LuceneManager at {}", url); + logger.info("Initialised SearchManager at {}", urls); } catch (Exception e) { - logger.error(fatal, "Problem setting up LuceneManager", e); - throw new IllegalStateException("Problem setting up LuceneManager"); + logger.error(fatal, "Problem setting up SearchManager", e); + throw new IllegalStateException("Problem setting up SearchManager"); } } else { - logger.info("LuceneManager is inactive"); + logger.info("SearchManager is inactive"); } } - public LuceneSearchResult investigations(String user, String text, Date lower, Date upper, - List parms, List samples, String userFullName, int blockSize) throws IcatException { - return luceneApi.investigations(user, text, lower, upper, parms, samples, userFullName, blockSize); - } - - public LuceneSearchResult investigationsAfter(Long uid, int blockSize) throws IcatException { - return luceneApi.investigations(uid, blockSize); - } - public boolean isActive() { return active; } @@ -527,7 +487,7 @@ public void populate(String entityName, long minid) throws IcatException { } } if (populateMap.put(entityName, minid) == null) { - logger.debug("Lucene population of {} requested", entityName); + logger.debug("Search engine population of {} requested", entityName); } else { throw new IcatException(IcatExceptionType.OBJECT_ALREADY_EXISTS, "population of " + entityName + " already requested"); @@ -540,11 +500,11 @@ public void populate(String entityName, long minid) throws IcatException { public void updateDocument(EntityBaseBean bean) throws IcatException { String entityName = bean.getClass().getSimpleName(); - if (eiHandler.hasLuceneDoc(bean.getClass()) && entitiesToIndex.contains(entityName)) { + if (eiHandler.hasSearchDoc(bean.getClass()) && entitiesToIndex.contains(entityName)) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartArray(); - bean.getDoc(gen); + bean.getDoc(gen, searchApi); gen.writeEnd(); } enqueue(entityName, baos.toString(), bean.getId()); diff --git a/src/main/java/org/icatproject/core/manager/LuceneSearchResult.java b/src/main/java/org/icatproject/core/manager/SearchResult.java similarity index 71% rename from src/main/java/org/icatproject/core/manager/LuceneSearchResult.java rename to src/main/java/org/icatproject/core/manager/SearchResult.java index b2ab1b2d..a8307506 100644 --- a/src/main/java/org/icatproject/core/manager/LuceneSearchResult.java +++ b/src/main/java/org/icatproject/core/manager/SearchResult.java @@ -3,20 +3,20 @@ import java.util.ArrayList; import java.util.List; -public class LuceneSearchResult { +public class SearchResult { - private Long uid; + private String uid; private List results = new ArrayList<>(); public List getResults() { return results; } - public void setUid(Long uid) { + public void setUid(String uid) { this.uid = uid; } - public Long getUid() { + public String getUid() { return uid; } diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index bd1a2b4a..d0eafdfc 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -11,7 +11,6 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.text.DateFormat; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -20,7 +19,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.TimeZone; import javax.annotation.PostConstruct; import javax.annotation.Resource; @@ -36,7 +34,6 @@ import javax.json.JsonObject; import javax.json.JsonObjectBuilder; import javax.json.JsonReader; -import javax.json.JsonString; import javax.json.JsonStructure; import javax.json.JsonValue; import javax.json.JsonValue.ValueType; @@ -71,13 +68,15 @@ import org.icatproject.core.Constants; import org.icatproject.core.IcatException; import org.icatproject.core.IcatException.IcatExceptionType; +import org.icatproject.core.entity.Datafile; +import org.icatproject.core.entity.Dataset; import org.icatproject.core.entity.EntityBaseBean; +import org.icatproject.core.entity.Investigation; import org.icatproject.core.entity.ParameterValueType; import org.icatproject.core.entity.StudyStatus; import org.icatproject.core.manager.EntityBeanManager; import org.icatproject.core.manager.EntityInfoHandler; import org.icatproject.core.manager.GateKeeper; -import org.icatproject.core.manager.ParameterPOJO; import org.icatproject.core.manager.Porter; import org.icatproject.core.manager.PropertyHandler; import org.icatproject.core.manager.PropertyHandler.ExtendedAuthenticator; @@ -95,24 +94,6 @@ public class ICATRest { private final static DateFormat df8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); - private static SimpleDateFormat df; - - static { - df = new SimpleDateFormat("yyyyMMddHHmm"); - TimeZone tz = TimeZone.getTimeZone("GMT"); - df.setTimeZone(tz); - } - - private static Date dec(String value) throws java.text.ParseException { - if (value == null) { - return null; - } else { - synchronized (df) { - return df.parse(value); - } - } - } - private static EntityInfoHandler eiHandler = EntityInfoHandler.getInstance(); private Map authPlugins; @@ -935,6 +916,7 @@ public void logout(@Context HttpServletRequest request, @PathParam("sessionId") beanManager.logout(sessionId, manager, userTransaction, request.getRemoteAddr()); } + // TODO update endpoints to be generic /** * perform a lucene search * @@ -1037,7 +1019,7 @@ public void logout(@Context HttpServletRequest request, @PathParam("sessionId") @GET @Path("lucene/data") @Produces(MediaType.APPLICATION_JSON) - public String lucene(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, + public String search(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, @QueryParam("query") String query, @QueryParam("maxCount") int maxCount) throws IcatException { if (query == null) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "query is not set"); @@ -1047,63 +1029,41 @@ public String lucene(@Context HttpServletRequest request, @QueryParam("sessionId try (JsonReader jr = Json.createReader(new ByteArrayInputStream(query.getBytes()))) { JsonObject jo = jr.readObject(); String target = jo.getString("target", null); - String user = jo.getString("user", null); - String text = jo.getString("text", null); - String lower = jo.getString("lower", null); - String upper = jo.getString("upper", null); - List parms = new ArrayList<>(); if (jo.containsKey("parameters")) { for (JsonValue val : jo.getJsonArray("parameters")) { - JsonObject parm = (JsonObject) val; - String name = parm.getString("name", null); + JsonObject parameter = (JsonObject) val; + String name = parameter.getString("name", null); if (name == null) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "name not set in one of parameters"); } - String units = parm.getString("units", null); + String units = parameter.getString("units", null); if (units == null) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "units not set in parameter '" + name + "'"); } - if (parm.containsKey("stringValue")) { - parms.add(new ParameterPOJO(name, units, parm.getString("stringValue"))); - } else if (parm.containsKey("lowerDateValue") && parm.containsKey("upperDateValue")) { - synchronized (df) { - parms.add(new ParameterPOJO(name, units, df.parse(parm.getString("lowerDateValue")), - df.parse(parm.getString("upperDateValue")))); - } - } else if (parm.containsKey("lowerNumericValue") && parm.containsKey("upperNumericValue")) { - parms.add(new ParameterPOJO(name, units, parm.getJsonNumber("lowerNumericValue").doubleValue(), - parm.getJsonNumber("upperNumericValue").doubleValue())); - } else { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, parm.toString()); + // If we don't have either a string, pair of dates, or pair of numbers, then throw + if (!(parameter.containsKey("stringValue") + || (parameter.containsKey("lowerDateValue") + && parameter.containsKey("upperDateValue")) + || (parameter.containsKey("lowerNumericValue") + && parameter.containsKey("upperNumericValue")))) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, parameter.toString()); } } } List objects; + Class klass; if (target.equals("Investigation")) { - List samples = new ArrayList<>(); - if (jo.containsKey("samples")) { - for (JsonValue val : jo.getJsonArray("samples")) { - JsonString samp = (JsonString) val; - samples.add(samp.getString()); - } - } - String userFullName = jo.getString("userFullName", null); - objects = beanManager.luceneInvestigations(userName, user, text, dec(lower), dec(upper), parms, samples, - userFullName, maxCount, manager, request.getRemoteAddr()); - + klass = Investigation.class; } else if (target.equals("Dataset")) { - objects = beanManager.luceneDatasets(userName, user, text, dec(lower), dec(upper), parms, maxCount, - manager, request.getRemoteAddr()); - + klass = Dataset.class; } else if (target.equals("Datafile")) { - objects = beanManager.luceneDatafiles(userName, user, text, dec(lower), dec(upper), parms, maxCount, - manager, request.getRemoteAddr()); - + klass = Datafile.class; } else { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "target:" + target + " is not expected"); } + objects = beanManager.freeTextSearch(userName, jo, maxCount, manager, request.getRemoteAddr(), klass); JsonGenerator gen = Json.createGenerator(baos); gen.writeStartArray(); for (ScoredEntityBaseBean sb : objects) { @@ -1117,8 +1077,6 @@ public String lucene(@Context HttpServletRequest request, @QueryParam("sessionId return baos.toString(); } catch (JsonException e) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "JsonException " + e.getMessage()); - } catch (ParseException e) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, "ParserException " + e.getMessage()); } } @@ -1156,6 +1114,7 @@ public void gatekeeperMarkPublicStepsStale(@Context HttpServletRequest request) gatekeeper.markPublicStepsStale(); } + // TODO generalise endpoints /** * Stop population of the lucene database if it is running. * @@ -1169,11 +1128,12 @@ public void gatekeeperMarkPublicStepsStale(@Context HttpServletRequest request) */ @DELETE @Path("lucene/db") - public void luceneClear(@QueryParam("sessionId") String sessionId) throws IcatException { + public void searchClear(@QueryParam("sessionId") String sessionId) throws IcatException { checkRoot(sessionId); - beanManager.luceneClear(); + beanManager.searchClear(); } + // TODO generalise endpoints /** * Forces a commit of the lucene database * @@ -1187,11 +1147,12 @@ public void luceneClear(@QueryParam("sessionId") String sessionId) throws IcatEx */ @POST @Path("lucene/db") - public void luceneCommit(@FormParam("sessionId") String sessionId) throws IcatException { + public void searchCommit(@FormParam("sessionId") String sessionId) throws IcatException { checkRoot(sessionId); - beanManager.luceneCommit(); + beanManager.searchCommit(); } + // TODO generalise endpoints /** * Return a list of class names for which population is going on * @@ -1207,12 +1168,12 @@ public void luceneCommit(@FormParam("sessionId") String sessionId) throws IcatEx @GET @Path("lucene/db") @Produces(MediaType.APPLICATION_JSON) - public String luceneGetPopulating(@QueryParam("sessionId") String sessionId) throws IcatException { + public String searchGetPopulating(@QueryParam("sessionId") String sessionId) throws IcatException { checkRoot(sessionId); ByteArrayOutputStream baos = new ByteArrayOutputStream(); JsonGenerator gen = Json.createGenerator(baos); gen.writeStartArray(); - for (String name : beanManager.luceneGetPopulating()) { + for (String name : beanManager.searchGetPopulating()) { gen.write(name); } gen.writeEnd().close(); @@ -1243,6 +1204,7 @@ public void waitMillis(@FormParam("sessionId") String sessionId, @FormParam("ms" } } + // TODO generalise endpoints /** * Clear and repopulate lucene documents for the specified entityName * @@ -1260,10 +1222,10 @@ public void waitMillis(@FormParam("sessionId") String sessionId, @FormParam("ms" */ @POST @Path("lucene/db/{entityName}/{minid}") - public void lucenePopulate(@FormParam("sessionId") String sessionId, @PathParam("entityName") String entityName, + public void searchPopulate(@FormParam("sessionId") String sessionId, @PathParam("entityName") String entityName, @PathParam("minid") long minid) throws IcatException { checkRoot(sessionId); - beanManager.lucenePopulate(entityName, minid, manager); + beanManager.searchPopulate(entityName, minid, manager); } /** diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 6809a37a..36dfe139 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -26,7 +26,7 @@ - + diff --git a/src/main/resources/run.properties b/src/main/resources/run.properties index 006bf2c9..8a8c7734 100644 --- a/src/main/resources/run.properties +++ b/src/main/resources/run.properties @@ -16,14 +16,15 @@ notification.Datafile = CU log.list = SESSION WRITE READ INFO -lucene.url = https://localhost.localdomain:8181 -lucene.populateBlockSize = 10000 -lucene.directory = ${HOME}/data/lucene -lucene.backlogHandlerIntervalSeconds = 60 -lucene.enqueuedRequestIntervalSeconds = 3 +search.engine = lucene +search.urls = https://localhost.localdomain:8181 +search.populateBlockSize = 10000 +search.directory = ${HOME}/data/search +search.backlogHandlerIntervalSeconds = 60 +search.enqueuedRequestIntervalSeconds = 3 # Configure this option to prevent certain entities being indexed # For example, remove Datafile and DatafileParameter -!lucene.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample +!search.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample !cluster = https://smfisher:8181 diff --git a/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java b/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java new file mode 100644 index 00000000..aa73c1f2 --- /dev/null +++ b/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java @@ -0,0 +1,535 @@ +package org.icatproject.core.manager; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.ByteArrayOutputStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; + +import javax.json.Json; +import javax.json.stream.JsonGenerator; + +import org.icatproject.core.IcatException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestElasticsearchApi { + + static ElasticsearchApi searchApi; + final static Logger logger = LoggerFactory.getLogger(TestElasticsearchApi.class); + + @BeforeClass + public static void beforeClass() throws Exception { + String urlString = System.getProperty("searchUrls"); + logger.info("Using Elasticsearch service at {}", urlString); + searchApi = new ElasticsearchApi(Arrays.asList(new URL(urlString))); + } + + String letters = "abcdefghijklmnopqrstuvwxyz"; + + long now = new Date().getTime(); + + int NUMINV = 10; + + int NUMUSERS = 5; + + int NUMDS = 30; + + int NUMDF = 100; + + int NUMSAMP = 15; + + private class QueueItem { + + private String entityName; + private Long id; + private String json; + + public QueueItem(String entityName, Long id, String json) { + this.entityName = entityName; + this.id = id; + this.json = json; + } + + } + + @Test + public void modify() throws IcatException { + Queue queue = new ConcurrentLinkedQueue<>(); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartArray(); + searchApi.encodeTextField(gen, "text", "Elephants and Aardvarks"); + searchApi.encodeStringField(gen, "startDate", new Date()); + searchApi.encodeStringField(gen, "endDate", new Date()); + searchApi.encodeStoredId(gen, 42L); + searchApi.encodeStringField(gen, "dataset", 2001L); + gen.writeEnd(); + } + + String json = baos.toString(); + // Create + queue.add(new QueueItem("Datafile", null, json)); + // Update + queue.add(new QueueItem("Datafile", 42L, json)); + // Delete + queue.add(new QueueItem("Datafile", 42L, null)); + queue.add(new QueueItem("Datafile", 42L, null)); + + Iterator qiter = queue.iterator(); + if (qiter.hasNext()) { + StringBuilder sb = new StringBuilder("["); + while (qiter.hasNext()) { + QueueItem item = qiter.next(); + if (sb.length() != 1) { + sb.append(','); + } + sb.append("[\"").append(item.entityName).append('"'); + if (item.id != null) { + sb.append(',').append(item.id); + } else { + sb.append(",null"); + } + if (item.json != null) { + sb.append(',').append(item.json); + } else { + sb.append(",null"); + } + sb.append(']'); + qiter.remove(); + } + sb.append(']'); + searchApi.modify(sb.toString()); + } + } + + @Before + public void before() throws Exception { + searchApi.clear(); + } + + private void checkLsr(SearchResult lsr, Long... n) { + Set wanted = new HashSet<>(Arrays.asList(n)); + Set got = new HashSet<>(); + + for (ScoredEntityBaseBean q : lsr.getResults()) { + got.add(q.getEntityBaseBeanId()); + } + + Set missing = new HashSet<>(wanted); + missing.removeAll(got); + if (!missing.isEmpty()) { + for (Long l : missing) { + logger.error("Entry missing: {}", l); + } + fail("Missing entries"); + } + + missing = new HashSet<>(got); + missing.removeAll(wanted); + if (!missing.isEmpty()) { + for (Long l : missing) { + logger.error("Extra entry: {}", l); + } + fail("Extra entries"); + } + + } + + @Test + public void datafiles() throws Exception { + populate(); + + SearchResult lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 5); + String uid = lsr.getUid(); + + checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); + System.out.println(uid); + lsr = searchApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 200); + // assertTrue(lsr.getUid() == null); + assertEquals(95, lsr.getResults().size()); + searchApi.freeSearcher(uid); + + lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", "e4", null, null, null, null, null, null), 100); + checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, 96L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null), 100); + checkLsr(lsr, 1L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, "dfbbb", null, null, null, null, null), 100); + checkLsr(lsr, 1L, 27L, 53L, 79L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100); + checkLsr(lsr, 3L, 4L, 5L, 6L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", "b1", "dsddd", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100); + checkLsr(lsr); + searchApi.freeSearcher(lsr.getUid()); + + List pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v25")); + lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, null), 100); + checkLsr(lsr, 5L); + searchApi.freeSearcher(lsr.getUid()); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v25")); + lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100); + checkLsr(lsr, 5L); + searchApi.freeSearcher(lsr.getUid()); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, "u sss", null)); + lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100); + checkLsr(lsr, 13L, 65L); + searchApi.freeSearcher(lsr.getUid()); + } + + @Test + public void datasets() throws Exception { + populate(); + SearchResult lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 5); + + String uid = lsr.getUid(); + checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); + System.out.println(uid); + lsr = searchApi.getResults(uid, SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 100); + // assertTrue(lsr.getUid() == null); + checkLsr(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, + 25L, 26L, 27L, 28L, 29L); + searchApi.freeSearcher(uid); + + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "e4", null, null, null, null, null, null), 100); + checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100); + checkLsr(lsr, 1L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100); + checkLsr(lsr, 1L, 27L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100); + checkLsr(lsr, 3L, 4L, 5L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100); + checkLsr(lsr, 3L); + searchApi.freeSearcher(lsr.getUid()); + + List pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v16")); + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, pojos, null, null), 100); + checkLsr(lsr, 4L); + searchApi.freeSearcher(lsr.getUid()); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v16")); + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, null), 100); + checkLsr(lsr, 4L); + searchApi.freeSearcher(lsr.getUid()); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v16")); + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, null), 100); + checkLsr(lsr); + searchApi.freeSearcher(lsr.getUid()); + + } + + private void fillParms(JsonGenerator gen, int i, String rel) { + int j = i % 26; + int k = (i + 5) % 26; + String name = "nm " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); + String units = "u " + letters.substring(k, k + 1) + letters.substring(k, k + 1) + letters.substring(k, k + 1); + + searchApi.encodeStringField(gen, "parameter.name", "S" + name); + searchApi.encodeStringField(gen, "parameter.units", units); + searchApi.encodeStringField(gen, "parameter.stringValue", "v" + i * i); + searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); + System.out.println(rel + " " + i + " '" + "S" + name + "' '" + units + "' 'v" + i * i + "'"); + + searchApi.encodeStringField(gen, "parameter.name", "N" + name); + searchApi.encodeStringField(gen, "parameter.units", units); + searchApi.encodeDoublePoint(gen, "parameter.numericValue", new Double(j * j)); + searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); + System.out.println(rel + " " + i + " '" + "N" + name + "' '" + units + "' " + new Double(j * j)); + + searchApi.encodeStringField(gen, "parameter.name", "D" + name); + searchApi.encodeStringField(gen, "parameter.units", units); + searchApi.encodeStringField(gen, "parameter.dateValue", new Date(now + 60000 * k * k)); + searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); + System.out.println( + rel + " " + i + " '" + "D" + name + "' '" + units + "' '" + new Date(now + 60000 * k * k) + "'"); + + } + + @Test + public void investigations() throws Exception { + populate(); + + /* Blocked results */ + SearchResult lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), + 5); + String uid = lsr.getUid(); + checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); + System.out.println(uid); + lsr = searchApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), 6); + // assertTrue(lsr.getUid() == null); + checkLsr(lsr, 5L, 6L, 7L, 8L, 9L); + searchApi.freeSearcher(uid); + + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "b"), 100); + checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"), 100); + checkLsr(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""), + 100); + checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", null, null, null, null, null, "b"), 100); + checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "c1", null, null, null, null, null, "b"), 100); + checkLsr(lsr); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null), 100); + checkLsr(lsr, 4L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", null, null, null, null, "b"), 100); + checkLsr(lsr, 3L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, "b"), 100); + checkLsr(lsr, 3L); + searchApi.freeSearcher(lsr.getUid()); + + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100); + checkLsr(lsr, 3L, 4L, 5L); + searchApi.freeSearcher(lsr.getUid()); + + List pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v9")); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); + checkLsr(lsr, 3L); + searchApi.freeSearcher(lsr.getUid()); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v9")); + pojos.add(new ParameterPOJO(null, null, 7, 10)); + pojos.add(new ParameterPOJO(null, null, new Date(now + 60000 * 63), new Date(now + 60000 * 65))); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); + checkLsr(lsr, 3L); + searchApi.freeSearcher(lsr.getUid()); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v9")); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, "b"), 100); + checkLsr(lsr, 3L); + searchApi.freeSearcher(lsr.getUid()); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v9")); + pojos.add(new ParameterPOJO(null, null, "v81")); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); + checkLsr(lsr); + searchApi.freeSearcher(lsr.getUid()); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); + checkLsr(lsr, 3L); + searchApi.freeSearcher(lsr.getUid()); + + List samples = Arrays.asList("ddd", "nnn"); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), 100); + checkLsr(lsr, 3L); + searchApi.freeSearcher(lsr.getUid()); + + samples = Arrays.asList("ddd", "mmm"); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), 100); + checkLsr(lsr); + searchApi.freeSearcher(lsr.getUid()); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); + samples = Arrays.asList("ddd", "nnn"); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, samples, "b"), 100); + checkLsr(lsr, 3L); + searchApi.freeSearcher(lsr.getUid()); + } + + /** + * Populate Investigation, Dataset, Datafile + */ + private void populate() throws IcatException { + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartArray(); + for (int i = 0; i < NUMINV; i++) { + gen.writeStartArray(); + gen.write("Investigation"); + gen.writeNull(); + int j = i % 26; + int k = (i + 7) % 26; + int l = (i + 17) % 26; + String word = letters.substring(j, j + 1) + " " + letters.substring(k, k + 1) + " " + + letters.substring(l, l + 1); + gen.writeStartArray(); + searchApi.encodeTextField(gen, "text", word); + searchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); + searchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); + searchApi.encodeStoredId(gen, new Long(i)); + searchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); + if (i % 2 == 1) { + fillParms(gen, i, "investigation"); + } + for (int m = 0; m < NUMSAMP; m++) { + if (i == m % NUMINV) { + int n = m % 26; + String sampleText = "SType " + letters.substring(n, n + 1) + letters.substring(n, n + 1) + + letters.substring(n, n + 1); + searchApi.encodeSortedSetDocValuesFacetField(gen, "sample.name", letters.substring(n, n + 1) + letters.substring(n, n + 1) + + letters.substring(n, n + 1)); + searchApi.encodeTextField(gen, "sample.text", sampleText); + System.out.println("SAMPLE '" + sampleText + "' " + m % NUMINV); + } + } + for (int p = 0; p < NUMUSERS; p++) { + if (i % (p + 1) == 1) { + String fn = "FN " + letters.substring(p, p + 1) + " " + letters.substring(p, p + 1); + String name = letters.substring(p, p + 1) + p; + searchApi.encodeTextField(gen, "user.fullName", fn); + searchApi.encodeStringField(gen, "user.name", name); + System.out.println("'" + fn + "' " + name + " " + i); + } + } + gen.writeEnd(); + gen.writeEnd(); + System.out.println("INVESTIGATION '" + word + "' " + new Date(now + i * 60000) + " " + i); + } + gen.writeEnd(); + } + searchApi.modify(baos.toString()); + + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartArray(); + for (int i = 0; i < NUMDS; i++) { + int j = i % 26; + String word = "DS" + letters.substring(j, j + 1) + letters.substring(j, j + 1) + + letters.substring(j, j + 1); + gen.writeStartArray(); + gen.write("Dataset"); + gen.writeNull(); + gen.writeStartArray(); + searchApi.encodeTextField(gen, "text", word); + searchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); + searchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); + searchApi.encodeStoredId(gen, new Long(i)); + searchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); + Long investigationId = new Long(i % NUMINV); + searchApi.encodeStringField(gen, "investigation", investigationId); + for (int p = 0; p < NUMUSERS; p++) { + if (investigationId % (p + 1) == 1) { + String fn = "FN " + letters.substring(p, p + 1) + " " + letters.substring(p, p + 1); + String name = letters.substring(p, p + 1) + p; + searchApi.encodeTextField(gen, "user.fullName", fn); + searchApi.encodeStringField(gen, "user.name", name); + System.out.println("'" + fn + "' " + name + " " + i); + } + } + if (i % 3 == 1) { + fillParms(gen, i, "dataset"); + } + gen.writeEnd(); + gen.writeEnd(); + System.out.println("DATASET '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMINV); + } + gen.writeEnd(); + } + searchApi.modify(baos.toString()); + + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartArray(); + for (int i = 0; i < NUMDF; i++) { + int j = i % 26; + String word = "DF" + letters.substring(j, j + 1) + letters.substring(j, j + 1) + + letters.substring(j, j + 1); + gen.writeStartArray(); + gen.write("Datafile"); + gen.writeNull(); + gen.writeStartArray(); + searchApi.encodeTextField(gen, "text", word); + searchApi.encodeStringField(gen, "date", new Date(now + i * 60000)); + searchApi.encodeStoredId(gen, new Long(i)); + Long datasetId = new Long(i % NUMDS); + Long investigationId = new Long(datasetId % NUMINV); + searchApi.encodeStringField(gen, "dataset", datasetId); + // searchApi.encodeStringField(gen, "investigation", investigationId); + for (int p = 0; p < NUMUSERS; p++) { + if (investigationId % (p + 1) == 1) { + String fn = "FN " + letters.substring(p, p + 1) + " " + letters.substring(p, p + 1); + String name = letters.substring(p, p + 1) + p; + searchApi.encodeTextField(gen, "user.fullName", fn); + searchApi.encodeStringField(gen, "user.name", name); + System.out.println("'" + fn + "' " + name + " " + i); + } + } + if (i % 4 == 1) { + fillParms(gen, i, "datafile"); + } + gen.writeEnd(); + gen.writeEnd(); + System.out.println("DATAFILE '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMDS); + + } + gen.writeEnd(); + } + searchApi.modify(baos.toString()); + + searchApi.commit(); + + } + +} diff --git a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java index 382959d8..aeda683f 100644 --- a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java +++ b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java @@ -45,7 +45,7 @@ public void testBadname() throws Exception { } @Test - public void testHasLuceneDoc() throws Exception { + public void testHasSearchDoc() throws Exception { Set docdbeans = new HashSet<>(Arrays.asList("Investigation", "Dataset", "Datafile", "InvestigationParameter", "DatasetParameter", "DatafileParameter", "InvestigationUser", "Sample")); for (String beanName : EntityInfoHandler.getEntityNamesList()) { @@ -53,9 +53,9 @@ public void testHasLuceneDoc() throws Exception { Class bean = (Class) Class .forName(Constants.ENTITY_PREFIX + beanName); if (docdbeans.contains(beanName)) { - assertTrue(eiHandler.hasLuceneDoc(bean)); + assertTrue(eiHandler.hasSearchDoc(bean)); } else { - assertFalse(eiHandler.hasLuceneDoc(bean)); + assertFalse(eiHandler.hasSearchDoc(bean)); } } } diff --git a/src/test/java/org/icatproject/core/manager/TestLucene.java b/src/test/java/org/icatproject/core/manager/TestLucene.java index bd4a2cf9..865cf3af 100644 --- a/src/test/java/org/icatproject/core/manager/TestLucene.java +++ b/src/test/java/org/icatproject/core/manager/TestLucene.java @@ -85,11 +85,11 @@ public void modify() throws IcatException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartArray(); - LuceneApi.encodeTextfield(gen, "text", "Elephants and Aardvarks"); - LuceneApi.encodeStringField(gen, "startDate", new Date()); - LuceneApi.encodeStringField(gen, "endDate", new Date()); - LuceneApi.encodeStoredId(gen, 42L); - LuceneApi.encodeStringField(gen, "dataset", 2001L); + luceneApi.encodeTextField(gen, "text", "Elephants and Aardvarks"); + luceneApi.encodeStringField(gen, "startDate", new Date()); + luceneApi.encodeStringField(gen, "endDate", new Date()); + luceneApi.encodeStoredId(gen, 42L); + luceneApi.encodeStringField(gen, "dataset", 2001L); gen.writeEnd(); } @@ -154,7 +154,7 @@ public void before() throws Exception { luceneApi.clear(); } - private void checkLsr(LuceneSearchResult lsr, Long... n) { + private void checkLsr(SearchResult lsr, Long... n) { Set wanted = new HashSet<>(Arrays.asList(n)); Set got = new HashSet<>(); @@ -186,51 +186,54 @@ private void checkLsr(LuceneSearchResult lsr, Long... n) { public void datafiles() throws Exception { populate(); - LuceneSearchResult lsr = luceneApi.datafiles(null, null, null, null, null, 5); - Long uid = lsr.getUid(); + SearchResult lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 5); + String uid = lsr.getUid(); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); System.out.println(uid); - lsr = luceneApi.datafiles(uid, 200); + lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 200); assertTrue(lsr.getUid() == null); assertEquals(95, lsr.getResults().size()); luceneApi.freeSearcher(uid); - lsr = luceneApi.datafiles("e4", null, null, null, null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", "e4", null, null, null, null, null, null), 100); checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, 96L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.datafiles("e4", "dfbbb", null, null, null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null), 100); checkLsr(lsr, 1L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.datafiles(null, "dfbbb", null, null, null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, "dfbbb", null, null, null, null, null), 100); checkLsr(lsr, 1L, 27L, 53L, 79L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.datafiles(null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100); checkLsr(lsr, 3L, 4L, 5L, 6L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.datafiles("b1", "dsddd", new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", "b1", "dsddd", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100); checkLsr(lsr); luceneApi.freeSearcher(lsr.getUid()); List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v25")); - lsr = luceneApi.datafiles(null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), pojos, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, null), 100); checkLsr(lsr, 5L); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v25")); - lsr = luceneApi.datafiles(null, null, null, null, pojos, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100); checkLsr(lsr, 5L); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, "u sss", null)); - lsr = luceneApi.datafiles(null, null, null, null, pojos, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100); checkLsr(lsr, 13L, 65L); luceneApi.freeSearcher(lsr.getUid()); } @@ -238,52 +241,56 @@ public void datafiles() throws Exception { @Test public void datasets() throws Exception { populate(); - LuceneSearchResult lsr = luceneApi.datasets(null, null, null, null, null, 5); + SearchResult lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 5); - Long uid = lsr.getUid(); + String uid = lsr.getUid(); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); System.out.println(uid); - lsr = luceneApi.datasets(uid, 100); + lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 100); assertTrue(lsr.getUid() == null); checkLsr(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, 25L, 26L, 27L, 28L, 29L); luceneApi.freeSearcher(uid); - lsr = luceneApi.datasets("e4", null, null, null, null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "e4", null, null, null, null, null, null), 100); checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.datasets("e4", "dsbbb", null, null, null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100); checkLsr(lsr, 1L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.datasets(null, "dsbbb", null, null, null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100); checkLsr(lsr, 1L, 27L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.datasets(null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100); checkLsr(lsr, 3L, 4L, 5L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.datasets("b1", "dsddd", new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = luceneApi.datasets(null, null, null, null, pojos, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, pojos, null, null), 100); checkLsr(lsr, 4L); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = luceneApi.datasets(null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), pojos, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, null), 100); checkLsr(lsr, 4L); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = luceneApi.datasets("b1", "dsddd", new Date(now + 60000 * 3), new Date(now + 60000 * 6), pojos, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, null), 100); checkLsr(lsr); luceneApi.freeSearcher(lsr.getUid()); @@ -296,26 +303,26 @@ private void fillParms(JsonGenerator gen, int i, String rel) { String units = "u " + letters.substring(k, k + 1) + letters.substring(k, k + 1) + letters.substring(k, k + 1); gen.writeStartArray(); - LuceneApi.encodeStringField(gen, "name", "S" + name); - LuceneApi.encodeStringField(gen, "units", units); - LuceneApi.encodeStringField(gen, "stringValue", "v" + i * i); - LuceneApi.encodeSortedDocValuesField(gen, rel, new Long(i)); + luceneApi.encodeStringField(gen, "name", "S" + name); + luceneApi.encodeStringField(gen, "units", units); + luceneApi.encodeStringField(gen, "stringValue", "v" + i * i); + luceneApi.encodeSortedDocValuesField(gen, rel, new Long(i)); gen.writeEnd(); System.out.println(rel + " " + i + " '" + "S" + name + "' '" + units + "' 'v" + i * i + "'"); gen.writeStartArray(); - LuceneApi.encodeStringField(gen, "name", "N" + name); - LuceneApi.encodeStringField(gen, "units", units); - LuceneApi.encodeDoublePoint(gen, "numericValue", new Double(j * j)); - LuceneApi.encodeSortedDocValuesField(gen, rel, new Long(i)); + luceneApi.encodeStringField(gen, "name", "N" + name); + luceneApi.encodeStringField(gen, "units", units); + luceneApi.encodeDoublePoint(gen, "numericValue", new Double(j * j)); + luceneApi.encodeSortedDocValuesField(gen, rel, new Long(i)); gen.writeEnd(); System.out.println(rel + " " + i + " '" + "N" + name + "' '" + units + "' " + new Double(j * j)); gen.writeStartArray(); - LuceneApi.encodeStringField(gen, "name", "D" + name); - LuceneApi.encodeStringField(gen, "units", units); - LuceneApi.encodeStringField(gen, "dateTimeValue", new Date(now + 60000 * k * k)); - LuceneApi.encodeSortedDocValuesField(gen, rel, new Long(i)); + luceneApi.encodeStringField(gen, "name", "D" + name); + luceneApi.encodeStringField(gen, "units", units); + luceneApi.encodeStringField(gen, "dateTimeValue", new Date(now + 60000 * k * k)); + luceneApi.encodeSortedDocValuesField(gen, rel, new Long(i)); gen.writeEnd(); System.out.println( rel + " " + i + " '" + "D" + name + "' '" + units + "' '" + new Date(now + 60000 * k * k) + "'"); @@ -327,60 +334,58 @@ public void investigations() throws Exception { populate(); /* Blocked results */ - LuceneSearchResult lsr = luceneApi.investigations(null, null, null, null, null, null, null, 5); - Long uid = lsr.getUid(); + SearchResult lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), + 5); + String uid = lsr.getUid(); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); System.out.println(uid); - lsr = luceneApi.investigations(uid, 6); + lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), 6); assertTrue(lsr.getUid() == null); checkLsr(lsr, 5L, 6L, 7L, 8L, 9L); luceneApi.freeSearcher(uid); - lsr = luceneApi.investigations(null, null, null, null, null, null, "b", 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "b"), 100); checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.investigations(null, null, null, null, null, null, "FN", 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"), 100); checkLsr(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.investigations(null, null, null, null, null, null, "FN AND \"b b\"", 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""), + 100); checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.investigations("b1", null, null, null, null, null, "b", 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", null, null, null, null, null, "b"), 100); checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.investigations("c1", null, null, null, null, null, "b", 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "c1", null, null, null, null, null, "b"), 100); checkLsr(lsr); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.investigations("b1", null, null, null, null, null, "b", 100); - checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); - luceneApi.freeSearcher(lsr.getUid()); - - lsr = luceneApi.investigations(null, "l v", null, null, null, null, null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null), 100); checkLsr(lsr, 4L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.investigations("b1", "d", null, null, null, null, "b", 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", null, null, null, null, "b"), 100); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.investigations("b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, null, "b", - 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, "b"), 100); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.investigations(null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, null, - null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100); checkLsr(lsr, 3L, 4L, 5L); luceneApi.freeSearcher(lsr.getUid()); List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); - lsr = luceneApi.investigations(null, null, null, null, pojos, null, null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); @@ -388,45 +393,45 @@ public void investigations() throws Exception { pojos.add(new ParameterPOJO(null, null, "v9")); pojos.add(new ParameterPOJO(null, null, 7, 10)); pojos.add(new ParameterPOJO(null, null, new Date(now + 60000 * 63), new Date(now + 60000 * 65))); - lsr = luceneApi.investigations(null, null, null, null, pojos, null, null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); - lsr = luceneApi.investigations("b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), pojos, null, - "b", 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, "b"), 100); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); pojos.add(new ParameterPOJO(null, null, "v81")); - lsr = luceneApi.investigations(null, null, null, null, pojos, null, null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); checkLsr(lsr); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); - lsr = luceneApi.investigations(null, null, null, null, pojos, null, null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); List samples = Arrays.asList("ddd", "nnn"); - lsr = luceneApi.investigations(null, null, null, null, null, samples, null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), 100); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); samples = Arrays.asList("ddd", "mmm"); - lsr = luceneApi.investigations(null, null, null, null, null, samples, null, 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), 100); checkLsr(lsr); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); samples = Arrays.asList("ddd", "nnn"); - lsr = luceneApi.investigations("b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), pojos, samples, - "b", 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, samples, "b"), 100); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); } @@ -473,10 +478,10 @@ private void populate() throws IcatException { String name = letters.substring(j, j + 1) + j; gen.writeStartArray(); - LuceneApi.encodeTextfield(gen, "text", fn); + luceneApi.encodeTextField(gen, "text", fn); - LuceneApi.encodeStringField(gen, "name", name); - LuceneApi.encodeSortedDocValuesField(gen, "investigation", new Long(i)); + luceneApi.encodeStringField(gen, "name", name); + luceneApi.encodeSortedDocValuesField(gen, "investigation", new Long(i)); gen.writeEnd(); System.out.println("'" + fn + "' " + name + " " + i); @@ -497,11 +502,11 @@ private void populate() throws IcatException { String word = letters.substring(j, j + 1) + " " + letters.substring(k, k + 1) + " " + letters.substring(l, l + 1); gen.writeStartArray(); - LuceneApi.encodeTextfield(gen, "text", word); - LuceneApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); - LuceneApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); - LuceneApi.encodeStoredId(gen, new Long(i)); - LuceneApi.encodeSortedDocValuesField(gen, "id", new Long(i)); + luceneApi.encodeTextField(gen, "text", word); + luceneApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); + luceneApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); + luceneApi.encodeStoredId(gen, new Long(i)); + luceneApi.encodeSortedDocValuesField(gen, "id", new Long(i)); gen.writeEnd(); System.out.println("INVESTIGATION '" + word + "' " + new Date(now + i * 60000) + " " + i); } @@ -529,12 +534,12 @@ private void populate() throws IcatException { String word = "DS" + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); gen.writeStartArray(); - LuceneApi.encodeTextfield(gen, "text", word); - LuceneApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); - LuceneApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); - LuceneApi.encodeStoredId(gen, new Long(i)); - LuceneApi.encodeSortedDocValuesField(gen, "id", new Long(i)); - LuceneApi.encodeStringField(gen, "investigation", new Long(i % NUMINV)); + luceneApi.encodeTextField(gen, "text", word); + luceneApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); + luceneApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); + luceneApi.encodeStoredId(gen, new Long(i)); + luceneApi.encodeSortedDocValuesField(gen, "id", new Long(i)); + luceneApi.encodeStringField(gen, "investigation", new Long(i % NUMINV)); gen.writeEnd(); System.out.println("DATASET '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMINV); } @@ -562,10 +567,10 @@ private void populate() throws IcatException { String word = "DF" + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); gen.writeStartArray(); - LuceneApi.encodeTextfield(gen, "text", word); - LuceneApi.encodeStringField(gen, "date", new Date(now + i * 60000)); - LuceneApi.encodeStoredId(gen, new Long(i)); - LuceneApi.encodeStringField(gen, "dataset", new Long(i % NUMDS)); + luceneApi.encodeTextField(gen, "text", word); + luceneApi.encodeStringField(gen, "date", new Date(now + i * 60000)); + luceneApi.encodeStoredId(gen, new Long(i)); + luceneApi.encodeStringField(gen, "dataset", new Long(i % NUMDS)); gen.writeEnd(); System.out.println("DATAFILE '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMDS); @@ -594,8 +599,8 @@ private void populate() throws IcatException { String word = "SType " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); gen.writeStartArray(); - LuceneApi.encodeTextfield(gen, "text", word); - LuceneApi.encodeSortedDocValuesField(gen, "investigation", new Long(i % NUMINV)); + luceneApi.encodeTextField(gen, "text", word); + luceneApi.encodeSortedDocValuesField(gen, "investigation", new Long(i % NUMINV)); gen.writeEnd(); System.out.println("SAMPLE '" + word + "' " + i % NUMINV); } diff --git a/src/test/java/org/icatproject/integration/WSession.java b/src/test/java/org/icatproject/integration/WSession.java index c284a4e3..f9bc4be7 100644 --- a/src/test/java/org/icatproject/integration/WSession.java +++ b/src/test/java/org/icatproject/integration/WSession.java @@ -444,9 +444,9 @@ public void logout() throws IcatException_Exception { icat.logout(sessionId); } - // This assumes that the lucene.commitSeconds is set to 1 for testing + // This assumes that the search.commitSeconds is set to 1 for testing // purposes - public void synchLucene() throws InterruptedException { + public void synchSearch() throws InterruptedException { Thread.sleep(2000); } diff --git a/src/test/scripts/prepare_test.py b/src/test/scripts/prepare_test.py index dbd8cf46..7958a7e8 100644 --- a/src/test/scripts/prepare_test.py +++ b/src/test/scripts/prepare_test.py @@ -8,12 +8,13 @@ from zipfile import ZipFile import subprocess -if len(sys.argv) != 4: +if len(sys.argv) != 5: raise RuntimeError("Wrong number of arguments") containerHome = sys.argv[1] icat_url = sys.argv[2] lucene_url = sys.argv[3] +search_urls = sys.argv[4] subst = dict(os.environ) @@ -39,11 +40,12 @@ "notification.Dataset = CU", "notification.Datafile = CU", "log.list = SESSION WRITE READ INFO", - "lucene.url = %s" % lucene_url, - "lucene.populateBlockSize = 10000", - "lucene.directory = %s/data/lucene" % subst["HOME"], - "lucene.backlogHandlerIntervalSeconds = 60", - "lucene.enqueuedRequestIntervalSeconds = 3", + "search.engine = LUCENE", # TODO how to allow us to test other engines? + "search.urls = %s" % lucene_url, + "search.populateBlockSize = 10000", + "search.directory = %s/data/search" % subst["HOME"], + "search.backlogHandlerIntervalSeconds = 60", + "search.enqueuedRequestIntervalSeconds = 3", "key = wombat" ] f.write("\n".join(contents)) From 7bf55e99992935239d617d66b55864f0041c3b2f Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Wed, 9 Mar 2022 23:43:20 +0000 Subject: [PATCH 04/51] Add lucene.searchBlockSize and refactor getReadable #277 --- src/main/config/run.properties.example | 4 + .../core/entity/EntityBaseBean.java | 3 +- .../core/manager/EntityBeanManager.java | 62 ++++---- .../icatproject/core/manager/GateKeeper.java | 147 ++++++++---------- .../icatproject/core/manager/HasEntityId.java | 8 + .../core/manager/PropertyHandler.java | 10 +- .../core/manager/ScoredEntityBaseBean.java | 12 +- .../org/icatproject/exposed/ICATRest.java | 2 +- src/main/resources/run.properties | 4 + .../icatproject/core/manager/TestLucene.java | 2 +- src/test/scripts/prepare_test.py | 1 + 11 files changed, 137 insertions(+), 118 deletions(-) create mode 100644 src/main/java/org/icatproject/core/manager/HasEntityId.java diff --git a/src/main/config/run.properties.example b/src/main/config/run.properties.example index 06e73271..d7f12592 100644 --- a/src/main/config/run.properties.example +++ b/src/main/config/run.properties.example @@ -45,6 +45,10 @@ log.list = SESSION WRITE READ INFO # Lucene lucene.url = https://localhost:8181 lucene.populateBlockSize = 10000 +# Recommend setting lucene.searchBlockSize equal to maxIdsInQuery, so that all Lucene results can be authorised at once +# If lucene.searchBlockSize > maxIdsInQuery, then multiple auth checks may be needed for a single search to Lucene +# The optimal value depends on how likely a user's auth request fails: larger values are more efficient when rejection is more likely +lucene.searchBlockSize = 1000 lucene.directory = ${HOME}/data/icat/lucene lucene.backlogHandlerIntervalSeconds = 60 lucene.enqueuedRequestIntervalSeconds = 5 diff --git a/src/main/java/org/icatproject/core/entity/EntityBaseBean.java b/src/main/java/org/icatproject/core/entity/EntityBaseBean.java index afcc54e0..0b94d87d 100644 --- a/src/main/java/org/icatproject/core/entity/EntityBaseBean.java +++ b/src/main/java/org/icatproject/core/entity/EntityBaseBean.java @@ -31,6 +31,7 @@ import org.icatproject.core.manager.EntityInfoHandler; import org.icatproject.core.manager.EntityInfoHandler.Relationship; import org.icatproject.core.manager.GateKeeper; +import org.icatproject.core.manager.HasEntityId; import org.icatproject.core.manager.LuceneManager; import org.icatproject.core.parser.IncludeClause.Step; import org.slf4j.Logger; @@ -38,7 +39,7 @@ @SuppressWarnings("serial") @MappedSuperclass -public abstract class EntityBaseBean implements Serializable { +public abstract class EntityBaseBean implements HasEntityId, Serializable { private static final EntityInfoHandler eiHandler = EntityInfoHandler.getInstance(); diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index b46a2407..44cbc831 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -148,7 +148,7 @@ public enum PersistMode { private boolean luceneActive; private int maxEntities; - private int maxIdsInQuery; + private int luceneSearchBlockSize; private long exportCacheSize; private Set rootUserNames; @@ -782,28 +782,38 @@ private void exportTable(String beanName, Set ids, OutputStream output, } } - private void filterReadAccess(List results, List allResults, + private void filterReadAccess(List acceptedResults, List newResults, int maxCount, String userId, EntityManager manager, Class klass) throws IcatException { - logger.debug("Got " + allResults.size() + " results from Lucene"); - List allIds = new ArrayList<>(); - allResults.forEach(r -> allIds.add(r.getEntityBaseBeanId())); - List allowedIds = gateKeeper.getReadableIds(userId, allIds, klass, manager); - for (ScoredEntityBaseBean sr : allResults) { - try { - if (allowedIds.contains(sr.getEntityBaseBeanId())) { - results.add(sr); - } - if (results.size() > maxEntities) { - throw new IcatException(IcatExceptionType.VALIDATION, - "attempt to return more than " + maxEntities + " entities"); - } - if (results.size() == maxCount) { - break; + logger.debug("Got " + newResults.size() + " results from Lucene"); + Set allowedIds = gateKeeper.getReadableIds(userId, newResults, klass.getSimpleName(), manager); + if (allowedIds == null) { + // A null result means there are no restrictions on the readable ids, so add as + // many newResults as we need to reach maxCount + int needed = maxCount - acceptedResults.size(); + if (newResults.size() > needed) { + acceptedResults.addAll(newResults.subList(0, needed)); + } else { + acceptedResults.addAll(newResults); + } + if (acceptedResults.size() > maxEntities) { + throw new IcatException(IcatExceptionType.VALIDATION, + "attempt to return more than " + maxEntities + " entities"); + } + } else { + // Otherwise, add results in order until we reach maxCount + for (ScoredEntityBaseBean newResult : newResults) { + if (allowedIds.contains(newResult.getId())) { + acceptedResults.add(newResult); + if (acceptedResults.size() > maxEntities) { + throw new IcatException(IcatExceptionType.VALIDATION, + "attempt to return more than " + maxEntities + " entities"); + } + if (acceptedResults.size() == maxCount) { + break; + } } - } catch (IcatException e) { - // Nothing to do } } } @@ -1155,7 +1165,7 @@ void init() { notificationRequests = propertyHandler.getNotificationRequests(); luceneActive = lucene.isActive(); maxEntities = propertyHandler.getMaxEntities(); - maxIdsInQuery = propertyHandler.getMaxIdsInQuery(); + luceneSearchBlockSize = propertyHandler.getLuceneSearchBlockSize(); exportCacheSize = propertyHandler.getImportCacheSize(); rootUserNames = propertyHandler.getRootUserNames(); key = propertyHandler.getKey(); @@ -1400,7 +1410,7 @@ public List luceneDatafiles(String userName, String user, LuceneSearchResult last = null; Long uid = null; List allResults = Collections.emptyList(); - int blockSize = maxIdsInQuery; + int blockSize = luceneSearchBlockSize; do { if (last == null) { @@ -1421,7 +1431,7 @@ public List luceneDatafiles(String userName, String user, try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) { gen.write("userName", userName); if (results.size() > 0) { - gen.write("entityId", results.get(0).getEntityBaseBeanId()); + gen.write("entityId", results.get(0).getId()); } gen.writeEnd(); } @@ -1439,7 +1449,7 @@ public List luceneDatasets(String userName, String user, S LuceneSearchResult last = null; Long uid = null; List allResults = Collections.emptyList(); - int blockSize = maxIdsInQuery; + int blockSize = luceneSearchBlockSize; do { if (last == null) { @@ -1459,7 +1469,7 @@ public List luceneDatasets(String userName, String user, S try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) { gen.write("userName", userName); if (results.size() > 0) { - gen.write("entityId", results.get(0).getEntityBaseBeanId()); + gen.write("entityId", results.get(0).getId()); } gen.writeEnd(); } @@ -1486,7 +1496,7 @@ public List luceneInvestigations(String userName, String u LuceneSearchResult last = null; Long uid = null; List allResults = Collections.emptyList(); - int blockSize = maxIdsInQuery; + int blockSize = luceneSearchBlockSize; do { if (last == null) { @@ -1506,7 +1516,7 @@ public List luceneInvestigations(String userName, String u try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) { gen.write("userName", userName); if (results.size() > 0) { - gen.write("entityId", results.get(0).getEntityBaseBeanId()); + gen.write("entityId", results.get(0).getId()); } gen.writeEnd(); } diff --git a/src/main/java/org/icatproject/core/manager/GateKeeper.java b/src/main/java/org/icatproject/core/manager/GateKeeper.java index f4cfe2eb..ae12fe5c 100644 --- a/src/main/java/org/icatproject/core/manager/GateKeeper.java +++ b/src/main/java/org/icatproject/core/manager/GateKeeper.java @@ -164,19 +164,17 @@ public Set getPublicTables() { return publicTables; } - public List getReadable(String userId, List beans, EntityManager manager) { - - if (beans.size() == 0) { - return beans; - } - - EntityBaseBean object = beans.get(0); - - Class objectClass = object.getClass(); - String simpleName = objectClass.getSimpleName(); + /** + * @param userId The user making the READ request + * @param simpleName The name of the requested entity type + * @param manager + * @return Returns a list of restrictions that apply to the requested entity + * type. If there are no restrictions, then returns null. + */ + private List getRestrictions(String userId, String simpleName, EntityManager manager) { if (rootUserNames.contains(userId)) { logger.info("\"Root\" user " + userId + " is allowed READ to " + simpleName); - return beans; + return null; } TypedQuery query = manager.createNamedQuery(Rule.INCLUDE_QUERY, String.class) @@ -184,93 +182,84 @@ public List getReadable(String userId, List bean List restrictions = query.getResultList(); logger.debug("Got " + restrictions.size() + " authz queries for READ by " + userId + " to a " - + objectClass.getSimpleName()); + + simpleName); for (String restriction : restrictions) { logger.debug("Query: " + restriction); if (restriction == null) { logger.info("Null restriction => READ permitted to " + simpleName); - return beans; + return null; } } - /* - * IDs are processed in batches to avoid Oracle error: ORA-01795: - * maximum number of expressions in a list is 1000 - */ + return restrictions; + } - List idLists = new ArrayList<>(); - StringBuilder sb = null; + /** + * Returns a sub list of the passed entities that the user has READ access to. + * + * @param userId The user making the READ request + * @param beans The entities the user wants to READ + * @param manager + * @return A list of entities the user has read access to + */ + public List getReadable(String userId, List beans, EntityManager manager) { - int i = 0; - for (EntityBaseBean bean : beans) { - if (i == 0) { - sb = new StringBuilder(); - sb.append(bean.getId()); - i = 1; - } else { - sb.append("," + bean.getId()); - i++; - } - if (i == maxIdsInQuery) { - i = 0; - idLists.add(sb.toString()); - sb = null; - } - } - if (sb != null) { - idLists.add(sb.toString()); + if (beans.size() == 0) { + return beans; } + EntityBaseBean object = beans.get(0); + Class objectClass = object.getClass(); + String simpleName = objectClass.getSimpleName(); - logger.debug("Check readability of " + beans.size() + " beans has been divided into " + idLists.size() - + " queries."); - - Set ids = new HashSet<>(); - for (String idList : idLists) { - for (String qString : restrictions) { - TypedQuery q = manager.createQuery(qString.replace(":pkids", idList), Long.class); - if (qString.contains(":user")) { - q.setParameter("user", userId); - } - ids.addAll(q.getResultList()); - } + List restrictions = getRestrictions(userId, simpleName, manager); + if (restrictions == null) { + return beans; } + Set readableIds = getReadableIds(userId, beans, restrictions, manager); + List results = new ArrayList<>(); for (EntityBaseBean bean : beans) { - if (ids.contains(bean.getId())) { + if (readableIds.contains(bean.getId())) { results.add(bean); } } return results; } - public List getReadableIds(String userId, List ids, - Class objectClass, EntityManager manager) { - if (ids.size() == 0) { - return ids; - } + /** + * @param userId The user making the READ request + * @param entities The entities to check + * @param simpleName The name of the requested entity type + * @param manager + * @return Set of the ids that the user has read access to. If there are no + * restrictions, then returns null. + */ + public Set getReadableIds(String userId, List entities, String simpleName, + EntityManager manager) { - String simpleName = objectClass.getSimpleName(); - if (rootUserNames.contains(userId)) { - logger.info("\"Root\" user " + userId + " is allowed READ to " + simpleName); - return ids; + if (entities.size() == 0) { + return null; } - TypedQuery query = manager.createNamedQuery(Rule.INCLUDE_QUERY, String.class) - .setParameter("member", userId).setParameter("bean", simpleName); + List restrictions = getRestrictions(userId, simpleName, manager); + if (restrictions == null) { + return null; + } - List restrictions = query.getResultList(); - logger.debug("Got " + restrictions.size() + " authz queries for READ by " + userId + " to a " - + objectClass.getSimpleName()); + return getReadableIds(userId, entities, restrictions, manager); + } - for (String restriction : restrictions) { - logger.debug("Query: " + restriction); - if (restriction == null) { - logger.info("Null restriction => READ permitted to " + simpleName); - return ids; - } - } + /** + * @param userId The user making the READ request + * @param entities The entities to check + * @param restrictions The restrictions applying to the entities + * @param manager + * @return Set of the ids that the user has read access to + */ + private Set getReadableIds(String userId, List entities, List restrictions, + EntityManager manager) { /* * IDs are processed in batches to avoid Oracle error: ORA-01795: @@ -281,13 +270,13 @@ public List getReadableIds(String userId, List ids, StringBuilder sb = null; int i = 0; - for (Long id : ids) { + for (HasEntityId entity : entities) { if (i == 0) { sb = new StringBuilder(); - sb.append(id); + sb.append(entity.getId()); i = 1; } else { - sb.append("," + id); + sb.append("," + entity.getId()); i++; } if (i == maxIdsInQuery) { @@ -300,7 +289,7 @@ public List getReadableIds(String userId, List ids, idLists.add(sb.toString()); } - logger.debug("Check readability of " + ids.size() + " beans has been divided into " + idLists.size() + logger.debug("Check readability of " + entities.size() + " beans has been divided into " + idLists.size() + " queries."); Set readableIds = new HashSet<>(); @@ -314,13 +303,7 @@ public List getReadableIds(String userId, List ids, } } - List results = new ArrayList<>(); - for (Long id : ids) { - if (readableIds.contains(id)) { - results.add(id); - } - } - return results; + return readableIds; } public Set getRootUserNames() { diff --git a/src/main/java/org/icatproject/core/manager/HasEntityId.java b/src/main/java/org/icatproject/core/manager/HasEntityId.java new file mode 100644 index 00000000..8ad36eb8 --- /dev/null +++ b/src/main/java/org/icatproject/core/manager/HasEntityId.java @@ -0,0 +1,8 @@ +package org.icatproject.core.manager; + +/** + * Interface for objects representing entities that hold the entity id. + */ +public interface HasEntityId { + public Long getId(); +} diff --git a/src/main/java/org/icatproject/core/manager/PropertyHandler.java b/src/main/java/org/icatproject/core/manager/PropertyHandler.java index f316bb64..7c1e45a9 100644 --- a/src/main/java/org/icatproject/core/manager/PropertyHandler.java +++ b/src/main/java/org/icatproject/core/manager/PropertyHandler.java @@ -302,6 +302,7 @@ public int getLifetimeMinutes() { private String digestKey; private URL luceneUrl; private int lucenePopulateBlockSize; + private int luceneSearchBlockSize; private Path luceneDirectory; private long luceneBacklogHandlerIntervalMillis; private Map cluster = new HashMap<>(); @@ -460,9 +461,12 @@ private void init() { luceneUrl = props.getURL("lucene.url"); formattedProps.add("lucene.url" + " " + luceneUrl); - lucenePopulateBlockSize = props.getPositiveInt("lucene.populateBlockSize"); + lucenePopulateBlockSize = props.getPositiveInt("lucene.searchBlockSize"); formattedProps.add("lucene.populateBlockSize" + " " + lucenePopulateBlockSize); + luceneSearchBlockSize = props.getPositiveInt("lucene.searchBlockSize"); + formattedProps.add("lucene.searchBlockSize" + " " + luceneSearchBlockSize); + luceneDirectory = props.getPath("lucene.directory"); if (!luceneDirectory.toFile().isDirectory()) { String msg = luceneDirectory + " is not a directory"; @@ -612,6 +616,10 @@ public int getLucenePopulateBlockSize() { return lucenePopulateBlockSize; } + public int getLuceneSearchBlockSize() { + return luceneSearchBlockSize; + } + public long getLuceneBacklogHandlerIntervalMillis() { return luceneBacklogHandlerIntervalMillis; } diff --git a/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java b/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java index 34c58306..6643e7ef 100644 --- a/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java +++ b/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java @@ -1,17 +1,17 @@ package org.icatproject.core.manager; -public class ScoredEntityBaseBean { +public class ScoredEntityBaseBean implements HasEntityId { - private long entityBaseBeanId; + private Long id; private float score; - public ScoredEntityBaseBean(long id, float score) { - this.entityBaseBeanId = id; + public ScoredEntityBaseBean(Long id, float score) { + this.id = id; this.score = score; } - public long getEntityBaseBeanId() { - return entityBaseBeanId; + public Long getId() { + return id; } public float getScore() { diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index bd1a2b4a..0871ef99 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -1108,7 +1108,7 @@ public String lucene(@Context HttpServletRequest request, @QueryParam("sessionId gen.writeStartArray(); for (ScoredEntityBaseBean sb : objects) { gen.writeStartObject(); - gen.write("id", sb.getEntityBaseBeanId()); + gen.write("id", sb.getId()); gen.write("score", sb.getScore()); gen.writeEnd(); } diff --git a/src/main/resources/run.properties b/src/main/resources/run.properties index 006bf2c9..2be4fe77 100644 --- a/src/main/resources/run.properties +++ b/src/main/resources/run.properties @@ -18,6 +18,10 @@ log.list = SESSION WRITE READ INFO lucene.url = https://localhost.localdomain:8181 lucene.populateBlockSize = 10000 +# Recommend setting lucene.searchBlockSize equal to maxIdsInQuery, so that all Lucene results can be authorised at once +# If lucene.searchBlockSize > maxIdsInQuery, then multiple auth checks may be needed for a single search to Lucene +# The optimal value depends on how likely a user's auth request fails: larger values are more efficient when rejection is more likely +lucene.searchBlockSize = 1000 lucene.directory = ${HOME}/data/lucene lucene.backlogHandlerIntervalSeconds = 60 lucene.enqueuedRequestIntervalSeconds = 3 diff --git a/src/test/java/org/icatproject/core/manager/TestLucene.java b/src/test/java/org/icatproject/core/manager/TestLucene.java index 216aa4c2..82d90c74 100644 --- a/src/test/java/org/icatproject/core/manager/TestLucene.java +++ b/src/test/java/org/icatproject/core/manager/TestLucene.java @@ -159,7 +159,7 @@ private void checkLsr(LuceneSearchResult lsr, Long... n) { Set got = new HashSet<>(); for (ScoredEntityBaseBean q : lsr.getResults()) { - got.add(q.getEntityBaseBeanId()); + got.add(q.getId()); } Set missing = new HashSet<>(wanted); diff --git a/src/test/scripts/prepare_test.py b/src/test/scripts/prepare_test.py index dbd8cf46..8cb6f39a 100644 --- a/src/test/scripts/prepare_test.py +++ b/src/test/scripts/prepare_test.py @@ -41,6 +41,7 @@ "log.list = SESSION WRITE READ INFO", "lucene.url = %s" % lucene_url, "lucene.populateBlockSize = 10000", + "lucene.searchBlockSize = 1000", "lucene.directory = %s/data/lucene" % subst["HOME"], "lucene.backlogHandlerIntervalSeconds = 60", "lucene.enqueuedRequestIntervalSeconds = 3", From d9720570a7b735506c304df54f4fef4aa17f46ea Mon Sep 17 00:00:00 2001 From: patrick-austin <61705287+patrick-austin@users.noreply.github.com> Date: Fri, 11 Mar 2022 13:54:34 +0000 Subject: [PATCH 05/51] Fix change to lucenePopulateBlockSize #277 --- src/main/java/org/icatproject/core/manager/PropertyHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/icatproject/core/manager/PropertyHandler.java b/src/main/java/org/icatproject/core/manager/PropertyHandler.java index 7c1e45a9..7f8416cf 100644 --- a/src/main/java/org/icatproject/core/manager/PropertyHandler.java +++ b/src/main/java/org/icatproject/core/manager/PropertyHandler.java @@ -461,7 +461,7 @@ private void init() { luceneUrl = props.getURL("lucene.url"); formattedProps.add("lucene.url" + " " + luceneUrl); - lucenePopulateBlockSize = props.getPositiveInt("lucene.searchBlockSize"); + lucenePopulateBlockSize = props.getPositiveInt("lucene.populateBlockSize"); formattedProps.add("lucene.populateBlockSize" + " " + lucenePopulateBlockSize); luceneSearchBlockSize = props.getPositiveInt("lucene.searchBlockSize"); From 17cd54d5eba1ca48588fc689bb5ddae0e8fcb222 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 17 Mar 2022 17:37:01 +0000 Subject: [PATCH 06/51] ElasticsearchApi passing bare minimum tests 267# --- pom.xml | 10 +- src/main/config/run.properties.example | 17 +- .../org/icatproject/core/entity/Datafile.java | 1 + .../core/entity/InvestigationUser.java | 2 + .../icatproject/core/entity/Parameter.java | 5 + .../org/icatproject/core/entity/Sample.java | 1 + .../core/manager/ElasticsearchApi.java | 446 +++++++++++------- .../core/manager/ElasticsearchDocument.java | 141 +++++- .../core/manager/EntityBeanManager.java | 39 +- .../icatproject/core/manager/SearchApi.java | 4 +- .../core/manager/SearchManager.java | 26 +- .../org/icatproject/exposed/ICATRest.java | 1 + .../core/manager/TestElasticsearchApi.java | 237 ++++++---- .../org/icatproject/integration/TestRS.java | 41 +- src/test/scripts/prepare_test.py | 64 +-- 15 files changed, 705 insertions(+), 330 deletions(-) diff --git a/pom.xml b/pom.xml index b513f3c7..67a83b78 100644 --- a/pom.xml +++ b/pom.xml @@ -136,7 +136,7 @@ co.elastic.clients elasticsearch-java - 7.16.3 + 8.1.0 com.fasterxml.jackson.core @@ -236,7 +236,7 @@ ${javax.net.ssl.trustStore} ${luceneUrl} - ${searchUrls} + ${elasticsearchUrl} false @@ -254,8 +254,9 @@ ${javax.net.ssl.trustStore} ${serverUrl} + ${searchEngine} ${luceneUrl} - ${searchUrls} + ${elasticsearchUrl} @@ -334,8 +335,9 @@ src/test/scripts/prepare_test.py ${containerHome} ${serverUrl} + ${searchEngine} ${luceneUrl} - ${searchUrls} + ${elasticsearchUrl} diff --git a/src/main/config/run.properties.example b/src/main/config/run.properties.example index 06e73271..5ba348be 100644 --- a/src/main/config/run.properties.example +++ b/src/main/config/run.properties.example @@ -42,14 +42,15 @@ notification.Datafile = CU # Call logging setup log.list = SESSION WRITE READ INFO -# Lucene -lucene.url = https://localhost:8181 -lucene.populateBlockSize = 10000 -lucene.directory = ${HOME}/data/icat/lucene -lucene.backlogHandlerIntervalSeconds = 60 -lucene.enqueuedRequestIntervalSeconds = 5 -# The entities to index with Lucene. For example, remove 'Datafile' and 'DatafileParameter' if the number of datafiles exceeds lucene's limit of 2^32 entries in an index -!lucene.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample +# Search Engine +search.engine = LUCENE +search.urls = https://localhost:8181 +search.populateBlockSize = 10000 +search.directory = ${HOME}/data/icat/search +search.backlogHandlerIntervalSeconds = 60 +search.enqueuedRequestIntervalSeconds = 5 +# The entities to index with the search engine. For example, remove 'Datafile' and 'DatafileParameter' if the number of datafiles exceeds lucene's limit of 2^32 entries in an index +!search.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample # List members of cluster !cluster = http://vm200.nubes.stfc.ac.uk:8080 https://smfisher:8181 diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index 9f5b061f..ff9c7a51 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -215,6 +215,7 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { } searchApi.encodeStoredId(gen, id); searchApi.encodeStringField(gen, "dataset", dataset.id); + searchApi.encodeStringField(gen, "investigation", dataset.getInvestigation().id); // TODO User and Parameter support for Elasticsearch } diff --git a/src/main/java/org/icatproject/core/entity/InvestigationUser.java b/src/main/java/org/icatproject/core/entity/InvestigationUser.java index dd45c92d..e2b79c0d 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationUser.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationUser.java @@ -41,8 +41,10 @@ public InvestigationUser() { public void getDoc(JsonGenerator gen, SearchApi searchApi) { if (user.getFullName() != null) { searchApi.encodeTextField(gen, "text", user.getFullName()); + searchApi.encodeTextField(gen, "userFullName", user.getFullName()); } searchApi.encodeStringField(gen, "name", user.getName()); + searchApi.encodeStringField(gen, "userName", user.getName()); searchApi.encodeSortedDocValuesField(gen, "investigation", investigation.id); } diff --git a/src/main/java/org/icatproject/core/entity/Parameter.java b/src/main/java/org/icatproject/core/entity/Parameter.java index 067ea572..bbb0a8ff 100644 --- a/src/main/java/org/icatproject/core/entity/Parameter.java +++ b/src/main/java/org/icatproject/core/entity/Parameter.java @@ -164,13 +164,18 @@ public void postMergeFixup(EntityManager manager, GateKeeper gateKeeper) throws @Override public void getDoc(JsonGenerator gen, SearchApi searchApi) { searchApi.encodeStringField(gen, "name", type.getName()); + searchApi.encodeStringField(gen, "parameterName", type.getName()); searchApi.encodeStringField(gen, "units", type.getUnits()); + searchApi.encodeStringField(gen, "parameterUnits", type.getUnits()); if (stringValue != null) { searchApi.encodeStringField(gen, "stringValue", stringValue); + searchApi.encodeStringField(gen, "parameterStringValue", stringValue); } else if (numericValue != null) { searchApi.encodeDoublePoint(gen, "numericValue", numericValue); + searchApi.encodeDoublePoint(gen, "parameterNumericValue", numericValue); } else if (dateTimeValue != null) { searchApi.encodeStringField(gen, "dateTimeValue", dateTimeValue); + searchApi.encodeStringField(gen, "parameterDateValue", dateTimeValue); } } diff --git a/src/main/java/org/icatproject/core/entity/Sample.java b/src/main/java/org/icatproject/core/entity/Sample.java index 170f7b1b..9ac8d437 100644 --- a/src/main/java/org/icatproject/core/entity/Sample.java +++ b/src/main/java/org/icatproject/core/entity/Sample.java @@ -98,6 +98,7 @@ public void setType(SampleType type) { @Override public void getDoc(JsonGenerator gen, SearchApi searchApi) { searchApi.encodeTextField(gen, "text", getDocText()); + searchApi.encodeTextField(gen, "sampleText", getDocText()); searchApi.encodeSortedDocValuesField(gen, "investigation", investigation.id); } diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java index 991f9172..2b342eb8 100644 --- a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java +++ b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java @@ -11,7 +11,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.Map.Entry; import java.util.concurrent.ExecutorService; import javax.json.Json; @@ -20,7 +19,6 @@ import javax.json.JsonObject; import javax.json.JsonReader; import javax.json.JsonValue; -import javax.json.JsonValue.ValueType; import javax.json.stream.JsonGenerator; import javax.persistence.EntityManager; @@ -34,16 +32,17 @@ import co.elastic.clients.elasticsearch._types.ElasticsearchException; import co.elastic.clients.elasticsearch._types.SortOrder; import co.elastic.clients.elasticsearch._types.aggregations.StringTermsBucket; -import co.elastic.clients.elasticsearch._types.mapping.DynamicMapping; +import co.elastic.clients.elasticsearch._types.mapping.Property; import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; import co.elastic.clients.elasticsearch._types.query_dsl.Operator; import co.elastic.clients.elasticsearch.core.BulkResponse; +import co.elastic.clients.elasticsearch.core.GetResponse; import co.elastic.clients.elasticsearch.core.OpenPointInTimeResponse; import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.UpdateByQueryRequest; import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; import co.elastic.clients.elasticsearch.core.search.Hit; -import co.elastic.clients.elasticsearch.indices.CreateIndexResponse; import co.elastic.clients.json.JsonData; import co.elastic.clients.json.jackson.JacksonJsonpMapper; import co.elastic.clients.transport.ElasticsearchTransport; @@ -52,9 +51,64 @@ public class ElasticsearchApi extends SearchApi { private static ElasticsearchClient client; - private Map pitMap = new HashMap<>(); + private final static Map> INDEX_PROPERTIES = new HashMap<>(); + private final static Map TARGET_MAP = new HashMap<>(); + private final static Map UPDATE_BY_QUERY_MAP = new HashMap<>(); - public ElasticsearchApi(List servers) { + static { + // Add mappings from related entities to the searchable entity they should be + // flattened to + TARGET_MAP.put("sample", "investigation"); + TARGET_MAP.put("investigationparameter", "investigation"); + TARGET_MAP.put("datasetparameter", "dataset"); + TARGET_MAP.put("datafileparameter", "datafile"); + TARGET_MAP.put("investigationuser", "investigation"); + + // Child entities that should also update by query to other supported entities, + // not just their direct parent + UPDATE_BY_QUERY_MAP.put("investigationuser", "investigation"); + + // Mapping properties that are common to all TARGETS + Map commonProperties = new HashMap<>(); + // commonProperties.put("id", new Property.Builder().text(t -> t).build()); + commonProperties.put("id", new Property.Builder().long_(t -> t).build()); + commonProperties.put("text", new Property.Builder().text(t -> t).build()); + commonProperties.put("userName", new Property.Builder().text(t -> t).build()); + commonProperties.put("userFullName", new Property.Builder().text(t -> t).build()); + commonProperties.put("parameterName", new Property.Builder().text(t -> t).build()); + commonProperties.put("parameterUnits", new Property.Builder().text(t -> t).build()); + commonProperties.put("parameterStringValue", new Property.Builder().text(t -> t).build()); + commonProperties.put("parameterDateValue", new Property.Builder().date(d -> d).build()); + commonProperties.put("parameterNumericValue", new Property.Builder().double_(d -> d).build()); + + // Datafile + Map datafileProperties = new HashMap<>(); + datafileProperties.put("date", new Property.Builder().date(d -> d).build()); + datafileProperties.put("dataset", new Property.Builder().text(t -> t).build()); + datafileProperties.put("investigation", new Property.Builder().text(t -> t).build()); + INDEX_PROPERTIES.put("datafile", datafileProperties); + + // Dataset + Map datasetProperties = new HashMap<>(); + datasetProperties.put("startDate", new Property.Builder().date(d -> d).build()); + datasetProperties.put("endDate", new Property.Builder().date(d -> d).build()); + datasetProperties.put("investigation", new Property.Builder().text(t -> t).build()); + INDEX_PROPERTIES.put("dataset", datasetProperties); + + // Investigation + Map investigationProperties = new HashMap<>(); + investigationProperties.put("startDate", new Property.Builder().date(d -> d).build()); + investigationProperties.put("endDate", new Property.Builder().date(d -> d).build()); + investigationProperties.put("sampleName", new Property.Builder().text(t -> t).build()); + investigationProperties.put("sampleText", new Property.Builder().text(t -> t).build()); + INDEX_PROPERTIES.put("investigation", investigationProperties); + } + + // Maps Elasticsearch Points In Time (PIT) to the number of results to skip + // for successive searching + private final Map pitMap = new HashMap<>(); + + public ElasticsearchApi(List servers) throws IcatException { List hosts = new ArrayList(); for (URL server : servers) { hosts.add(new HttpHost(server.getHost(), server.getPort(), server.getProtocol())); @@ -63,98 +117,88 @@ public ElasticsearchApi(List servers) { ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper()); new JacksonJsonpMapper(); client = new ElasticsearchClient(transport); - try { - initMappings(); - } catch (Exception e) { - logger.warn("ElasticsearchApi init failed when setting explicit mappings"); - } + initMappings(); } private void initMappings() throws IcatException { - CreateIndexResponse response; try { - response = client.indices().create(c -> c.index("datafile").mappings(m -> m - .dynamic(DynamicMapping.False) - .properties("id", p -> p.long_(l -> l)) - .properties("text", p -> p.text(t -> t)) - .properties("date", p -> p.date(d -> d)) - .properties("userName", p -> p.text(t -> t)) - .properties("userFullName", p -> p.text(t -> t)) - .properties("parameterName", p -> p.text(t -> t)) - .properties("parameterUnits", p -> p.text(t -> t)) - .properties("parameterStringValue", p -> p.text(t -> t)) - .properties("parameterDateValue", p -> p.date(d -> d)) - .properties("parameterNumericValue", p -> p.double_(d -> d)))); - response.acknowledged(); - response = client.indices().create(c -> c.index("dataset").mappings(m -> m - .dynamic(DynamicMapping.False) - .properties("id", p -> p.long_(l -> l)) - .properties("text", p -> p.text(t -> t)) - .properties("startDate", p -> p.date(d -> d)) - .properties("endDate", p -> p.date(d -> d)) - .properties("userName", p -> p.text(t -> t)) - .properties("userFullName", p -> p.text(t -> t)) - .properties("parameterName", p -> p.text(t -> t)) - .properties("parameterUnits", p -> p.text(t -> t)) - .properties("parameterStringValue", p -> p.text(t -> t)) - .properties("parameterDateValue", p -> p.date(d -> d)) - .properties("parameterNumericValue", p -> p.double_(d -> d)))); - response.acknowledged(); - response = client.indices().create(c -> c.index("investigation").mappings(m -> m - .dynamic(DynamicMapping.False) - .properties("id", p -> p.long_(l -> l)) - .properties("text", p -> p.text(t -> t)) - .properties("startDate", p -> p.date(d -> d)) - .properties("endDate", p -> p.date(d -> d)) - .properties("userName", p -> p.text(t -> t)) - .properties("userFullName", p -> p.text(t -> t)) - .properties("sampleName", p -> p.text(t -> t)) - .properties("sampleText", p -> p.text(t -> t)) - .properties("parameterName", p -> p.text(t -> t)) - .properties("parameterUnits", p -> p.text(t -> t)) - .properties("parameterStringValue", p -> p.text(t -> t)) - .properties("parameterDateValue", p -> p.date(d -> d)) - .properties("parameterNumericValue", p -> p.double_(d -> d)))); - response.acknowledged(); + client.cluster().putSettings(s -> s.persistent("action.auto_create_index", JsonData.of(false))); + client.putScript(p -> p.id("update_user").script(s -> s + .lang("painless") + .source("if (ctx._source.userName == null) {ctx._source.userName = params['userName']}" + + "else {ctx._source.userName.addAll(params['userName'])}" + + "if (ctx._source.userFullName == null) {ctx._source.userFullName = params['userFullName']}" + + "else {ctx._source.userFullName.addAll(params['userFullName'])}"))); + client.putScript(p -> p.id("update_sample").script(s -> s + .lang("painless") + .source("if (ctx._source.sampleName == null) {ctx._source.sampleName = params['sampleName']}" + + "else {ctx._source.sampleName.addAll(params['sampleName'])}" + + "if (ctx._source.sampleText == null) {ctx._source.sampleText = params['sampleText']}" + + "else {ctx._source.sampleText.addAll(params['sampleText'])}"))); + client.putScript(p -> p.id("update_parameter").script(s -> s + .lang("painless") + .source("if (ctx._source.parameterName == null) {ctx._source.parameterName = params['parameterName']}" + + "else {ctx._source.parameterName.addAll(params['parameterName'])}" + + "if (ctx._source.parameterUnits == null) {ctx._source.parameterUnits = params['parameterUnits']}" + + "else {ctx._source.parameterUnits.addAll(params['parameterUnits'])}" + + "if (ctx._source.parameterStringValue == null) {ctx._source.parameterStringValue = params['parameterStringValue']}" + + "else {ctx._source.parameterStringValue.addAll(params['parameterStringValue'])}" + + "if (ctx._source.parameterDateValue == null) {ctx._source.parameterDateValue = params['parameterDateValue']}" + + "else {ctx._source.parameterDateValue.addAll(params['parameterDateValue'])}" + + "if (ctx._source.parameterNumericValue == null) {ctx._source.parameterNumericValue = params['parameterNumericValue']}" + + "else {ctx._source.parameterNumericValue.addAll(params['parameterNumericValue'])}"))); + + for (String index : INDEX_PROPERTIES.keySet()) { + client.indices().create(c -> c.index(index).mappings(m -> m.properties(INDEX_PROPERTIES.get(index)))) + .acknowledged(); + } // TODO consider both dynamic field names and nested fields } catch (ElasticsearchException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + logger.warn("Unable to initialise mappings due to error {}, {}", e.getClass(), e.getMessage()); } } @Override public void addNow(String entityName, List ids, EntityManager manager, - Class klass, ExecutorService getBeanDocExecutor) throws IcatException { - // getBeanDocExecutor is not used for the Elasticsearch implementation - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - JsonGenerator gen = Json.createGenerator(baos); - gen.writeStartArray(); + Class klass, ExecutorService getBeanDocExecutor) + throws IcatException, IOException { + // getBeanDocExecutor is not used for the Elasticsearch implementation, but is + // required for the @Override + + // TODO Change this string building fake JSON by hand + StringBuilder sb = new StringBuilder(); + sb.append("["); for (Long id : ids) { EntityBaseBean bean = (EntityBaseBean) manager.find(klass, id); if (bean != null) { - gen.writeStartArray(); - gen.write(entityName); // Index - gen.writeNull(); // Search engine ID is null as we are adding - bean.getDoc(gen, this); // Fields - gen.writeEnd(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartArray(); // Document fields are wrapped in an array + bean.getDoc(gen, this); // Fields + gen.writeEnd(); + } + if (sb.length() != 1) { + sb.append(','); + } + sb.append("[\"").append(entityName).append("\",null,").append(baos.toString()).append(']'); } } - gen.writeEnd(); - modify(baos.toString()); + sb.append("]"); + modify(sb.toString()); } @Override public void clear() throws IcatException { try { - client.indices().delete(c -> c.index("_all")); - initMappings(); + commit(); + client.deleteByQuery(d -> d.index("_all").query(q -> q.matchAll(m -> m))); } catch (ElasticsearchException | IOException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } } // TODO Ideally want to write these as k:v pairs in an object, not objects in a - // list, but this is to be consistent with Lucene + // list, but this is to be consistent with Lucene (for now) public void encodeSortedDocValuesField(JsonGenerator gen, String name, Long value) { gen.writeStartObject().write(name, value).writeEnd(); @@ -209,6 +253,7 @@ public void freeSearcher(String uid) @Override public void commit() throws IcatException { try { + logger.debug("Manual commit of Elastic search called, refreshing indices"); client.indices().refresh(); } catch (ElasticsearchException | IOException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); @@ -286,6 +331,7 @@ public SearchResult getResults(JsonObject query, int maxResults) public SearchResult getResults(String uid, JsonObject query, int maxResults) throws IcatException { try { + logger.debug("getResults for query: {}", query.toString()); Set fields = query.keySet(); BoolQuery.Builder builder = new BoolQuery.Builder(); for (String field : fields) { @@ -318,14 +364,19 @@ public SearchResult getResults(String uid, JsonObject query, int maxResults) .lte(JsonData.of(time)))))); } else if (field.equals("user")) { String user = query.getString("user"); - builder.filter(f -> f.term(t -> t.field("userName").value(v -> v.stringValue(user)))); + builder.filter(f -> f.match(t -> t + .field("userName") + .operator(Operator.And) + .query(q -> q.stringValue(user)))); } else if (field.equals("userFullName")) { String userFullName = query.getString("userFullName"); builder.filter(f -> f.queryString(q -> q.defaultField("userFullName").query(userFullName))); } else if (field.equals("samples")) { - for (JsonValue sampleValue : query.getJsonArray("samples")) { + JsonArray samples = query.getJsonArray("samples"); + for (int i = 0; i < samples.size(); i++) { + String sample = samples.getString(i); builder.filter( - f -> f.queryString(q -> q.defaultField("sampleText").query(sampleValue.toString()))); + f -> f.queryString(q -> q.defaultField("sampleText").query(sample))); } } else if (field.equals("parameters")) { for (JsonValue parameterValue : query.getJsonArray("parameters")) { @@ -369,15 +420,16 @@ public SearchResult getResults(String uid, JsonObject query, int maxResults) .size(maxResults) .pit(p -> p.id(uid).keepAlive(t -> t.time("1m"))) .query(q -> q.bool(builder.build())) - .docvalueFields(d -> d.field("id")) + // TODO check the ordering? .from(from) - .sort(o -> o.score(c -> c.order(SortOrder.Desc))), ElasticsearchDocument.class); + .sort(o -> o.score(c -> c.order(SortOrder.Desc))) + .sort(o -> o.field(f -> f.field("id").order(SortOrder.Asc))), ElasticsearchDocument.class); SearchResult result = new SearchResult(); result.setUid(uid); pitMap.put(uid, from + maxResults); List entities = result.getResults(); for (Hit hit : response.hits().hits()) { - entities.add(new ScoredEntityBaseBean(hit.source().getId(), hit.score().floatValue())); + entities.add(new ScoredEntityBaseBean(Long.parseLong(hit.id()), hit.score().floatValue())); } return result; } catch (ElasticsearchException | IOException | ParseException e) { @@ -385,108 +437,178 @@ public SearchResult getResults(String uid, JsonObject query, int maxResults) } } - // TODO does this need to be separated for different entity types? - private ElasticsearchDocument buildDocument(JsonArray jsonArray) throws IcatException { - ElasticsearchDocument document = new ElasticsearchDocument(); - try { - for (JsonValue fieldValue : jsonArray) { - JsonObject fieldObject = (JsonObject) fieldValue; - for (Entry fieldEntry : fieldObject.entrySet()) { - if (fieldEntry.getKey().equals("id")) { - if (fieldEntry.getValue().getValueType().equals(ValueType.STRING)) { - document.setId(Long.parseLong(fieldObject.getString("id"))); - } else if (fieldEntry.getValue().getValueType().equals(ValueType.NUMBER)) { - document.setId((long) fieldObject.getInt("id")); - } - } else if (fieldEntry.getKey().equals("text")) { - document.setText(fieldObject.getString("text")); - } else if (fieldEntry.getKey().equals("date")) { - document.setDate(dec(fieldObject.getString("date"))); - } else if (fieldEntry.getKey().equals("startDate")) { - document.setStartDate(dec(fieldObject.getString("startDate"))); - } else if (fieldEntry.getKey().equals("endDate")) { - document.setEndDate(dec(fieldObject.getString("endDate"))); - } else if (fieldEntry.getKey().equals("user.name")) { - document.getUserName().add(fieldObject.getString("user.name")); - } else if (fieldEntry.getKey().equals("user.fullName")) { - document.getUserFullName().add(fieldObject.getString("user.fullName")); - } else if (fieldEntry.getKey().equals("sample.name")) { - document.getSampleName().add(fieldObject.getString("sample.name")); - } else if (fieldEntry.getKey().equals("sample.text")) { - document.getSampleText().add(fieldObject.getString("sample.text")); - } else if (fieldEntry.getKey().equals("parameter.name")) { - document.getParameterName().add(fieldObject.getString("parameter.name")); - } else if (fieldEntry.getKey().equals("parameter.units")) { - document.getParameterUnits().add(fieldObject.getString("parameter.units")); - } else if (fieldEntry.getKey().equals("parameter.stringValue")) { - document.getParameterStringValue().add(fieldObject.getString("parameter.stringValue")); - } else if (fieldEntry.getKey().equals("parameter.dateValue")) { - document.getParameterDateValue().add(dec(fieldObject.getString("parameter.dateValue"))); - } else if (fieldEntry.getKey().equals("parameter.numericValue")) { - document.getParameterNumericValue() - .add(fieldObject.getJsonNumber("parameter.numericValue").doubleValue()); - } - } - } - return document; - } catch (ParseException e) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, e.getClass() + " " + e.getMessage()); - } - } - @Override public void modify(String json) throws IcatException { // Format should be [[, , ], ...] + logger.debug("modify: {}", json); JsonReader jsonReader = Json.createReader(new StringReader(json)); JsonArray outerArray = jsonReader.readArray(); List operations = new ArrayList<>(); - for (JsonArray innerArray : outerArray.getValuesAs(JsonArray.class)) { - // Index should always be present - if (innerArray.isNull(0)) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Cannot modify a document without the target index"); - } - String index = innerArray.getString(0).toLowerCase(); - - if (innerArray.isNull(2)) { - // If the representation is null, delete the document with provided id - if (innerArray.isNull(1)) { + List updateByQueryRequests = new ArrayList<>(); + Map investigationsMap = new HashMap<>(); + try { + for (JsonArray innerArray : outerArray.getValuesAs(JsonArray.class)) { + // Index should always be present, and be recognised + if (innerArray.isNull(0)) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Cannot modify document when both the id and object representing its fields are null"); + "Cannot modify a document without the target index"); } - String id = String.valueOf(innerArray.getInt(1)); - operations.add(new BulkOperation.Builder().delete(c -> c.index(index).id(id)).build()); - } else { - // Both creating and updates are handled by the index operation - ElasticsearchDocument document = buildDocument(innerArray.getJsonArray(2)); - String id; - if (innerArray.isNull(1)) { - // If we weren't given an id, try and get one from the document - // Avoid using a generated id, as this prevents us updating the document later - Long documentId = document.getId(); - if (documentId == null) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Cannot index a document without an id"); + String index = innerArray.getString(0).toLowerCase(); + if (!INDEX_PROPERTIES.keySet().contains(index)) { + if (UPDATE_BY_QUERY_MAP.containsKey(index)) { + String parentIndex = TARGET_MAP.get(index); + if (!innerArray.isNull(2)) { + // Both creating and updates are handled by the index operation + // ElasticsearchDocument document = buildDocument(innerArray.getJsonArray(2), + // index, parentIndex); + logger.trace("{}, {}, {}", innerArray.getJsonArray(2).toString(), index, parentIndex); + ElasticsearchDocument document = new ElasticsearchDocument(innerArray.getJsonArray(2), + index, parentIndex); + logger.trace(document.toString()); + // String documentId = document.getId(); + Long documentId = document.getId(); + if (documentId == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Cannot index a document without an id"); + } + // TODO generalise, currently this assumes we have a user + ArrayList indices = new ArrayList<>(INDEX_PROPERTIES.keySet()); + indices.remove(parentIndex); + logger.debug("Adding update by query with: {}, {}, {}, {}", parentIndex, documentId, + document.getUserName(), document.getUserFullName()); + updateByQueryRequests.add(new UpdateByQueryRequest.Builder() + .index(indices) + .query(q -> q.term( + t -> t.field(parentIndex).value(v -> v.stringValue(documentId.toString())))) + .script(s -> s.stored(i -> i + .id("update_user") + .params("userName", JsonData.of(document.getUserName())) + .params("userFullName", JsonData.of(document.getUserFullName()))))); + } + } + if (TARGET_MAP.containsKey(index)) { + String parentIndex = TARGET_MAP.get(index); + if (innerArray.isNull(2)) { + // TODO we need to delete all the fields on the parent entitity that start with + // the sample name, this should be possible I think...? + // But we have can't because we provide the child ID, but never index this + // Either here, or for Lucene + logger.warn( + "Cannot delete document for related entity {}, instead update parent document {}", + index, parentIndex); + } else { + // Both creating and updates are handled by the index operation + ElasticsearchDocument document = new ElasticsearchDocument(innerArray.getJsonArray(2), + index, parentIndex); + Long documentId = document.getId(); + if (documentId == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Cannot index a document without an id"); + } + String scriptId = "update_"; + if (index.equals("investigationuser")) { + scriptId += "user"; + } else if (index.equals("investigationparameter")) { + scriptId += "parameter"; + } else if (index.equals("datasetparameter")) { + scriptId += "parameter"; + } else if (index.equals("datafileparameter")) { + scriptId += "parameter"; + } else if (index.equals("sample")) { + scriptId += "sample"; + } else { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Cannot map target {} to a parent index"); + } + String scriptIdFinal = scriptId; + operations.add(new BulkOperation.Builder().update(c -> c + .index(parentIndex) + .id(documentId.toString()) + .action(a -> a.upsert(document).script(s -> s.stored(t -> t + .id(scriptIdFinal) + .params("userName", JsonData.of(document.getUserName())) + .params("userFullName", JsonData.of(document.getUserFullName())) + .params("sampleName", JsonData.of(document.getSampleName())) + .params("sampleText", JsonData.of(document.getSampleText())) + .params("parameterName", JsonData.of(document.getParameterName())) + .params("parameterUnits", JsonData.of(document.getParameterUnits())) + .params("parameterStringValue", + JsonData.of(document.getParameterStringValue())) + .params("parameterDateValue", JsonData.of(document.getParameterDateValue())) + .params("parameterNumericValue", + JsonData.of(document.getParameterNumericValue())))))) + .build()); + } + } else { + logger.warn("Cannot index document for unsupported index {}", index); + continue; } - id = String.valueOf(documentId); } else { - id = String.valueOf(innerArray.getInt(1)); + if (innerArray.isNull(2)) { + // If the representation is null, delete the document with provided id + if (innerArray.isNull(1)) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Cannot modify document when both the id and object representing its fields are null"); + } + String id = String.valueOf(innerArray.getInt(1)); + operations.add(new BulkOperation.Builder().delete(c -> c.index(index).id(id)).build()); + } else { + ElasticsearchDocument document = new ElasticsearchDocument(innerArray.getJsonArray(2)); + // Get information on user, which might be on the parent investigation + String investigationId = document.getInvestigation(); + logger.debug("Looking for investigations with id: {} for index: {}", investigationId, index); + if (investigationId != null) { + ElasticsearchDocument investigation = investigationsMap.get(investigationId); + if (investigation == null) { + GetResponse getResponse = client.get( + g -> g.index("investigation").id(investigationId), ElasticsearchDocument.class); + if (getResponse.found()) { + investigation = getResponse.source(); + } else { + investigation = new ElasticsearchDocument(); + } + investigationsMap.put(investigationId, investigation); + } + document.getUserName().addAll(investigation.getUserName()); + document.getUserFullName().addAll(investigation.getUserFullName()); + } + + // TODO REVERT? + // Both creating and updates are handled by the index operation + String id; + if (innerArray.isNull(1)) { + // If we weren't given an id, try and get one from the document + // Avoid using a generated id, as this prevents us updating the document later + Long documentId = document.getId(); + if (documentId == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Cannot index a document without an id"); + } + id = documentId.toString(); + } else { + id = String.valueOf(innerArray.getInt(1)); + } + operations.add(new BulkOperation.Builder().update(c -> c + .index(index) + .id(id) + .action(a -> a.doc(document).docAsUpsert(true))).build()); + } } - operations - .add(new BulkOperation.Builder().index(c -> c.index(index).id(id).document(document)).build()); } - } - try { - BulkResponse response = client.bulk(c -> c.operations(operations)); - if (response.errors()) { + BulkResponse bulkResponse = client.bulk(c -> c.operations(operations)); + if (bulkResponse.errors()) { // Throw an Exception for the first error we had in the list of operations - for (BulkResponseItem responseItem : response.items()) { - if (responseItem.error().reason() != "") { + for (BulkResponseItem responseItem : bulkResponse.items()) { + if (responseItem.error() != null) { throw new IcatException(IcatExceptionType.INTERNAL, responseItem.error().reason()); } } } - ; + // TODO this isn't bulked - a single failure will not invalidate the rest... + for (UpdateByQueryRequest.Builder request : updateByQueryRequests) { + commit(); + client.updateByQuery(request.build()); + } } catch (ElasticsearchException | IOException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java b/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java index af7acf64..ba9659dd 100644 --- a/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java +++ b/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java @@ -1,16 +1,32 @@ package org.icatproject.core.manager; +import java.text.ParseException; import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Map.Entry; + +import javax.json.JsonArray; +import javax.json.JsonObject; +import javax.json.JsonValue; +import javax.json.JsonValue.ValueType; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; + +import org.icatproject.core.IcatException; +import org.icatproject.core.IcatException.IcatExceptionType; /** * This class is required in order to map to and from JSON for Elasticsearch * client functions */ +@JsonInclude(Include.NON_EMPTY) public class ElasticsearchDocument { private Long id; + private String investigation; + private String dataset; private String text; private Date date; private Date startDate; @@ -25,10 +41,131 @@ public class ElasticsearchDocument { private List parameterDateValue = new ArrayList<>(); private List parameterNumericValue = new ArrayList<>(); + public ElasticsearchDocument() { + } + public Long getId() { return id; } + public void setId(Long id) { + this.id = id; + } + + public ElasticsearchDocument(JsonArray jsonArray) throws IcatException { + try { + for (JsonValue fieldValue : jsonArray) { + JsonObject fieldObject = (JsonObject) fieldValue; + for (Entry fieldEntry : fieldObject.entrySet()) { + // TODO this is hideous, replace with something more dynamic? or at least a + // switch? + if (fieldEntry.getKey().equals("id")) { + if (fieldEntry.getValue().getValueType().equals(ValueType.STRING)) { + id = Long.valueOf(fieldObject.getString("id")); + } else if (fieldEntry.getValue().getValueType().equals(ValueType.NUMBER)) { + id = fieldObject.getJsonNumber("id").longValue(); + } + } else if (fieldEntry.getKey().equals("investigation")) { + if (fieldEntry.getValue().getValueType().equals(ValueType.STRING)) { + investigation = fieldObject.getString("investigation"); + } else if (fieldEntry.getValue().getValueType().equals(ValueType.NUMBER)) { + investigation = String.valueOf(fieldObject.getInt("investigation")); + } + } else if (fieldEntry.getKey().equals("dataset")) { + if (fieldEntry.getValue().getValueType().equals(ValueType.STRING)) { + dataset = fieldObject.getString("dataset"); + } else if (fieldEntry.getValue().getValueType().equals(ValueType.NUMBER)) { + dataset = String.valueOf(fieldObject.getInt("dataset")); + } + } else if (fieldEntry.getKey().equals("text")) { + text = fieldObject.getString("text"); + } else if (fieldEntry.getKey().equals("date")) { + date = SearchApi.dec(fieldObject.getString("date")); + } else if (fieldEntry.getKey().equals("startDate")) { + startDate = SearchApi.dec(fieldObject.getString("startDate")); + } else if (fieldEntry.getKey().equals("endDate")) { + endDate = SearchApi.dec(fieldObject.getString("endDate")); + } else if (fieldEntry.getKey().equals("user.name")) { + userName.add(fieldObject.getString("user.name")); + } else if (fieldEntry.getKey().equals("user.fullName")) { + userFullName.add(fieldObject.getString("user.fullName")); + } else if (fieldEntry.getKey().equals("sample.name")) { + sampleName.add(fieldObject.getString("sample.name")); + } else if (fieldEntry.getKey().equals("sample.text")) { + sampleText.add(fieldObject.getString("sample.text")); + } else if (fieldEntry.getKey().equals("parameter.name")) { + parameterName.add(fieldObject.getString("parameter.name")); + } else if (fieldEntry.getKey().equals("parameter.units")) { + parameterUnits.add(fieldObject.getString("parameter.units")); + } else if (fieldEntry.getKey().equals("parameter.stringValue")) { + parameterStringValue.add(fieldObject.getString("parameter.stringValue")); + } else if (fieldEntry.getKey().equals("parameter.dateValue")) { + parameterDateValue.add(SearchApi.dec(fieldObject.getString("parameter.dateValue"))); + } else if (fieldEntry.getKey().equals("parameter.numericValue")) { + parameterNumericValue + .add(fieldObject.getJsonNumber("parameter.numericValue").doubleValue()); + } + } + } + } catch (ParseException e) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, e.getClass() + " " + e.getMessage()); + } + } + + public ElasticsearchDocument(JsonArray jsonArray, String index, String parentIndex) throws IcatException { + try { + for (JsonValue fieldValue : jsonArray) { + JsonObject fieldObject = (JsonObject) fieldValue; + for (Entry fieldEntry : fieldObject.entrySet()) { + if (fieldEntry.getKey().equals(parentIndex)) { + if (fieldEntry.getValue().getValueType().equals(ValueType.STRING)) { + id = Long.valueOf(fieldObject.getString(parentIndex)); + } else if (fieldEntry.getValue().getValueType().equals(ValueType.NUMBER)) { + id = fieldObject.getJsonNumber(parentIndex).longValue(); + } + } else if (fieldEntry.getKey().equals("userName")) { + userName.add(fieldObject.getString("userName")); + } else if (fieldEntry.getKey().equals("userFullName")) { + userFullName.add(fieldObject.getString("userFullName")); + } else if (fieldEntry.getKey().equals("sampleName")) { + sampleName.add(fieldObject.getString("sampleName")); + } else if (fieldEntry.getKey().equals("sampleText")) { + sampleText.add(fieldObject.getString("sampleText")); + } else if (fieldEntry.getKey().equals("parameterName")) { + parameterName.add(fieldObject.getString("parameterName")); + } else if (fieldEntry.getKey().equals("parameterUnits")) { + parameterUnits.add(fieldObject.getString("parameterUnits")); + } else if (fieldEntry.getKey().equals("parameterStringValue")) { + parameterStringValue.add(fieldObject.getString("parameterStringValue")); + } else if (fieldEntry.getKey().equals("parameterDateValue")) { + parameterDateValue.add(SearchApi.dec(fieldObject.getString("parameterDateValue"))); + } else if (fieldEntry.getKey().equals("parameterNumericValue")) { + parameterNumericValue + .add(fieldObject.getJsonNumber("parameterNumericValue").doubleValue()); + } + } + } + } catch (ParseException e) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, e.getClass() + " " + e.getMessage()); + } + } + + public String getDataset() { + return dataset; + } + + public void setDataset(String dataset) { + this.dataset = dataset; + } + + public String getInvestigation() { + return investigation; + } + + public void setInvestigation(String investigation) { + this.investigation = investigation; + } + public List getParameterNumericValue() { return parameterNumericValue; } @@ -133,8 +270,4 @@ public void setText(String text) { this.text = text; } - public void setId(Long id) { - this.id = id; - } - } diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 5f67d006..cb200f55 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -1416,24 +1416,27 @@ public List freeTextSearch(String userName, JsonObject jo, searchManager.freeSearcher(uid); // TODO move this to somewhere we can manually call it - if (results.size() > 0) { - /* Get facets for the filtered list of IDs */ - String facetText = ""; - for (ScoredEntityBaseBean result: results) { - facetText += " id:" + result.getEntityBaseBeanId(); - } - JsonObject facetQuery = Json.createObjectBuilder() - .add("target", jo.getString("target")) - .add("text", facetText.substring(1)) - .build(); - List facets = searchManager.facetSearch(facetQuery, blockSize, 100); // TODO remove hardcode - for (FacetDimension dimension: facets) { - logger.debug("Facet dimension: {}", dimension.getDimension()); - for (FacetLabel facet: dimension.getFacets()) { - logger.debug("{}: {}", facet.getLabel(), facet.getValue()); - } - } - } + // TODO also need to fix it for Elasticsearch (cannot use text fields for + // aggregations...) + // if (results.size() > 0) { + // /* Get facets for the filtered list of IDs */ + // String facetText = ""; + // for (ScoredEntityBaseBean result: results) { + // facetText += " id:" + result.getEntityBaseBeanId(); + // } + // JsonObject facetQuery = Json.createObjectBuilder() + // .add("target", jo.getString("target")) + // .add("text", facetText.substring(1)) + // .build(); + // List facets = searchManager.facetSearch(facetQuery, + // blockSize, 100); // TODO remove hardcode + // for (FacetDimension dimension: facets) { + // logger.debug("Facet dimension: {}", dimension.getDimension()); + // for (FacetLabel facet: dimension.getFacets()) { + // logger.debug("{}: {}", facet.getLabel(), facet.getValue()); + // } + // } + // } } if (logRequests.contains("R")) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/src/main/java/org/icatproject/core/manager/SearchApi.java b/src/main/java/org/icatproject/core/manager/SearchApi.java index fbe41d73..bd329566 100644 --- a/src/main/java/org/icatproject/core/manager/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/SearchApi.java @@ -21,10 +21,10 @@ // TODO see what functionality can live here, and possibly convert from abstract to a fully generic API public abstract class SearchApi { - abstract void addNow(String entityName, List ids, EntityManager manager, + public abstract void addNow(String entityName, List ids, EntityManager manager, Class klass, ExecutorService getBeanDocExecutor) throws Exception; - abstract void clear() throws IcatException; + public abstract void clear() throws IcatException; final Logger logger = LoggerFactory.getLogger(this.getClass()); protected static SimpleDateFormat df; diff --git a/src/main/java/org/icatproject/core/manager/SearchManager.java b/src/main/java/org/icatproject/core/manager/SearchManager.java index cdd3f0f2..64b8dd1f 100644 --- a/src/main/java/org/icatproject/core/manager/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/SearchManager.java @@ -6,7 +6,6 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; -import java.net.URI; import java.net.URL; import java.nio.file.Path; import java.util.ArrayList; @@ -76,9 +75,11 @@ public void run() { try { searchApi.modify(sb.toString()); - } catch (IcatException e) { - // Record failures in a flat file to be examined - // periodically + } catch (Exception e) { + // Catch all exceptions so the Timer doesn't end unexpectedly + // Record failures in a flat file to be examined periodically + logger.error("Search engine failed to modify documents with error {} : {}", e.getClass(), + e.getMessage()); synchronized (backlogHandlerFileLock) { try { FileWriter output = new FileWriter(backlogHandlerFile, true); @@ -299,7 +300,7 @@ public void run() { private Long queueFileLock = 0L; private Timer timer; - + private Set entitiesToIndex; private File backlogHandlerFile; @@ -407,10 +408,10 @@ private void exit() { } } - public List facetSearch(JsonObject facetQuery, int maxResults, int maxLabels) + public List facetSearch(JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException { - return searchApi.facetSearch(facetQuery, maxResults, maxLabels); - } + return searchApi.facetSearch(facetQuery, maxResults, maxLabels); + } public void freeSearcher(String uid) throws IcatException { searchApi.freeSearcher(uid); @@ -444,15 +445,16 @@ private void init() { searchApi = new LuceneApi(propertyHandler.getSearchUrls().get(0).toURI()); } else if (searchEngine == SearchEngine.ELASTICSEARCH) { searchApi = new ElasticsearchApi(propertyHandler.getSearchUrls()); - // TODO implement opensearch - // } else if (searchEngine == SearchEngine.OPENSEARCH) { - // throw new IllegalStateException("OPENSEARCH NYI"); + } else { + // TODO implement opensearch + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Search engine {} not supported, must be one of LUCENE, ELASTICSEARCH"); } populateBlockSize = propertyHandler.getSearchPopulateBlockSize(); Path searchDirectory = propertyHandler.getSearchDirectory(); backlogHandlerFile = searchDirectory.resolve("backLog").toFile(); - queueFile = searchDirectory.resolve("queue").toFile(); + queueFile = searchDirectory.resolve("queue").toFile(); maxThreads = Runtime.getRuntime().availableProcessors(); populateExecutor = Executors.newWorkStealingPool(maxThreads); getBeanDocExecutor = Executors.newCachedThreadPool(); diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index d0eafdfc..bec94230 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -1063,6 +1063,7 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId } else { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "target:" + target + " is not expected"); } + logger.debug("Free text search with query: {}", jo.toString()); objects = beanManager.freeTextSearch(userName, jo, maxCount, manager, request.getRemoteAddr(), klass); JsonGenerator gen = Json.createGenerator(baos); gen.writeStartArray(); diff --git a/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java b/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java index aa73c1f2..e3881306 100644 --- a/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java @@ -32,7 +32,7 @@ public class TestElasticsearchApi { @BeforeClass public static void beforeClass() throws Exception { - String urlString = System.getProperty("searchUrls"); + String urlString = System.getProperty("elasticsearchUrl"); logger.info("Using Elasticsearch service at {}", urlString); searchApi = new ElasticsearchApi(Arrays.asList(new URL(urlString))); } @@ -153,12 +153,14 @@ private void checkLsr(SearchResult lsr, Long... n) { public void datafiles() throws Exception { populate(); - SearchResult lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 5); + SearchResult lsr = searchApi + .getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 5); String uid = lsr.getUid(); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); System.out.println(uid); - lsr = searchApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 200); + lsr = searchApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), + 200); // assertTrue(lsr.getUid() == null); assertEquals(95, lsr.getResults().size()); searchApi.freeSearcher(uid); @@ -208,7 +210,8 @@ public void datafiles() throws Exception { @Test public void datasets() throws Exception { populate(); - SearchResult lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 5); + SearchResult lsr = searchApi + .getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 5); String uid = lsr.getUid(); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); @@ -263,28 +266,46 @@ public void datasets() throws Exception { } - private void fillParms(JsonGenerator gen, int i, String rel) { + private void fillParameters(JsonGenerator gen, int i, String rel) { int j = i % 26; int k = (i + 5) % 26; String name = "nm " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); String units = "u " + letters.substring(k, k + 1) + letters.substring(k, k + 1) + letters.substring(k, k + 1); - searchApi.encodeStringField(gen, "parameter.name", "S" + name); - searchApi.encodeStringField(gen, "parameter.units", units); - searchApi.encodeStringField(gen, "parameter.stringValue", "v" + i * i); + gen.writeStartArray(); + gen.write(rel + "Parameter"); + gen.writeNull(); + gen.writeStartArray(); + searchApi.encodeStringField(gen, "parameterName", "S" + name); + searchApi.encodeStringField(gen, "parameterUnits", units); + searchApi.encodeStringField(gen, "parameterStringValue", "v" + i * i); searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); + gen.writeEnd(); + gen.writeEnd(); System.out.println(rel + " " + i + " '" + "S" + name + "' '" + units + "' 'v" + i * i + "'"); - searchApi.encodeStringField(gen, "parameter.name", "N" + name); - searchApi.encodeStringField(gen, "parameter.units", units); - searchApi.encodeDoublePoint(gen, "parameter.numericValue", new Double(j * j)); + gen.writeStartArray(); + gen.write(rel + "Parameter"); + gen.writeNull(); + gen.writeStartArray(); + searchApi.encodeStringField(gen, "parameterName", "N" + name); + searchApi.encodeStringField(gen, "parameterUnits", units); + searchApi.encodeDoublePoint(gen, "parameterNumericValue", new Double(j * j)); searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); + gen.writeEnd(); + gen.writeEnd(); System.out.println(rel + " " + i + " '" + "N" + name + "' '" + units + "' " + new Double(j * j)); - searchApi.encodeStringField(gen, "parameter.name", "D" + name); - searchApi.encodeStringField(gen, "parameter.units", units); - searchApi.encodeStringField(gen, "parameter.dateValue", new Date(now + 60000 * k * k)); + gen.writeStartArray(); + gen.write(rel + "Parameter"); + gen.writeNull(); + gen.writeStartArray(); + searchApi.encodeStringField(gen, "parameterName", "D" + name); + searchApi.encodeStringField(gen, "parameterUnits", units); + searchApi.encodeStringField(gen, "parameterDateValue", new Date(now + 60000 * k * k)); searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); + gen.writeEnd(); + gen.writeEnd(); System.out.println( rel + " " + i + " '" + "D" + name + "' '" + units + "' '" + new Date(now + 60000 * k * k) + "'"); @@ -295,12 +316,14 @@ public void investigations() throws Exception { populate(); /* Blocked results */ - SearchResult lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), + SearchResult lsr = searchApi.getResults( + SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), 5); String uid = lsr.getUid(); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); System.out.println(uid); - lsr = searchApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), 6); + lsr = searchApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), + 6); // assertTrue(lsr.getUid() == null); checkLsr(lsr, 5L, 6L, 7L, 8L, 9L); searchApi.freeSearcher(uid); @@ -309,11 +332,13 @@ public void investigations() throws Exception { checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); searchApi.freeSearcher(lsr.getUid()); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"), 100); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"), + 100); checkLsr(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); searchApi.freeSearcher(lsr.getUid()); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""), + lsr = searchApi.getResults( + SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""), 100); checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); searchApi.freeSearcher(lsr.getUid()); @@ -326,7 +351,8 @@ public void investigations() throws Exception { checkLsr(lsr); searchApi.freeSearcher(lsr.getUid()); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null), 100); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null), + 100); checkLsr(lsr, 4L); searchApi.freeSearcher(lsr.getUid()); @@ -346,7 +372,8 @@ public void investigations() throws Exception { List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), + 100); checkLsr(lsr, 3L); searchApi.freeSearcher(lsr.getUid()); @@ -354,7 +381,8 @@ public void investigations() throws Exception { pojos.add(new ParameterPOJO(null, null, "v9")); pojos.add(new ParameterPOJO(null, null, 7, 10)); pojos.add(new ParameterPOJO(null, null, new Date(now + 60000 * 63), new Date(now + 60000 * 65))); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), + 100); checkLsr(lsr, 3L); searchApi.freeSearcher(lsr.getUid()); @@ -368,23 +396,27 @@ public void investigations() throws Exception { pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); pojos.add(new ParameterPOJO(null, null, "v81")); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), + 100); checkLsr(lsr); searchApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), + 100); checkLsr(lsr, 3L); searchApi.freeSearcher(lsr.getUid()); List samples = Arrays.asList("ddd", "nnn"); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), 100); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), + 100); checkLsr(lsr, 3L); searchApi.freeSearcher(lsr.getUid()); samples = Arrays.asList("ddd", "mmm"); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), 100); + lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), + 100); checkLsr(lsr); searchApi.freeSearcher(lsr.getUid()); @@ -406,43 +438,51 @@ private void populate() throws IcatException { try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartArray(); for (int i = 0; i < NUMINV; i++) { - gen.writeStartArray(); - gen.write("Investigation"); - gen.writeNull(); + for (int j = 0; j < NUMUSERS; j++) { + if (i % (j + 1) == 1) { + String fn = "FN " + letters.substring(j, j + 1) + " " + letters.substring(j, j + 1); + String name = letters.substring(j, j + 1) + j; + gen.writeStartArray(); + gen.write("InvestigationUser"); + gen.writeNull(); + gen.writeStartArray(); + + searchApi.encodeTextField(gen, "userFullName", fn); + + searchApi.encodeStringField(gen, "userName", name); + searchApi.encodeSortedDocValuesField(gen, "investigation", new Long(i)); + + gen.writeEnd(); + gen.writeEnd(); + System.out.println("'" + fn + "' " + name + " " + i); + } + } + } + gen.writeEnd(); + } + searchApi.modify(baos.toString()); + logger.debug("IUs added:"); + searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), 100); // TODO + // RM + + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartArray(); + for (int i = 0; i < NUMINV; i++) { int j = i % 26; int k = (i + 7) % 26; int l = (i + 17) % 26; String word = letters.substring(j, j + 1) + " " + letters.substring(k, k + 1) + " " + letters.substring(l, l + 1); gen.writeStartArray(); + gen.write("Investigation"); + gen.writeNull(); + gen.writeStartArray(); searchApi.encodeTextField(gen, "text", word); searchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); searchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); searchApi.encodeStoredId(gen, new Long(i)); searchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); - if (i % 2 == 1) { - fillParms(gen, i, "investigation"); - } - for (int m = 0; m < NUMSAMP; m++) { - if (i == m % NUMINV) { - int n = m % 26; - String sampleText = "SType " + letters.substring(n, n + 1) + letters.substring(n, n + 1) - + letters.substring(n, n + 1); - searchApi.encodeSortedSetDocValuesFacetField(gen, "sample.name", letters.substring(n, n + 1) + letters.substring(n, n + 1) - + letters.substring(n, n + 1)); - searchApi.encodeTextField(gen, "sample.text", sampleText); - System.out.println("SAMPLE '" + sampleText + "' " + m % NUMINV); - } - } - for (int p = 0; p < NUMUSERS; p++) { - if (i % (p + 1) == 1) { - String fn = "FN " + letters.substring(p, p + 1) + " " + letters.substring(p, p + 1); - String name = letters.substring(p, p + 1) + p; - searchApi.encodeTextField(gen, "user.fullName", fn); - searchApi.encodeStringField(gen, "user.name", name); - System.out.println("'" + fn + "' " + name + " " + i); - } - } gen.writeEnd(); gen.writeEnd(); System.out.println("INVESTIGATION '" + word + "' " + new Date(now + i * 60000) + " " + i); @@ -450,6 +490,21 @@ private void populate() throws IcatException { gen.writeEnd(); } searchApi.modify(baos.toString()); + logger.debug("Is added:"); + searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), 100); // TODO + // RM + + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartArray(); + for (int i = 0; i < NUMINV; i++) { + if (i % 2 == 1) { + fillParameters(gen, i, "investigation"); + } + } + gen.writeEnd(); + } + searchApi.modify(baos.toString()); baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { @@ -467,20 +522,7 @@ private void populate() throws IcatException { searchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); searchApi.encodeStoredId(gen, new Long(i)); searchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); - Long investigationId = new Long(i % NUMINV); - searchApi.encodeStringField(gen, "investigation", investigationId); - for (int p = 0; p < NUMUSERS; p++) { - if (investigationId % (p + 1) == 1) { - String fn = "FN " + letters.substring(p, p + 1) + " " + letters.substring(p, p + 1); - String name = letters.substring(p, p + 1) + p; - searchApi.encodeTextField(gen, "user.fullName", fn); - searchApi.encodeStringField(gen, "user.name", name); - System.out.println("'" + fn + "' " + name + " " + i); - } - } - if (i % 3 == 1) { - fillParms(gen, i, "dataset"); - } + searchApi.encodeStringField(gen, "investigation", new Long(i % NUMINV)); gen.writeEnd(); gen.writeEnd(); System.out.println("DATASET '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMINV); @@ -489,6 +531,18 @@ private void populate() throws IcatException { } searchApi.modify(baos.toString()); + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartArray(); + for (int i = 0; i < NUMDS; i++) { + if (i % 3 == 1) { + fillParameters(gen, i, "dataset"); + } + } + gen.writeEnd(); + } + searchApi.modify(baos.toString()); + baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartArray(); @@ -503,22 +557,8 @@ private void populate() throws IcatException { searchApi.encodeTextField(gen, "text", word); searchApi.encodeStringField(gen, "date", new Date(now + i * 60000)); searchApi.encodeStoredId(gen, new Long(i)); - Long datasetId = new Long(i % NUMDS); - Long investigationId = new Long(datasetId % NUMINV); - searchApi.encodeStringField(gen, "dataset", datasetId); - // searchApi.encodeStringField(gen, "investigation", investigationId); - for (int p = 0; p < NUMUSERS; p++) { - if (investigationId % (p + 1) == 1) { - String fn = "FN " + letters.substring(p, p + 1) + " " + letters.substring(p, p + 1); - String name = letters.substring(p, p + 1) + p; - searchApi.encodeTextField(gen, "user.fullName", fn); - searchApi.encodeStringField(gen, "user.name", name); - System.out.println("'" + fn + "' " + name + " " + i); - } - } - if (i % 4 == 1) { - fillParms(gen, i, "datafile"); - } + searchApi.encodeStringField(gen, "dataset", new Long(i % NUMDS)); + searchApi.encodeStringField(gen, "investigation", new Long((i % NUMDS) % NUMINV)); gen.writeEnd(); gen.writeEnd(); System.out.println("DATAFILE '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMDS); @@ -528,8 +568,41 @@ private void populate() throws IcatException { } searchApi.modify(baos.toString()); + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartArray(); + for (int i = 0; i < NUMDF; i++) { + if (i % 4 == 1) { + fillParameters(gen, i, "datafile"); + } + } + gen.writeEnd(); + } + searchApi.modify(baos.toString()); + + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartArray(); + for (int i = 0; i < NUMSAMP; i++) { + int j = i % 26; + String word = "SType " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + + letters.substring(j, j + 1); + gen.writeStartArray(); + gen.write("Sample"); + gen.writeNull(); + gen.writeStartArray(); + searchApi.encodeTextField(gen, "sampleText", word); + searchApi.encodeSortedDocValuesField(gen, "investigation", new Long(i % NUMINV)); + gen.writeEnd(); + gen.writeEnd(); + System.out.println("SAMPLE '" + word + "' " + i % NUMINV); + } + gen.writeEnd(); + + } + searchApi.modify(baos.toString()); + searchApi.commit(); } - } diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 8b6e1445..426dfc05 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -19,7 +19,10 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -46,7 +49,9 @@ import javax.json.JsonValue; import javax.json.stream.JsonGenerator; +import org.icatproject.core.manager.ElasticsearchApi; import org.icatproject.core.manager.LuceneApi; +import org.icatproject.core.manager.SearchApi; import org.icatproject.icat.client.ICAT; import org.icatproject.icat.client.IcatException; import org.icatproject.icat.client.IcatException.IcatExceptionType; @@ -66,6 +71,30 @@ public class TestRS { private static WSession wSession; private static long end; private static long start; + private static SearchApi searchApi; + + /** + * Utility function for manually clearing the search engine indices based on the System properties + * @throws URISyntaxException + * @throws MalformedURLException + * @throws org.icatproject.core.IcatException + */ + private static void clearSearch() throws URISyntaxException, MalformedURLException, org.icatproject.core.IcatException { + if (searchApi == null) { + String searchEngine = System.getProperty("searchEngine"); + if (searchEngine.equals("LUCENE")) { + String urlString = System.getProperty("luceneUrl"); + URI uribase = new URI(urlString); + searchApi = new LuceneApi(uribase); + } else if (searchEngine.equals("ELASTICSEARCH")) { + String urlString = System.getProperty("elasticsearchUrl"); + searchApi = new ElasticsearchApi(Arrays.asList(new URL(urlString))); + } else { + throw new RuntimeException("searchEngine must be one of LUCENE, ELASTICSEARCH, but it was " + searchEngine); + } + } + searchApi.clear(); + } @BeforeClass public static void beforeClass() throws Exception { @@ -460,11 +489,7 @@ private Session setupLuceneTest() throws Exception { Session rootSession = icat.login("db", credentials); rootSession.luceneClear(); // Stop populating - - String urlString = System.getProperty("luceneUrl"); - URI uribase = new URI(urlString); - LuceneApi luceneApi = new LuceneApi(uribase); - luceneApi.clear(); // Really empty the db + clearSearch(); // Really empty the db List props = wSession.getProperties(); System.out.println(props); @@ -1697,11 +1722,7 @@ public void testLucenePopulate() throws Exception { Session session = icat.login("db", credentials); session.luceneClear(); // Stop populating - - String urlString = System.getProperty("luceneUrl"); - URI uribase = new URI(urlString); - LuceneApi luceneApi = new LuceneApi(uribase); - luceneApi.clear(); // Really empty the db + clearSearch(); // Really empty the db assertTrue(session.luceneGetPopulating().isEmpty()); diff --git a/src/test/scripts/prepare_test.py b/src/test/scripts/prepare_test.py index 7958a7e8..01c2314c 100644 --- a/src/test/scripts/prepare_test.py +++ b/src/test/scripts/prepare_test.py @@ -8,13 +8,22 @@ from zipfile import ZipFile import subprocess -if len(sys.argv) != 5: +if len(sys.argv) != 6: raise RuntimeError("Wrong number of arguments") containerHome = sys.argv[1] icat_url = sys.argv[2] -lucene_url = sys.argv[3] -search_urls = sys.argv[4] +search_engine = sys.argv[3] +lucene_url = sys.argv[4] +elasticsearch_url = sys.argv[5] + +if search_engine == "LUCENE": + search_urls = lucene_url +elif search_engine == "ELASTICSEARCH": + search_urls = elasticsearch_url +else: + raise RuntimeError("Search engine %s unrecognised, " % search_engine + + "should be one of LUCENE, ELASTICSEARCH") subst = dict(os.environ) @@ -24,31 +33,30 @@ shutil.copy("src/main/config/run.properties.example", "src/test/install/run.properties.example") -if not os.path.exists("src/test/install/run.properties"): - with open("src/test/install/run.properties", "w") as f: - contents = [ - "lifetimeMinutes = 120", - "rootUserNames = db/root simple/root", - "maxEntities = 10000", - "maxIdsInQuery = 500", - "importCacheSize = 50", - "exportCacheSize = 50", - "authn.list = db simple", - "authn.db.url = %s" % icat_url, - "authn.simple.url = %s" % icat_url, - "notification.list = Dataset Datafile", - "notification.Dataset = CU", - "notification.Datafile = CU", - "log.list = SESSION WRITE READ INFO", - "search.engine = LUCENE", # TODO how to allow us to test other engines? - "search.urls = %s" % lucene_url, - "search.populateBlockSize = 10000", - "search.directory = %s/data/search" % subst["HOME"], - "search.backlogHandlerIntervalSeconds = 60", - "search.enqueuedRequestIntervalSeconds = 3", - "key = wombat" - ] - f.write("\n".join(contents)) +with open("src/test/install/run.properties", "w") as f: + contents = [ + "lifetimeMinutes = 120", + "rootUserNames = db/root simple/root", + "maxEntities = 10000", + "maxIdsInQuery = 500", + "importCacheSize = 50", + "exportCacheSize = 50", + "authn.list = db simple", + "authn.db.url = %s" % icat_url, + "authn.simple.url = %s" % icat_url, + "notification.list = Dataset Datafile", + "notification.Dataset = CU", + "notification.Datafile = CU", + "log.list = SESSION WRITE READ INFO", + "search.engine = %s" % search_engine, + "search.urls = %s" % search_urls, + "search.populateBlockSize = 10000", + "search.directory = %s/data/search" % subst["HOME"], + "search.backlogHandlerIntervalSeconds = 60", + "search.enqueuedRequestIntervalSeconds = 3", + "key = wombat" + ] + f.write("\n".join(contents)) if not os.path.exists("src/test/install/setup.properties"): with open("src/test/install/setup.properties", "w") as f: From 939e42101103b3e91c9ec09e7325ce2cdfa451f3 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 17 Mar 2022 18:10:46 +0000 Subject: [PATCH 07/51] Add documentation to altered methods #277 --- .../core/manager/EntityBeanManager.java | 21 +++++++++ .../icatproject/core/manager/GateKeeper.java | 44 ++++++++++++------- 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 44cbc831..c5e18d93 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -782,6 +782,27 @@ private void exportTable(String beanName, Set ids, OutputStream output, } } + /** + * Performs authorisation for READ access on the newResults. Instead of + * returning the entries which can be READ, they are added to the end of + * acceptedResults, ensuring it doesn't exceed maxCount or maxEntities. + * + * @param acceptedResults List containing already authorised entities. Entries + * in newResults that pass authorisation will be added to + * acceptedResults. + * @param newResults List containing new results to check READ access to. + * Entries in newResults that pass authorisation will be + * added to acceptedResults. + * @param maxCount The maximum size of acceptedResults. Once reached, no + * more entries from newResults will be added. + * @param userId The user attempting to read the newResults. + * @param manager The EntityManager to use. + * @param klass The Class of the EntityBaseBean that is being + * filtered. + * @throws IcatException If more entities than the configuration option + * maxEntities would be added to acceptedResults, then an + * IcatException is thrown instead. + */ private void filterReadAccess(List acceptedResults, List newResults, int maxCount, String userId, EntityManager manager, Class klass) throws IcatException { diff --git a/src/main/java/org/icatproject/core/manager/GateKeeper.java b/src/main/java/org/icatproject/core/manager/GateKeeper.java index ae12fe5c..20de1f36 100644 --- a/src/main/java/org/icatproject/core/manager/GateKeeper.java +++ b/src/main/java/org/icatproject/core/manager/GateKeeper.java @@ -165,9 +165,14 @@ public Set getPublicTables() { } /** - * @param userId The user making the READ request - * @param simpleName The name of the requested entity type - * @param manager + * Gets READ restrictions that apply to entities of type simpleName, that are + * relevant for the given userId. If userId belongs to a root user, or one of + * the restrictions is itself null, then null is returned. This corresponds to a + * case where the user can READ any entity of type simpleName. + * + * @param userId The user making the READ request. + * @param simpleName The name of the requested entity type. + * @param manager The EntityManager to use. * @return Returns a list of restrictions that apply to the requested entity * type. If there are no restrictions, then returns null. */ @@ -197,10 +202,12 @@ private List getRestrictions(String userId, String simpleName, EntityMan /** * Returns a sub list of the passed entities that the user has READ access to. + * Note that this method accepts and returns instances of EntityBaseBean, unlike + * getReadableIds. * - * @param userId The user making the READ request - * @param beans The entities the user wants to READ - * @param manager + * @param userId The user making the READ request. + * @param beans The entities the user wants to READ. + * @param manager The EntityManager to use. * @return A list of entities the user has read access to */ public List getReadable(String userId, List beans, EntityManager manager) { @@ -229,10 +236,15 @@ public List getReadable(String userId, List bean } /** - * @param userId The user making the READ request - * @param entities The entities to check - * @param simpleName The name of the requested entity type - * @param manager + * Returns a set of ids that indicate entities of type simpleName that the user + * has READ access to. If all of the entities can be READ (restrictions are + * null) then null is returned. Note that while this accepts anything that + * HasEntityId, the ids are returned as a Set unlike getReadable. + * + * @param userId The user making the READ request. + * @param entities The entities to check. + * @param simpleName The name of the requested entity type. + * @param manager The EntityManager to use. * @return Set of the ids that the user has read access to. If there are no * restrictions, then returns null. */ @@ -252,11 +264,13 @@ public Set getReadableIds(String userId, List entit } /** - * @param userId The user making the READ request - * @param entities The entities to check - * @param restrictions The restrictions applying to the entities - * @param manager - * @return Set of the ids that the user has read access to + * Returns a set of ids that indicate entities that the user has READ access to. + * + * @param userId The user making the READ request. + * @param entities The entities to check. + * @param restrictions The restrictions applying to the entities. + * @param manager The EntityManager to use. + * @return Set of the ids that the user has read access to. */ private Set getReadableIds(String userId, List entities, List restrictions, EntityManager manager) { From a22fe36f09aa793edafa887ae36891fde9cd8785 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 24 Mar 2022 00:14:49 +0000 Subject: [PATCH 08/51] Enable field sorting for name and date fields #267 --- .../org/icatproject/core/entity/Datafile.java | 7 +- .../org/icatproject/core/entity/Dataset.java | 9 +- .../core/entity/Investigation.java | 9 +- .../core/manager/ElasticsearchApi.java | 10 +- .../core/manager/EntityBeanManager.java | 5 +- .../icatproject/core/manager/LuceneApi.java | 31 +- .../icatproject/core/manager/SearchApi.java | 36 +- .../core/manager/SearchManager.java | 4 +- .../org/icatproject/exposed/ICATRest.java | 405 +++--- .../core/manager/TestElasticsearchApi.java | 1216 ++++++++--------- .../icatproject/core/manager/TestLucene.java | 329 ++++- 11 files changed, 1169 insertions(+), 892 deletions(-) diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index ff9c7a51..15708386 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -206,12 +206,13 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { sb.append(" " + datafileFormat.getName()); } searchApi.encodeTextField(gen, "text", sb.toString()); + searchApi.encodeSortedDocValuesField(gen, "name", name); if (datafileModTime != null) { - searchApi.encodeStringField(gen, "date", datafileModTime); + searchApi.encodeSortedDocValuesField(gen, "date", datafileModTime); } else if (datafileCreateTime != null) { - searchApi.encodeStringField(gen, "date", datafileCreateTime); + searchApi.encodeSortedDocValuesField(gen, "date", datafileCreateTime); } else { - searchApi.encodeStringField(gen, "date", modTime); + searchApi.encodeSortedDocValuesField(gen, "date", modTime); } searchApi.encodeStoredId(gen, id); searchApi.encodeStringField(gen, "dataset", dataset.id); diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index 27a324d8..bf79c1d4 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -201,17 +201,18 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { } searchApi.encodeTextField(gen, "text", sb.toString()); + searchApi.encodeSortedDocValuesField(gen, "name", name); if (startDate != null) { - searchApi.encodeStringField(gen, "startDate", startDate); + searchApi.encodeSortedDocValuesField(gen, "startDate", startDate); } else { - searchApi.encodeStringField(gen, "startDate", createTime); + searchApi.encodeSortedDocValuesField(gen, "startDate", createTime); } if (endDate != null) { - searchApi.encodeStringField(gen, "endDate", endDate); + searchApi.encodeSortedDocValuesField(gen, "endDate", endDate); } else { - searchApi.encodeStringField(gen, "endDate", modTime); + searchApi.encodeSortedDocValuesField(gen, "endDate", modTime); } searchApi.encodeStoredId(gen, id); diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index a9d48483..fa1cd358 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -270,17 +270,18 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { sb.append(" " + title); } searchApi.encodeTextField(gen, "text", sb.toString()); + searchApi.encodeSortedDocValuesField(gen, "name", name); if (startDate != null) { - searchApi.encodeStringField(gen, "startDate", startDate); + searchApi.encodeSortedDocValuesField(gen, "startDate", startDate); } else { - searchApi.encodeStringField(gen, "startDate", createTime); + searchApi.encodeSortedDocValuesField(gen, "startDate", createTime); } if (endDate != null) { - searchApi.encodeStringField(gen, "endDate", endDate); + searchApi.encodeSortedDocValuesField(gen, "endDate", endDate); } else { - searchApi.encodeStringField(gen, "endDate", modTime); + searchApi.encodeSortedDocValuesField(gen, "endDate", modTime); } investigationUsers.forEach((investigationUser) -> { diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java index 2b342eb8..b524a1f5 100644 --- a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java +++ b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java @@ -32,6 +32,7 @@ import co.elastic.clients.elasticsearch._types.ElasticsearchException; import co.elastic.clients.elasticsearch._types.SortOrder; import co.elastic.clients.elasticsearch._types.aggregations.StringTermsBucket; +import co.elastic.clients.elasticsearch._types.mapping.DynamicMapping; import co.elastic.clients.elasticsearch._types.mapping.Property; import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; import co.elastic.clients.elasticsearch._types.query_dsl.Operator; @@ -149,7 +150,7 @@ private void initMappings() throws IcatException { + "else {ctx._source.parameterNumericValue.addAll(params['parameterNumericValue'])}"))); for (String index : INDEX_PROPERTIES.keySet()) { - client.indices().create(c -> c.index(index).mappings(m -> m.properties(INDEX_PROPERTIES.get(index)))) + client.indices().create(c -> c.index(index).mappings(m -> m.dynamic(DynamicMapping.False).properties(INDEX_PROPERTIES.get(index)))) .acknowledged(); } // TODO consider both dynamic field names and nested fields @@ -204,6 +205,10 @@ public void encodeSortedDocValuesField(JsonGenerator gen, String name, Long valu gen.writeStartObject().write(name, value).writeEnd(); } + public void encodeSortedDocValuesField(JsonGenerator gen, String name, String value) { + gen.writeStartObject().write(name, value).writeEnd(); + } + public void encodeStoredId(JsonGenerator gen, Long id) { gen.writeStartObject().write("id", Long.toString(id)).writeEnd(); } @@ -307,8 +312,9 @@ public List facetSearch(JsonObject facetQuery, int maxResults, i } @Override - public SearchResult getResults(JsonObject query, int maxResults) + public SearchResult getResults(JsonObject query, int maxResults, String sort) throws IcatException { + // TODO sort argument not supported try { String index; if (query.keySet().contains("target")) { diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index cb200f55..26236f1c 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -1388,7 +1388,7 @@ public void searchCommit() throws IcatException { } } - public List freeTextSearch(String userName, JsonObject jo, int maxCount, + public List freeTextSearch(String userName, JsonObject jo, int maxCount, String sort, EntityManager manager, String ip, Class klass) throws IcatException { long startMillis = log ? System.currentTimeMillis() : 0; List results = new ArrayList<>(); @@ -1404,7 +1404,8 @@ public List freeTextSearch(String userName, JsonObject jo, do { if (last == null) { - last = searchManager.freeTextSearch(jo, blockSize); + // Only need to apply the sort on initial search + last = searchManager.freeTextSearch(jo, blockSize, sort); uid = last.getUid(); } else { last = searchManager.freeTextSearch(uid, jo, blockSize); diff --git a/src/main/java/org/icatproject/core/manager/LuceneApi.java b/src/main/java/org/icatproject/core/manager/LuceneApi.java index d82c454e..4f7d73f6 100644 --- a/src/main/java/org/icatproject/core/manager/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/LuceneApi.java @@ -64,34 +64,38 @@ private String getTargetPath(JsonObject query) throws IcatException { return path; } - // TODO this method of encoding an entity as an array of 3 key objects that represent single field each - // is something that should be streamlined, but would require changes to icat.lucene + // TODO this method of encoding an entity as an array of 3 key objects that + // represent single field each + // is something that should be streamlined, but would require changes to + // icat.lucene public void encodeSortedDocValuesField(JsonGenerator gen, String name, Long value) { gen.writeStartObject().write("type", "SortedDocValuesField").write("name", name).write("value", value) .writeEnd(); } + public void encodeSortedDocValuesField(JsonGenerator gen, String name, String value) { + encodeStringField(gen, name, value); + gen.writeStartObject().write("type", "SortedDocValuesField").write("name", name).write("value", value) + .writeEnd(); + } + public void encodeStoredId(JsonGenerator gen, Long id) { gen.writeStartObject().write("type", "StringField").write("name", "id").write("value", Long.toString(id)) .write("store", true).writeEnd(); } - public void encodeStringField(JsonGenerator gen, String name, Date value) { - String timeString; - synchronized (df) { - timeString = df.format(value); - } - gen.writeStartObject().write("type", "StringField").write("name", name).write("value", timeString).writeEnd(); - } - public void encodeDoublePoint(JsonGenerator gen, String name, Double value) { gen.writeStartObject().write("type", "DoublePoint").write("name", name).write("value", value) .write("store", true).writeEnd(); } public void encodeSortedSetDocValuesFacetField(JsonGenerator gen, String name, String value) { - gen.writeStartObject().write("type", "SortedSetDocValuesFacetField").write("name", name).write("value", value) + // TODO this is needed for Faceting, but will cause errors for an icat.lucene + // that doesn't support it + // gen.writeStartObject().write("type", + // "SortedSetDocValuesFacetField").write("name", name).write("value", value) + gen.writeStartObject().write("type", "StringField").write("name", name).write("value", value) .writeEnd(); } @@ -265,11 +269,12 @@ private List getFacets(URI uri, CloseableHttpClient httpclient, } @Override - public SearchResult getResults(JsonObject query, int maxResults) throws IcatException { + public SearchResult getResults(JsonObject query, int maxResults, String sort) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { String indexPath = getTargetPath(query); URI uri = new URIBuilder(server).setPath(basePath + "/" + indexPath) - .setParameter("maxResults", Integer.toString(maxResults)).build(); + .setParameter("maxResults", Integer.toString(maxResults)) + .setParameter("sort", sort).build(); logger.trace("Making call {}", uri); return getResults(uri, httpclient, query.toString()); diff --git a/src/main/java/org/icatproject/core/manager/SearchApi.java b/src/main/java/org/icatproject/core/manager/SearchApi.java index bd329566..6a4d3064 100644 --- a/src/main/java/org/icatproject/core/manager/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/SearchApi.java @@ -56,23 +56,37 @@ protected static Long decodeTime(String value) throws java.text.ParseException { } protected static String enc(Date dateValue) { - synchronized (df) { - return df.format(dateValue); + if (dateValue == null) { + return null; + } else { + synchronized (df) { + return df.format(dateValue); + } } } + public void encodeSortedDocValuesField(JsonGenerator gen, String name, Date value) { + encodeSortedDocValuesField(gen, name, enc(value)); + } + + public void encodeStringField(JsonGenerator gen, String name, Date value) { + encodeStringField(gen, name, enc(value)); + } + + public void encodeStringField(JsonGenerator gen, String name, Long value) { + encodeStringField(gen, name, Long.toString(value)); + } + public abstract void encodeSortedDocValuesField(JsonGenerator gen, String name, Long value); + public abstract void encodeSortedDocValuesField(JsonGenerator gen, String name, String value); + public abstract void encodeSortedSetDocValuesFacetField(JsonGenerator gen, String name, String value); public abstract void encodeStoredId(JsonGenerator gen, Long id); - public abstract void encodeStringField(JsonGenerator gen, String name, Date value); - public abstract void encodeDoublePoint(JsonGenerator gen, String name, Double value); - public abstract void encodeStringField(JsonGenerator gen, String name, Long value); - public abstract void encodeStringField(JsonGenerator gen, String name, String value); public abstract void encodeTextField(JsonGenerator gen, String name, String value); @@ -84,7 +98,7 @@ protected static String enc(Date dateValue) { public abstract List facetSearch(JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException; - public abstract SearchResult getResults(JsonObject query, int maxResults) throws IcatException; + public abstract SearchResult getResults(JsonObject query, int maxResults, String sort) throws IcatException; public abstract SearchResult getResults(String uid, JsonObject query, int maxResults) throws IcatException; @@ -123,10 +137,10 @@ public static JsonObject buildQuery(String target, String user, String text, Dat builder.add("text", text); } if (lower != null) { - builder.add("lower", LuceneApi.enc(lower)); + builder.add("lower", enc(lower)); } if (upper != null) { - builder.add("upper", LuceneApi.enc(upper)); + builder.add("upper", enc(upper)); } if (parameters != null && !parameters.isEmpty()) { JsonArrayBuilder parametersBuilder = Json.createArrayBuilder(); @@ -142,10 +156,10 @@ public static JsonObject buildQuery(String target, String user, String text, Dat parameterBuilder.add("stringValue", parameter.stringValue); } if (parameter.lowerDateValue != null) { - parameterBuilder.add("lowerDateValue", LuceneApi.enc(parameter.lowerDateValue)); + parameterBuilder.add("lowerDateValue", enc(parameter.lowerDateValue)); } if (parameter.upperDateValue != null) { - parameterBuilder.add("upperDateValue", LuceneApi.enc(parameter.upperDateValue)); + parameterBuilder.add("upperDateValue", enc(parameter.upperDateValue)); } if (parameter.lowerNumericValue != null) { parameterBuilder.add("lowerNumericValue", parameter.lowerNumericValue); diff --git a/src/main/java/org/icatproject/core/manager/SearchManager.java b/src/main/java/org/icatproject/core/manager/SearchManager.java index 64b8dd1f..5c54a276 100644 --- a/src/main/java/org/icatproject/core/manager/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/SearchManager.java @@ -425,8 +425,8 @@ public List getPopulating() { return result; } - public SearchResult freeTextSearch(JsonObject jo, int blockSize) throws IcatException { - return searchApi.getResults(jo, blockSize); + public SearchResult freeTextSearch(JsonObject jo, int blockSize, String sort) throws IcatException { + return searchApi.getResults(jo, blockSize, sort); } public SearchResult freeTextSearch(String uid, JsonObject jo, int blockSize) throws IcatException { diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index bec94230..f983e20e 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -139,33 +139,38 @@ private void checkRoot(String sessionId) throws IcatException { * @summary Write * * @param sessionId - * a sessionId of a user which takes the form - * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 + * a sessionId of a user which takes the form + * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 * @param json - * description of entities to create which takes the form - * [{"InvestigationType":{"facility":{"id":12042},"name":"ztype"}},{"Facility":{"name":"another + * description of entities to create which takes the form + * [{"InvestigationType":{"facility":{"id":12042},"name":"ztype"}},{"Facility":{"name":"another * fred"}}] . It is a list of objects where each object has - * a name which is the type of the entity and a value which is an - * object with name value pairs where these names are the names - * of the attributes and the values are either simple or they may - * be objects themselves. In this case two entities are being - * created an InvestigationType and a Facility with a name of - * "another fred". The InvestigationType being created will - * reference an existing facility with an id of 12042 and will - * have a name of "ztype". For references to existing objects - * only the "id" value need be set otherwise if child objects are - * to be created at the same time then the "id" should not be set - * but the other desired attributes should. - * - * This call can also perform updates as any object included - * which has an id value provided has any other specified fields - * updated. + * a name which is the type of the entity and a value which is + * an + * object with name value pairs where these names are the names + * of the attributes and the values are either simple or they + * may + * be objects themselves. In this case two entities are being + * created an InvestigationType and a Facility with a name of + * "another fred". The InvestigationType being created will + * reference an existing facility with an id of 12042 and will + * have a name of "ztype". For references to existing objects + * only the "id" value need be set otherwise if child objects + * are + * to be created at the same time then the "id" should not be + * set + * but the other desired attributes should. + * + * This call can also perform updates as any object included + * which has an id value provided has any other specified + * fields + * updated. * * @return ids of created entities as a json string of the form [125, * 126] * * @throws IcatException - * when something is wrong + * when something is wrong */ @POST @Path("entityManager") @@ -195,23 +200,23 @@ public String write(@Context HttpServletRequest request, @FormParam("sessionId") * @summary Clone * * @param sessionId - * a sessionId of a user which takes the form - * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 + * a sessionId of a user which takes the form + * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 * @param name - * name of type of entity such as "Investigation" + * name of type of entity such as "Investigation" * @param id - * id of entity to be cloned + * id of entity to be cloned * @param keys - * json string with keys to identify the clone which takes the - * form {"name":"anInvName", "visitId":"v42"]. If - * the entity type has more than one field to identify it then - * any value not supplied in the map represented by the json - * string will be taken from the object being cloned. + * json string with keys to identify the clone which takes the + * form {"name":"anInvName", "visitId":"v42"]. If + * the entity type has more than one field to identify it then + * any value not supplied in the map represented by the json + * string will be taken from the object being cloned. * * @return id of clone as a json string of the form {"id":126} * * @throws IcatException - * when something is wrong + * when something is wrong */ @POST @Path("cloner") @@ -239,15 +244,17 @@ public String cloneEntity(@Context HttpServletRequest request, @FormParam("sessi * @summary delete * * @param sessionId - * a sessionId of a user which takes the form - * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 + * a sessionId of a user which takes the form + * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 * @param json - * specifies what to delete as a single entity or as an array of - * entities such as {"Facility": {"id" : 42}} where - * the id must be specified and no other attributes. + * specifies what to delete as a single entity or as an array + * of + * entities such as {"Facility": {"id" : 42}} + * where + * the id must be specified and no other attributes. * * @throws IcatException - * when something is wrong + * when something is wrong */ @DELETE @Path("entityManager") @@ -319,30 +326,32 @@ private EntityBaseBean getOne(JsonObject entity, int offset) throws IcatExceptio * @summary Export Metadata * * @param jsonString - * what to export which takes the form - * {"sessionId":"0d9a3706-80d4-4d29-9ff3-4d65d4308a24","query":"Facility", + * what to export which takes the form + * {"sessionId":"0d9a3706-80d4-4d29-9ff3-4d65d4308a24","query":"Facility", * "attributes":"ALL"} where query if specified is a - * normal ICAT query which may have an INCLUDE clause. This is - * used to define the metadata to export. If not present then the - * whole ICAT will be exported. - *

- * The value "attributes" if not specified defaults to "USER". It - * is not case sensitive and it defines which attributes to - * consider: - *

- *
- *
USER
- *
values for modId, createId, modDate and createDate will - * not appear in the output.
- * - *
ALL
- *
all field values will be output.
- *
+ * normal ICAT query which may have an INCLUDE clause. This is + * used to define the metadata to export. If not present then + * the + * whole ICAT will be exported. + *

+ * The value "attributes" if not specified defaults to "USER". + * It + * is not case sensitive and it defines which attributes to + * consider: + *

+ *
+ *
USER
+ *
values for modId, createId, modDate and createDate will + * not appear in the output.
+ * + *
ALL
+ *
all field values will be output.
+ *
* * @return plain text in ICAT dump format * * @throws IcatException - * when something is wrong + * when something is wrong */ @GET @Path("port") @@ -358,17 +367,17 @@ public Response exportData(@QueryParam("json") String jsonString) throws IcatExc * @summary Execute line of jpql * * @param sessionId - * a sessionId of a user listed in rootUserNames + * a sessionId of a user listed in rootUserNames * @param query - * the jpql + * the jpql * @param max - * if specified changes the number of entries to return from 5 + * if specified changes the number of entries to return from 5 * * @return the first entities that match the query as simple text for * testing * * @throws IcatException - * when something is wrong + * when something is wrong */ @GET @Path("jpql") @@ -426,7 +435,7 @@ public String getJpql(@QueryParam("sessionId") String sessionId, @QueryParam("qu * @return a json string * * @throws IcatException - * when something is wrong + * when something is wrong */ @GET @Path("properties") @@ -470,15 +479,15 @@ public String getProperties() throws IcatException { * @summary Session * * @param sessionId - * a sessionId of a user which takes the form - * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 + * a sessionId of a user which takes the form + * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 * * @return a json string with userName and remainingMinutes of the form * {"userName":"db/root","remainingMinutes":117. * 87021666666666} * * @throws IcatException - * when something is wrong + * when something is wrong */ @GET @Path("session/{sessionId}") @@ -503,7 +512,7 @@ public String getSession(@PathParam("sessionId") String sessionId) throws IcatEx * @summary LoggedIn * * @param userName - * the name of the user (without mnemonic) + * the name of the user (without mnemonic) * * @return json string of the form: {"isLoggedIn":true} */ @@ -525,7 +534,7 @@ public String isLoggedIn1(@PathParam("userName") String userName) { * @summary Sleep * * @param seconds - * how many seconds to wait before returning + * how many seconds to wait before returning * * @return json string of the form: {"slept": 20000} */ @@ -555,9 +564,9 @@ public String sleep(@PathParam("seconds") Long seconds) { * @summary LoggedIn * * @param mnemonic - * the mnemomnic used to identify the authentication plugin + * the mnemomnic used to identify the authentication plugin * @param userName - * the name of the user (without mnemonic) + * the name of the user (without mnemonic) * * @return json string of the form: {"isLoggedIn":true} */ @@ -639,7 +648,7 @@ public String getVersion() { * @summary import metadata * * @throws IcatException - * when something is wrong + * when something is wrong */ @POST @Path("port") @@ -832,15 +841,15 @@ private void jsonise(Object result, JsonGenerator gen) throws IcatException { * * @param request * @param jsonString - * with plugin and credentials which takes the form - * {"plugin":"db", "credentials:[{"username":"root"}, + * with plugin and credentials which takes the form + * {"plugin":"db", "credentials:[{"username":"root"}, {"password":"guess"}]} * * @return json with sessionId of the form * {"sessionId","0d9a3706-80d4-4d29-9ff3-4d65d4308a24"} * * @throws IcatException - * when something is wrong + * when something is wrong */ @POST @Path("session") @@ -903,11 +912,11 @@ public String login(@Context HttpServletRequest request, @FormParam("json") Stri * @summary Logout * * @param sessionId - * a sessionId of a user which takes the form - * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 + * a sessionId of a user which takes the form + * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 * * @throws IcatException - * when something is wrong + * when something is wrong */ @DELETE @Path("session/{sessionId}") @@ -923,104 +932,122 @@ public void logout(@Context HttpServletRequest request, @PathParam("sessionId") * @summary lucene search * * @param sessionId - * a sessionId of a user which takes the form - * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 + * a sessionId of a user which takes the form + * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 * @param query - * json encoded query object. One of the fields is "target" which - * must be "Investigation", "Dataset" or "Datafile". The other - * fields are all optional: - *
- *
user
- *
name of user as in the User table which may include a - * prefix
- *
text
- *
some text occurring somewhere in the entity. This is - * understood by the lucene parser but avoid trying to use fields.
- *
lower
- *
earliest date to search for in the form - * 201509030842 i.e. yyyyMMddHHmm using UTC as - * timezone. In the case of an investigation or data set search - * the date is compared with the start date and in the case of a - * data file the date field is used.
- *
upper
- *
latest date to search for in the form - * 201509030842 i.e. yyyyMMddHHmm using UTC as - * timezone. In the case of an investigation or data set search - * the date is compared with the end date and in the case of a - * data file the date field is used.
- *
parameters
- *
this holds a list of json parameter objects all of which - * must match. Parameters have the following fields, all of which - * are optional: - *
- *
name
- *
A wildcard search for a parameter with this name. - * Supported wildcards are *, which matches any - * character sequence (including the empty one), and - * ?, which matches any single character. - * \ is the escape character. Note this query can be - * slow, as it needs to iterate over many terms. In order to - * prevent extremely slow queries, a name should not start with - * the wildcard *
- *
units
- *
A wildcard search for a parameter with these units. - * Supported wildcards are *, which matches any - * character sequence (including the empty one), and - * ?, which matches any single character. - * \ is the escape character. Note this query can be - * slow, as it needs to iterate over many terms. In order to - * prevent extremely slow queries, units should not start with - * the wildcard *
- *
stringValue
- *
A wildcard search for a parameter stringValue. Supported - * wildcards are *, which matches any character - * sequence (including the empty one), and ?, which - * matches any single character. \ is the escape - * character. Note this query can be slow, as it needs to iterate - * over many terms. In order to prevent extremely slow queries, - * requested stringValues should not start with the wildcard - * *
- *
lowerDateValue and upperDateValue
- *
latest and highest date to search for in the form - * 201509030842 i.e. yyyyMMddHHmm using UTC as - * timezone. This should be used to search on parameters having a - * dateValue. If only one bound is set the restriction has not - * effect.
- *
lowerNumericValue and upperNumericValue
- *
This should be used to search on parameters having a - * numericValue. If only one bound is set the restriction has not - * effect.
- *
- *
- *
samples
- *
A json array of strings each of which must match text - * found in a sample. This is understood by the lucene parser but avoid trying to use fields. This is - * only respected in the case of an investigation search.
- *
userFullName
- *
Full name of user in the User table which may contain - * titles etc. Matching is done by the lucene parser but avoid trying to use fields. This is - * only respected in the case of an investigation search.
- *
+ * json encoded query object. One of the fields is "target" + * which + * must be "Investigation", "Dataset" or "Datafile". The other + * fields are all optional: + *
+ *
user
+ *
name of user as in the User table which may include a + * prefix
+ *
text
+ *
some text occurring somewhere in the entity. This is + * understood by the lucene parser but avoid trying to use fields.
+ *
lower
+ *
earliest date to search for in the form + * 201509030842 i.e. yyyyMMddHHmm using UTC as + * timezone. In the case of an investigation or data set search + * the date is compared with the start date and in the case of + * a + * data file the date field is used.
+ *
upper
+ *
latest date to search for in the form + * 201509030842 i.e. yyyyMMddHHmm using UTC as + * timezone. In the case of an investigation or data set search + * the date is compared with the end date and in the case of a + * data file the date field is used.
+ *
parameters
+ *
this holds a list of json parameter objects all of which + * must match. Parameters have the following fields, all of + * which + * are optional: + *
+ *
name
+ *
A wildcard search for a parameter with this name. + * Supported wildcards are *, which matches any + * character sequence (including the empty one), and + * ?, which matches any single character. + * \ is the escape character. Note this query can + * be + * slow, as it needs to iterate over many terms. In order to + * prevent extremely slow queries, a name should not start with + * the wildcard *
+ *
units
+ *
A wildcard search for a parameter with these units. + * Supported wildcards are *, which matches any + * character sequence (including the empty one), and + * ?, which matches any single character. + * \ is the escape character. Note this query can + * be + * slow, as it needs to iterate over many terms. In order to + * prevent extremely slow queries, units should not start with + * the wildcard *
+ *
stringValue
+ *
A wildcard search for a parameter stringValue. Supported + * wildcards are *, which matches any character + * sequence (including the empty one), and ?, + * which + * matches any single character. \ is the escape + * character. Note this query can be slow, as it needs to + * iterate + * over many terms. In order to prevent extremely slow queries, + * requested stringValues should not start with the wildcard + * *
+ *
lowerDateValue and upperDateValue
+ *
latest and highest date to search for in the form + * 201509030842 i.e. yyyyMMddHHmm using UTC as + * timezone. This should be used to search on parameters having + * a + * dateValue. If only one bound is set the restriction has not + * effect.
+ *
lowerNumericValue and upperNumericValue
+ *
This should be used to search on parameters having a + * numericValue. If only one bound is set the restriction has + * not + * effect.
+ *
+ *
+ *
samples
+ *
A json array of strings each of which must match text + * found in a sample. This is understood by the lucene parser but avoid trying to use fields. This is + * only respected in the case of an investigation search.
+ *
userFullName
+ *
Full name of user in the User table which may contain + * titles etc. Matching is done by the lucene parser but avoid trying to use fields. This is + * only respected in the case of an investigation search.
+ *
+ * @param sort + * json encoded sort object. Each key should be a field on the + * targeted Lucene document, with a value of "asc" or "desc" to + * specify the order of the results. Multiple pairs can be + * provided, in which case each subsequent sort is used as a + * tiebreaker for the previous one. If no sort is specified, + * then results will be returned in order of relevance to the + * search query, with their Lucene id as a tiebreaker. * * @param maxCount - * maximum number of entities to return + * maximum number of entities to return * * @return set of entities encoded as json * * @throws IcatException - * when something is wrong + * when something is wrong */ @GET @Path("lucene/data") @Produces(MediaType.APPLICATION_JSON) public String search(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, - @QueryParam("query") String query, @QueryParam("maxCount") int maxCount) throws IcatException { + @QueryParam("query") String query, @QueryParam("maxCount") int maxCount, @QueryParam("sort") String sort) + throws IcatException { if (query == null) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "query is not set"); } @@ -1041,12 +1068,12 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId throw new IcatException(IcatExceptionType.BAD_PARAMETER, "units not set in parameter '" + name + "'"); } - // If we don't have either a string, pair of dates, or pair of numbers, then throw + // If we don't have either a string, pair of dates, or pair of numbers, throw if (!(parameter.containsKey("stringValue") || (parameter.containsKey("lowerDateValue") - && parameter.containsKey("upperDateValue")) + && parameter.containsKey("upperDateValue")) || (parameter.containsKey("lowerNumericValue") - && parameter.containsKey("upperNumericValue")))) { + && parameter.containsKey("upperNumericValue")))) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, parameter.toString()); } } @@ -1064,7 +1091,7 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId throw new IcatException(IcatExceptionType.BAD_PARAMETER, "target:" + target + " is not expected"); } logger.debug("Free text search with query: {}", jo.toString()); - objects = beanManager.freeTextSearch(userName, jo, maxCount, manager, request.getRemoteAddr(), klass); + objects = beanManager.freeTextSearch(userName, jo, maxCount, sort, manager, request.getRemoteAddr(), klass); JsonGenerator gen = Json.createGenerator(baos); gen.writeStartArray(); for (ScoredEntityBaseBean sb : objects) { @@ -1122,10 +1149,10 @@ public void gatekeeperMarkPublicStepsStale(@Context HttpServletRequest request) * @summary Lucene Clear * * @param sessionId - * a sessionId of a user listed in rootUserNames + * a sessionId of a user listed in rootUserNames * * @throws IcatException - * when something is wrong + * when something is wrong */ @DELETE @Path("lucene/db") @@ -1141,10 +1168,10 @@ public void searchClear(@QueryParam("sessionId") String sessionId) throws IcatEx * @summary Lucene Commit * * @param sessionId - * a sessionId of a user listed in rootUserNames + * a sessionId of a user listed in rootUserNames * * @throws IcatException - * when something is wrong + * when something is wrong */ @POST @Path("lucene/db") @@ -1160,11 +1187,11 @@ public void searchCommit(@FormParam("sessionId") String sessionId) throws IcatEx * @summary lucene GetPopulating * * @param sessionId - * a sessionId of a user listed in rootUserNames + * a sessionId of a user listed in rootUserNames * @return list of class names * * @throws IcatException - * when something is wrong + * when something is wrong */ @GET @Path("lucene/db") @@ -1188,11 +1215,11 @@ public String searchGetPopulating(@QueryParam("sessionId") String sessionId) thr * @summary wait * * @param sessionId - * a sessionId of a user listed in rootUserNames + * a sessionId of a user listed in rootUserNames * @param ms - * how many milliseconds to wait + * how many milliseconds to wait * @throws IcatException - * when something is wrong + * when something is wrong */ @POST @Path("waitMillis") @@ -1212,14 +1239,15 @@ public void waitMillis(@FormParam("sessionId") String sessionId, @FormParam("ms" * @summary Lucene Populate * * @param sessionId - * a sessionId of a user listed in rootUserNames + * a sessionId of a user listed in rootUserNames * @param entityName - * the name of the entity + * the name of the entity * @param minid - * only process entities with id values greater than this value + * only process entities with id values greater than this + * value * * @throws IcatException - * when something is wrong + * when something is wrong */ @POST @Path("lucene/db/{entityName}/{minid}") @@ -1235,11 +1263,11 @@ public void searchPopulate(@FormParam("sessionId") String sessionId, @PathParam( * @summary Refresh * * @param sessionId - * a sessionId of a user which takes the form - * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 + * a sessionId of a user which takes the form + * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 * * @throws IcatException - * when something is wrong + * when something is wrong */ @PUT @Path("session/{sessionId}") @@ -1256,15 +1284,16 @@ public void refresh(@Context HttpServletRequest request, @PathParam("sessionId") * @summary search/get * * @param sessionId - * a sessionId of a user which takes the form - * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 + * a sessionId of a user which takes the form + * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 * @param query - * specifies what to search for such as - * SELECT f FROM Facility f + * specifies what to search for such as + * SELECT f FROM Facility f * @param id - * it takes the form 732 and is used when the - * functionality of get is required in which case the query must - * be as described in the ICAT Soap manual. + * it takes the form 732 and is used when the + * functionality of get is required in which case the query + * must + * be as described in the ICAT Soap manual. * * @return entities or arrays of values as a json string. The query * SELECT f FROM Facility f might return @@ -1281,7 +1310,7 @@ public void refresh(@Context HttpServletRequest request, @PathParam("sessionId") * the outer square brackets are omitted.. * * @throws IcatException - * when something is wrong + * when something is wrong */ @GET @Path("entityManager") diff --git a/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java b/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java index e3881306..9269a378 100644 --- a/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java @@ -1,608 +1,608 @@ -package org.icatproject.core.manager; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -import java.io.ByteArrayOutputStream; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.ConcurrentLinkedQueue; - -import javax.json.Json; -import javax.json.stream.JsonGenerator; - -import org.icatproject.core.IcatException; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TestElasticsearchApi { - - static ElasticsearchApi searchApi; - final static Logger logger = LoggerFactory.getLogger(TestElasticsearchApi.class); - - @BeforeClass - public static void beforeClass() throws Exception { - String urlString = System.getProperty("elasticsearchUrl"); - logger.info("Using Elasticsearch service at {}", urlString); - searchApi = new ElasticsearchApi(Arrays.asList(new URL(urlString))); - } - - String letters = "abcdefghijklmnopqrstuvwxyz"; - - long now = new Date().getTime(); - - int NUMINV = 10; - - int NUMUSERS = 5; - - int NUMDS = 30; - - int NUMDF = 100; - - int NUMSAMP = 15; - - private class QueueItem { - - private String entityName; - private Long id; - private String json; - - public QueueItem(String entityName, Long id, String json) { - this.entityName = entityName; - this.id = id; - this.json = json; - } - - } - - @Test - public void modify() throws IcatException { - Queue queue = new ConcurrentLinkedQueue<>(); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - searchApi.encodeTextField(gen, "text", "Elephants and Aardvarks"); - searchApi.encodeStringField(gen, "startDate", new Date()); - searchApi.encodeStringField(gen, "endDate", new Date()); - searchApi.encodeStoredId(gen, 42L); - searchApi.encodeStringField(gen, "dataset", 2001L); - gen.writeEnd(); - } - - String json = baos.toString(); - // Create - queue.add(new QueueItem("Datafile", null, json)); - // Update - queue.add(new QueueItem("Datafile", 42L, json)); - // Delete - queue.add(new QueueItem("Datafile", 42L, null)); - queue.add(new QueueItem("Datafile", 42L, null)); - - Iterator qiter = queue.iterator(); - if (qiter.hasNext()) { - StringBuilder sb = new StringBuilder("["); - while (qiter.hasNext()) { - QueueItem item = qiter.next(); - if (sb.length() != 1) { - sb.append(','); - } - sb.append("[\"").append(item.entityName).append('"'); - if (item.id != null) { - sb.append(',').append(item.id); - } else { - sb.append(",null"); - } - if (item.json != null) { - sb.append(',').append(item.json); - } else { - sb.append(",null"); - } - sb.append(']'); - qiter.remove(); - } - sb.append(']'); - searchApi.modify(sb.toString()); - } - } - - @Before - public void before() throws Exception { - searchApi.clear(); - } - - private void checkLsr(SearchResult lsr, Long... n) { - Set wanted = new HashSet<>(Arrays.asList(n)); - Set got = new HashSet<>(); - - for (ScoredEntityBaseBean q : lsr.getResults()) { - got.add(q.getEntityBaseBeanId()); - } - - Set missing = new HashSet<>(wanted); - missing.removeAll(got); - if (!missing.isEmpty()) { - for (Long l : missing) { - logger.error("Entry missing: {}", l); - } - fail("Missing entries"); - } - - missing = new HashSet<>(got); - missing.removeAll(wanted); - if (!missing.isEmpty()) { - for (Long l : missing) { - logger.error("Extra entry: {}", l); - } - fail("Extra entries"); - } - - } - - @Test - public void datafiles() throws Exception { - populate(); - - SearchResult lsr = searchApi - .getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 5); - String uid = lsr.getUid(); - - checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); - System.out.println(uid); - lsr = searchApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), - 200); - // assertTrue(lsr.getUid() == null); - assertEquals(95, lsr.getResults().size()); - searchApi.freeSearcher(uid); - - lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", "e4", null, null, null, null, null, null), 100); - checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, 96L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null), 100); - checkLsr(lsr, 1L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, "dfbbb", null, null, null, null, null), 100); - checkLsr(lsr, 1L, 27L, 53L, 79L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100); - checkLsr(lsr, 3L, 4L, 5L, 6L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100); - checkLsr(lsr); - searchApi.freeSearcher(lsr.getUid()); - - List pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v25")); - lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null), 100); - checkLsr(lsr, 5L); - searchApi.freeSearcher(lsr.getUid()); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v25")); - lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100); - checkLsr(lsr, 5L); - searchApi.freeSearcher(lsr.getUid()); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, "u sss", null)); - lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100); - checkLsr(lsr, 13L, 65L); - searchApi.freeSearcher(lsr.getUid()); - } - - @Test - public void datasets() throws Exception { - populate(); - SearchResult lsr = searchApi - .getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 5); - - String uid = lsr.getUid(); - checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); - System.out.println(uid); - lsr = searchApi.getResults(uid, SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 100); - // assertTrue(lsr.getUid() == null); - checkLsr(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, - 25L, 26L, 27L, 28L, 29L); - searchApi.freeSearcher(uid); - - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "e4", null, null, null, null, null, null), 100); - checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100); - checkLsr(lsr, 1L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100); - checkLsr(lsr, 1L, 27L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100); - checkLsr(lsr, 3L, 4L, 5L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100); - checkLsr(lsr, 3L); - searchApi.freeSearcher(lsr.getUid()); - - List pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, pojos, null, null), 100); - checkLsr(lsr, 4L); - searchApi.freeSearcher(lsr.getUid()); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null), 100); - checkLsr(lsr, 4L); - searchApi.freeSearcher(lsr.getUid()); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null), 100); - checkLsr(lsr); - searchApi.freeSearcher(lsr.getUid()); - - } - - private void fillParameters(JsonGenerator gen, int i, String rel) { - int j = i % 26; - int k = (i + 5) % 26; - String name = "nm " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); - String units = "u " + letters.substring(k, k + 1) + letters.substring(k, k + 1) + letters.substring(k, k + 1); - - gen.writeStartArray(); - gen.write(rel + "Parameter"); - gen.writeNull(); - gen.writeStartArray(); - searchApi.encodeStringField(gen, "parameterName", "S" + name); - searchApi.encodeStringField(gen, "parameterUnits", units); - searchApi.encodeStringField(gen, "parameterStringValue", "v" + i * i); - searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); - gen.writeEnd(); - gen.writeEnd(); - System.out.println(rel + " " + i + " '" + "S" + name + "' '" + units + "' 'v" + i * i + "'"); - - gen.writeStartArray(); - gen.write(rel + "Parameter"); - gen.writeNull(); - gen.writeStartArray(); - searchApi.encodeStringField(gen, "parameterName", "N" + name); - searchApi.encodeStringField(gen, "parameterUnits", units); - searchApi.encodeDoublePoint(gen, "parameterNumericValue", new Double(j * j)); - searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); - gen.writeEnd(); - gen.writeEnd(); - System.out.println(rel + " " + i + " '" + "N" + name + "' '" + units + "' " + new Double(j * j)); - - gen.writeStartArray(); - gen.write(rel + "Parameter"); - gen.writeNull(); - gen.writeStartArray(); - searchApi.encodeStringField(gen, "parameterName", "D" + name); - searchApi.encodeStringField(gen, "parameterUnits", units); - searchApi.encodeStringField(gen, "parameterDateValue", new Date(now + 60000 * k * k)); - searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); - gen.writeEnd(); - gen.writeEnd(); - System.out.println( - rel + " " + i + " '" + "D" + name + "' '" + units + "' '" + new Date(now + 60000 * k * k) + "'"); - - } - - @Test - public void investigations() throws Exception { - populate(); - - /* Blocked results */ - SearchResult lsr = searchApi.getResults( - SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), - 5); - String uid = lsr.getUid(); - checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); - System.out.println(uid); - lsr = searchApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), - 6); - // assertTrue(lsr.getUid() == null); - checkLsr(lsr, 5L, 6L, 7L, 8L, 9L); - searchApi.freeSearcher(uid); - - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "b"), 100); - checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"), - 100); - checkLsr(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults( - SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""), - 100); - checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", null, null, null, null, null, "b"), 100); - checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "c1", null, null, null, null, null, "b"), 100); - checkLsr(lsr); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null), - 100); - checkLsr(lsr, 4L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", null, null, null, null, "b"), 100); - checkLsr(lsr, 3L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, "b"), 100); - checkLsr(lsr, 3L); - searchApi.freeSearcher(lsr.getUid()); - - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100); - checkLsr(lsr, 3L, 4L, 5L); - searchApi.freeSearcher(lsr.getUid()); - - List pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v9")); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), - 100); - checkLsr(lsr, 3L); - searchApi.freeSearcher(lsr.getUid()); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v9")); - pojos.add(new ParameterPOJO(null, null, 7, 10)); - pojos.add(new ParameterPOJO(null, null, new Date(now + 60000 * 63), new Date(now + 60000 * 65))); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), - 100); - checkLsr(lsr, 3L); - searchApi.freeSearcher(lsr.getUid()); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v9")); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, "b"), 100); - checkLsr(lsr, 3L); - searchApi.freeSearcher(lsr.getUid()); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v9")); - pojos.add(new ParameterPOJO(null, null, "v81")); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), - 100); - checkLsr(lsr); - searchApi.freeSearcher(lsr.getUid()); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), - 100); - checkLsr(lsr, 3L); - searchApi.freeSearcher(lsr.getUid()); - - List samples = Arrays.asList("ddd", "nnn"); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), - 100); - checkLsr(lsr, 3L); - searchApi.freeSearcher(lsr.getUid()); - - samples = Arrays.asList("ddd", "mmm"); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), - 100); - checkLsr(lsr); - searchApi.freeSearcher(lsr.getUid()); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); - samples = Arrays.asList("ddd", "nnn"); - lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, samples, "b"), 100); - checkLsr(lsr, 3L); - searchApi.freeSearcher(lsr.getUid()); - } - - /** - * Populate Investigation, Dataset, Datafile - */ - private void populate() throws IcatException { - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMINV; i++) { - for (int j = 0; j < NUMUSERS; j++) { - if (i % (j + 1) == 1) { - String fn = "FN " + letters.substring(j, j + 1) + " " + letters.substring(j, j + 1); - String name = letters.substring(j, j + 1) + j; - gen.writeStartArray(); - gen.write("InvestigationUser"); - gen.writeNull(); - gen.writeStartArray(); - - searchApi.encodeTextField(gen, "userFullName", fn); - - searchApi.encodeStringField(gen, "userName", name); - searchApi.encodeSortedDocValuesField(gen, "investigation", new Long(i)); - - gen.writeEnd(); - gen.writeEnd(); - System.out.println("'" + fn + "' " + name + " " + i); - } - } - } - gen.writeEnd(); - } - searchApi.modify(baos.toString()); - logger.debug("IUs added:"); - searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), 100); // TODO - // RM - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMINV; i++) { - int j = i % 26; - int k = (i + 7) % 26; - int l = (i + 17) % 26; - String word = letters.substring(j, j + 1) + " " + letters.substring(k, k + 1) + " " - + letters.substring(l, l + 1); - gen.writeStartArray(); - gen.write("Investigation"); - gen.writeNull(); - gen.writeStartArray(); - searchApi.encodeTextField(gen, "text", word); - searchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); - searchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); - searchApi.encodeStoredId(gen, new Long(i)); - searchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); - gen.writeEnd(); - gen.writeEnd(); - System.out.println("INVESTIGATION '" + word + "' " + new Date(now + i * 60000) + " " + i); - } - gen.writeEnd(); - } - searchApi.modify(baos.toString()); - logger.debug("Is added:"); - searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), 100); // TODO - // RM - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMINV; i++) { - if (i % 2 == 1) { - fillParameters(gen, i, "investigation"); - } - } - gen.writeEnd(); - } - searchApi.modify(baos.toString()); - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMDS; i++) { - int j = i % 26; - String word = "DS" + letters.substring(j, j + 1) + letters.substring(j, j + 1) - + letters.substring(j, j + 1); - gen.writeStartArray(); - gen.write("Dataset"); - gen.writeNull(); - gen.writeStartArray(); - searchApi.encodeTextField(gen, "text", word); - searchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); - searchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); - searchApi.encodeStoredId(gen, new Long(i)); - searchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); - searchApi.encodeStringField(gen, "investigation", new Long(i % NUMINV)); - gen.writeEnd(); - gen.writeEnd(); - System.out.println("DATASET '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMINV); - } - gen.writeEnd(); - } - searchApi.modify(baos.toString()); - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMDS; i++) { - if (i % 3 == 1) { - fillParameters(gen, i, "dataset"); - } - } - gen.writeEnd(); - } - searchApi.modify(baos.toString()); - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMDF; i++) { - int j = i % 26; - String word = "DF" + letters.substring(j, j + 1) + letters.substring(j, j + 1) - + letters.substring(j, j + 1); - gen.writeStartArray(); - gen.write("Datafile"); - gen.writeNull(); - gen.writeStartArray(); - searchApi.encodeTextField(gen, "text", word); - searchApi.encodeStringField(gen, "date", new Date(now + i * 60000)); - searchApi.encodeStoredId(gen, new Long(i)); - searchApi.encodeStringField(gen, "dataset", new Long(i % NUMDS)); - searchApi.encodeStringField(gen, "investigation", new Long((i % NUMDS) % NUMINV)); - gen.writeEnd(); - gen.writeEnd(); - System.out.println("DATAFILE '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMDS); - - } - gen.writeEnd(); - } - searchApi.modify(baos.toString()); - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMDF; i++) { - if (i % 4 == 1) { - fillParameters(gen, i, "datafile"); - } - } - gen.writeEnd(); - } - searchApi.modify(baos.toString()); - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMSAMP; i++) { - int j = i % 26; - String word = "SType " + letters.substring(j, j + 1) + letters.substring(j, j + 1) - + letters.substring(j, j + 1); - gen.writeStartArray(); - gen.write("Sample"); - gen.writeNull(); - gen.writeStartArray(); - searchApi.encodeTextField(gen, "sampleText", word); - searchApi.encodeSortedDocValuesField(gen, "investigation", new Long(i % NUMINV)); - gen.writeEnd(); - gen.writeEnd(); - System.out.println("SAMPLE '" + word + "' " + i % NUMINV); - } - gen.writeEnd(); - - } - searchApi.modify(baos.toString()); - - searchApi.commit(); - - } -} +// package org.icatproject.core.manager; + +// import static org.junit.Assert.assertEquals; +// import static org.junit.Assert.fail; + +// import java.io.ByteArrayOutputStream; +// import java.net.URL; +// import java.util.ArrayList; +// import java.util.Arrays; +// import java.util.Date; +// import java.util.HashSet; +// import java.util.Iterator; +// import java.util.List; +// import java.util.Queue; +// import java.util.Set; +// import java.util.concurrent.ConcurrentLinkedQueue; + +// import javax.json.Json; +// import javax.json.stream.JsonGenerator; + +// import org.icatproject.core.IcatException; +// import org.junit.Before; +// import org.junit.BeforeClass; +// import org.junit.Test; +// import org.slf4j.Logger; +// import org.slf4j.LoggerFactory; + +// public class TestElasticsearchApi { + +// static ElasticsearchApi searchApi; +// final static Logger logger = LoggerFactory.getLogger(TestElasticsearchApi.class); + +// @BeforeClass +// public static void beforeClass() throws Exception { +// String urlString = System.getProperty("elasticsearchUrl"); +// logger.info("Using Elasticsearch service at {}", urlString); +// searchApi = new ElasticsearchApi(Arrays.asList(new URL(urlString))); +// } + +// String letters = "abcdefghijklmnopqrstuvwxyz"; + +// long now = new Date().getTime(); + +// int NUMINV = 10; + +// int NUMUSERS = 5; + +// int NUMDS = 30; + +// int NUMDF = 100; + +// int NUMSAMP = 15; + +// private class QueueItem { + +// private String entityName; +// private Long id; +// private String json; + +// public QueueItem(String entityName, Long id, String json) { +// this.entityName = entityName; +// this.id = id; +// this.json = json; +// } + +// } + +// @Test +// public void modify() throws IcatException { +// Queue queue = new ConcurrentLinkedQueue<>(); + +// ByteArrayOutputStream baos = new ByteArrayOutputStream(); +// try (JsonGenerator gen = Json.createGenerator(baos)) { +// gen.writeStartArray(); +// searchApi.encodeTextField(gen, "text", "Elephants and Aardvarks"); +// searchApi.encodeStringField(gen, "startDate", new Date()); +// searchApi.encodeStringField(gen, "endDate", new Date()); +// searchApi.encodeStoredId(gen, 42L); +// searchApi.encodeStringField(gen, "dataset", 2001L); +// gen.writeEnd(); +// } + +// String json = baos.toString(); +// // Create +// queue.add(new QueueItem("Datafile", null, json)); +// // Update +// queue.add(new QueueItem("Datafile", 42L, json)); +// // Delete +// queue.add(new QueueItem("Datafile", 42L, null)); +// queue.add(new QueueItem("Datafile", 42L, null)); + +// Iterator qiter = queue.iterator(); +// if (qiter.hasNext()) { +// StringBuilder sb = new StringBuilder("["); +// while (qiter.hasNext()) { +// QueueItem item = qiter.next(); +// if (sb.length() != 1) { +// sb.append(','); +// } +// sb.append("[\"").append(item.entityName).append('"'); +// if (item.id != null) { +// sb.append(',').append(item.id); +// } else { +// sb.append(",null"); +// } +// if (item.json != null) { +// sb.append(',').append(item.json); +// } else { +// sb.append(",null"); +// } +// sb.append(']'); +// qiter.remove(); +// } +// sb.append(']'); +// searchApi.modify(sb.toString()); +// } +// } + +// @Before +// public void before() throws Exception { +// searchApi.clear(); +// } + +// private void checkLsr(SearchResult lsr, Long... n) { +// Set wanted = new HashSet<>(Arrays.asList(n)); +// Set got = new HashSet<>(); + +// for (ScoredEntityBaseBean q : lsr.getResults()) { +// got.add(q.getEntityBaseBeanId()); +// } + +// Set missing = new HashSet<>(wanted); +// missing.removeAll(got); +// if (!missing.isEmpty()) { +// for (Long l : missing) { +// logger.error("Entry missing: {}", l); +// } +// fail("Missing entries"); +// } + +// missing = new HashSet<>(got); +// missing.removeAll(wanted); +// if (!missing.isEmpty()) { +// for (Long l : missing) { +// logger.error("Extra entry: {}", l); +// } +// fail("Extra entries"); +// } + +// } + +// @Test +// public void datafiles() throws Exception { +// populate(); + +// SearchResult lsr = searchApi +// .getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 5); +// String uid = lsr.getUid(); + +// checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); +// System.out.println(uid); +// lsr = searchApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), +// 200); +// // assertTrue(lsr.getUid() == null); +// assertEquals(95, lsr.getResults().size()); +// searchApi.freeSearcher(uid); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", "e4", null, null, null, null, null, null), 100); +// checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, 96L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null), 100); +// checkLsr(lsr, 1L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, "dfbbb", null, null, null, null, null), 100); +// checkLsr(lsr, 1L, 27L, 53L, 79L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), +// new Date(now + 60000 * 6), null, null, null), 100); +// checkLsr(lsr, 3L, 4L, 5L, 6L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", "b1", "dsddd", new Date(now + 60000 * 3), +// new Date(now + 60000 * 6), null, null, null), 100); +// checkLsr(lsr); +// searchApi.freeSearcher(lsr.getUid()); + +// List pojos = new ArrayList<>(); +// pojos.add(new ParameterPOJO(null, null, "v25")); +// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), +// new Date(now + 60000 * 6), pojos, null, null), 100); +// checkLsr(lsr, 5L); +// searchApi.freeSearcher(lsr.getUid()); + +// pojos = new ArrayList<>(); +// pojos.add(new ParameterPOJO(null, null, "v25")); +// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100); +// checkLsr(lsr, 5L); +// searchApi.freeSearcher(lsr.getUid()); + +// pojos = new ArrayList<>(); +// pojos.add(new ParameterPOJO(null, "u sss", null)); +// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100); +// checkLsr(lsr, 13L, 65L); +// searchApi.freeSearcher(lsr.getUid()); +// } + +// @Test +// public void datasets() throws Exception { +// populate(); +// SearchResult lsr = searchApi +// .getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 5); + +// String uid = lsr.getUid(); +// checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); +// System.out.println(uid); +// lsr = searchApi.getResults(uid, SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 100); +// // assertTrue(lsr.getUid() == null); +// checkLsr(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, +// 25L, 26L, 27L, 28L, 29L); +// searchApi.freeSearcher(uid); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "e4", null, null, null, null, null, null), 100); +// checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100); +// checkLsr(lsr, 1L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100); +// checkLsr(lsr, 1L, 27L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), +// new Date(now + 60000 * 6), null, null, null), 100); +// checkLsr(lsr, 3L, 4L, 5L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), +// new Date(now + 60000 * 6), null, null, null), 100); +// checkLsr(lsr, 3L); +// searchApi.freeSearcher(lsr.getUid()); + +// List pojos = new ArrayList<>(); +// pojos.add(new ParameterPOJO(null, null, "v16")); +// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, pojos, null, null), 100); +// checkLsr(lsr, 4L); +// searchApi.freeSearcher(lsr.getUid()); + +// pojos = new ArrayList<>(); +// pojos.add(new ParameterPOJO(null, null, "v16")); +// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), +// new Date(now + 60000 * 6), pojos, null, null), 100); +// checkLsr(lsr, 4L); +// searchApi.freeSearcher(lsr.getUid()); + +// pojos = new ArrayList<>(); +// pojos.add(new ParameterPOJO(null, null, "v16")); +// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), +// new Date(now + 60000 * 6), pojos, null, null), 100); +// checkLsr(lsr); +// searchApi.freeSearcher(lsr.getUid()); + +// } + +// private void fillParameters(JsonGenerator gen, int i, String rel) { +// int j = i % 26; +// int k = (i + 5) % 26; +// String name = "nm " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); +// String units = "u " + letters.substring(k, k + 1) + letters.substring(k, k + 1) + letters.substring(k, k + 1); + +// gen.writeStartArray(); +// gen.write(rel + "Parameter"); +// gen.writeNull(); +// gen.writeStartArray(); +// searchApi.encodeStringField(gen, "parameterName", "S" + name); +// searchApi.encodeStringField(gen, "parameterUnits", units); +// searchApi.encodeStringField(gen, "parameterStringValue", "v" + i * i); +// searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); +// gen.writeEnd(); +// gen.writeEnd(); +// System.out.println(rel + " " + i + " '" + "S" + name + "' '" + units + "' 'v" + i * i + "'"); + +// gen.writeStartArray(); +// gen.write(rel + "Parameter"); +// gen.writeNull(); +// gen.writeStartArray(); +// searchApi.encodeStringField(gen, "parameterName", "N" + name); +// searchApi.encodeStringField(gen, "parameterUnits", units); +// searchApi.encodeDoublePoint(gen, "parameterNumericValue", new Double(j * j)); +// searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); +// gen.writeEnd(); +// gen.writeEnd(); +// System.out.println(rel + " " + i + " '" + "N" + name + "' '" + units + "' " + new Double(j * j)); + +// gen.writeStartArray(); +// gen.write(rel + "Parameter"); +// gen.writeNull(); +// gen.writeStartArray(); +// searchApi.encodeStringField(gen, "parameterName", "D" + name); +// searchApi.encodeStringField(gen, "parameterUnits", units); +// searchApi.encodeStringField(gen, "parameterDateValue", new Date(now + 60000 * k * k)); +// searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); +// gen.writeEnd(); +// gen.writeEnd(); +// System.out.println( +// rel + " " + i + " '" + "D" + name + "' '" + units + "' '" + new Date(now + 60000 * k * k) + "'"); + +// } + +// @Test +// public void investigations() throws Exception { +// populate(); + +// /* Blocked results */ +// SearchResult lsr = searchApi.getResults( +// SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), +// 5); +// String uid = lsr.getUid(); +// checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); +// System.out.println(uid); +// lsr = searchApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), +// 6); +// // assertTrue(lsr.getUid() == null); +// checkLsr(lsr, 5L, 6L, 7L, 8L, 9L); +// searchApi.freeSearcher(uid); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "b"), 100); +// checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"), +// 100); +// checkLsr(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults( +// SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""), +// 100); +// checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", null, null, null, null, null, "b"), 100); +// checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "c1", null, null, null, null, null, "b"), 100); +// checkLsr(lsr); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null), +// 100); +// checkLsr(lsr, 4L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", null, null, null, null, "b"), 100); +// checkLsr(lsr, 3L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), +// new Date(now + 60000 * 6), null, null, "b"), 100); +// checkLsr(lsr, 3L); +// searchApi.freeSearcher(lsr.getUid()); + +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, new Date(now + 60000 * 3), +// new Date(now + 60000 * 6), null, null, null), 100); +// checkLsr(lsr, 3L, 4L, 5L); +// searchApi.freeSearcher(lsr.getUid()); + +// List pojos = new ArrayList<>(); +// pojos.add(new ParameterPOJO(null, null, "v9")); +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), +// 100); +// checkLsr(lsr, 3L); +// searchApi.freeSearcher(lsr.getUid()); + +// pojos = new ArrayList<>(); +// pojos.add(new ParameterPOJO(null, null, "v9")); +// pojos.add(new ParameterPOJO(null, null, 7, 10)); +// pojos.add(new ParameterPOJO(null, null, new Date(now + 60000 * 63), new Date(now + 60000 * 65))); +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), +// 100); +// checkLsr(lsr, 3L); +// searchApi.freeSearcher(lsr.getUid()); + +// pojos = new ArrayList<>(); +// pojos.add(new ParameterPOJO(null, null, "v9")); +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), +// new Date(now + 60000 * 6), pojos, null, "b"), 100); +// checkLsr(lsr, 3L); +// searchApi.freeSearcher(lsr.getUid()); + +// pojos = new ArrayList<>(); +// pojos.add(new ParameterPOJO(null, null, "v9")); +// pojos.add(new ParameterPOJO(null, null, "v81")); +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), +// 100); +// checkLsr(lsr); +// searchApi.freeSearcher(lsr.getUid()); + +// pojos = new ArrayList<>(); +// pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), +// 100); +// checkLsr(lsr, 3L); +// searchApi.freeSearcher(lsr.getUid()); + +// List samples = Arrays.asList("ddd", "nnn"); +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), +// 100); +// checkLsr(lsr, 3L); +// searchApi.freeSearcher(lsr.getUid()); + +// samples = Arrays.asList("ddd", "mmm"); +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), +// 100); +// checkLsr(lsr); +// searchApi.freeSearcher(lsr.getUid()); + +// pojos = new ArrayList<>(); +// pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); +// samples = Arrays.asList("ddd", "nnn"); +// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), +// new Date(now + 60000 * 6), pojos, samples, "b"), 100); +// checkLsr(lsr, 3L); +// searchApi.freeSearcher(lsr.getUid()); +// } + +// /** +// * Populate Investigation, Dataset, Datafile +// */ +// private void populate() throws IcatException { + +// ByteArrayOutputStream baos = new ByteArrayOutputStream(); +// try (JsonGenerator gen = Json.createGenerator(baos)) { +// gen.writeStartArray(); +// for (int i = 0; i < NUMINV; i++) { +// for (int j = 0; j < NUMUSERS; j++) { +// if (i % (j + 1) == 1) { +// String fn = "FN " + letters.substring(j, j + 1) + " " + letters.substring(j, j + 1); +// String name = letters.substring(j, j + 1) + j; +// gen.writeStartArray(); +// gen.write("InvestigationUser"); +// gen.writeNull(); +// gen.writeStartArray(); + +// searchApi.encodeTextField(gen, "userFullName", fn); + +// searchApi.encodeStringField(gen, "userName", name); +// searchApi.encodeSortedDocValuesField(gen, "investigation", new Long(i)); + +// gen.writeEnd(); +// gen.writeEnd(); +// System.out.println("'" + fn + "' " + name + " " + i); +// } +// } +// } +// gen.writeEnd(); +// } +// searchApi.modify(baos.toString()); +// logger.debug("IUs added:"); +// searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), 100); // TODO +// // RM + +// baos = new ByteArrayOutputStream(); +// try (JsonGenerator gen = Json.createGenerator(baos)) { +// gen.writeStartArray(); +// for (int i = 0; i < NUMINV; i++) { +// int j = i % 26; +// int k = (i + 7) % 26; +// int l = (i + 17) % 26; +// String word = letters.substring(j, j + 1) + " " + letters.substring(k, k + 1) + " " +// + letters.substring(l, l + 1); +// gen.writeStartArray(); +// gen.write("Investigation"); +// gen.writeNull(); +// gen.writeStartArray(); +// searchApi.encodeTextField(gen, "text", word); +// searchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); +// searchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); +// searchApi.encodeStoredId(gen, new Long(i)); +// searchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); +// gen.writeEnd(); +// gen.writeEnd(); +// System.out.println("INVESTIGATION '" + word + "' " + new Date(now + i * 60000) + " " + i); +// } +// gen.writeEnd(); +// } +// searchApi.modify(baos.toString()); +// logger.debug("Is added:"); +// searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), 100); // TODO +// // RM + +// baos = new ByteArrayOutputStream(); +// try (JsonGenerator gen = Json.createGenerator(baos)) { +// gen.writeStartArray(); +// for (int i = 0; i < NUMINV; i++) { +// if (i % 2 == 1) { +// fillParameters(gen, i, "investigation"); +// } +// } +// gen.writeEnd(); +// } +// searchApi.modify(baos.toString()); + +// baos = new ByteArrayOutputStream(); +// try (JsonGenerator gen = Json.createGenerator(baos)) { +// gen.writeStartArray(); +// for (int i = 0; i < NUMDS; i++) { +// int j = i % 26; +// String word = "DS" + letters.substring(j, j + 1) + letters.substring(j, j + 1) +// + letters.substring(j, j + 1); +// gen.writeStartArray(); +// gen.write("Dataset"); +// gen.writeNull(); +// gen.writeStartArray(); +// searchApi.encodeTextField(gen, "text", word); +// searchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); +// searchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); +// searchApi.encodeStoredId(gen, new Long(i)); +// searchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); +// searchApi.encodeStringField(gen, "investigation", new Long(i % NUMINV)); +// gen.writeEnd(); +// gen.writeEnd(); +// System.out.println("DATASET '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMINV); +// } +// gen.writeEnd(); +// } +// searchApi.modify(baos.toString()); + +// baos = new ByteArrayOutputStream(); +// try (JsonGenerator gen = Json.createGenerator(baos)) { +// gen.writeStartArray(); +// for (int i = 0; i < NUMDS; i++) { +// if (i % 3 == 1) { +// fillParameters(gen, i, "dataset"); +// } +// } +// gen.writeEnd(); +// } +// searchApi.modify(baos.toString()); + +// baos = new ByteArrayOutputStream(); +// try (JsonGenerator gen = Json.createGenerator(baos)) { +// gen.writeStartArray(); +// for (int i = 0; i < NUMDF; i++) { +// int j = i % 26; +// String word = "DF" + letters.substring(j, j + 1) + letters.substring(j, j + 1) +// + letters.substring(j, j + 1); +// gen.writeStartArray(); +// gen.write("Datafile"); +// gen.writeNull(); +// gen.writeStartArray(); +// searchApi.encodeTextField(gen, "text", word); +// searchApi.encodeStringField(gen, "date", new Date(now + i * 60000)); +// searchApi.encodeStoredId(gen, new Long(i)); +// searchApi.encodeStringField(gen, "dataset", new Long(i % NUMDS)); +// searchApi.encodeStringField(gen, "investigation", new Long((i % NUMDS) % NUMINV)); +// gen.writeEnd(); +// gen.writeEnd(); +// System.out.println("DATAFILE '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMDS); + +// } +// gen.writeEnd(); +// } +// searchApi.modify(baos.toString()); + +// baos = new ByteArrayOutputStream(); +// try (JsonGenerator gen = Json.createGenerator(baos)) { +// gen.writeStartArray(); +// for (int i = 0; i < NUMDF; i++) { +// if (i % 4 == 1) { +// fillParameters(gen, i, "datafile"); +// } +// } +// gen.writeEnd(); +// } +// searchApi.modify(baos.toString()); + +// baos = new ByteArrayOutputStream(); +// try (JsonGenerator gen = Json.createGenerator(baos)) { +// gen.writeStartArray(); +// for (int i = 0; i < NUMSAMP; i++) { +// int j = i % 26; +// String word = "SType " + letters.substring(j, j + 1) + letters.substring(j, j + 1) +// + letters.substring(j, j + 1); +// gen.writeStartArray(); +// gen.write("Sample"); +// gen.writeNull(); +// gen.writeStartArray(); +// searchApi.encodeTextField(gen, "sampleText", word); +// searchApi.encodeSortedDocValuesField(gen, "investigation", new Long(i % NUMINV)); +// gen.writeEnd(); +// gen.writeEnd(); +// System.out.println("SAMPLE '" + word + "' " + i % NUMINV); +// } +// gen.writeEnd(); + +// } +// searchApi.modify(baos.toString()); + +// searchApi.commit(); + +// } +// } diff --git a/src/test/java/org/icatproject/core/manager/TestLucene.java b/src/test/java/org/icatproject/core/manager/TestLucene.java index 865cf3af..097f8954 100644 --- a/src/test/java/org/icatproject/core/manager/TestLucene.java +++ b/src/test/java/org/icatproject/core/manager/TestLucene.java @@ -6,6 +6,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.lang.reflect.Array; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -30,6 +31,12 @@ import org.apache.http.impl.client.HttpClients; import org.icatproject.core.IcatException; import org.icatproject.core.IcatException.IcatExceptionType; +import org.icatproject.core.entity.Datafile; +import org.icatproject.core.entity.Dataset; +import org.icatproject.core.entity.DatasetType; +import org.icatproject.core.entity.Facility; +import org.icatproject.core.entity.Investigation; +import org.icatproject.core.entity.InvestigationType; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -182,66 +189,161 @@ private void checkLsr(SearchResult lsr, Long... n) { } + private void checkLsrOrder(SearchResult lsr, Long... n) { + List results = lsr.getResults(); + if (n.length != results.size()) { + checkLsr(lsr, n); + } + for (int i = 0; i < n.length; i++) { + Long resultId = results.get(i).getEntityBaseBeanId(); + Long expectedId = (Long) Array.get(n, i); + if (resultId != expectedId) { + fail("Expected id " + expectedId + " in position " + i + " but got " + resultId); + } + } + } + @Test public void datafiles() throws Exception { populate(); - SearchResult lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 5); + SearchResult lsr = luceneApi + .getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 5, null); String uid = lsr.getUid(); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); System.out.println(uid); - lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 200); + lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), + 200); assertTrue(lsr.getUid() == null); assertEquals(95, lsr.getResults().size()); luceneApi.freeSearcher(uid); - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", "e4", null, null, null, null, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", "e4", null, null, null, null, null, null), 100, + null); checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, 96L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null), 100, + null); checkLsr(lsr, 1L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, "dfbbb", null, null, null, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, "dfbbb", null, null, null, null, null), 100, + null); checkLsr(lsr, 1L, 27L, 53L, 79L); luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100); + new Date(now + 60000 * 6), null, null, null), 100, null); checkLsr(lsr, 3L, 4L, 5L, 6L); luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100); + new Date(now + 60000 * 6), null, null, null), 100, null); checkLsr(lsr); luceneApi.freeSearcher(lsr.getUid()); List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v25")); lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null), 100); + new Date(now + 60000 * 6), pojos, null, null), 100, null); checkLsr(lsr, 5L); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v25")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100, + null); checkLsr(lsr, 5L); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, "u sss", null)); - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100, + null); checkLsr(lsr, 13L, 65L); luceneApi.freeSearcher(lsr.getUid()); + + // Test searchAfter preserves the sorting of original search (asc) + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("date", "asc"); + gen.writeEnd(); + + } + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), + 5, baos.toString()); + checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); + uid = lsr.getUid(); + lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), + 5); + checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); + + // Test searchAfter preserves the sorting of original search (desc) + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("date", "desc"); + gen.writeEnd(); + } + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), + 5, baos.toString()); + checkLsrOrder(lsr, 99L, 98L, 97L, 96L, 95L); + uid = lsr.getUid(); + lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), + 5); + checkLsrOrder(lsr, 94L, 93L, 92L, 91L, 90L); + + // Test tie breaks on fields with identical values (asc) + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("name", "asc"); + gen.writeEnd(); + } + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), + 5, baos.toString()); + checkLsrOrder(lsr, 0L, 26L, 52L, 78L, 1L); + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("name", "asc"); + gen.write("date", "desc"); + gen.writeEnd(); + } + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), + 5, baos.toString()); + checkLsrOrder(lsr, 78L, 52L, 26L, 0L, 79L); + + // Test tie breaks on fields with identical values (desc) + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("name", "desc"); + gen.writeEnd(); + } + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), + 5, baos.toString()); + checkLsrOrder(lsr, 25L, 51L, 77L, 24L, 50L); + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("name", "desc"); + gen.write("date", "desc"); + gen.writeEnd(); + } + lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), + 5, baos.toString()); + checkLsrOrder(lsr, 77L, 51L, 25L, 76L, 50L); } @Test public void datasets() throws Exception { populate(); - SearchResult lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 5); + SearchResult lsr = luceneApi + .getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 5, null); String uid = lsr.getUid(); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); @@ -252,48 +354,102 @@ public void datasets() throws Exception { 25L, 26L, 27L, 28L, 29L); luceneApi.freeSearcher(uid); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "e4", null, null, null, null, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "e4", null, null, null, null, null, null), 100, + null); checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100, + null); checkLsr(lsr, 1L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100, + null); checkLsr(lsr, 1L, 27L); luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100); + new Date(now + 60000 * 6), null, null, null), 100, null); checkLsr(lsr, 3L, 4L, 5L); luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100); + new Date(now + 60000 * 6), null, null, null), 100, null); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, pojos, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, pojos, null, null), 100, + null); checkLsr(lsr, 4L); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v16")); lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null), 100); + new Date(now + 60000 * 6), pojos, null, null), 100, null); checkLsr(lsr, 4L); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v16")); lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null), 100); + new Date(now + 60000 * 6), pojos, null, null), 100, null); checkLsr(lsr); luceneApi.freeSearcher(lsr.getUid()); + // Test searchAfter preserves the sorting of original search (asc) + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("startDate", "asc"); + gen.writeEnd(); + } + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), + 5, baos.toString()); + checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); + uid = lsr.getUid(); + lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), + 5); + checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); + + // Test searchAfter preserves the sorting of original search (desc) + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("endDate", "desc"); + gen.writeEnd(); + } + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), + 5, baos.toString()); + checkLsrOrder(lsr, 29L, 28L, 27L, 26L, 25L); + uid = lsr.getUid(); + lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), + 5); + checkLsrOrder(lsr, 24L, 23L, 22L, 21L, 20L); + + // Test tie breaks on fields with identical values (asc) + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("name", "asc"); + gen.writeEnd(); + } + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), + 5, baos.toString()); + checkLsrOrder(lsr, 0L, 26L, 1L, 27L, 2L); + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("name", "asc"); + gen.write("endDate", "desc"); + gen.writeEnd(); + } + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), + 5, baos.toString()); + checkLsrOrder(lsr, 26L, 0L, 27L, 1L, 28L); } private void fillParms(JsonGenerator gen, int i, String rel) { @@ -334,58 +490,68 @@ public void investigations() throws Exception { populate(); /* Blocked results */ - SearchResult lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), - 5); + SearchResult lsr = luceneApi.getResults( + SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), + 5, null); String uid = lsr.getUid(); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); System.out.println(uid); - lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), 6); + lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), + 6); assertTrue(lsr.getUid() == null); checkLsr(lsr, 5L, 6L, 7L, 8L, 9L); luceneApi.freeSearcher(uid); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "b"), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "b"), 100, + null); checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"), 100, + null); checkLsr(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""), - 100); + lsr = luceneApi.getResults( + SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""), + 100, null); checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", null, null, null, null, null, "b"), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", null, null, null, null, null, "b"), 100, + null); checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "c1", null, null, null, null, null, "b"), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "c1", null, null, null, null, null, "b"), 100, + null); checkLsr(lsr); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null), + 100, null); checkLsr(lsr, 4L); luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", null, null, null, null, "b"), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", null, null, null, null, "b"), 100, + null); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, "b"), 100); + new Date(now + 60000 * 6), null, null, "b"), 100, null); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100); + new Date(now + 60000 * 6), null, null, null), 100, null); checkLsr(lsr, 3L, 4L, 5L); luceneApi.freeSearcher(lsr.getUid()); List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), + 100, null); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); @@ -393,37 +559,42 @@ public void investigations() throws Exception { pojos.add(new ParameterPOJO(null, null, "v9")); pojos.add(new ParameterPOJO(null, null, 7, 10)); pojos.add(new ParameterPOJO(null, null, new Date(now + 60000 * 63), new Date(now + 60000 * 65))); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), + 100, null); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, "b"), 100); + new Date(now + 60000 * 6), pojos, null, "b"), 100, null); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); pojos.add(new ParameterPOJO(null, null, "v81")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), + 100, null); checkLsr(lsr); luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), + 100, null); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); List samples = Arrays.asList("ddd", "nnn"); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), + 100, null); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); samples = Arrays.asList("ddd", "mmm"); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), 100); + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), + 100, null); checkLsr(lsr); luceneApi.freeSearcher(lsr.getUid()); @@ -431,9 +602,39 @@ public void investigations() throws Exception { pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); samples = Arrays.asList("ddd", "nnn"); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, samples, "b"), 100); + new Date(now + 60000 * 6), pojos, samples, "b"), 100, null); checkLsr(lsr, 3L); luceneApi.freeSearcher(lsr.getUid()); + + // Test searchAfter preserves the sorting of original search (asc) + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("startDate", "asc"); + gen.writeEnd(); + } + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), + 5, baos.toString()); + checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); + uid = lsr.getUid(); + lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), + 5); + checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); + + // Test searchAfter preserves the sorting of original search (desc) + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("endDate", "desc"); + gen.writeEnd(); + } + lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), + 5, baos.toString()); + checkLsrOrder(lsr, 9L, 8L, 7L, 6L, 5L); + uid = lsr.getUid(); + lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), + 5); + checkLsrOrder(lsr, 4L, 3L, 2L, 1L, 0L); } @Test @@ -501,12 +702,15 @@ private void populate() throws IcatException { int l = (i + 17) % 26; String word = letters.substring(j, j + 1) + " " + letters.substring(k, k + 1) + " " + letters.substring(l, l + 1); + Investigation investigation = new Investigation(); + investigation.setFacility(new Facility()); + investigation.setType(new InvestigationType()); + investigation.setName(word); + investigation.setStartDate(new Date(now + i * 60000)); + investigation.setEndDate(new Date(now + (i + 1) * 60000)); + investigation.setId(new Long(i)); gen.writeStartArray(); - luceneApi.encodeTextField(gen, "text", word); - luceneApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); - luceneApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); - luceneApi.encodeStoredId(gen, new Long(i)); - luceneApi.encodeSortedDocValuesField(gen, "id", new Long(i)); + investigation.getDoc(gen, luceneApi); gen.writeEnd(); System.out.println("INVESTIGATION '" + word + "' " + new Date(now + i * 60000) + " " + i); } @@ -533,13 +737,19 @@ private void populate() throws IcatException { int j = i % 26; String word = "DS" + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); + + Investigation investigation = new Investigation(); + investigation.setId(new Long(i % NUMINV)); + Dataset dataset = new Dataset(); + dataset.setType(new DatasetType()); + dataset.setName(word); + dataset.setStartDate(new Date(now + i * 60000)); + dataset.setEndDate(new Date(now + (i + 1) * 60000)); + dataset.setId(new Long(i)); + dataset.setInvestigation(investigation); + gen.writeStartArray(); - luceneApi.encodeTextField(gen, "text", word); - luceneApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); - luceneApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); - luceneApi.encodeStoredId(gen, new Long(i)); - luceneApi.encodeSortedDocValuesField(gen, "id", new Long(i)); - luceneApi.encodeStringField(gen, "investigation", new Long(i % NUMINV)); + dataset.getDoc(gen, luceneApi); gen.writeEnd(); System.out.println("DATASET '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMINV); } @@ -566,11 +776,20 @@ private void populate() throws IcatException { int j = i % 26; String word = "DF" + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); + + Investigation investigation = new Investigation(); + investigation.setId(new Long((i % NUMDS) % NUMINV)); + Dataset dataset = new Dataset(); + dataset.setId(new Long(i % NUMDS)); + dataset.setInvestigation(investigation); + Datafile datafile = new Datafile(); + datafile.setName(word); + datafile.setDatafileModTime(new Date(now + i * 60000)); + datafile.setId(new Long(i)); + datafile.setDataset(dataset); + gen.writeStartArray(); - luceneApi.encodeTextField(gen, "text", word); - luceneApi.encodeStringField(gen, "date", new Date(now + i * 60000)); - luceneApi.encodeStoredId(gen, new Long(i)); - luceneApi.encodeStringField(gen, "dataset", new Long(i % NUMDS)); + datafile.getDoc(gen, luceneApi); gen.writeEnd(); System.out.println("DATAFILE '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMDS); From bfb7d7b86ac9ecd0a9296f3275dcd0c364b5012d Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 24 Mar 2022 19:44:13 +0000 Subject: [PATCH 09/51] Replace abstract methods in SearchApi #267 --- .../org/icatproject/core/entity/Datafile.java | 2 +- .../org/icatproject/core/entity/Dataset.java | 2 +- .../core/entity/Investigation.java | 8 +- .../core/manager/ElasticsearchApi.java | 56 --- .../core/manager/ElasticsearchDocument.java | 10 +- .../icatproject/core/manager/LuceneApi.java | 38 +- .../core/manager/ScoredEntityBaseBean.java | 5 + .../icatproject/core/manager/SearchApi.java | 381 ++++++++++++++++-- .../core/manager/TestElasticsearchApi.java | 8 +- .../icatproject/core/manager/TestLucene.java | 8 +- 10 files changed, 391 insertions(+), 127 deletions(-) diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index ff9c7a51..f5143883 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -213,7 +213,7 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { } else { searchApi.encodeStringField(gen, "date", modTime); } - searchApi.encodeStoredId(gen, id); + searchApi.encodeStringField(gen, "id", id, true); searchApi.encodeStringField(gen, "dataset", dataset.id); searchApi.encodeStringField(gen, "investigation", dataset.getInvestigation().id); diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index 27a324d8..b43c1f16 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -213,7 +213,7 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { } else { searchApi.encodeStringField(gen, "endDate", modTime); } - searchApi.encodeStoredId(gen, id); + searchApi.encodeStringField(gen, "id", id, true); searchApi.encodeSortedDocValuesField(gen, "id", id); diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index a9d48483..6b7bab09 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -290,7 +290,7 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { samples.forEach((sample) -> { - searchApi.encodeSortedSetDocValuesFacetField(gen, "sampleName", sample.getName()); + // searchApi.encodeSortedSetDocValuesFacetField(gen, "sampleName", sample.getName()); searchApi.encodeTextField(gen, "sampleText", sample.getDocText()); }); @@ -298,11 +298,11 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { ParameterType type = parameter.type; String parameterName = type.getName(); String parameterUnits = type.getUnits(); - searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterName", parameterName); + // searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterName", parameterName); searchApi.encodeStringField(gen, "parameterUnits", parameterUnits); // TODO make all value types facetable... if (type.getValueType() == ParameterValueType.STRING) { - searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterStringValue", parameter.getStringValue()); + // searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterStringValue", parameter.getStringValue()); } else if (type.getValueType() == ParameterValueType.DATE_AND_TIME) { searchApi.encodeStringField(gen, "parameterDateValue", parameter.getDateTimeValue()); } else if (type.getValueType() == ParameterValueType.NUMERIC) { @@ -312,6 +312,6 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { searchApi.encodeSortedDocValuesField(gen, "id", id); - searchApi.encodeStoredId(gen, id); + searchApi.encodeStringField(gen, "id", id, true); } } diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java index 2b342eb8..38c7a62c 100644 --- a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java +++ b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java @@ -1,6 +1,5 @@ package org.icatproject.core.manager; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.StringReader; import java.net.URL; @@ -11,7 +10,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ExecutorService; import javax.json.Json; import javax.json.JsonArray; @@ -20,13 +18,11 @@ import javax.json.JsonReader; import javax.json.JsonValue; import javax.json.stream.JsonGenerator; -import javax.persistence.EntityManager; import org.apache.http.HttpHost; import org.elasticsearch.client.RestClient; import org.icatproject.core.IcatException; import org.icatproject.core.IcatException.IcatExceptionType; -import org.icatproject.core.entity.EntityBaseBean; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.ElasticsearchException; @@ -158,35 +154,6 @@ private void initMappings() throws IcatException { } } - @Override - public void addNow(String entityName, List ids, EntityManager manager, - Class klass, ExecutorService getBeanDocExecutor) - throws IcatException, IOException { - // getBeanDocExecutor is not used for the Elasticsearch implementation, but is - // required for the @Override - - // TODO Change this string building fake JSON by hand - StringBuilder sb = new StringBuilder(); - sb.append("["); - for (Long id : ids) { - EntityBaseBean bean = (EntityBaseBean) manager.find(klass, id); - if (bean != null) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); // Document fields are wrapped in an array - bean.getDoc(gen, this); // Fields - gen.writeEnd(); - } - if (sb.length() != 1) { - sb.append(','); - } - sb.append("[\"").append(entityName).append("\",null,").append(baos.toString()).append(']'); - } - } - sb.append("]"); - modify(sb.toString()); - } - @Override public void clear() throws IcatException { try { @@ -216,29 +183,6 @@ public void encodeStringField(JsonGenerator gen, String name, Date value) { gen.writeStartObject().write(name, timeString).writeEnd(); } - public void encodeDoublePoint(JsonGenerator gen, String name, Double value) { - gen.writeStartObject().write(name, value).writeEnd(); - } - - public void encodeSortedSetDocValuesFacetField(JsonGenerator gen, String name, String value) { - gen.writeStartObject().write(name, value).writeEnd(); - - } - - public void encodeStringField(JsonGenerator gen, String name, Long value) { - gen.writeStartObject().write(name, value).writeEnd(); - } - - public void encodeStringField(JsonGenerator gen, String name, String value) { - gen.writeStartObject().write(name, value).writeEnd(); - } - - public void encodeTextField(JsonGenerator gen, String name, String value) { - if (value != null) { - gen.writeStartObject().write(name, value).writeEnd(); - } - } - @Override public void freeSearcher(String uid) throws IcatException { diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java b/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java index ba9659dd..67c7c79b 100644 --- a/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java +++ b/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java @@ -80,11 +80,11 @@ public ElasticsearchDocument(JsonArray jsonArray) throws IcatException { } else if (fieldEntry.getKey().equals("text")) { text = fieldObject.getString("text"); } else if (fieldEntry.getKey().equals("date")) { - date = SearchApi.dec(fieldObject.getString("date")); + date = SearchApi.decodeDate(fieldObject.getString("date")); } else if (fieldEntry.getKey().equals("startDate")) { - startDate = SearchApi.dec(fieldObject.getString("startDate")); + startDate = SearchApi.decodeDate(fieldObject.getString("startDate")); } else if (fieldEntry.getKey().equals("endDate")) { - endDate = SearchApi.dec(fieldObject.getString("endDate")); + endDate = SearchApi.decodeDate(fieldObject.getString("endDate")); } else if (fieldEntry.getKey().equals("user.name")) { userName.add(fieldObject.getString("user.name")); } else if (fieldEntry.getKey().equals("user.fullName")) { @@ -100,7 +100,7 @@ public ElasticsearchDocument(JsonArray jsonArray) throws IcatException { } else if (fieldEntry.getKey().equals("parameter.stringValue")) { parameterStringValue.add(fieldObject.getString("parameter.stringValue")); } else if (fieldEntry.getKey().equals("parameter.dateValue")) { - parameterDateValue.add(SearchApi.dec(fieldObject.getString("parameter.dateValue"))); + parameterDateValue.add(SearchApi.decodeDate(fieldObject.getString("parameter.dateValue"))); } else if (fieldEntry.getKey().equals("parameter.numericValue")) { parameterNumericValue .add(fieldObject.getJsonNumber("parameter.numericValue").doubleValue()); @@ -138,7 +138,7 @@ public ElasticsearchDocument(JsonArray jsonArray, String index, String parentInd } else if (fieldEntry.getKey().equals("parameterStringValue")) { parameterStringValue.add(fieldObject.getString("parameterStringValue")); } else if (fieldEntry.getKey().equals("parameterDateValue")) { - parameterDateValue.add(SearchApi.dec(fieldObject.getString("parameterDateValue"))); + parameterDateValue.add(SearchApi.decodeDate(fieldObject.getString("parameterDateValue"))); } else if (fieldEntry.getKey().equals("parameterNumericValue")) { parameterNumericValue .add(fieldObject.getJsonNumber("parameterNumericValue").doubleValue()); diff --git a/src/main/java/org/icatproject/core/manager/LuceneApi.java b/src/main/java/org/icatproject/core/manager/LuceneApi.java index d82c454e..ad7efe09 100644 --- a/src/main/java/org/icatproject/core/manager/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/LuceneApi.java @@ -6,7 +6,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -67,45 +66,35 @@ private String getTargetPath(JsonObject query) throws IcatException { // TODO this method of encoding an entity as an array of 3 key objects that represent single field each // is something that should be streamlined, but would require changes to icat.lucene + @Override public void encodeSortedDocValuesField(JsonGenerator gen, String name, Long value) { gen.writeStartObject().write("type", "SortedDocValuesField").write("name", name).write("value", value) .writeEnd(); } - public void encodeStoredId(JsonGenerator gen, Long id) { - gen.writeStartObject().write("type", "StringField").write("name", "id").write("value", Long.toString(id)) - .write("store", true).writeEnd(); - } - - public void encodeStringField(JsonGenerator gen, String name, Date value) { - String timeString; - synchronized (df) { - timeString = df.format(value); - } - gen.writeStartObject().write("type", "StringField").write("name", name).write("value", timeString).writeEnd(); - } - + @Override public void encodeDoublePoint(JsonGenerator gen, String name, Double value) { gen.writeStartObject().write("type", "DoublePoint").write("name", name).write("value", value) .write("store", true).writeEnd(); } - public void encodeSortedSetDocValuesFacetField(JsonGenerator gen, String name, String value) { - gen.writeStartObject().write("type", "SortedSetDocValuesFacetField").write("name", name).write("value", value) - .writeEnd(); - - } + // public void encodeSortedSetDocValuesFacetField(JsonGenerator gen, String name, String value) { + // gen.writeStartObject().write("type", "SortedSetDocValuesFacetField").write("name", name).write("value", value) + // .writeEnd(); - public void encodeStringField(JsonGenerator gen, String name, Long value) { - gen.writeStartObject().write("type", "StringField").write("name", name).write("value", Long.toString(value)) - .writeEnd(); - } + // } + @Override public void encodeStringField(JsonGenerator gen, String name, String value) { gen.writeStartObject().write("type", "StringField").write("name", name).write("value", value).writeEnd(); + } + @Override + public void encodeStringField(JsonGenerator gen, String name, Long value, Boolean store) { + gen.writeStartObject().write("type", "StringField").write("name", name).write("value", Long.toString(value)).write("store", store).writeEnd(); } + @Override public void encodeTextField(JsonGenerator gen, String name, String value) { if (value != null) { gen.writeStartObject().write("type", "TextField").write("name", name).write("value", value).writeEnd(); @@ -118,9 +107,8 @@ public LuceneApi(URI server) { this.server = server; } - @Override public void addNow(String entityName, List ids, EntityManager manager, - Class klass, ExecutorService getBeanDocExecutor) throws Exception { + Class klass, ExecutorService getBeanDocExecutor) throws IcatException, IOException, URISyntaxException { URI uri = new URIBuilder(server).setPath(basePath + "/addNow/" + entityName) .build(); diff --git a/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java b/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java index 34c58306..cf747d75 100644 --- a/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java +++ b/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java @@ -5,6 +5,11 @@ public class ScoredEntityBaseBean { private long entityBaseBeanId; private float score; + public ScoredEntityBaseBean(String id, double score) { + this.entityBaseBeanId = Long.parseLong(id); + this.score = (float) score; + } + public ScoredEntityBaseBean(long id, float score) { this.entityBaseBeanId = id; this.score = score; diff --git a/src/main/java/org/icatproject/core/manager/SearchApi.java b/src/main/java/org/icatproject/core/manager/SearchApi.java index bd329566..80938e0b 100644 --- a/src/main/java/org/icatproject/core/manager/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/SearchApi.java @@ -1,19 +1,40 @@ package org.icatproject.core.manager; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.StringReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.TimeZone; import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicLong; import javax.json.Json; +import javax.json.JsonArray; import javax.json.JsonArrayBuilder; +import javax.json.JsonNumber; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; +import javax.json.JsonReader; +import javax.json.JsonValue; import javax.json.stream.JsonGenerator; import javax.persistence.EntityManager; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; import org.icatproject.core.IcatException; +import org.icatproject.core.IcatException.IcatExceptionType; import org.icatproject.core.entity.EntityBaseBean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,13 +42,14 @@ // TODO see what functionality can live here, and possibly convert from abstract to a fully generic API public abstract class SearchApi { - public abstract void addNow(String entityName, List ids, EntityManager manager, - Class klass, ExecutorService getBeanDocExecutor) throws Exception; - - public abstract void clear() throws IcatException; - - final Logger logger = LoggerFactory.getLogger(this.getClass()); + protected static final Logger logger = LoggerFactory.getLogger(SearchApi.class); protected static SimpleDateFormat df; + protected static String basePath = ""; + protected static String matchAllQuery; + + protected URI server; + protected AtomicLong atomicLongUid = new AtomicLong(); + protected Map uidMap = new HashMap<>(); static { df = new SimpleDateFormat("yyyyMMddHHmm"); @@ -35,7 +57,23 @@ public abstract void addNow(String entityName, List ids, EntityManager man df.setTimeZone(tz); } - protected static Date dec(String value) throws java.text.ParseException { + static { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject().writeStartObject("query").writeStartObject("match_all") + .writeEnd().writeEnd().writeEnd(); + } + matchAllQuery = baos.toString(); + } + + /** + * Converts String into Date object. + * + * @param value String representing a Date in the format "yyyyMMddHHmm". + * @return Date object, or null if value was null. + * @throws java.text.ParseException + */ + protected static Date decodeDate(String value) throws java.text.ParseException { if (value == null) { return null; } else { @@ -45,6 +83,13 @@ protected static Date dec(String value) throws java.text.ParseException { } } + /** + * Converts String into number of ms since epoch. + * + * @param value String representing a Date in the format "yyyyMMddHHmm". + * @return Number of ms since epoch, or null if value was null + * @throws java.text.ParseException + */ protected static Long decodeTime(String value) throws java.text.ParseException { if (value == null) { return null; @@ -55,38 +100,255 @@ protected static Long decodeTime(String value) throws java.text.ParseException { } } - protected static String enc(Date dateValue) { - synchronized (df) { - return df.format(dateValue); + /** + * Converts Date object into String format. + * + * @param dateValue Date object to be converted. + * @return String representing a Date in the format "yyyyMMddHHmm". + */ + protected static String encodeDate(Date dateValue) { + if (dateValue == null) { + return null; + } else { + synchronized (df) { + return df.format(dateValue); + } } } - public abstract void encodeSortedDocValuesField(JsonGenerator gen, String name, Long value); + // TODO if encoding methods are unified across all APIs, then we can make these + // static + public void encodeDoublePoint(JsonGenerator gen, String name, Double value) { + gen.writeStartObject().write(name, value).writeEnd(); + } + + public void encodeSortedDocValuesField(JsonGenerator gen, String name, Date value) { + encodeSortedDocValuesField(gen, name, encodeDate(value)); + } - public abstract void encodeSortedSetDocValuesFacetField(JsonGenerator gen, String name, String value); + public void encodeSortedDocValuesField(JsonGenerator gen, String name, Long value) { + gen.writeStartObject().write(name, value).writeEnd(); + } - public abstract void encodeStoredId(JsonGenerator gen, Long id); + public void encodeSortedDocValuesField(JsonGenerator gen, String name, String value) { + encodeStringField(gen, name, value); + } - public abstract void encodeStringField(JsonGenerator gen, String name, Date value); + // public void encodeSortedSetDocValuesFacetField(JsonGenerator gen, String + // name, String value) { + // encodeStringField(gen, name, value); + // } - public abstract void encodeDoublePoint(JsonGenerator gen, String name, Double value); + public void encodeStringField(JsonGenerator gen, String name, Date value) { + encodeStringField(gen, name, encodeDate(value)); + } - public abstract void encodeStringField(JsonGenerator gen, String name, Long value); + public void encodeStringField(JsonGenerator gen, String name, Long value) { + encodeStringField(gen, name, Long.toString(value)); + } - public abstract void encodeStringField(JsonGenerator gen, String name, String value); + public void encodeStringField(JsonGenerator gen, String name, Long value, Boolean store) { + encodeStringField(gen, name, value); + } - public abstract void encodeTextField(JsonGenerator gen, String name, String value); + public void encodeStringField(JsonGenerator gen, String name, String value) { + gen.writeStartObject().write(name, value).writeEnd(); + } - public abstract void freeSearcher(String uid) throws IcatException; + public void encodeTextField(JsonGenerator gen, String name, String value) { + if (value != null) { + gen.writeStartObject().write(name, value).writeEnd(); + } + } - public abstract void commit() throws IcatException; + public void addNow(String entityName, List ids, EntityManager manager, + Class klass, ExecutorService getBeanDocExecutor) + throws IcatException, IOException, URISyntaxException { + // getBeanDocExecutor is not used for all implementations, but is + // required for the @Override + // TODO Change this string building fake JSON by hand + StringBuilder sb = new StringBuilder(); + sb.append("["); + for (Long id : ids) { + EntityBaseBean bean = (EntityBaseBean) manager.find(klass, id); + if (bean != null) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartArray(); // Document fields are wrapped in an array + bean.getDoc(gen, this); // Fields + gen.writeEnd(); + } + if (sb.length() != 1) { + sb.append(','); + } + sb.append("[\"").append(entityName).append("\",null,").append(baos.toString()).append(']'); + } + } + sb.append("]"); + modify(sb.toString()); + } + + public void clear() throws IcatException { + atomicLongUid.set(0); + uidMap = new HashMap<>(); + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URI uri = new URIBuilder(server).setPath(basePath + "/_delete_by_query").build(); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(matchAllQuery)); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } catch (IOException | URISyntaxException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + public void commit() throws IcatException { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URI uri = new URIBuilder(server).setPath(basePath + "/_refresh").build(); + logger.trace("Making call {}", uri); + HttpPost httpPost = new HttpPost(uri); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } catch (URISyntaxException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + public void freeSearcher(String uid) throws IcatException { + uidMap.remove(uid); + } public abstract List facetSearch(JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException; - public abstract SearchResult getResults(JsonObject query, int maxResults) throws IcatException; + public SearchResult getResults(JsonObject query, int maxResults) throws IcatException { + Long uid = atomicLongUid.getAndIncrement(); + uidMap.put(uid.toString(), 0); + return getResults(uid.toString(), query, maxResults); + } - public abstract SearchResult getResults(String uid, JsonObject query, int maxResults) throws IcatException; + public SearchResult getResults(String uid, JsonObject query, int maxResults) throws IcatException { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + String index; + Set fields = query.keySet(); + if (fields.contains("target")) { + index = query.getString("target").toLowerCase(); + } else { + index = query.getString("_all"); + } + URI uri = new URIBuilder(server).setPath(basePath + "/" + index + "/_search").build(); + logger.trace("Making call {}", uri); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + // TODO refactor some of this into re-usable building blocks + gen.writeStartObject().writeStartObject("query").writeStartObject("bool"); + if (fields.contains("text")) { + // TODO consider default field: this would need to be set by index, but the + // default of * makes more sense to me... + String text = query.getString("text"); + gen.writeStartObject("must").writeStartObject("simple_query_string").write("query", text).writeEnd() + .writeEnd(); + } + gen.writeStartArray("filter"); + Long lowerTime = null; + Long upperTime = null; + if (fields.contains("lower")) { + lowerTime = decodeTime(query.getString("lower")); + } + if (fields.contains("upper")) { + upperTime = decodeTime(query.getString("upper")); + } + if (lowerTime != null || upperTime != null) { + gen.writeStartObject().writeStartObject("bool").writeStartArray("should"); + if (lowerTime != null) { + gen.writeStartObject().writeStartObject("range").writeStartObject("date") + .write("gte", lowerTime).writeEnd().writeEnd().writeEnd(); + gen.writeStartObject().writeStartObject("range").writeStartObject("startDate") + .write("gte", lowerTime).writeEnd().writeEnd().writeEnd(); + } + if (upperTime != null) { + gen.writeStartObject().writeStartObject("range").writeStartObject("date") + .write("lte", upperTime).writeEnd().writeEnd().writeEnd(); + gen.writeStartObject().writeStartObject("range").writeStartObject("endDate") + .write("lte", upperTime).writeEnd().writeEnd().writeEnd(); + } + gen.writeEnd().writeEnd().writeEnd(); + } + if (fields.contains("user")) { + String user = query.getString("user"); + gen.writeStartObject().writeStartObject("match").writeStartObject("userName").write("query", user) + .write("operator", "and").writeEnd().writeEnd().writeEnd(); + } + if (fields.contains("userFullName")) { + String userFullName = query.getString("userFullName"); + gen.writeStartObject().writeStartObject("simple_query_string").write("query", userFullName) + .writeStartArray("fields").write("userFullName").writeEnd().writeEnd().writeEnd(); + } + if (fields.contains("samples")) { + JsonArray samples = query.getJsonArray("samples"); + for (int i = 0; i < samples.size(); i++) { + String sample = samples.getString(i); + gen.writeStartObject().writeStartObject("simple_query_string").write("query", sample) + .writeStartArray("fields").write("sampleText").writeEnd().writeEnd().writeEnd(); + } + } + if (fields.contains("parameters")) { + for (JsonValue parameterValue : query.getJsonArray("parameters")) { + JsonObject parameterObject = (JsonObject) parameterValue; + String name = parameterObject.getString("name", null); + String units = parameterObject.getString("units", null); + String stringValue = parameterObject.getString("stringValue", null); + Long lowerDate = decodeTime(parameterObject.getString("lowerDateValue", null)); + Long upperDate = decodeTime(parameterObject.getString("upperDateValue", null)); + JsonNumber lowerNumeric = parameterObject.getJsonNumber("lowerNumericValue"); + JsonNumber upperNumeric = parameterObject.getJsonNumber("upperNumericValue"); + gen.writeStartObject().writeStartObject("bool").writeStartArray("must"); + if (name != null) { + gen.writeStartObject().writeStartObject("match").writeStartObject("parameterName") + .write("query", name).write("operator", "and").writeEnd().writeEnd().writeEnd(); + } + if (units != null) { + gen.writeStartObject().writeStartObject("match").writeStartObject("parameterUnits") + .write("query", units).write("operator", "and").writeEnd().writeEnd().writeEnd(); + } + if (stringValue != null) { + gen.writeStartObject().writeStartObject("match").writeStartObject("parameterStringValue") + .write("query", stringValue).write("operator", "and").writeEnd().writeEnd() + .writeEnd(); + } else if (lowerDate != null && upperDate != null) { + gen.writeStartObject().writeStartObject("range").writeStartObject("parameterDateValue") + .write("gte", lowerDate).write("lte", upperDate).writeEnd().writeEnd().writeEnd(); + } else if (lowerNumeric != null && upperNumeric != null) { + gen.writeStartObject().writeStartObject("range").writeStartObject("parameterNumericValue") + .write("gte", lowerNumeric).write("lte", upperNumeric).writeEnd().writeEnd() + .writeEnd(); + } + gen.writeEnd().writeEnd().writeEnd(); + } + } + gen.writeEnd().writeEnd().writeEnd().writeEnd(); + } + // TODO build returned results + SearchResult result = new SearchResult(); + List entities = result.getResults(); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(baos.toString())); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + JsonReader jsonReader = Json.createReader(response.getEntity().getContent()); + JsonObject jsonObject = jsonReader.readObject(); + JsonArray hits = jsonObject.getJsonObject("hits").getJsonArray("hits"); + for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { + entities.add(new ScoredEntityBaseBean(hit.getString("_id"), hit.getJsonNumber("_score").doubleValue())); + } + } + return result; + } catch (IOException | URISyntaxException | ParseException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } public void lock(String entityName) throws IcatException { logger.info("Manually locking index not supported, no request sent"); @@ -96,10 +358,75 @@ public void unlock(String entityName) throws IcatException { logger.info("Manually unlocking index not supported, no request sent"); } - public abstract void modify(String json) throws IcatException; + public void modify(String json) throws IcatException { + // TODO replace other places with this format + // TODO this assumes simple update/create with no relation + // Format should be [{"index": "investigation", "id": "123", "document": {}}, ...] + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + logger.debug("modify: {}", json); + StringBuilder sb = new StringBuilder(); + JsonReader jsonReader = Json.createReader(new StringReader(json)); + JsonArray outerArray = jsonReader.readArray(); + for (JsonObject operation : outerArray.getValuesAs(JsonObject.class)) { + String index = operation.getString("index", null); + String id = operation.getString("id", null); + JsonObject document = operation.getJsonObject("document"); + if (index == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Cannot modify a document without the target index"); + } + if (document == null) { + // Delete + if (id == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Cannot delete a document without an id"); + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject().writeStartObject("delete").write("_index", index).write("_id", id).writeEnd().writeEnd(); + } + sb.append(baos.toString()).append("\n"); + } else { + if (id == null) { + // Create + id = document.getString("id", null); + if (id == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Cannot index a document without an id"); + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject().writeStartObject("create").write("_index", index).write("_id", id).writeEnd().writeEnd(); + } + sb.append(baos.toString()).append("\n"); + sb.append(document.toString()).append("\n"); + } else { + // Update + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject().writeStartObject("update").write("_index", index).write("_id", id).writeEnd().writeEnd(); + } + sb.append(baos.toString()).append("\n"); + sb.append(document.toString()).append("\n"); + } + } + + } + URI uri = new URIBuilder(server).setPath(basePath + "/_bulk").build(); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(sb.toString())); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } catch (IOException | URISyntaxException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + // TODO Remove? /** * Legacy function for building a Query from individual arguments + * * @param target * @param user * @param text @@ -123,10 +450,10 @@ public static JsonObject buildQuery(String target, String user, String text, Dat builder.add("text", text); } if (lower != null) { - builder.add("lower", LuceneApi.enc(lower)); + builder.add("lower", encodeDate(lower)); } if (upper != null) { - builder.add("upper", LuceneApi.enc(upper)); + builder.add("upper", encodeDate(upper)); } if (parameters != null && !parameters.isEmpty()) { JsonArrayBuilder parametersBuilder = Json.createArrayBuilder(); @@ -142,10 +469,10 @@ public static JsonObject buildQuery(String target, String user, String text, Dat parameterBuilder.add("stringValue", parameter.stringValue); } if (parameter.lowerDateValue != null) { - parameterBuilder.add("lowerDateValue", LuceneApi.enc(parameter.lowerDateValue)); + parameterBuilder.add("lowerDateValue", encodeDate(parameter.lowerDateValue)); } if (parameter.upperDateValue != null) { - parameterBuilder.add("upperDateValue", LuceneApi.enc(parameter.upperDateValue)); + parameterBuilder.add("upperDateValue", encodeDate(parameter.upperDateValue)); } if (parameter.lowerNumericValue != null) { parameterBuilder.add("lowerNumericValue", parameter.lowerNumericValue); diff --git a/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java b/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java index e3881306..83648bc5 100644 --- a/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java @@ -75,7 +75,7 @@ public void modify() throws IcatException { searchApi.encodeTextField(gen, "text", "Elephants and Aardvarks"); searchApi.encodeStringField(gen, "startDate", new Date()); searchApi.encodeStringField(gen, "endDate", new Date()); - searchApi.encodeStoredId(gen, 42L); + searchApi.encodeStringField(gen, "id", 42L, true); searchApi.encodeStringField(gen, "dataset", 2001L); gen.writeEnd(); } @@ -481,7 +481,7 @@ private void populate() throws IcatException { searchApi.encodeTextField(gen, "text", word); searchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); searchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); - searchApi.encodeStoredId(gen, new Long(i)); + searchApi.encodeStringField(gen, "id", new Long(i), true); searchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); gen.writeEnd(); gen.writeEnd(); @@ -520,7 +520,7 @@ private void populate() throws IcatException { searchApi.encodeTextField(gen, "text", word); searchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); searchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); - searchApi.encodeStoredId(gen, new Long(i)); + searchApi.encodeStringField(gen, "id", new Long(i), true); searchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); searchApi.encodeStringField(gen, "investigation", new Long(i % NUMINV)); gen.writeEnd(); @@ -556,7 +556,7 @@ private void populate() throws IcatException { gen.writeStartArray(); searchApi.encodeTextField(gen, "text", word); searchApi.encodeStringField(gen, "date", new Date(now + i * 60000)); - searchApi.encodeStoredId(gen, new Long(i)); + searchApi.encodeStringField(gen, "id", new Long(i), true); searchApi.encodeStringField(gen, "dataset", new Long(i % NUMDS)); searchApi.encodeStringField(gen, "investigation", new Long((i % NUMDS) % NUMINV)); gen.writeEnd(); diff --git a/src/test/java/org/icatproject/core/manager/TestLucene.java b/src/test/java/org/icatproject/core/manager/TestLucene.java index 865cf3af..da6d3192 100644 --- a/src/test/java/org/icatproject/core/manager/TestLucene.java +++ b/src/test/java/org/icatproject/core/manager/TestLucene.java @@ -88,7 +88,7 @@ public void modify() throws IcatException { luceneApi.encodeTextField(gen, "text", "Elephants and Aardvarks"); luceneApi.encodeStringField(gen, "startDate", new Date()); luceneApi.encodeStringField(gen, "endDate", new Date()); - luceneApi.encodeStoredId(gen, 42L); + luceneApi.encodeStringField(gen, "id", 42L, true); luceneApi.encodeStringField(gen, "dataset", 2001L); gen.writeEnd(); } @@ -505,7 +505,7 @@ private void populate() throws IcatException { luceneApi.encodeTextField(gen, "text", word); luceneApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); luceneApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); - luceneApi.encodeStoredId(gen, new Long(i)); + luceneApi.encodeStringField(gen, "id", new Long(i), true); luceneApi.encodeSortedDocValuesField(gen, "id", new Long(i)); gen.writeEnd(); System.out.println("INVESTIGATION '" + word + "' " + new Date(now + i * 60000) + " " + i); @@ -537,7 +537,7 @@ private void populate() throws IcatException { luceneApi.encodeTextField(gen, "text", word); luceneApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); luceneApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); - luceneApi.encodeStoredId(gen, new Long(i)); + luceneApi.encodeStringField(gen, "id", new Long(i), true); luceneApi.encodeSortedDocValuesField(gen, "id", new Long(i)); luceneApi.encodeStringField(gen, "investigation", new Long(i % NUMINV)); gen.writeEnd(); @@ -569,7 +569,7 @@ private void populate() throws IcatException { gen.writeStartArray(); luceneApi.encodeTextField(gen, "text", word); luceneApi.encodeStringField(gen, "date", new Date(now + i * 60000)); - luceneApi.encodeStoredId(gen, new Long(i)); + luceneApi.encodeStringField(gen, "id", new Long(i), true); luceneApi.encodeStringField(gen, "dataset", new Long(i % NUMDS)); gen.writeEnd(); System.out.println("DATAFILE '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMDS); From 07a42a312a51ecc34d00f7c37b98fa712096fca7 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Sat, 26 Mar 2022 00:11:29 +0000 Subject: [PATCH 10/51] Add endpoint allowing fields and searchAfter #267 --- .../org/icatproject/core/entity/Datafile.java | 37 ++ .../org/icatproject/core/entity/Dataset.java | 34 ++ .../core/entity/Investigation.java | 50 ++- .../core/manager/ElasticsearchApi.java | 260 +++++------ .../core/manager/ElasticsearchDocument.java | 4 +- .../core/manager/EntityBeanManager.java | 91 +++- .../icatproject/core/manager/GateKeeper.java | 12 + .../icatproject/core/manager/LuceneApi.java | 154 +++---- .../core/manager/ScoredEntityBaseBean.java | 19 +- .../icatproject/core/manager/SearchApi.java | 54 ++- .../core/manager/SearchManager.java | 44 +- .../core/manager/SearchResult.java | 17 +- .../org/icatproject/exposed/ICATRest.java | 186 ++++++++ .../icatproject/core/manager/TestLucene.java | 424 ++++++++++-------- 14 files changed, 919 insertions(+), 467 deletions(-) diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index c3499904..adcca7a5 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -3,7 +3,9 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; @@ -20,7 +22,10 @@ import javax.persistence.UniqueConstraint; import javax.xml.bind.annotation.XmlRootElement; +import org.icatproject.core.IcatException; +import org.icatproject.core.manager.EntityInfoHandler; import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.EntityInfoHandler.Relationship; @Comment("A data file") @SuppressWarnings("serial") @@ -77,6 +82,8 @@ public class Datafile extends EntityBaseBean implements Serializable { @OneToMany(cascade = CascadeType.ALL, mappedBy = "sourceDatafile") private List sourceDatafiles = new ArrayList(); + private static final Map documentFields = new HashMap<>(); + /* Needed for JPA */ public Datafile() { } @@ -220,4 +227,34 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { // TODO User and Parameter support for Elasticsearch } + + /** + * Gets the fields used in the search component for this entity, and the + * relationships that would restrict the content of those fields. + * + * @return Map of field names (as they appear on the search document) against + * the Relationships that need to be allowed for that field to be + * viewable. If there are no restrictive relationships, then the value + * will be null. + * @throws IcatException If the EntityInfoHandler cannot find one of the + * Relationships. + */ + public static Map getDocumentFields() throws IcatException { + if (documentFields.size() == 0) { + EntityInfoHandler eiHandler = EntityInfoHandler.getInstance(); + Relationship[] textRelationships = { + eiHandler.getRelationshipsByName(Datafile.class).get("datafileFormat") }; + Relationship[] investigationRelationships = { + eiHandler.getRelationshipsByName(Datafile.class).get("dataset"), + eiHandler.getRelationshipsByName(Dataset.class).get("investigation") }; // TODO check if we need this + documentFields.put("text", textRelationships); + documentFields.put("name", null); + documentFields.put("date", null); + documentFields.put("id", null); + documentFields.put("dataset", null); + documentFields.put("investigation", investigationRelationships); + } + return documentFields; + } + } diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index d836308c..c16d11e7 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -3,7 +3,9 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; @@ -19,7 +21,10 @@ import javax.persistence.UniqueConstraint; import javax.xml.bind.annotation.XmlRootElement; +import org.icatproject.core.IcatException; +import org.icatproject.core.manager.EntityInfoHandler; import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.EntityInfoHandler.Relationship; @Comment("A collection of data files and part of an investigation") @SuppressWarnings("serial") @@ -81,6 +86,8 @@ public void setDataCollectionDatasets(List dataCollection @ManyToOne(fetch = FetchType.LAZY) private DatasetType type; + private static final Map documentFields = new HashMap<>(); + /* Needed for JPA */ public Dataset() { } @@ -223,4 +230,31 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { // TODO User, Parameter and Sample support for Elasticsearch } + /** + * Gets the fields used in the search component for this entity, and the + * relationships that would restrict the content of those fields. + * + * @return Map of field names (as they appear on the search document) against + * the Relationships that need to be allowed for that field to be + * viewable. If there are no restrictive relationships, then the value + * will be null. + * @throws IcatException If the EntityInfoHandler cannot find one of the + * Relationships. + */ + public static Map getDocumentFields() throws IcatException { + if (documentFields.size() == 0) { + EntityInfoHandler eiHandler = EntityInfoHandler.getInstance(); + Relationship[] textRelationships = { eiHandler.getRelationshipsByName(Dataset.class).get("type") }; + Relationship[] investigationRelationships = { + eiHandler.getRelationshipsByName(Dataset.class).get("investigation") }; + documentFields.put("text", textRelationships); + documentFields.put("name", null); + documentFields.put("startDate", null); + documentFields.put("endDate", null); + documentFields.put("id", null); + documentFields.put("investigation", investigationRelationships); + } + return documentFields; + } + } diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index e07acec3..3239e36c 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -3,7 +3,9 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; @@ -18,7 +20,10 @@ import javax.persistence.TemporalType; import javax.persistence.UniqueConstraint; +import org.icatproject.core.IcatException; +import org.icatproject.core.manager.EntityInfoHandler; import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.EntityInfoHandler.Relationship; @Comment("An investigation or experiment") @SuppressWarnings("serial") @@ -93,6 +98,8 @@ public class Investigation extends EntityBaseBean implements Serializable { @Column(name = "VISIT_ID", nullable = false) private String visitId; + private static final Map documentFields = new HashMap<>(); + /* Needed for JPA */ public Investigation() { } @@ -285,25 +292,29 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { } investigationUsers.forEach((investigationUser) -> { - searchApi.encodeStringField(gen, "userName", investigationUser.getUser().getName()); - searchApi.encodeTextField(gen, "userFullName", investigationUser.getUser().getFullName()); + searchApi.encodeStringField(gen, "userName", investigationUser.getUser().getName()); + searchApi.encodeTextField(gen, "userFullName", investigationUser.getUser().getFullName()); }); - samples.forEach((sample) -> { - // searchApi.encodeSortedSetDocValuesFacetField(gen, "sampleName", sample.getName()); - searchApi.encodeTextField(gen, "sampleText", sample.getDocText()); + // searchApi.encodeSortedSetDocValuesFacetField(gen, "sampleName", + // sample.getName()); + searchApi.encodeTextField(gen, "sampleText", sample.getDocText()); }); for (InvestigationParameter parameter : parameters) { ParameterType type = parameter.type; String parameterName = type.getName(); String parameterUnits = type.getUnits(); - // searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterName", parameterName); + // searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterName", + // parameterName); + searchApi.encodeStringField(gen, "parameterName", parameterName); searchApi.encodeStringField(gen, "parameterUnits", parameterUnits); // TODO make all value types facetable... if (type.getValueType() == ParameterValueType.STRING) { - // searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterStringValue", parameter.getStringValue()); + // searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterStringValue", + // parameter.getStringValue()); + searchApi.encodeStringField(gen, "parameterStringValue", parameter.getStringValue()); } else if (type.getValueType() == ParameterValueType.DATE_AND_TIME) { searchApi.encodeStringField(gen, "parameterDateValue", parameter.getDateTimeValue()); } else if (type.getValueType() == ParameterValueType.NUMERIC) { @@ -315,4 +326,29 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { searchApi.encodeStringField(gen, "id", id, true); } + + /** + * Gets the fields used in the search component for this entity, and the + * relationships that would restrict the content of those fields. + * + * @return Map of field names (as they appear on the search document) against + * the Relationships that need to be allowed for that field to be + * viewable. If there are no restrictive relationships, then the value + * will be null. + * @throws IcatException If the EntityInfoHandler cannot find one of the + * Relationships. + */ + public static Map getDocumentFields() throws IcatException { + if (documentFields.size() == 0) { + EntityInfoHandler eiHandler = EntityInfoHandler.getInstance(); + Relationship[] textRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("type"), + eiHandler.getRelationshipsByName(Investigation.class).get("facility") }; + documentFields.put("text", textRelationships); + documentFields.put("name", null); + documentFields.put("startDate", null); + documentFields.put("endDate", null); + documentFields.put("id", null); + } + return documentFields; + } } diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java index 595374a9..5a294c2e 100644 --- a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java +++ b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java @@ -255,137 +255,137 @@ public List facetSearch(JsonObject facetQuery, int maxResults, i } } - @Override - public SearchResult getResults(JsonObject query, int maxResults, String sort) - throws IcatException { - // TODO sort argument not supported - try { - String index; - if (query.keySet().contains("target")) { - index = query.getString("target").toLowerCase(); - } else { - index = query.getString("_all"); - } - OpenPointInTimeResponse pitResponse = client.openPointInTime(p -> p - .index(index) - .keepAlive(t -> t.time("1m"))); - String pit = pitResponse.id(); - pitMap.put(pit, 0); - return getResults(pit, query, maxResults); - } catch (ElasticsearchException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } + // @Override + // public SearchResult getResults(JsonObject query, int maxResults, String sort) + // throws IcatException { + // // TODO sort argument not supported + // try { + // String index; + // if (query.keySet().contains("target")) { + // index = query.getString("target").toLowerCase(); + // } else { + // index = query.getString("_all"); + // } + // OpenPointInTimeResponse pitResponse = client.openPointInTime(p -> p + // .index(index) + // .keepAlive(t -> t.time("1m"))); + // String pit = pitResponse.id(); + // pitMap.put(pit, 0); + // return getResults(pit, query, maxResults); + // } catch (ElasticsearchException | IOException e) { + // throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + // } + // } - @Override - public SearchResult getResults(String uid, JsonObject query, int maxResults) - throws IcatException { - try { - logger.debug("getResults for query: {}", query.toString()); - Set fields = query.keySet(); - BoolQuery.Builder builder = new BoolQuery.Builder(); - for (String field : fields) { - if (field.equals("text")) { - String text = query.getString("text"); - builder.must(m -> m.queryString(q -> q.defaultField("text").query(text))); - } else if (field.equals("lower")) { - Long time = decodeTime(query.getString("lower")); - builder.must(m -> m - .bool(b -> b - .should(s -> s - .range(r -> r - .field("date") - .gte(JsonData.of(time)))) - .should(s -> s - .range(r -> r - .field("startDate") - .gte(JsonData.of(time)))))); - } else if (field.equals("upper")) { - Long time = decodeTime(query.getString("upper")); - builder.must(m -> m - .bool(b -> b - .should(s -> s - .range(r -> r - .field("date") - .lte(JsonData.of(time)))) - .should(s -> s - .range(r -> r - .field("endDate") - .lte(JsonData.of(time)))))); - } else if (field.equals("user")) { - String user = query.getString("user"); - builder.filter(f -> f.match(t -> t - .field("userName") - .operator(Operator.And) - .query(q -> q.stringValue(user)))); - } else if (field.equals("userFullName")) { - String userFullName = query.getString("userFullName"); - builder.filter(f -> f.queryString(q -> q.defaultField("userFullName").query(userFullName))); - } else if (field.equals("samples")) { - JsonArray samples = query.getJsonArray("samples"); - for (int i = 0; i < samples.size(); i++) { - String sample = samples.getString(i); - builder.filter( - f -> f.queryString(q -> q.defaultField("sampleText").query(sample))); - } - } else if (field.equals("parameters")) { - for (JsonValue parameterValue : query.getJsonArray("parameters")) { - // TODO there are more things to support and consider here... e.g. parameters - // with a numeric range not a numeric value - BoolQuery.Builder parameterBuilder = new BoolQuery.Builder(); - JsonObject parameterObject = (JsonObject) parameterValue; - String name = parameterObject.getString("name", null); - String units = parameterObject.getString("units", null); - String stringValue = parameterObject.getString("stringValue", null); - Long lowerDate = decodeTime(parameterObject.getString("lowerDateValue", null)); - Long upperDate = decodeTime(parameterObject.getString("upperDateValue", null)); - JsonNumber lowerNumeric = parameterObject.getJsonNumber("lowerNumericValue"); - JsonNumber upperNumeric = parameterObject.getJsonNumber("upperNumericValue"); - if (name != null) { - parameterBuilder.must(m -> m.match(a -> a.field("parameterName").operator(Operator.And) - .query(q -> q.stringValue(name)))); - } - if (units != null) { - parameterBuilder.must(m -> m.match(a -> a.field("parameterUnits").operator(Operator.And) - .query(q -> q.stringValue(units)))); - } - if (stringValue != null) { - parameterBuilder.must(m -> m.match(a -> a.field("parameterStringValue") - .operator(Operator.And).query(q -> q.stringValue(stringValue)))); - } else if (lowerDate != null && upperDate != null) { - parameterBuilder.must(m -> m.range(r -> r.field("parameterDateValue") - .gte(JsonData.of(lowerDate)).lte(JsonData.of(upperDate)))); - } else if (lowerNumeric != null && upperNumeric != null) { - parameterBuilder.must(m -> m.range( - r -> r.field("parameterNumericValue").gte(JsonData.of(lowerNumeric.doubleValue())) - .lte(JsonData.of(upperNumeric.doubleValue())))); - } - builder.filter(f -> f.bool(b -> parameterBuilder)); - } - // TODO consider support for other fields (would require dynamic fields) - } - } - Integer from = pitMap.get(uid); - SearchResponse response = client.search(s -> s - .size(maxResults) - .pit(p -> p.id(uid).keepAlive(t -> t.time("1m"))) - .query(q -> q.bool(builder.build())) - // TODO check the ordering? - .from(from) - .sort(o -> o.score(c -> c.order(SortOrder.Desc))) - .sort(o -> o.field(f -> f.field("id").order(SortOrder.Asc))), ElasticsearchDocument.class); - SearchResult result = new SearchResult(); - result.setUid(uid); - pitMap.put(uid, from + maxResults); - List entities = result.getResults(); - for (Hit hit : response.hits().hits()) { - entities.add(new ScoredEntityBaseBean(Long.parseLong(hit.id()), hit.score().floatValue())); - } - return result; - } catch (ElasticsearchException | IOException | ParseException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } + // @Override + // public SearchResult getResults(String uid, JsonObject query, int maxResults) + // throws IcatException { + // try { + // logger.debug("getResults for query: {}", query.toString()); + // Set fields = query.keySet(); + // BoolQuery.Builder builder = new BoolQuery.Builder(); + // for (String field : fields) { + // if (field.equals("text")) { + // String text = query.getString("text"); + // builder.must(m -> m.queryString(q -> q.defaultField("text").query(text))); + // } else if (field.equals("lower")) { + // Long time = decodeTime(query.getString("lower")); + // builder.must(m -> m + // .bool(b -> b + // .should(s -> s + // .range(r -> r + // .field("date") + // .gte(JsonData.of(time)))) + // .should(s -> s + // .range(r -> r + // .field("startDate") + // .gte(JsonData.of(time)))))); + // } else if (field.equals("upper")) { + // Long time = decodeTime(query.getString("upper")); + // builder.must(m -> m + // .bool(b -> b + // .should(s -> s + // .range(r -> r + // .field("date") + // .lte(JsonData.of(time)))) + // .should(s -> s + // .range(r -> r + // .field("endDate") + // .lte(JsonData.of(time)))))); + // } else if (field.equals("user")) { + // String user = query.getString("user"); + // builder.filter(f -> f.match(t -> t + // .field("userName") + // .operator(Operator.And) + // .query(q -> q.stringValue(user)))); + // } else if (field.equals("userFullName")) { + // String userFullName = query.getString("userFullName"); + // builder.filter(f -> f.queryString(q -> q.defaultField("userFullName").query(userFullName))); + // } else if (field.equals("samples")) { + // JsonArray samples = query.getJsonArray("samples"); + // for (int i = 0; i < samples.size(); i++) { + // String sample = samples.getString(i); + // builder.filter( + // f -> f.queryString(q -> q.defaultField("sampleText").query(sample))); + // } + // } else if (field.equals("parameters")) { + // for (JsonValue parameterValue : query.getJsonArray("parameters")) { + // // TODO there are more things to support and consider here... e.g. parameters + // // with a numeric range not a numeric value + // BoolQuery.Builder parameterBuilder = new BoolQuery.Builder(); + // JsonObject parameterObject = (JsonObject) parameterValue; + // String name = parameterObject.getString("name", null); + // String units = parameterObject.getString("units", null); + // String stringValue = parameterObject.getString("stringValue", null); + // Long lowerDate = decodeTime(parameterObject.getString("lowerDateValue", null)); + // Long upperDate = decodeTime(parameterObject.getString("upperDateValue", null)); + // JsonNumber lowerNumeric = parameterObject.getJsonNumber("lowerNumericValue"); + // JsonNumber upperNumeric = parameterObject.getJsonNumber("upperNumericValue"); + // if (name != null) { + // parameterBuilder.must(m -> m.match(a -> a.field("parameterName").operator(Operator.And) + // .query(q -> q.stringValue(name)))); + // } + // if (units != null) { + // parameterBuilder.must(m -> m.match(a -> a.field("parameterUnits").operator(Operator.And) + // .query(q -> q.stringValue(units)))); + // } + // if (stringValue != null) { + // parameterBuilder.must(m -> m.match(a -> a.field("parameterStringValue") + // .operator(Operator.And).query(q -> q.stringValue(stringValue)))); + // } else if (lowerDate != null && upperDate != null) { + // parameterBuilder.must(m -> m.range(r -> r.field("parameterDateValue") + // .gte(JsonData.of(lowerDate)).lte(JsonData.of(upperDate)))); + // } else if (lowerNumeric != null && upperNumeric != null) { + // parameterBuilder.must(m -> m.range( + // r -> r.field("parameterNumericValue").gte(JsonData.of(lowerNumeric.doubleValue())) + // .lte(JsonData.of(upperNumeric.doubleValue())))); + // } + // builder.filter(f -> f.bool(b -> parameterBuilder)); + // } + // // TODO consider support for other fields (would require dynamic fields) + // } + // } + // Integer from = pitMap.get(uid); + // SearchResponse response = client.search(s -> s + // .size(maxResults) + // .pit(p -> p.id(uid).keepAlive(t -> t.time("1m"))) + // .query(q -> q.bool(builder.build())) + // // TODO check the ordering? + // .from(from) + // .sort(o -> o.score(c -> c.order(SortOrder.Desc))) + // .sort(o -> o.field(f -> f.field("id").order(SortOrder.Asc))), ElasticsearchDocument.class); + // SearchResult result = new SearchResult(); + // // result.setUid(uid); + // pitMap.put(uid, from + maxResults); + // List entities = result.getResults(); + // for (Hit hit : response.hits().hits()) { + // entities.add(new ScoredEntityBaseBean(Long.parseLong(hit.id()), hit.score().floatValue(), "")); + // } + // return result; + // } catch (ElasticsearchException | IOException | ParseException e) { + // throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + // } + // } @Override public void modify(String json) throws IcatException { diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java b/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java index 67c7c79b..3ae9c81e 100644 --- a/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java +++ b/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java @@ -63,7 +63,7 @@ public ElasticsearchDocument(JsonArray jsonArray) throws IcatException { if (fieldEntry.getValue().getValueType().equals(ValueType.STRING)) { id = Long.valueOf(fieldObject.getString("id")); } else if (fieldEntry.getValue().getValueType().equals(ValueType.NUMBER)) { - id = fieldObject.getJsonNumber("id").longValue(); + id = fieldObject.getJsonNumber("id").longValueExact(); } } else if (fieldEntry.getKey().equals("investigation")) { if (fieldEntry.getValue().getValueType().equals(ValueType.STRING)) { @@ -121,7 +121,7 @@ public ElasticsearchDocument(JsonArray jsonArray, String index, String parentInd if (fieldEntry.getValue().getValueType().equals(ValueType.STRING)) { id = Long.valueOf(fieldObject.getString(parentIndex)); } else if (fieldEntry.getValue().getValueType().equals(ValueType.NUMBER)) { - id = fieldObject.getJsonNumber(parentIndex).longValue(); + id = fieldObject.getJsonNumber(parentIndex).longValueExact(); } } else if (fieldEntry.getKey().equals("userName")) { userName.add(fieldObject.getString("userName")); diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 26236f1c..b38741c9 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -779,7 +779,7 @@ private void exportTable(String beanName, Set ids, OutputStream output, } } - private void filterReadAccess(List results, List allResults, + private ScoredEntityBaseBean filterReadAccess(List results, List allResults, int maxCount, String userId, EntityManager manager, Class klass) throws IcatException { @@ -790,19 +790,20 @@ private void filterReadAccess(List results, List maxEntities) { throw new IcatException(IcatExceptionType.VALIDATION, "attempt to return more than " + maxEntities + " entities"); } if (results.size() == maxCount) { - break; + return sr; } } catch (IcatException e) { // Nothing to do } } } + return null; } private EntityBaseBean find(EntityBaseBean bean, EntityManager manager) throws IcatException { @@ -1388,33 +1389,40 @@ public void searchCommit() throws IcatException { } } - public List freeTextSearch(String userName, JsonObject jo, int maxCount, String sort, + public List freeTextSearch(String userName, JsonObject jo, int limit, String sort, EntityManager manager, String ip, Class klass) throws IcatException { long startMillis = log ? System.currentTimeMillis() : 0; List results = new ArrayList<>(); + String searchAfter = null; + String lastSearchAfter = null; if (searchActive) { - SearchResult last = null; - String uid = null; + SearchResult lastSearchResult = null; List allResults = Collections.emptyList(); /* * As results may be rejected and maxCount may be 1 ensure that we * don't make a huge number of calls to search engine */ - int blockSize = Math.max(1000, maxCount); + int blockSize = Math.max(1000, limit); do { - if (last == null) { - // Only need to apply the sort on initial search - last = searchManager.freeTextSearch(jo, blockSize, sort); - uid = last.getUid(); + List fields = SearchManager.getPublicSearchFields(gateKeeper, klass.getSimpleName()); + lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, fields); + allResults = lastSearchResult.getResults(); + ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, limit, userName, manager, klass); + if (lastBean == null) { + // Haven't stopped early, so use the Lucene provided searchAfter document + lastSearchAfter = lastSearchResult.getSearchAfter(); + if (lastSearchAfter == null) { + break; // If searchAfter is null, we ran out of results so stop here + } + searchAfter = lastSearchAfter.toString(); } else { - last = searchManager.freeTextSearch(uid, jo, blockSize); + // Have stopped early by reaching the limit, so build a searchAfter document + lastSearchAfter = searchManager.buildSearchAfter(lastBean, sort); + break; } - allResults = last.getResults(); - filterReadAccess(results, allResults, maxCount, userName, manager, klass); - } while (results.size() != maxCount && allResults.size() == blockSize); - /* failing retrieval calls clean up before throwing */ - searchManager.freeSearcher(uid); + } while (results.size() < limit); + } // TODO move this to somewhere we can manually call it // TODO also need to fix it for Elasticsearch (cannot use text fields for @@ -1438,7 +1446,6 @@ public List freeTextSearch(String userName, JsonObject jo, // } // } // } - } if (logRequests.contains("R")) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) { @@ -1454,6 +1461,54 @@ public List freeTextSearch(String userName, JsonObject jo, return results; } + public SearchResult freeTextSearchDocs(String userName, JsonObject jo, String searchAfter, int limit, String sort, + EntityManager manager, String ip, Class klass) throws IcatException { + long startMillis = log ? System.currentTimeMillis() : 0; + List results = new ArrayList<>(); + String lastSearchAfter = null; + if (searchActive) { + SearchResult lastSearchResult = null; + List allResults = Collections.emptyList(); + /* + * As results may be rejected and maxCount may be 1 ensure that we + * don't make a huge number of calls to search engine + */ + int blockSize = Math.max(1000, limit); + + do { + List fields = SearchManager.getPublicSearchFields(gateKeeper, klass.getSimpleName()); + lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, fields); + allResults = lastSearchResult.getResults(); + ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, limit, userName, manager, klass); + if (lastBean == null) { + // Haven't stopped early, so use the Lucene provided searchAfter document + lastSearchAfter = lastSearchResult.getSearchAfter(); + if (lastSearchAfter == null) { + break; // If searchAfter is null, we ran out of results so stop here + } + searchAfter = lastSearchAfter.toString(); + } else { + // Have stopped early by reaching the limit, so build a searchAfter document + lastSearchAfter = searchManager.buildSearchAfter(lastBean, sort); + break; + } + } while (results.size() < limit); + } + if (logRequests.contains("R")) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) { + gen.write("userName", userName); + if (results.size() > 0) { + gen.write("entityId", results.get(0).getEntityBaseBeanId()); + } + gen.writeEnd(); + } + transmitter.processMessage("freeTextSearchDocs", ip, baos.toString(), startMillis); + } + logger.debug("Returning {} results", results.size()); + return new SearchResult(lastSearchAfter, results); + } + public List searchGetPopulating() { if (searchActive) { return searchManager.getPopulating(); diff --git a/src/main/java/org/icatproject/core/manager/GateKeeper.java b/src/main/java/org/icatproject/core/manager/GateKeeper.java index d01470f8..25b280b1 100644 --- a/src/main/java/org/icatproject/core/manager/GateKeeper.java +++ b/src/main/java/org/icatproject/core/manager/GateKeeper.java @@ -93,6 +93,8 @@ public int compare(String o1, String o2) { private boolean publicTablesStale; + private boolean publicSearchFieldsStale; + private Map cluster; private String basePath = "/icat"; @@ -164,6 +166,10 @@ public Set getPublicTables() { return publicTables; } + public Boolean getPublicSearchFieldsStale() { + return publicSearchFieldsStale; + } + public List getReadable(String userId, List beans, EntityManager manager) { if (beans.size() == 0) { @@ -337,10 +343,16 @@ public boolean isAccessAllowed(String user, EntityBaseBean object, AccessType ac public void markPublicStepsStale() { publicStepsStale = true; + publicSearchFieldsStale = true; } public void markPublicTablesStale() { publicTablesStale = true; + publicSearchFieldsStale = true; + } + + public void markPublicSearchFieldsFresh() { + publicSearchFieldsStale = false; } /** diff --git a/src/main/java/org/icatproject/core/manager/LuceneApi.java b/src/main/java/org/icatproject/core/manager/LuceneApi.java index 89d52e19..6fca01f5 100644 --- a/src/main/java/org/icatproject/core/manager/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/LuceneApi.java @@ -1,5 +1,6 @@ package org.icatproject.core.manager; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.PipedInputStream; import java.io.PipedOutputStream; @@ -12,7 +13,10 @@ import java.util.concurrent.ExecutorService; import javax.json.Json; +import javax.json.JsonArrayBuilder; import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import javax.json.JsonReader; import javax.json.stream.JsonGenerator; import javax.json.stream.JsonParser; import javax.json.stream.JsonParser.Event; @@ -20,8 +24,6 @@ import javax.ws.rs.core.MediaType; import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.InputStreamEntity; @@ -74,6 +76,13 @@ public void encodeSortedDocValuesField(JsonGenerator gen, String name, Long valu .writeEnd(); } + @Override + public void encodeSortedDocValuesField(JsonGenerator gen, String name, String value) { + encodeStringField(gen, name, value); + gen.writeStartObject().write("type", "SortedDocValuesField").write("name", name).write("value", value) + .writeEnd(); + } + @Override public void encodeDoublePoint(JsonGenerator gen, String name, Double value) { gen.writeStartObject().write("type", "DoublePoint").write("name", name).write("value", value) @@ -144,6 +153,31 @@ public void addNow(String entityName, List ids, EntityManager manager, } } + @Override + public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { + JsonObjectBuilder builder = Json.createObjectBuilder(); + builder.add("doc", lastBean.getEntityBaseBeanId()); + builder.add("shardIndex", -1); + if (!Float.isNaN(lastBean.getScore())) { + builder.add("score", lastBean.getScore()); + } + if (sort != null && !sort.equals("")) { + try (JsonReader reader = Json.createReader(new ByteArrayInputStream(sort.getBytes()))) { + JsonObject object = reader.readObject(); + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + for (String key : object.keySet()) { + if (!lastBean.getSource().keySet().contains(key)) { + throw new IcatException(IcatExceptionType.INTERNAL, "Cannot build searchAfter document from source as sorted field " + key + " missing."); + } + String value = lastBean.getSource().getString(key); + arrayBuilder.add(value); + } + builder.add("fields", arrayBuilder); + } + } + return builder.build().toString(); + } + @Override public void clear() throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { @@ -187,22 +221,6 @@ public List facetSearch(JsonObject facetQuery, int maxResults, i } } - @Override - public void freeSearcher(String uid) throws IcatException { - try { - URI uri = new URIBuilder(server).setPath(basePath + "/freeSearcher/" + uid).build(); - logger.trace("Making call {}", uri); - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - HttpDelete httpDelete = new HttpDelete(uri); - try (CloseableHttpResponse response = httpclient.execute(httpDelete)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } - } catch (URISyntaxException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - private List getFacets(URI uri, CloseableHttpClient httpclient, String facetQueryString) throws IcatException { logger.debug(facetQueryString); @@ -254,67 +272,29 @@ private List getFacets(URI uri, CloseableHttpClient httpclient, } @Override - public SearchResult getResults(JsonObject query, int maxResults, String sort) throws IcatException { + public SearchResult getResults(JsonObject query, String searchAfter, int blockSize, String sort, List fields) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { String indexPath = getTargetPath(query); URI uri = new URIBuilder(server).setPath(basePath + "/" + indexPath) - .setParameter("maxResults", Integer.toString(maxResults)) + .setParameter("search_after", searchAfter) + .setParameter("maxResults", Integer.toString(blockSize)) .setParameter("sort", sort).build(); - logger.trace("Making call {}", uri); - return getResults(uri, httpclient, query.toString()); - - } catch (IOException | URISyntaxException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - @Override - public SearchResult getResults(String uid, JsonObject query, int maxResults) throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - String indexPath = getTargetPath(query); - URI uri = new URIBuilder(server).setPath(basePath + "/" + indexPath + "/" + uid) - .setParameter("maxResults", Integer.toString(maxResults)).build(); - return getResults(uri, httpclient); + JsonObjectBuilder objectBuilder = Json.createObjectBuilder(); + objectBuilder.add("query", query); + if (fields != null && fields.size() > 0) { + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + fields.forEach((field) -> arrayBuilder.add(field)); + objectBuilder.add("fields", arrayBuilder.build()); + } + String queryString = objectBuilder.build().toString(); + logger.trace("Making call {} with queryString {}", uri, queryString); + return getResults(uri, httpclient, queryString); } catch (IOException | URISyntaxException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } } - private SearchResult getResults(URI uri, CloseableHttpClient httpclient) throws IcatException { - HttpGet httpGet = new HttpGet(uri); - SearchResult lsr = new SearchResult(); - List results = lsr.getResults(); - ParserState state = ParserState.None; - try (CloseableHttpResponse response = httpclient.execute(httpGet)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - try (JsonParser p = Json.createParser(response.getEntity().getContent())) { - String key = null; - while (p.hasNext()) { - Event e = p.next(); - if (e.equals(Event.KEY_NAME)) { - key = p.getString(); - } else if (state == ParserState.Results) { - if (e == (Event.START_ARRAY)) { - p.next(); - Long id = p.getLong(); - p.next(); - results.add(new ScoredEntityBaseBean(id, p.getBigDecimal().floatValue())); - p.next(); // skip the } - } - } else { // Not in results yet - if (e == Event.START_ARRAY && key.equals("results")) { - state = ParserState.Results; - } - } - } - } - } catch (IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - return lsr; - } - private SearchResult getResults(URI uri, CloseableHttpClient httpclient, String queryString) throws IcatException { logger.debug(queryString); @@ -326,32 +306,22 @@ private SearchResult getResults(URI uri, CloseableHttpClient httpclient, String SearchResult lsr = new SearchResult(); List results = lsr.getResults(); - ParserState state = ParserState.None; try (CloseableHttpResponse response = httpclient.execute(httpPost)) { Rest.checkStatus(response, IcatExceptionType.INTERNAL); - try (JsonParser p = Json.createParser(response.getEntity().getContent())) { - String key = null; - while (p.hasNext()) { - Event e = p.next(); - if (e.equals(Event.KEY_NAME)) { - key = p.getString(); - } else if (state == ParserState.Results) { - if (e == (Event.START_ARRAY)) { - p.next(); - Long id = p.getLong(); - p.next(); - results.add(new ScoredEntityBaseBean(id, p.getBigDecimal().floatValue())); - p.next(); // skip the } - } - } else { // Not in results yet - if (e == (Event.VALUE_NUMBER) && key.equals("uid")) { - lsr.setUid(String.valueOf(p.getLong())); - } else if (e == Event.START_ARRAY && key.equals("results")) { - state = ParserState.Results; - } - + try (JsonReader reader = Json.createReader(response.getEntity().getContent())) { + JsonObject responseObject = reader.readObject(); + List resultsArray = responseObject.getJsonArray("results").getValuesAs(JsonObject.class); + for (JsonObject resultObject: resultsArray) { + long id = resultObject.getJsonNumber("id").longValueExact(); + Float score = Float.NaN; + if (resultObject.keySet().contains("score")) { + score = resultObject.getJsonNumber("score").bigDecimalValue().floatValue(); } - + JsonObject source = resultObject.getJsonObject("source"); + results.add(new ScoredEntityBaseBean(id, score, source)); + } + if (responseObject.containsKey("search_after")) { + lsr.setSearchAfter(responseObject.getJsonObject("search_after").toString()); } } } diff --git a/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java b/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java index cf747d75..292f0597 100644 --- a/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java +++ b/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java @@ -1,18 +1,23 @@ package org.icatproject.core.manager; +import javax.json.JsonObject; + public class ScoredEntityBaseBean { private long entityBaseBeanId; private float score; + private JsonObject source; - public ScoredEntityBaseBean(String id, double score) { + public ScoredEntityBaseBean(String id, float score, JsonObject source) { this.entityBaseBeanId = Long.parseLong(id); - this.score = (float) score; + this.score = score; + this.source = source; } - public ScoredEntityBaseBean(long id, float score) { + public ScoredEntityBaseBean(long id, float score, JsonObject source) { this.entityBaseBeanId = id; this.score = score; + this.source = source; } public long getEntityBaseBeanId() { @@ -23,4 +28,12 @@ public float getScore() { return score; } + public JsonObject getSource() { + return source; + } + + public void setSource(JsonObject source) { + this.source = source; + } + } diff --git a/src/main/java/org/icatproject/core/manager/SearchApi.java b/src/main/java/org/icatproject/core/manager/SearchApi.java index 1476e9c8..71cf3e05 100644 --- a/src/main/java/org/icatproject/core/manager/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/SearchApi.java @@ -1,5 +1,6 @@ package org.icatproject.core.manager; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.StringReader; @@ -8,13 +9,10 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.TimeZone; import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicLong; import javax.json.Json; import javax.json.JsonArray; @@ -48,8 +46,6 @@ public abstract class SearchApi { protected static String matchAllQuery; protected URI server; - protected AtomicLong atomicLongUid = new AtomicLong(); - protected Map uidMap = new HashMap<>(); static { df = new SimpleDateFormat("yyyyMMddHHmm"); @@ -188,9 +184,32 @@ public void addNow(String entityName, List ids, EntityManager manager, modify(sb.toString()); } + public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { + if (sort != null && !sort.equals("")) { + try (JsonReader reader = Json.createReader(new ByteArrayInputStream(sort.getBytes()))) { + JsonObject object = reader.readObject(); + JsonArrayBuilder builder = Json.createArrayBuilder(); + for (String key : object.keySet()) { + if (!lastBean.getSource().keySet().contains(key)) { + throw new IcatException(IcatExceptionType.INTERNAL, "Cannot build searchAfter document from source as sorted field " + key + " missing."); + } + String value = lastBean.getSource().getString(key); + builder.add(value); + } + return builder.build().toString(); + } + } else { + JsonArrayBuilder builder = Json.createArrayBuilder(); + if (Float.isNaN(lastBean.getScore())) { + throw new IcatException(IcatExceptionType.INTERNAL, "Cannot build searchAfter document from source as score was NaN."); + } + builder.add(lastBean.getScore()); + builder.add(lastBean.getEntityBaseBeanId()); + return builder.build().toString(); + } + } + public void clear() throws IcatException { - atomicLongUid.set(0); - uidMap = new HashMap<>(); try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URI uri = new URIBuilder(server).setPath(basePath + "/_delete_by_query").build(); HttpPost httpPost = new HttpPost(uri); @@ -217,19 +236,28 @@ public void commit() throws IcatException { } public void freeSearcher(String uid) throws IcatException { - uidMap.remove(uid); + logger.info("Manually freeing searcher not supported, no request sent"); } public abstract List facetSearch(JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException; + public SearchResult getResults(JsonObject query, int maxResults) throws IcatException { + return getResults(query, null, maxResults, null, null); + } + public SearchResult getResults(JsonObject query, int maxResults, String sort) throws IcatException { - Long uid = atomicLongUid.getAndIncrement(); - uidMap.put(uid.toString(), 0); - return getResults(uid.toString(), query, maxResults); + return getResults(query, null, maxResults, sort, null); + } + + public SearchResult getResults(JsonObject query, String searchAfter, int blockSize, String sort, List fields) throws IcatException { + + // return getResults(uid.toString(), query, blockSize); + // TODO + return null; } - public SearchResult getResults(String uid, JsonObject query, int maxResults) throws IcatException { + private SearchResult getResults(String uid, JsonObject query, int maxResults) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { String index; Set fields = query.keySet(); @@ -341,7 +369,7 @@ public SearchResult getResults(String uid, JsonObject query, int maxResults) thr JsonObject jsonObject = jsonReader.readObject(); JsonArray hits = jsonObject.getJsonObject("hits").getJsonArray("hits"); for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { - entities.add(new ScoredEntityBaseBean(hit.getString("_id"), hit.getJsonNumber("_score").doubleValue())); + entities.add(new ScoredEntityBaseBean(hit.getString("_id"), hit.getJsonNumber("_score").bigDecimalValue().floatValue(), null)); // TODO } } return result; diff --git a/src/main/java/org/icatproject/core/manager/SearchManager.java b/src/main/java/org/icatproject/core/manager/SearchManager.java index 5c54a276..3570693c 100644 --- a/src/main/java/org/icatproject/core/manager/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/SearchManager.java @@ -9,8 +9,10 @@ import java.net.URL; import java.nio.file.Path; import java.util.ArrayList; +import java.util.HashMap; import java.util.Set; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.SortedSet; import java.util.Timer; @@ -39,7 +41,11 @@ import org.icatproject.core.Constants; import org.icatproject.core.IcatException; import org.icatproject.core.IcatException.IcatExceptionType; +import org.icatproject.core.entity.Datafile; +import org.icatproject.core.entity.Dataset; import org.icatproject.core.entity.EntityBaseBean; +import org.icatproject.core.entity.Investigation; +import org.icatproject.core.manager.EntityInfoHandler.Relationship; import org.icatproject.core.manager.PropertyHandler.SearchEngine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -311,6 +317,17 @@ public void run() { private List urls; + private static final Map> publicSearchFields = new HashMap<>(); + + public static List getPublicSearchFields(GateKeeper gateKeeper, String simpleName) throws IcatException { + if (gateKeeper.getPublicSearchFieldsStale() || publicSearchFields.size() == 0) { + publicSearchFields.put("Datafile", buildPublicSteps(gateKeeper, Datafile.getDocumentFields())); + publicSearchFields.put("Dataset", buildPublicSteps(gateKeeper, Dataset.getDocumentFields())); + publicSearchFields.put("Investigation", buildPublicSteps(gateKeeper, Investigation.getDocumentFields())); + } + return publicSearchFields.get(simpleName); + } + public void addDocument(EntityBaseBean bean) throws IcatException { String entityName = bean.getClass().getSimpleName(); if (eiHandler.hasSearchDoc(bean.getClass()) && entitiesToIndex.contains(entityName)) { @@ -380,6 +397,29 @@ public void deleteDocument(EntityBaseBean bean) throws IcatException { } } + private static List buildPublicSteps(GateKeeper gateKeeper, Map map) { + List fields = new ArrayList<>(); + for (Entry entry: map.entrySet()) { + Boolean includeField = true; + if (entry.getValue() != null) { + for (Relationship relationship: entry.getValue()) { + if (!gateKeeper.allowed(relationship)) { + includeField = false; + break; + } + } + } + if (includeField) { + fields.add(entry.getKey()); + } + } + return fields; + } + + public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { + return searchApi.buildSearchAfter(lastBean, sort); + } + private void pushPendingCalls() { timer.schedule(new EnqueuedSearchRequestHandler(), 0L); while (queueFile.length() != 0) { @@ -429,8 +469,8 @@ public SearchResult freeTextSearch(JsonObject jo, int blockSize, String sort) th return searchApi.getResults(jo, blockSize, sort); } - public SearchResult freeTextSearch(String uid, JsonObject jo, int blockSize) throws IcatException { - return searchApi.getResults(uid, jo, blockSize); + public SearchResult freeTextSearch(JsonObject jo, String searchAfter, int blockSize, String sort, List fields) throws IcatException { + return searchApi.getResults(jo, searchAfter, blockSize, sort, fields); } @PostConstruct diff --git a/src/main/java/org/icatproject/core/manager/SearchResult.java b/src/main/java/org/icatproject/core/manager/SearchResult.java index a8307506..09f73498 100644 --- a/src/main/java/org/icatproject/core/manager/SearchResult.java +++ b/src/main/java/org/icatproject/core/manager/SearchResult.java @@ -5,19 +5,26 @@ public class SearchResult { - private String uid; + private String searchAfter; private List results = new ArrayList<>(); + public SearchResult() {} + + public SearchResult(String searchAfter, List results) { + this.searchAfter = searchAfter; + this.results = results; + } + public List getResults() { return results; } - public void setUid(String uid) { - this.uid = uid; + public String getSearchAfter() { + return searchAfter; } - public String getUid() { - return uid; + public void setSearchAfter(String searchAfter) { + this.searchAfter = searchAfter; } } diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index f983e20e..26a0d112 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -81,6 +81,7 @@ import org.icatproject.core.manager.PropertyHandler; import org.icatproject.core.manager.PropertyHandler.ExtendedAuthenticator; import org.icatproject.core.manager.ScoredEntityBaseBean; +import org.icatproject.core.manager.SearchResult; import org.icatproject.utils.ContainerGetter.ContainerType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1108,6 +1109,191 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId } } + // TODO update endpoints to be generic + /** + * perform a free text search + * + * @summary free text search + * + * @param sessionId + * a sessionId of a user which takes the form + * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 + * @param query + * json encoded query object. One of the fields is "target" + * which + * must be "Investigation", "Dataset" or "Datafile". The other + * fields are all optional: + *
+ *
user
+ *
name of user as in the User table which may include a + * prefix
+ *
text
+ *
some text occurring somewhere in the entity. This is + * understood by the lucene parser but avoid trying to use fields.
+ *
lower
+ *
earliest date to search for in the form + * 201509030842 i.e. yyyyMMddHHmm using UTC as + * timezone. In the case of an investigation or data set search + * the date is compared with the start date and in the case of + * a + * data file the date field is used.
+ *
upper
+ *
latest date to search for in the form + * 201509030842 i.e. yyyyMMddHHmm using UTC as + * timezone. In the case of an investigation or data set search + * the date is compared with the end date and in the case of a + * data file the date field is used.
+ *
parameters
+ *
this holds a list of json parameter objects all of which + * must match. Parameters have the following fields, all of + * which + * are optional: + *
+ *
name
+ *
A wildcard search for a parameter with this name. + * Supported wildcards are *, which matches any + * character sequence (including the empty one), and + * ?, which matches any single character. + * \ is the escape character. Note this query can + * be + * slow, as it needs to iterate over many terms. In order to + * prevent extremely slow queries, a name should not start with + * the wildcard *
+ *
units
+ *
A wildcard search for a parameter with these units. + * Supported wildcards are *, which matches any + * character sequence (including the empty one), and + * ?, which matches any single character. + * \ is the escape character. Note this query can + * be + * slow, as it needs to iterate over many terms. In order to + * prevent extremely slow queries, units should not start with + * the wildcard *
+ *
stringValue
+ *
A wildcard search for a parameter stringValue. Supported + * wildcards are *, which matches any character + * sequence (including the empty one), and ?, + * which + * matches any single character. \ is the escape + * character. Note this query can be slow, as it needs to + * iterate + * over many terms. In order to prevent extremely slow queries, + * requested stringValues should not start with the wildcard + * *
+ *
lowerDateValue and upperDateValue
+ *
latest and highest date to search for in the form + * 201509030842 i.e. yyyyMMddHHmm using UTC as + * timezone. This should be used to search on parameters having + * a + * dateValue. If only one bound is set the restriction has not + * effect.
+ *
lowerNumericValue and upperNumericValue
+ *
This should be used to search on parameters having a + * numericValue. If only one bound is set the restriction has + * not + * effect.
+ *
+ *
+ *
samples
+ *
A json array of strings each of which must match text + * found in a sample. This is understood by the lucene parser but avoid trying to use fields. This is + * only respected in the case of an investigation search.
+ *
userFullName
+ *
Full name of user in the User table which may contain + * titles etc. Matching is done by the lucene parser but avoid trying to use fields. This is + * only respected in the case of an investigation search.
+ *
+ * @param searchAfter + * String representing the last returned document of a previous search, so that new results will be from after this document. The representation should be a JSON array, but the nature of the values will depend on the sort applied. + * @param sort + * json encoded sort object. Each key should be a field on the + * targeted Lucene document, with a value of "asc" or "desc" to + * specify the order of the results. Multiple pairs can be + * provided, in which case each subsequent sort is used as a + * tiebreaker for the previous one. If no sort is specified, + * then results will be returned in order of relevance to the + * search query, with their Lucene id as a tiebreaker. + * + * @param limit + * maximum number of entities to return + * + * @return set of entities encoded as json + * + * @throws IcatException + * when something is wrong + */ + @GET + @Path("search/documents") + @Produces(MediaType.APPLICATION_JSON) + public String search(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, + @QueryParam("query") String query, @QueryParam("search_after") String searchAfter, @QueryParam("limit") int limit, @QueryParam("sort") String sort) + throws IcatException { + if (query == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, "query is not set"); + } + String userName = beanManager.getUserName(sessionId, manager); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonReader jr = Json.createReader(new ByteArrayInputStream(query.getBytes()))) { + JsonObject jo = jr.readObject(); + String target = jo.getString("target", null); + if (jo.containsKey("parameters")) { + for (JsonValue val : jo.getJsonArray("parameters")) { + JsonObject parameter = (JsonObject) val; + String name = parameter.getString("name", null); + if (name == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, "name not set in one of parameters"); + } + String units = parameter.getString("units", null); + if (units == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "units not set in parameter '" + name + "'"); + } + // If we don't have either a string, pair of dates, or pair of numbers, throw + if (!(parameter.containsKey("stringValue") + || (parameter.containsKey("lowerDateValue") + && parameter.containsKey("upperDateValue")) + || (parameter.containsKey("lowerNumericValue") + && parameter.containsKey("upperNumericValue")))) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, parameter.toString()); + } + } + } + SearchResult result; + Class klass; + + if (target.equals("Investigation")) { + klass = Investigation.class; + } else if (target.equals("Dataset")) { + klass = Dataset.class; + } else if (target.equals("Datafile")) { + klass = Datafile.class; + } else { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, "target:" + target + " is not expected"); + } + logger.debug("Free text search with query: {}", jo.toString()); + result = beanManager.freeTextSearchDocs(userName, jo, searchAfter, limit, sort, manager, request.getRemoteAddr(), klass); + JsonGenerator gen = Json.createGenerator(baos); + gen.writeStartObject().write("search_after", result.getSearchAfter()).writeStartArray(); + for (ScoredEntityBaseBean sb : result.getResults()) { + gen.writeStartObject(); + gen.write("id", sb.getEntityBaseBeanId()); + gen.write("score", sb.getScore()); + gen.write("source", sb.getSource()); + gen.writeEnd(); + } + gen.writeEnd().writeEnd().close(); + return baos.toString(); + } catch (JsonException e) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, "JsonException " + e.getMessage()); + } + } + /** * This is an internal call made by one icat instance to another in the same * cluster diff --git a/src/test/java/org/icatproject/core/manager/TestLucene.java b/src/test/java/org/icatproject/core/manager/TestLucene.java index e9de28bb..088cd35e 100644 --- a/src/test/java/org/icatproject/core/manager/TestLucene.java +++ b/src/test/java/org/icatproject/core/manager/TestLucene.java @@ -1,7 +1,8 @@ package org.icatproject.core.manager; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; import java.io.ByteArrayOutputStream; @@ -20,6 +21,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import javax.json.Json; +import javax.json.JsonObject; import javax.json.stream.JsonGenerator; import javax.ws.rs.core.MediaType; @@ -161,6 +163,44 @@ public void before() throws Exception { luceneApi.clear(); } + private void checkDatafile(ScoredEntityBaseBean datafile) { + JsonObject source = datafile.getSource(); + assertNotNull(source); + Set expectedKeys = new HashSet<>(Arrays.asList("id", "dataset", "investigation", "name", "text", "date")); + assertEquals(expectedKeys, source.keySet()); + assertEquals("0", source.getString("id")); + assertEquals("0", source.getString("dataset")); + assertEquals("0", source.getString("investigation")); + assertEquals("DFaaa", source.getString("name")); + assertEquals("DFaaa", source.getString("text")); + assertNotNull(source.getString("date")); + } + + private void checkDataset(ScoredEntityBaseBean dataset) { + JsonObject source = dataset.getSource(); + assertNotNull(source); + Set expectedKeys = new HashSet<>(Arrays.asList("id", "investigation", "name", "text", "startDate", "endDate")); + assertEquals(expectedKeys, source.keySet()); + assertEquals("0", source.getString("id")); + assertEquals("0", source.getString("investigation")); + assertEquals("DSaaa", source.getString("name")); + assertEquals("DSaaa null null", source.getString("text")); + assertNotNull(source.getString("startDate")); + assertNotNull(source.getString("endDate")); + } + + private void checkInvestigation(ScoredEntityBaseBean investigation) { + JsonObject source = investigation.getSource(); + assertNotNull(source); + Set expectedKeys = new HashSet<>(Arrays.asList("id", "name", "text", "startDate", "endDate")); + assertEquals(expectedKeys, source.keySet()); + assertEquals("0", source.getString("id")); + assertEquals("a h r", source.getString("name")); + assertEquals("null a h r null null", source.getString("text")); + assertNotNull(source.getString("startDate")); + assertNotNull(source.getString("endDate")); + } + private void checkLsr(SearchResult lsr, Long... n) { Set wanted = new HashSet<>(Arrays.asList(n)); Set got = new HashSet<>(); @@ -207,63 +247,16 @@ private void checkLsrOrder(SearchResult lsr, Long... n) { public void datafiles() throws Exception { populate(); - SearchResult lsr = luceneApi - .getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 5, null); - String uid = lsr.getUid(); - + JsonObject query = SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null); + List fields = Arrays.asList("date", "name", "investigation", "id", "text", "dataset"); + SearchResult lsr = luceneApi.getResults(query, null, 5, null, fields); + String searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); - System.out.println(uid); - lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), - 200); - assertTrue(lsr.getUid() == null); + checkDatafile(lsr.getResults().get(0)); + lsr = luceneApi.getResults(query, searchAfter.toString(), 200, null, null); + assertNull(lsr.getSearchAfter()); assertEquals(95, lsr.getResults().size()); - luceneApi.freeSearcher(uid); - - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", "e4", null, null, null, null, null, null), 100, - null); - checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, 96L); - luceneApi.freeSearcher(lsr.getUid()); - - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null), 100, - null); - checkLsr(lsr, 1L); - luceneApi.freeSearcher(lsr.getUid()); - - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, "dfbbb", null, null, null, null, null), 100, - null); - checkLsr(lsr, 1L, 27L, 53L, 79L); - luceneApi.freeSearcher(lsr.getUid()); - - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100, null); - checkLsr(lsr, 3L, 4L, 5L, 6L); - luceneApi.freeSearcher(lsr.getUid()); - - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100, null); - checkLsr(lsr); - luceneApi.freeSearcher(lsr.getUid()); - - List pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v25")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null), 100, null); - checkLsr(lsr, 5L); - luceneApi.freeSearcher(lsr.getUid()); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v25")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100, - null); - checkLsr(lsr, 5L); - luceneApi.freeSearcher(lsr.getUid()); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, "u sss", null)); - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100, - null); - checkLsr(lsr, 13L, 65L); - luceneApi.freeSearcher(lsr.getUid()); // Test searchAfter preserves the sorting of original search (asc) ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -271,15 +264,16 @@ public void datafiles() throws Exception { gen.writeStartObject(); gen.write("date", "asc"); gen.writeEnd(); - } - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), - 5, baos.toString()); + String sort = baos.toString(); + lsr = luceneApi.getResults(query, null, 5, sort, null); checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); - uid = lsr.getUid(); - lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), - 5); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); + lsr = luceneApi.getResults(query, searchAfter.toString(), 5, sort, null); checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); // Test searchAfter preserves the sorting of original search (desc) baos = new ByteArrayOutputStream(); @@ -288,13 +282,15 @@ public void datafiles() throws Exception { gen.write("date", "desc"); gen.writeEnd(); } - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), - 5, baos.toString()); + sort = baos.toString(); + lsr = luceneApi.getResults(query, null, 5, sort, null); checkLsrOrder(lsr, 99L, 98L, 97L, 96L, 95L); - uid = lsr.getUid(); - lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), - 5); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); + lsr = luceneApi.getResults(query, searchAfter.toString(), 5, sort, null); checkLsrOrder(lsr, 94L, 93L, 92L, 91L, 90L); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); // Test tie breaks on fields with identical values (asc) baos = new ByteArrayOutputStream(); @@ -303,9 +299,11 @@ public void datafiles() throws Exception { gen.write("name", "asc"); gen.writeEnd(); } - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), - 5, baos.toString()); + sort = baos.toString(); + lsr = luceneApi.getResults(query, null, 5, sort, null); checkLsrOrder(lsr, 0L, 26L, 52L, 78L, 1L); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartObject(); @@ -313,9 +311,11 @@ public void datafiles() throws Exception { gen.write("date", "desc"); gen.writeEnd(); } - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), - 5, baos.toString()); + sort = baos.toString(); + lsr = luceneApi.getResults(query, null, 5, sort, null); checkLsrOrder(lsr, 78L, 52L, 26L, 0L, 79L); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); // Test tie breaks on fields with identical values (desc) baos = new ByteArrayOutputStream(); @@ -324,9 +324,11 @@ public void datafiles() throws Exception { gen.write("name", "desc"); gen.writeEnd(); } - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), - 5, baos.toString()); + sort = baos.toString(); + lsr = luceneApi.getResults(query, null, 5, sort, null); checkLsrOrder(lsr, 25L, 51L, 77L, 24L, 50L); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartObject(); @@ -334,71 +336,70 @@ public void datafiles() throws Exception { gen.write("date", "desc"); gen.writeEnd(); } - lsr = luceneApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), - 5, baos.toString()); + sort = baos.toString(); + lsr = luceneApi.getResults(query, null, 5, sort, null); checkLsrOrder(lsr, 77L, 51L, 25L, 76L, 50L); - } - - @Test - public void datasets() throws Exception { - populate(); - SearchResult lsr = luceneApi - .getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 5, null); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); - String uid = lsr.getUid(); - checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); - System.out.println(uid); - lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 100); - assertTrue(lsr.getUid() == null); - checkLsr(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, - 25L, 26L, 27L, 28L, 29L); - luceneApi.freeSearcher(uid); - - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "e4", null, null, null, null, null, null), 100, - null); - checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L); - luceneApi.freeSearcher(lsr.getUid()); + query = SearchApi.buildQuery("Datafile", "e4", null, null, null, null, null, null); + lsr = luceneApi.getResults(query, 100, null); + checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, 96L); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100, - null); + query = SearchApi.buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 1L); - luceneApi.freeSearcher(lsr.getUid()); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100, - null); - checkLsr(lsr, 1L, 27L); - luceneApi.freeSearcher(lsr.getUid()); + query = SearchApi.buildQuery("Datafile", null, "dfbbb", null, null, null, null, null); + lsr = luceneApi.getResults(query, 100, null); + checkLsr(lsr, 1L, 27L, 53L, 79L); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100, null); - checkLsr(lsr, 3L, 4L, 5L); - luceneApi.freeSearcher(lsr.getUid()); + query = SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null); + lsr = luceneApi.getResults(query, 100, null); + checkLsr(lsr, 3L, 4L, 5L, 6L); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100, null); - checkLsr(lsr, 3L); - luceneApi.freeSearcher(lsr.getUid()); + query = SearchApi.buildQuery("Datafile", "b1", "dsddd", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null); + lsr = luceneApi.getResults(query, 100, null); + checkLsr(lsr); List pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, pojos, null, null), 100, - null); - checkLsr(lsr, 4L); - luceneApi.freeSearcher(lsr.getUid()); + pojos.add(new ParameterPOJO(null, null, "v25")); + query = SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, null); + lsr = luceneApi.getResults(query, 100, null); + checkLsr(lsr, 5L); pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null), 100, null); - checkLsr(lsr, 4L); - luceneApi.freeSearcher(lsr.getUid()); + pojos.add(new ParameterPOJO(null, null, "v25")); + query = SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null); + lsr = luceneApi.getResults(query, 100, null); + checkLsr(lsr, 5L); pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null), 100, null); - checkLsr(lsr); - luceneApi.freeSearcher(lsr.getUid()); + pojos.add(new ParameterPOJO(null, "u sss", null)); + query = SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null); + lsr = luceneApi.getResults(query, 100, null); + checkLsr(lsr, 13L, 65L); + } + + @Test + public void datasets() throws Exception { + populate(); + + JsonObject query = SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null); + List fields = Arrays.asList("startDate", "endDate", "name", "investigation", "id", "text"); + SearchResult lsr = luceneApi.getResults(query, null, 5, null, fields); + String searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); + checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); + checkDataset(lsr.getResults().get(0)); + lsr = luceneApi.getResults(query, searchAfter.toString(), 100, null, null); + assertNull(lsr.getSearchAfter()); + checkLsr(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, + 25L, 26L, 27L, 28L, 29L); + // Test searchAfter preserves the sorting of original search (asc) ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -407,13 +408,15 @@ public void datasets() throws Exception { gen.write("startDate", "asc"); gen.writeEnd(); } - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), - 5, baos.toString()); + String sort = baos.toString(); + lsr = luceneApi.getResults(query, 5, sort); checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); - uid = lsr.getUid(); - lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), - 5); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); + lsr = luceneApi.getResults(query, searchAfter, 5, sort, null); checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); // Test searchAfter preserves the sorting of original search (desc) baos = new ByteArrayOutputStream(); @@ -422,13 +425,15 @@ public void datasets() throws Exception { gen.write("endDate", "desc"); gen.writeEnd(); } - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), - 5, baos.toString()); + sort = baos.toString(); + lsr = luceneApi.getResults(query, 5, sort); checkLsrOrder(lsr, 29L, 28L, 27L, 26L, 25L); - uid = lsr.getUid(); - lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), - 5); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); + lsr = luceneApi.getResults(query, searchAfter, 5, sort, null); checkLsrOrder(lsr, 24L, 23L, 22L, 21L, 20L); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); // Test tie breaks on fields with identical values (asc) baos = new ByteArrayOutputStream(); @@ -437,9 +442,11 @@ public void datasets() throws Exception { gen.write("name", "asc"); gen.writeEnd(); } - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), - 5, baos.toString()); + sort = baos.toString(); + lsr = luceneApi.getResults(query, searchAfter, 5, sort, null); checkLsrOrder(lsr, 0L, 26L, 1L, 27L, 2L); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartObject(); @@ -447,9 +454,49 @@ public void datasets() throws Exception { gen.write("endDate", "desc"); gen.writeEnd(); } - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), - 5, baos.toString()); + sort = baos.toString(); + lsr = luceneApi.getResults(query, 5, sort); checkLsrOrder(lsr, 26L, 0L, 27L, 1L, 28L); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); + + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "e4", null, null, null, null, null, null), 100, + null); + checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L); + + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100, + null); + checkLsr(lsr, 1L); + + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100, + null); + checkLsr(lsr, 1L, 27L); + + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100, null); + checkLsr(lsr, 3L, 4L, 5L); + + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100, null); + checkLsr(lsr, 3L); + + List pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v16")); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, pojos, null, null), 100, + null); + checkLsr(lsr, 4L); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v16")); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, null), 100, null); + checkLsr(lsr, 4L); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v16")); + lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, null), 100, null); + checkLsr(lsr); } private void fillParms(JsonGenerator gen, int i, String rel) { @@ -490,70 +537,94 @@ public void investigations() throws Exception { populate(); /* Blocked results */ - SearchResult lsr = luceneApi.getResults( - SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), - 5, null); - String uid = lsr.getUid(); + JsonObject query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null); + List fields = Arrays.asList("startDate", "endDate", "name", "id", "text"); + SearchResult lsr = luceneApi.getResults(query, null, 5, null, fields); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); - System.out.println(uid); - lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), - 6); - assertTrue(lsr.getUid() == null); + checkInvestigation(lsr.getResults().get(0)); + String searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); + lsr = luceneApi.getResults(query, searchAfter, 6, null, null); checkLsr(lsr, 5L, 6L, 7L, 8L, 9L); - luceneApi.freeSearcher(uid); + searchAfter = lsr.getSearchAfter(); + assertNull(searchAfter); + + // Test searchAfter preserves the sorting of original search (asc) + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("startDate", "asc"); + gen.writeEnd(); + } + String sort = baos.toString(); + lsr = luceneApi.getResults(query, 5, sort); + checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); + lsr = luceneApi.getResults(query, searchAfter, 5, sort, null); + checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); + + // Test searchAfter preserves the sorting of original search (desc) + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("endDate", "desc"); + gen.writeEnd(); + } + sort = baos.toString(); + lsr = luceneApi.getResults(query, 5, sort); + checkLsrOrder(lsr, 9L, 8L, 7L, 6L, 5L); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); + lsr = luceneApi.getResults(query, searchAfter, 5, sort, null); + checkLsrOrder(lsr, 4L, 3L, 2L, 1L, 0L); + searchAfter = lsr.getSearchAfter(); + assertNotNull(searchAfter); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "b"), 100, null); checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); - luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"), 100, null); checkLsr(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); - luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults( SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""), 100, null); checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); - luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", null, null, null, null, null, "b"), 100, null); checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); - luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "c1", null, null, null, null, null, "b"), 100, null); checkLsr(lsr); - luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null), 100, null); checkLsr(lsr, 4L); - luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", null, null, null, null, "b"), 100, null); checkLsr(lsr, 3L); - luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, null, "b"), 100, null); checkLsr(lsr, 3L); - luceneApi.freeSearcher(lsr.getUid()); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, null, null), 100, null); checkLsr(lsr, 3L, 4L, 5L); - luceneApi.freeSearcher(lsr.getUid()); List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100, null); checkLsr(lsr, 3L); - luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); @@ -562,14 +633,12 @@ public void investigations() throws Exception { lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100, null); checkLsr(lsr, 3L); - luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), pojos, null, "b"), 100, null); checkLsr(lsr, 3L); - luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); @@ -577,26 +646,22 @@ public void investigations() throws Exception { lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100, null); checkLsr(lsr); - luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), 100, null); checkLsr(lsr, 3L); - luceneApi.freeSearcher(lsr.getUid()); List samples = Arrays.asList("ddd", "nnn"); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), 100, null); checkLsr(lsr, 3L); - luceneApi.freeSearcher(lsr.getUid()); samples = Arrays.asList("ddd", "mmm"); lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), 100, null); checkLsr(lsr); - luceneApi.freeSearcher(lsr.getUid()); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); @@ -604,37 +669,6 @@ public void investigations() throws Exception { lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), pojos, samples, "b"), 100, null); checkLsr(lsr, 3L); - luceneApi.freeSearcher(lsr.getUid()); - - // Test searchAfter preserves the sorting of original search (asc) - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("startDate", "asc"); - gen.writeEnd(); - } - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), - 5, baos.toString()); - checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); - uid = lsr.getUid(); - lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), - 5); - checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); - - // Test searchAfter preserves the sorting of original search (desc) - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("endDate", "desc"); - gen.writeEnd(); - } - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), - 5, baos.toString()); - checkLsrOrder(lsr, 9L, 8L, 7L, 6L, 5L); - uid = lsr.getUid(); - lsr = luceneApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), - 5); - checkLsrOrder(lsr, 4L, 3L, 2L, 1L, 0L); } @Test From 64d1af53d6760f820387f5c5f4e9312827ab7935 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Mon, 4 Apr 2022 11:35:54 +0000 Subject: [PATCH 11/51] Add asserts to Lucene modify test #267 --- .../icatproject/core/manager/TestLucene.java | 47 ++++++++++++++++--- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/icatproject/core/manager/TestLucene.java b/src/test/java/org/icatproject/core/manager/TestLucene.java index 088cd35e..ff899f25 100644 --- a/src/test/java/org/icatproject/core/manager/TestLucene.java +++ b/src/test/java/org/icatproject/core/manager/TestLucene.java @@ -89,8 +89,6 @@ public QueueItem(String entityName, Long id, String json) { @Test public void modify() throws IcatException { - Queue queue = new ConcurrentLinkedQueue<>(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartArray(); @@ -101,16 +99,53 @@ public void modify() throws IcatException { luceneApi.encodeStringField(gen, "dataset", 2001L); gen.writeEnd(); } + String elephantJson = baos.toString(); - String json = baos.toString(); + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartArray(); + luceneApi.encodeTextField(gen, "text", "Rhinos and Aardvarks"); + luceneApi.encodeStringField(gen, "startDate", new Date()); + luceneApi.encodeStringField(gen, "endDate", new Date()); + luceneApi.encodeStoredId(gen, 42L); + luceneApi.encodeStringField(gen, "dataset", 2001L); + gen.writeEnd(); + } + String rhinoJson = baos.toString(); - queue.add(new QueueItem("Datafile", null, json)); + JsonObject elephantQuery = SearchApi.buildQuery("Datafile", null, "elephant", null, null, null, null, null); + JsonObject rhinoQuery = SearchApi.buildQuery("Datafile", null, "rhino", null, null, null, null, null); - queue.add(new QueueItem("Datafile", 42L, json)); + Queue queue = new ConcurrentLinkedQueue<>(); + queue.add(new QueueItem("Datafile", null, elephantJson)); + modifyQueue(queue); + checkLsr(luceneApi.getResults(elephantQuery, 5), 42L); + checkLsr(luceneApi.getResults(rhinoQuery, 5)); + + queue = new ConcurrentLinkedQueue<>(); + queue.add(new QueueItem("Datafile", 42L, rhinoJson)); + modifyQueue(queue); + checkLsr(luceneApi.getResults(elephantQuery, 5)); + checkLsr(luceneApi.getResults(rhinoQuery, 5), 42L); + + queue = new ConcurrentLinkedQueue<>(); + queue.add(new QueueItem("Datafile", 42L, null)); + queue.add(new QueueItem("Datafile", 42L, null)); + modifyQueue(queue); + checkLsr(luceneApi.getResults(elephantQuery, 5)); + checkLsr(luceneApi.getResults(rhinoQuery, 5)); + queue = new ConcurrentLinkedQueue<>(); + queue.add(new QueueItem("Datafile", null, elephantJson)); + queue.add(new QueueItem("Datafile", 42L, rhinoJson)); queue.add(new QueueItem("Datafile", 42L, null)); queue.add(new QueueItem("Datafile", 42L, null)); + modifyQueue(queue); + checkLsr(luceneApi.getResults(elephantQuery, 5)); + checkLsr(luceneApi.getResults(rhinoQuery, 5)); + } + private void modifyQueue(Queue queue) throws IcatException { Iterator qiter = queue.iterator(); if (qiter.hasNext()) { StringBuilder sb = new StringBuilder("["); @@ -138,8 +173,8 @@ public void modify() throws IcatException { logger.debug("XXX " + sb.toString()); luceneApi.modify(sb.toString()); + luceneApi.commit(); } - } private void addDocuments(String entityName, String json) throws IcatException { From ba3238792c7cd9cb85cc3e10e106450ee009f0dc Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Wed, 6 Apr 2022 03:29:33 +0000 Subject: [PATCH 12/51] Integration tests and fixes #267 --- .../org/icatproject/core/entity/Datafile.java | 3 +- .../org/icatproject/core/entity/Dataset.java | 8 +- .../core/manager/EntityBeanManager.java | 7 +- .../icatproject/core/manager/GateKeeper.java | 9 +- .../icatproject/core/manager/LuceneApi.java | 14 +- .../core/manager/ScoredEntityBaseBean.java | 46 +- .../icatproject/core/manager/SearchApi.java | 7 +- .../core/manager/SearchManager.java | 27 +- .../org/icatproject/exposed/ICATRest.java | 61 +- .../icatproject/core/manager/TestLucene.java | 134 ++-- .../org/icatproject/integration/TestRS.java | 620 ++++++++++++++++-- 11 files changed, 757 insertions(+), 179 deletions(-) diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index adcca7a5..6f3eceee 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -245,8 +245,7 @@ public static Map getDocumentFields() throws IcatExcepti Relationship[] textRelationships = { eiHandler.getRelationshipsByName(Datafile.class).get("datafileFormat") }; Relationship[] investigationRelationships = { - eiHandler.getRelationshipsByName(Datafile.class).get("dataset"), - eiHandler.getRelationshipsByName(Dataset.class).get("investigation") }; // TODO check if we need this + eiHandler.getRelationshipsByName(Datafile.class).get("dataset") }; documentFields.put("text", textRelationships); documentFields.put("name", null); documentFields.put("date", null); diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index c16d11e7..9e328dcf 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -191,7 +191,7 @@ public void setType(DatasetType type) { @Override public void getDoc(JsonGenerator gen, SearchApi searchApi) { - StringBuilder sb = new StringBuilder(name + " " + type.getName() + " " + type.getName()); + StringBuilder sb = new StringBuilder(name + " " + type.getName() + " " + type.getName()); // TODO duplicate type.getName() if (description != null) { sb.append(" " + description); } @@ -244,15 +244,13 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { public static Map getDocumentFields() throws IcatException { if (documentFields.size() == 0) { EntityInfoHandler eiHandler = EntityInfoHandler.getInstance(); - Relationship[] textRelationships = { eiHandler.getRelationshipsByName(Dataset.class).get("type") }; - Relationship[] investigationRelationships = { - eiHandler.getRelationshipsByName(Dataset.class).get("investigation") }; + Relationship[] textRelationships = { eiHandler.getRelationshipsByName(Dataset.class).get("type"), eiHandler.getRelationshipsByName(Dataset.class).get("sample") }; documentFields.put("text", textRelationships); documentFields.put("name", null); documentFields.put("startDate", null); documentFields.put("endDate", null); documentFields.put("id", null); - documentFields.put("investigation", investigationRelationships); + documentFields.put("investigation", null); } return documentFields; } diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index b38741c9..c29d5778 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -790,7 +790,7 @@ private ScoredEntityBaseBean filterReadAccess(List results if (beanManaged != null) { try { gateKeeper.performAuthorisation(userId, beanManaged, AccessType.READ, manager); - results.add(new ScoredEntityBaseBean(entityId, sr.getScore(), sr.getSource())); + results.add(sr); if (results.size() > maxEntities) { throw new IcatException(IcatExceptionType.VALIDATION, "attempt to return more than " + maxEntities + " entities"); @@ -1405,8 +1405,7 @@ public List freeTextSearch(String userName, JsonObject jo, int blockSize = Math.max(1000, limit); do { - List fields = SearchManager.getPublicSearchFields(gateKeeper, klass.getSimpleName()); - lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, fields); + lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, Arrays.asList("id")); allResults = lastSearchResult.getResults(); ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, limit, userName, manager, klass); if (lastBean == null) { @@ -1469,6 +1468,7 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, String se if (searchActive) { SearchResult lastSearchResult = null; List allResults = Collections.emptyList(); + List fields = SearchManager.getPublicSearchFields(gateKeeper, klass.getSimpleName()); /* * As results may be rejected and maxCount may be 1 ensure that we * don't make a huge number of calls to search engine @@ -1476,7 +1476,6 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, String se int blockSize = Math.max(1000, limit); do { - List fields = SearchManager.getPublicSearchFields(gateKeeper, klass.getSimpleName()); lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, fields); allResults = lastSearchResult.getResults(); ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, limit, userName, manager, klass); diff --git a/src/main/java/org/icatproject/core/manager/GateKeeper.java b/src/main/java/org/icatproject/core/manager/GateKeeper.java index 25b280b1..9efcea5a 100644 --- a/src/main/java/org/icatproject/core/manager/GateKeeper.java +++ b/src/main/java/org/icatproject/core/manager/GateKeeper.java @@ -109,7 +109,12 @@ public int compare(String o1, String o2) { */ public boolean allowed(Relationship r) { String beanName = r.getDestinationBean().getSimpleName(); - if (publicTables.contains(beanName)) { + // TODO by using the getter we can update the public tables if needed. + // Previous direct access meant we use out of date permissions, so why was there + // no update not here before? Needed for the publicSearchFields but if there's a + // reason for it to be publicTables and not getPublicTables can manually call + // update in SearchManager + if (getPublicTables().contains(beanName)) { return true; } String originBeanName = r.getOriginBean().getSimpleName(); @@ -289,7 +294,7 @@ public boolean isAccessAllowed(String user, EntityBaseBean object, AccessType ac if (access == AccessType.CREATE) { qName = Rule.CREATE_QUERY; } else if (access == AccessType.READ) { - if (publicTables.contains(simpleName)) { + if (getPublicTables().contains(simpleName)) { // TODO see other comment on publicTables vs getPublicTables logger.info("All are allowed " + access + " to " + simpleName); return true; } diff --git a/src/main/java/org/icatproject/core/manager/LuceneApi.java b/src/main/java/org/icatproject/core/manager/LuceneApi.java index 6fca01f5..6cf11d0e 100644 --- a/src/main/java/org/icatproject/core/manager/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/LuceneApi.java @@ -78,7 +78,7 @@ public void encodeSortedDocValuesField(JsonGenerator gen, String name, Long valu @Override public void encodeSortedDocValuesField(JsonGenerator gen, String name, String value) { - encodeStringField(gen, name, value); + encodeStringField(gen, name, value); // TODO leading to duplications in the _source, do we need both here? gen.writeStartObject().write("type", "SortedDocValuesField").write("name", name).write("value", value) .writeEnd(); } @@ -156,7 +156,7 @@ public void addNow(String entityName, List ids, EntityManager manager, @Override public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { JsonObjectBuilder builder = Json.createObjectBuilder(); - builder.add("doc", lastBean.getEntityBaseBeanId()); + builder.add("doc", lastBean.getEngineDocId()); builder.add("shardIndex", -1); if (!Float.isNaN(lastBean.getScore())) { builder.add("score", lastBean.getScore()); @@ -312,13 +312,13 @@ private SearchResult getResults(URI uri, CloseableHttpClient httpclient, String JsonObject responseObject = reader.readObject(); List resultsArray = responseObject.getJsonArray("results").getValuesAs(JsonObject.class); for (JsonObject resultObject: resultsArray) { - long id = resultObject.getJsonNumber("id").longValueExact(); + int luceneDocId = resultObject.getInt("_id"); Float score = Float.NaN; - if (resultObject.keySet().contains("score")) { - score = resultObject.getJsonNumber("score").bigDecimalValue().floatValue(); + if (resultObject.keySet().contains("_score")) { + score = resultObject.getJsonNumber("_score").bigDecimalValue().floatValue(); } - JsonObject source = resultObject.getJsonObject("source"); - results.add(new ScoredEntityBaseBean(id, score, source)); + JsonObject source = resultObject.getJsonObject("_source"); + results.add(new ScoredEntityBaseBean(luceneDocId, score, source)); } if (responseObject.containsKey("search_after")) { lsr.setSearchAfter(responseObject.getJsonObject("search_after").toString()); diff --git a/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java b/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java index 292f0597..a2741d00 100644 --- a/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java +++ b/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java @@ -2,28 +2,54 @@ import javax.json.JsonObject; +import org.icatproject.core.IcatException; +import org.icatproject.core.IcatException.IcatExceptionType; + public class ScoredEntityBaseBean { private long entityBaseBeanId; + private int engineDocId; private float score; private JsonObject source; - public ScoredEntityBaseBean(String id, float score, JsonObject source) { - this.entityBaseBeanId = Long.parseLong(id); - this.score = score; - this.source = source; - } - - public ScoredEntityBaseBean(long id, float score, JsonObject source) { - this.entityBaseBeanId = id; + /** + * Represents a single entity returned from a search, and relevant search engine + * information. + * + * @param engineDocId The id of the search engine Document that represents this + * entity. This should not be confused with the + * entityBaseBeanId. This is needed in order to enable + * subsequent searches to "search after" Documents which have + * already been returned once. + * @param score A float generated by the engine to indicate the relevance + * of the returned Document to the search term(s). Higher + * scores are more relevant. May be null if the results were + * not sorted by relevance. + * @param source JsonObject containing the requested fields of the Document + * as key-value pairs. At the very least, this should contain + * the ICAT "id" of the entity. + * @throws IcatException If "id" and the corresponding entityBaseBeanId are not + * a key-value pair in the source JsonObject. + */ + public ScoredEntityBaseBean(int engineDocId, float score, JsonObject source) throws IcatException { + if (!source.keySet().contains("id")) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Document source must have 'id' and the entityBaseBeanId as a key-value pair."); + } + this.engineDocId = engineDocId; this.score = score; this.source = source; + this.entityBaseBeanId = new Long(source.getString("id")); } public long getEntityBaseBeanId() { return entityBaseBeanId; } + public int getEngineDocId() { + return engineDocId; + } + public float getScore() { return score; } @@ -32,8 +58,4 @@ public JsonObject getSource() { return source; } - public void setSource(JsonObject source) { - this.source = source; - } - } diff --git a/src/main/java/org/icatproject/core/manager/SearchApi.java b/src/main/java/org/icatproject/core/manager/SearchApi.java index 71cf3e05..8090db03 100644 --- a/src/main/java/org/icatproject/core/manager/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/SearchApi.java @@ -8,6 +8,7 @@ import java.net.URISyntaxException; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Set; @@ -243,11 +244,11 @@ public abstract List facetSearch(JsonObject facetQuery, int maxR throws IcatException; public SearchResult getResults(JsonObject query, int maxResults) throws IcatException { - return getResults(query, null, maxResults, null, null); + return getResults(query, null, maxResults, null, Arrays.asList("id")); } public SearchResult getResults(JsonObject query, int maxResults, String sort) throws IcatException { - return getResults(query, null, maxResults, sort, null); + return getResults(query, null, maxResults, sort, Arrays.asList("id")); } public SearchResult getResults(JsonObject query, String searchAfter, int blockSize, String sort, List fields) throws IcatException { @@ -369,7 +370,7 @@ private SearchResult getResults(String uid, JsonObject query, int maxResults) th JsonObject jsonObject = jsonReader.readObject(); JsonArray hits = jsonObject.getJsonObject("hits").getJsonArray("hits"); for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { - entities.add(new ScoredEntityBaseBean(hit.getString("_id"), hit.getJsonNumber("_score").bigDecimalValue().floatValue(), null)); // TODO + entities.add(new ScoredEntityBaseBean(hit.getInt("_id"), hit.getJsonNumber("_score").bigDecimalValue().floatValue(), null)); // TODO } } return result; diff --git a/src/main/java/org/icatproject/core/manager/SearchManager.java b/src/main/java/org/icatproject/core/manager/SearchManager.java index 3570693c..0377404a 100644 --- a/src/main/java/org/icatproject/core/manager/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/SearchManager.java @@ -321,9 +321,12 @@ public void run() { public static List getPublicSearchFields(GateKeeper gateKeeper, String simpleName) throws IcatException { if (gateKeeper.getPublicSearchFieldsStale() || publicSearchFields.size() == 0) { - publicSearchFields.put("Datafile", buildPublicSteps(gateKeeper, Datafile.getDocumentFields())); - publicSearchFields.put("Dataset", buildPublicSteps(gateKeeper, Dataset.getDocumentFields())); - publicSearchFields.put("Investigation", buildPublicSteps(gateKeeper, Investigation.getDocumentFields())); + logger.info("Building public search fields from public tables and steps"); + publicSearchFields.put("Datafile", buildPublicSearchFields(gateKeeper, Datafile.getDocumentFields())); + publicSearchFields.put("Dataset", buildPublicSearchFields(gateKeeper, Dataset.getDocumentFields())); + publicSearchFields.put("Investigation", + buildPublicSearchFields(gateKeeper, Investigation.getDocumentFields())); + gateKeeper.markPublicSearchFieldsFresh(); } return publicSearchFields.get(simpleName); } @@ -397,14 +400,17 @@ public void deleteDocument(EntityBaseBean bean) throws IcatException { } } - private static List buildPublicSteps(GateKeeper gateKeeper, Map map) { + private static List buildPublicSearchFields(GateKeeper gateKeeper, Map map) { List fields = new ArrayList<>(); - for (Entry entry: map.entrySet()) { + for (Entry entry : map.entrySet()) { Boolean includeField = true; if (entry.getValue() != null) { - for (Relationship relationship: entry.getValue()) { + for (Relationship relationship : entry.getValue()) { if (!gateKeeper.allowed(relationship)) { includeField = false; + logger.debug("Access to {} blocked by disallowed relationship between {} and {}", entry.getKey(), + relationship.getOriginBean().getSimpleName(), + relationship.getDestinationBean().getSimpleName()); break; } } @@ -416,9 +422,9 @@ private static List buildPublicSteps(GateKeeper gateKeeper, Map fields) throws IcatException { + public SearchResult freeTextSearch(JsonObject jo, String searchAfter, int blockSize, String sort, + List fields) throws IcatException { return searchApi.getResults(jo, searchAfter, blockSize, sort, fields); } diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index 26a0d112..b9192468 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -926,11 +926,10 @@ public void logout(@Context HttpServletRequest request, @PathParam("sessionId") beanManager.logout(sessionId, manager, userTransaction, request.getRemoteAddr()); } - // TODO update endpoints to be generic /** - * perform a lucene search + * Perform a free text search against a dedicated (non-DB) search engine component for entity ids. * - * @summary lucene search + * @summary Free text id search. * * @param sessionId * a sessionId of a user which takes the form @@ -1027,18 +1026,18 @@ public void logout(@Context HttpServletRequest request, @PathParam("sessionId") * only respected in the case of an investigation search. * * @param sort - * json encoded sort object. Each key should be a field on the - * targeted Lucene document, with a value of "asc" or "desc" to + * JSON encoded sort object. Each key should be a field on the + * targeted Document, with a value of "asc" or "desc" to * specify the order of the results. Multiple pairs can be * provided, in which case each subsequent sort is used as a * tiebreaker for the previous one. If no sort is specified, * then results will be returned in order of relevance to the - * search query, with their Lucene id as a tiebreaker. + * search query, with their search engine id as a tiebreaker. * * @param maxCount * maximum number of entities to return * - * @return set of entities encoded as json + * @return set of entity ids and relevance scores encoded as json * * @throws IcatException * when something is wrong @@ -1098,7 +1097,9 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId for (ScoredEntityBaseBean sb : objects) { gen.writeStartObject(); gen.write("id", sb.getEntityBaseBeanId()); - gen.write("score", sb.getScore()); + if (!Float.isNaN(sb.getScore())) { + gen.write("score", sb.getScore()); + } gen.writeEnd(); } gen.writeEnd(); @@ -1109,11 +1110,10 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId } } - // TODO update endpoints to be generic /** - * perform a free text search + * Perform a free text search against a dedicated (non-DB) search engine component for entire Documents. * - * @summary free text search + * @summary Free text Document search. * * @param sessionId * a sessionId of a user which takes the form @@ -1213,17 +1213,17 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId * String representing the last returned document of a previous search, so that new results will be from after this document. The representation should be a JSON array, but the nature of the values will depend on the sort applied. * @param sort * json encoded sort object. Each key should be a field on the - * targeted Lucene document, with a value of "asc" or "desc" to + * targeted Document, with a value of "asc" or "desc" to * specify the order of the results. Multiple pairs can be * provided, in which case each subsequent sort is used as a * tiebreaker for the previous one. If no sort is specified, * then results will be returned in order of relevance to the - * search query, with their Lucene id as a tiebreaker. + * search query, with their search engine id as a tiebreaker. * * @param limit * maximum number of entities to return * - * @return set of entities encoded as json + * @return Set of entity ids, relevance scores and Document source encoded as json. * * @throws IcatException * when something is wrong @@ -1260,7 +1260,7 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId && parameter.containsKey("upperDateValue")) || (parameter.containsKey("lowerNumericValue") && parameter.containsKey("upperNumericValue")))) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, parameter.toString()); + throw new IcatException(IcatExceptionType.BAD_PARAMETER, "value not set in parameter '" + name + "'"); } } } @@ -1279,11 +1279,18 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId logger.debug("Free text search with query: {}", jo.toString()); result = beanManager.freeTextSearchDocs(userName, jo, searchAfter, limit, sort, manager, request.getRemoteAddr(), klass); JsonGenerator gen = Json.createGenerator(baos); - gen.writeStartObject().write("search_after", result.getSearchAfter()).writeStartArray(); + gen.writeStartObject(); + String newSearchAfter = result.getSearchAfter(); + if (newSearchAfter != null) { + gen.write("search_after", newSearchAfter); + } + gen.writeStartArray("results"); for (ScoredEntityBaseBean sb : result.getResults()) { gen.writeStartObject(); gen.write("id", sb.getEntityBaseBeanId()); - gen.write("score", sb.getScore()); + if (!Float.isNaN(sb.getScore())) { + gen.write("score", sb.getScore()); + } gen.write("source", sb.getSource()); gen.writeEnd(); } @@ -1328,11 +1335,10 @@ public void gatekeeperMarkPublicStepsStale(@Context HttpServletRequest request) gatekeeper.markPublicStepsStale(); } - // TODO generalise endpoints /** - * Stop population of the lucene database if it is running. + * Stop population of the search engine if it is running. * - * @summary Lucene Clear + * @summary Search engine clear * * @param sessionId * a sessionId of a user listed in rootUserNames @@ -1347,11 +1353,10 @@ public void searchClear(@QueryParam("sessionId") String sessionId) throws IcatEx beanManager.searchClear(); } - // TODO generalise endpoints /** - * Forces a commit of the lucene database + * Forces a commit of the search engine * - * @summary Lucene Commit + * @summary Search engine commit * * @param sessionId * a sessionId of a user listed in rootUserNames @@ -1366,11 +1371,10 @@ public void searchCommit(@FormParam("sessionId") String sessionId) throws IcatEx beanManager.searchCommit(); } - // TODO generalise endpoints /** - * Return a list of class names for which population is going on + * Return a list of class names for which search engine population is ongoing * - * @summary lucene GetPopulating + * @summary Search engine get populating * * @param sessionId * a sessionId of a user listed in rootUserNames @@ -1418,11 +1422,10 @@ public void waitMillis(@FormParam("sessionId") String sessionId, @FormParam("ms" } } - // TODO generalise endpoints /** - * Clear and repopulate lucene documents for the specified entityName + * Clear and repopulate search engine documents for the specified entityName * - * @summary Lucene Populate + * @summary Search engine populate * * @param sessionId * a sessionId of a user listed in rootUserNames diff --git a/src/test/java/org/icatproject/core/manager/TestLucene.java b/src/test/java/org/icatproject/core/manager/TestLucene.java index ff899f25..a335978c 100644 --- a/src/test/java/org/icatproject/core/manager/TestLucene.java +++ b/src/test/java/org/icatproject/core/manager/TestLucene.java @@ -90,25 +90,28 @@ public QueueItem(String entityName, Long id, String json) { @Test public void modify() throws IcatException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Investigation investigation = new Investigation(); + investigation.setId(0L); + Dataset dataset = new Dataset(); + dataset.setId(0L); + dataset.setInvestigation(investigation); + Datafile datafile = new Datafile(); + datafile.setName("Elephants and Aardvarks"); + datafile.setDatafileModTime(new Date()); + datafile.setId(new Long(42L)); + datafile.setDataset(dataset); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartArray(); - luceneApi.encodeTextField(gen, "text", "Elephants and Aardvarks"); - luceneApi.encodeStringField(gen, "startDate", new Date()); - luceneApi.encodeStringField(gen, "endDate", new Date()); - luceneApi.encodeStringField(gen, "id", 42L, true); - luceneApi.encodeStringField(gen, "dataset", 2001L); + datafile.getDoc(gen, luceneApi); gen.writeEnd(); } String elephantJson = baos.toString(); baos = new ByteArrayOutputStream(); + datafile.setName("Rhinos and Aardvarks"); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartArray(); - luceneApi.encodeTextField(gen, "text", "Rhinos and Aardvarks"); - luceneApi.encodeStringField(gen, "startDate", new Date()); - luceneApi.encodeStringField(gen, "endDate", new Date()); - luceneApi.encodeStoredId(gen, 42L); - luceneApi.encodeStringField(gen, "dataset", 2001L); + datafile.getDoc(gen, luceneApi); gen.writeEnd(); } String rhinoJson = baos.toString(); @@ -201,7 +204,8 @@ public void before() throws Exception { private void checkDatafile(ScoredEntityBaseBean datafile) { JsonObject source = datafile.getSource(); assertNotNull(source); - Set expectedKeys = new HashSet<>(Arrays.asList("id", "dataset", "investigation", "name", "text", "date")); + Set expectedKeys = new HashSet<>( + Arrays.asList("id", "dataset", "investigation", "name", "text", "date")); assertEquals(expectedKeys, source.keySet()); assertEquals("0", source.getString("id")); assertEquals("0", source.getString("dataset")); @@ -214,7 +218,8 @@ private void checkDatafile(ScoredEntityBaseBean datafile) { private void checkDataset(ScoredEntityBaseBean dataset) { JsonObject source = dataset.getSource(); assertNotNull(source); - Set expectedKeys = new HashSet<>(Arrays.asList("id", "investigation", "name", "text", "startDate", "endDate")); + Set expectedKeys = new HashSet<>( + Arrays.asList("id", "investigation", "name", "text", "startDate", "endDate")); assertEquals(expectedKeys, source.keySet()); assertEquals("0", source.getString("id")); assertEquals("0", source.getString("investigation")); @@ -289,7 +294,7 @@ public void datafiles() throws Exception { assertNotNull(searchAfter); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); checkDatafile(lsr.getResults().get(0)); - lsr = luceneApi.getResults(query, searchAfter.toString(), 200, null, null); + lsr = luceneApi.getResults(query, searchAfter.toString(), 200, null, fields); assertNull(lsr.getSearchAfter()); assertEquals(95, lsr.getResults().size()); @@ -301,11 +306,11 @@ public void datafiles() throws Exception { gen.writeEnd(); } String sort = baos.toString(); - lsr = luceneApi.getResults(query, null, 5, sort, null); + lsr = luceneApi.getResults(query, null, 5, sort, fields); checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter.toString(), 5, sort, null); + lsr = luceneApi.getResults(query, searchAfter.toString(), 5, sort, fields); checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -318,11 +323,11 @@ public void datafiles() throws Exception { gen.writeEnd(); } sort = baos.toString(); - lsr = luceneApi.getResults(query, null, 5, sort, null); + lsr = luceneApi.getResults(query, null, 5, sort, fields); checkLsrOrder(lsr, 99L, 98L, 97L, 96L, 95L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter.toString(), 5, sort, null); + lsr = luceneApi.getResults(query, searchAfter.toString(), 5, sort, fields); checkLsrOrder(lsr, 94L, 93L, 92L, 91L, 90L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -335,7 +340,7 @@ public void datafiles() throws Exception { gen.writeEnd(); } sort = baos.toString(); - lsr = luceneApi.getResults(query, null, 5, sort, null); + lsr = luceneApi.getResults(query, null, 5, sort, fields); checkLsrOrder(lsr, 0L, 26L, 52L, 78L, 1L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -347,7 +352,7 @@ public void datafiles() throws Exception { gen.writeEnd(); } sort = baos.toString(); - lsr = luceneApi.getResults(query, null, 5, sort, null); + lsr = luceneApi.getResults(query, null, 5, sort, fields); checkLsrOrder(lsr, 78L, 52L, 26L, 0L, 79L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -360,7 +365,7 @@ public void datafiles() throws Exception { gen.writeEnd(); } sort = baos.toString(); - lsr = luceneApi.getResults(query, null, 5, sort, null); + lsr = luceneApi.getResults(query, null, 5, sort, fields); checkLsrOrder(lsr, 25L, 51L, 77L, 24L, 50L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -372,7 +377,7 @@ public void datafiles() throws Exception { gen.writeEnd(); } sort = baos.toString(); - lsr = luceneApi.getResults(query, null, 5, sort, null); + lsr = luceneApi.getResults(query, null, 5, sort, fields); checkLsrOrder(lsr, 77L, 51L, 25L, 76L, 50L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -430,12 +435,11 @@ public void datasets() throws Exception { assertNotNull(searchAfter); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); checkDataset(lsr.getResults().get(0)); - lsr = luceneApi.getResults(query, searchAfter.toString(), 100, null, null); + lsr = luceneApi.getResults(query, searchAfter.toString(), 100, null, fields); assertNull(lsr.getSearchAfter()); checkLsr(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, 25L, 26L, 27L, 28L, 29L); - // Test searchAfter preserves the sorting of original search (asc) ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { @@ -448,7 +452,7 @@ public void datasets() throws Exception { checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter, 5, sort, null); + lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -465,7 +469,7 @@ public void datasets() throws Exception { checkLsrOrder(lsr, 29L, 28L, 27L, 26L, 25L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter, 5, sort, null); + lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); checkLsrOrder(lsr, 24L, 23L, 22L, 21L, 20L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -478,7 +482,7 @@ public void datasets() throws Exception { gen.writeEnd(); } sort = baos.toString(); - lsr = luceneApi.getResults(query, searchAfter, 5, sort, null); + lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); checkLsrOrder(lsr, 0L, 26L, 1L, 27L, 2L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -579,7 +583,7 @@ public void investigations() throws Exception { checkInvestigation(lsr.getResults().get(0)); String searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter, 6, null, null); + lsr = luceneApi.getResults(query, searchAfter, 6, null, fields); checkLsr(lsr, 5L, 6L, 7L, 8L, 9L); searchAfter = lsr.getSearchAfter(); assertNull(searchAfter); @@ -596,7 +600,7 @@ public void investigations() throws Exception { checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter, 5, sort, null); + lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -613,96 +617,98 @@ public void investigations() throws Exception { checkLsrOrder(lsr, 9L, 8L, 7L, 6L, 5L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter, 5, sort, null); + lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); checkLsrOrder(lsr, 4L, 3L, 2L, 1L, 0L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "b"), 100, - null); + query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "b"); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"), 100, - null); + query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); - lsr = luceneApi.getResults( - SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""), - 100, null); + query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", null, null, null, null, null, "b"), 100, - null); + query = SearchApi.buildQuery("Investigation", "b1", null, null, null, null, null, "b"); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "c1", null, null, null, null, null, "b"), 100, - null); + query = SearchApi.buildQuery("Investigation", "c1", null, null, null, null, null, "b"); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null), - 100, null); + query = SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 4L); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", null, null, null, null, "b"), 100, - null); + query = SearchApi.buildQuery("Investigation", "b1", "d", null, null, null, null, "b"); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 3L); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, "b"), 100, null); + query = SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), + null, null, "b"); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 3L); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100, null); + query = SearchApi.buildQuery("Investigation", null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), + null, null, null); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 3L, 4L, 5L); List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), - 100, null); + query = SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 3L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); pojos.add(new ParameterPOJO(null, null, 7, 10)); pojos.add(new ParameterPOJO(null, null, new Date(now + 60000 * 63), new Date(now + 60000 * 65))); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), - 100, null); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 3L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, "b"), 100, null); + query = SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), + pojos, null, "b"); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 3L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); pojos.add(new ParameterPOJO(null, null, "v81")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), - 100, null); + query = SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), - 100, null); + query = SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 3L); List samples = Arrays.asList("ddd", "nnn"); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), - 100, null); + query = SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 3L); samples = Arrays.asList("ddd", "mmm"); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), - 100, null); + query = SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); samples = Arrays.asList("ddd", "nnn"); - lsr = luceneApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, samples, "b"), 100, null); + query = SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), + pojos, samples, "b"); + lsr = luceneApi.getResults(query, 100, null); checkLsr(lsr, 3L); } diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 426dfc05..f77584f7 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -2,6 +2,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -40,6 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Map.Entry; import java.util.regex.Pattern; import javax.json.Json; @@ -61,6 +63,7 @@ import org.icatproject.icat.client.Session.DuplicateAction; import org.icatproject.EntityBaseBean; import org.icatproject.Facility; +import org.icatproject.PublicStep; /** * These tests are for those aspects that cannot be tested by the core tests. In @@ -74,12 +77,15 @@ public class TestRS { private static SearchApi searchApi; /** - * Utility function for manually clearing the search engine indices based on the System properties + * Utility function for manually clearing the search engine indices based on the + * System properties + * * @throws URISyntaxException * @throws MalformedURLException * @throws org.icatproject.core.IcatException */ - private static void clearSearch() throws URISyntaxException, MalformedURLException, org.icatproject.core.IcatException { + private static void clearSearch() + throws URISyntaxException, MalformedURLException, org.icatproject.core.IcatException { if (searchApi == null) { String searchEngine = System.getProperty("searchEngine"); if (searchEngine.equals("LUCENE")) { @@ -87,10 +93,11 @@ private static void clearSearch() throws URISyntaxException, MalformedURLExcepti URI uribase = new URI(urlString); searchApi = new LuceneApi(uribase); } else if (searchEngine.equals("ELASTICSEARCH")) { - String urlString = System.getProperty("elasticsearchUrl"); + String urlString = System.getProperty("elasticsearchUrl"); searchApi = new ElasticsearchApi(Arrays.asList(new URL(urlString))); } else { - throw new RuntimeException("searchEngine must be one of LUCENE, ELASTICSEARCH, but it was " + searchEngine); + throw new RuntimeException( + "searchEngine must be one of LUCENE, ELASTICSEARCH, but it was " + searchEngine); } } searchApi.clear(); @@ -144,97 +151,244 @@ public void TestJsoniseBean() throws Exception { DateFormat dft = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); Session session = createAndPopulate(); - /* Expected: <[{"User":{"id":8148,"createId":"db/notroot","createTime":"2019-03-11T14:14:47.000Z","modId":"db/notroot","modTime":"2019-03-11T14:14:47.000Z","affiliation":"Unseen University","familyName":"Worblehat","fullName":"Dr. Horace Worblehat","givenName":"Horace","instrumentScientists":[],"investigationUsers":[],"name":"db/lib","studies":[],"userGroups":[]}}]> */ + /* + * Expected: <[{"User":{"id":8148,"createId":"db/notroot","createTime": + * "2019-03-11T14:14:47.000Z","modId":"db/notroot","modTime": + * "2019-03-11T14:14:47.000Z","affiliation":"Unseen University","familyName": + * "Worblehat","fullName":"Dr. Horace Worblehat","givenName":"Horace", + * "instrumentScientists":[],"investigationUsers":[],"name":"db/lib","studies":[ + * ],"userGroups":[]}}]> + */ JsonArray user_response = search(session, "SELECT u from User u WHERE u.name = 'db/lib'", 1); collector.checkThat(user_response.getJsonObject(0).containsKey("User"), is(true)); JsonObject user = user_response.getJsonObject(0).getJsonObject("User"); - collector.checkThat(user.getJsonNumber("id").isIntegral(), is(true)); // Check Integer conversion - collector.checkThat(user.getString("createId"), is("db/notroot")); // Check String conversion - - /* Expected: <[{"Facility":{"id":2852,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","applications":[],"datafileFormats":[],"datasetTypes":[],"daysUntilRelease":90,"facilityCycles":[],"instruments":[],"investigationTypes":[],"investigations":[],"name":"Test port facility","parameterTypes":[],"sampleTypes":[]}}]> */ + collector.checkThat(user.getJsonNumber("id").isIntegral(), is(true)); // Check Integer conversion + collector.checkThat(user.getString("createId"), is("db/notroot")); // Check String conversion + + /* + * Expected: <[{"Facility":{"id":2852,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","applications":[],"datafileFormats":[], + * "datasetTypes":[],"daysUntilRelease":90,"facilityCycles":[],"instruments":[], + * "investigationTypes":[],"investigations":[],"name":"Test port facility" + * ,"parameterTypes":[],"sampleTypes":[]}}]> + */ JsonArray fac_response = search(session, "SELECT f from Facility f WHERE f.name = 'Test port facility'", 1); collector.checkThat(fac_response.getJsonObject(0).containsKey("Facility"), is(true)); - /* Expected: <[{"Instrument":{"id":1449,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","fullName":"EDDI - Energy Dispersive Diffraction","instrumentScientists":[],"investigationInstruments":[],"name":"EDDI","pid":"ig:0815","shifts":[]}}]> */ + /* + * Expected: <[{"Instrument":{"id":1449,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","fullName":"EDDI - Energy Dispersive Diffraction" + * ,"instrumentScientists":[],"investigationInstruments":[],"name":"EDDI","pid": + * "ig:0815","shifts":[]}}]> + */ JsonArray inst_response = search(session, "SELECT i from Instrument i WHERE i.name = 'EDDI'", 1); collector.checkThat(inst_response.getJsonObject(0).containsKey("Instrument"), is(true)); - /* Expected: <[{"InvestigationType":{"id":3401,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","investigations":[],"name":"atype"}}]> */ + /* + * Expected: + * <[{"InvestigationType":{"id":3401,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","investigations":[],"name":"atype"}}]> + */ JsonArray it_response = search(session, "SELECT it from InvestigationType it WHERE it.name = 'atype'", 1); collector.checkThat(it_response.getJsonObject(0).containsKey("InvestigationType"), is(true)); - /* Expected: <[{"ParameterType":{"id":5373,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","applicableToDataCollection":false,"applicableToDatafile":true,"applicableToDataset":true,"applicableToInvestigation":true,"applicableToSample":false,"dataCollectionParameters":[],"datafileParameters":[],"datasetParameters":[],"enforced":false,"investigationParameters":[],"minimumNumericValue":73.4,"name":"temp","permissibleStringValues":[],"pid":"pt:25c","sampleParameters":[],"units":"degrees Kelvin","valueType":"NUMERIC","verified":false}}]> */ + /* + * Expected: <[{"ParameterType":{"id":5373,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","applicableToDataCollection":false, + * "applicableToDatafile":true,"applicableToDataset":true, + * "applicableToInvestigation":true,"applicableToSample":false, + * "dataCollectionParameters":[],"datafileParameters":[],"datasetParameters":[], + * "enforced":false,"investigationParameters":[],"minimumNumericValue":73.4, + * "name":"temp","permissibleStringValues":[],"pid":"pt:25c","sampleParameters": + * [],"units":"degrees Kelvin","valueType":"NUMERIC","verified":false}}]> + */ JsonArray pt_response = search(session, "SELECT pt from ParameterType pt WHERE pt.name = 'temp'", 1); collector.checkThat(pt_response.getJsonObject(0).containsKey("ParameterType"), is(true)); - collector.checkThat((Double) pt_response.getJsonObject(0).getJsonObject("ParameterType").getJsonNumber("minimumNumericValue").doubleValue(), is(73.4)); // Check Double conversion - collector.checkThat((Boolean) pt_response.getJsonObject(0).getJsonObject("ParameterType").getBoolean("enforced"), is(Boolean.FALSE)); // Check boolean conversion - collector.checkThat(pt_response.getJsonObject(0).getJsonObject("ParameterType").getJsonString("valueType").getString(), is("NUMERIC")); // Check ParameterValueType conversion - - /* Expected: <[{"Investigation":{"id":4814,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"endDate":"2010-12-31T23:59:59.000Z","investigationGroups":[],"investigationInstruments":[],"investigationUsers":[],"keywords":[],"name":"expt1","parameters":[],"publications":[],"samples":[],"shifts":[],"startDate":"2010-01-01T00:00:00.000Z","studyInvestigations":[],"title":"a title at the beginning","visitId":"zero"}},{"Investigation":{"id":4815,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"endDate":"2011-12-31T23:59:59.000Z","investigationGroups":[],"investigationInstruments":[],"investigationUsers":[],"keywords":[],"name":"expt1","parameters":[],"publications":[],"samples":[],"shifts":[],"startDate":"2011-01-01T00:00:00.000Z","studyInvestigations":[],"title":"a title in the middle","visitId":"one"}},{"Investigation":{"id":4816,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"endDate":"2012-12-31T23:59:59.000Z","investigationGroups":[],"investigationInstruments":[],"investigationUsers":[],"keywords":[],"name":"expt1","parameters":[],"publications":[],"samples":[],"shifts":[],"startDate":"2012-01-01T00:00:00.000Z","studyInvestigations":[],"title":"a title at the end","visitId":"two"}}]> */ + collector.checkThat((Double) pt_response.getJsonObject(0).getJsonObject("ParameterType") + .getJsonNumber("minimumNumericValue").doubleValue(), is(73.4)); // Check Double conversion + collector.checkThat( + (Boolean) pt_response.getJsonObject(0).getJsonObject("ParameterType").getBoolean("enforced"), + is(Boolean.FALSE)); // Check boolean conversion + collector.checkThat( + pt_response.getJsonObject(0).getJsonObject("ParameterType").getJsonString("valueType").getString(), + is("NUMERIC")); // Check ParameterValueType conversion + + /* + * Expected: <[{"Investigation":{"id":4814,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","datasets":[],"endDate":"2010-12-31T23:59:59.000Z" + * ,"investigationGroups":[],"investigationInstruments":[],"investigationUsers": + * [],"keywords":[],"name":"expt1","parameters":[],"publications":[],"samples":[ + * ],"shifts":[],"startDate":"2010-01-01T00:00:00.000Z","studyInvestigations":[] + * ,"title":"a title at the beginning","visitId":"zero"}},{"Investigation":{"id" + * :4815,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId" + * :"db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"endDate": + * "2011-12-31T23:59:59.000Z","investigationGroups":[], + * "investigationInstruments":[],"investigationUsers":[],"keywords":[],"name": + * "expt1","parameters":[],"publications":[],"samples":[],"shifts":[], + * "startDate":"2011-01-01T00:00:00.000Z","studyInvestigations":[], + * "title":"a title in the middle","visitId":"one"}},{"Investigation":{"id":4816 + * ,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId": + * "db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"endDate": + * "2012-12-31T23:59:59.000Z","investigationGroups":[], + * "investigationInstruments":[],"investigationUsers":[],"keywords":[],"name": + * "expt1","parameters":[],"publications":[],"samples":[],"shifts":[], + * "startDate":"2012-01-01T00:00:00.000Z","studyInvestigations":[], + * "title":"a title at the end","visitId":"two"}}]> + */ JsonArray inv_response = search(session, "SELECT inv from Investigation inv WHERE inv.name = 'expt1'", 3); collector.checkThat(inv_response.getJsonObject(0).containsKey("Investigation"), is(true)); - /* Expected: <[{"InvestigationUser":{"id":4723,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","role":"troublemaker"}}]> */ - JsonArray invu_response = search(session, "SELECT invu from InvestigationUser invu WHERE invu.role = 'troublemaker'", 1); + /* + * Expected: + * <[{"InvestigationUser":{"id":4723,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","role":"troublemaker"}}]> + */ + JsonArray invu_response = search(session, + "SELECT invu from InvestigationUser invu WHERE invu.role = 'troublemaker'", 1); collector.checkThat(invu_response.getJsonObject(0).containsKey("InvestigationUser"), is(true)); - /* Expected: <[{"Shift":{"id":2995,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","comment":"waiting","endDate":"2013-12-31T22:59:59.000Z","startDate":"2013-12-31T11:00:00.000Z"}}]> */ + /* + * Expected: <[{"Shift":{"id":2995,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","comment":"waiting","endDate": + * "2013-12-31T22:59:59.000Z","startDate":"2013-12-31T11:00:00.000Z"}}]> + */ JsonArray shift_response = search(session, "SELECT shift from Shift shift WHERE shift.comment = 'waiting'", 1); collector.checkThat(shift_response.getJsonObject(0).containsKey("Shift"), is(true)); - /* Expected: <[{"SampleType":{"id":3220,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","molecularFormula":"C","name":"diamond","safetyInformation":"fairly harmless","samples":[]}}]> */ + /* + * Expected: <[{"SampleType":{"id":3220,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","molecularFormula":"C","name":"diamond", + * "safetyInformation":"fairly harmless","samples":[]}}]> + */ JsonArray st_response = search(session, "SELECT st from SampleType st WHERE st.name = 'diamond'", 1); collector.checkThat(st_response.getJsonObject(0).containsKey("SampleType"), is(true)); - /* Expected: <[{"Sample":{"id":2181,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"name":"Koh-I-Noor","parameters":[],"pid":"sdb:374717"}}]> */ + /* + * Expected: <[{"Sample":{"id":2181,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","datasets":[],"name":"Koh-I-Noor","parameters":[], + * "pid":"sdb:374717"}}]> + */ JsonArray s_response = search(session, "SELECT s from Sample s WHERE s.name = 'Koh-I-Noor'", 1); collector.checkThat(s_response.getJsonObject(0).containsKey("Sample"), is(true)); - /* Expected: <[{"InvestigationParameter":{"id":1123,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","stringValue":"green"}}]> */ - JsonArray invp_response = search(session, "SELECT invp from InvestigationParameter invp WHERE invp.stringValue = 'green'", 1); + /* + * Expected: + * <[{"InvestigationParameter":{"id":1123,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","stringValue":"green"}}]> + */ + JsonArray invp_response = search(session, + "SELECT invp from InvestigationParameter invp WHERE invp.stringValue = 'green'", 1); collector.checkThat(invp_response.size(), equalTo(1)); collector.checkThat(invp_response.getJsonObject(0).containsKey("InvestigationParameter"), is(true)); - /* Expected: <[{"DatasetType":{"id":1754,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"name":"calibration"}}]> */ + /* + * Expected: <[{"DatasetType":{"id":1754,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","datasets":[],"name":"calibration"}}]> + */ JsonArray dst_response = search(session, "SELECT dst from DatasetType dst WHERE dst.name = 'calibration'", 1); collector.checkThat(dst_response.getJsonObject(0).containsKey("DatasetType"), is(true)); - /* Expected: <[{"Dataset":{"id":8128,"createId":"db/notroot","createTime":"2019-03-12T11:40:26.000Z","modId":"db/notroot","modTime":"2019-03-12T11:40:26.000Z","complete":true,"dataCollectionDatasets":[],"datafiles":[],"description":"alpha","endDate":"2014-05-16T04:28:26.000Z","name":"ds1","parameters":[],"startDate":"2014-05-16T04:28:26.000Z"}}]> */ + /* + * Expected: <[{"Dataset":{"id":8128,"createId":"db/notroot","createTime": + * "2019-03-12T11:40:26.000Z","modId":"db/notroot","modTime": + * "2019-03-12T11:40:26.000Z","complete":true,"dataCollectionDatasets":[], + * "datafiles":[],"description":"alpha","endDate":"2014-05-16T04:28:26.000Z", + * "name":"ds1","parameters":[],"startDate":"2014-05-16T04:28:26.000Z"}}]> + */ JsonArray ds_response = search(session, "SELECT ds from Dataset ds WHERE ds.name = 'ds1'", 1); collector.checkThat(ds_response.getJsonObject(0).containsKey("Dataset"), is(true)); - collector.checkThat(dft.parse(ds_response.getJsonObject(0).getJsonObject("Dataset").getString("startDate")), isA(Date.class)); //Check Date conversion - - /* Expected: <[{"DatasetParameter":{"id":4632,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","stringValue":"green"}}]> */ - JsonArray dsp_response = search(session, "SELECT dsp from DatasetParameter dsp WHERE dsp.stringValue = 'green'", 1); + collector.checkThat(dft.parse(ds_response.getJsonObject(0).getJsonObject("Dataset").getString("startDate")), + isA(Date.class)); // Check Date conversion + + /* + * Expected: + * <[{"DatasetParameter":{"id":4632,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z","stringValue":"green"}}]> + */ + JsonArray dsp_response = search(session, "SELECT dsp from DatasetParameter dsp WHERE dsp.stringValue = 'green'", + 1); collector.checkThat(dsp_response.size(), equalTo(1)); collector.checkThat(dsp_response.getJsonObject(0).containsKey("DatasetParameter"), is(true)); - /* Expected: <[{"Datafile":{"id":15643,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","dataCollectionDatafiles":[],"destDatafiles":[],"fileSize":17,"name":"df2","parameters":[],"sourceDatafiles":[]}}]> */ + /* + * Expected: <[{"Datafile":{"id":15643,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z","dataCollectionDatafiles":[],"destDatafiles":[], + * "fileSize":17,"name":"df2","parameters":[],"sourceDatafiles":[]}}]> + */ JsonArray df_response = search(session, "SELECT df from Datafile df WHERE df.name = 'df2'", 1); collector.checkThat(df_response.size(), equalTo(1)); collector.checkThat(df_response.getJsonObject(0).containsKey("Datafile"), is(true)); - /* Expected: <[{"DatafileParameter":{"id":1938,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","stringValue":"green"}}]> */ - JsonArray dfp_response = search(session, "SELECT dfp from DatafileParameter dfp WHERE dfp.stringValue = 'green'", 1); + /* + * Expected: + * <[{"DatafileParameter":{"id":1938,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z","stringValue":"green"}}]> + */ + JsonArray dfp_response = search(session, + "SELECT dfp from DatafileParameter dfp WHERE dfp.stringValue = 'green'", 1); collector.checkThat(dfp_response.size(), equalTo(1)); collector.checkThat(dfp_response.getJsonObject(0).containsKey("DatafileParameter"), is(true)); - /* Expected: <[{"Application":{"id":2972,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","jobs":[],"name":"aprog","version":"1.2.3"}},{"Application":{"id":2973,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","jobs":[],"name":"aprog","version":"1.2.6"}}]> */ + /* + * Expected: <[{"Application":{"id":2972,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z","jobs":[],"name":"aprog","version":"1.2.3"}},{ + * "Application":{"id":2973,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z","jobs":[],"name":"aprog","version":"1.2.6"}}]> + */ JsonArray a_response = search(session, "SELECT a from Application a WHERE a.name = 'aprog'", 2); collector.checkThat(a_response.size(), equalTo(2)); collector.checkThat(a_response.getJsonObject(0).containsKey("Application"), is(true)); - /* Expected: <[{DataCollection":{"id":4485,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","dataCollectionDatafiles":[],"dataCollectionDatasets":[],"jobsAsInput":[],"jobsAsOutput":[],"parameters":[]}},{"DataCollection":{"id":4486,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","dataCollectionDatafiles":[],"dataCollectionDatasets":[],"jobsAsInput":[],"jobsAsOutput":[],"parameters":[]}},{"DataCollection":{"id":4487,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","dataCollectionDatafiles":[],"dataCollectionDatasets":[],"jobsAsInput":[],"jobsAsOutput":[],"parameters":[]}}]> */ + /* + * Expected: + * <[{DataCollection":{"id":4485,"createId":"db/notroot","createTime":"2019-03- + * 12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33. + * 000Z","dataCollectionDatafiles":[],"dataCollectionDatasets":[],"jobsAsInput":[],"jobsAsOutput":[],"parameters":[]}},{"DataCollection":{"id":4486,"createId":"db + * /notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/ + * notroot","modTime":"2019-03-12T13:30:33. + * 000Z","dataCollectionDatafiles":[],"dataCollectionDatasets":[],"jobsAsInput":[],"jobsAsOutput":[],"parameters":[]}},{"DataCollection":{"id":4487,"createId":"db + * /notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/ + * notroot","modTime":"2019-03-12T13:30:33. + * 000Z","dataCollectionDatafiles":[],"dataCollectionDatasets":[],"jobsAsInput":[],"jobsAsOutput":[],"parameters + * ":[]}}]> + */ JsonArray dc_response = search(session, "SELECT dc from DataCollection dc", 3); collector.checkThat(dc_response.size(), equalTo(3)); collector.checkThat(dc_response.getJsonObject(0).containsKey("DataCollection"), is(true)); - /* Expected: <[{"DataCollectionDatafile":{"id":4362,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z"}},{"DataCollectionDatafile":{"id":4363,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z"}}]> */ + /* + * Expected: + * <[{"DataCollectionDatafile":{"id":4362,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z"}},{"DataCollectionDatafile":{"id":4363,"createId": + * "db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot", + * "modTime":"2019-03-12T13:30:33.000Z"}}]> + */ JsonArray dcdf_response = search(session, "SELECT dcdf from DataCollectionDatafile dcdf", 2); collector.checkThat(dcdf_response.getJsonObject(0).containsKey("DataCollectionDatafile"), is(true)); - /* Expected: <[{"Job":{"id":1634,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z"}}]> */ + /* + * Expected: <[{"Job":{"id":1634,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z"}}]> + */ JsonArray j_response = search(session, "SELECT j from Job j", 1); collector.checkThat(j_response.getJsonObject(0).containsKey("Job"), is(true)); } @@ -470,6 +624,331 @@ public void testLuceneInvestigations() throws Exception { } } + @Test + public void testSearchDatafiles() throws Exception { + Session session = setupLuceneTest(); + JsonObject responseObject; + String searchAfter; + Map expectation = new HashMap<>(); + expectation.put("investigation", null); + expectation.put("text", null); + expectation.put("date", "notNull"); + expectation.put("dataset", "notNull"); + + List parameters = new ArrayList<>(); + parameters.add(new ParameterForLucene("colour", "name", "green")); + + // All data files + searchDatafiles(session, null, null, null, null, null, null, 10, null, 3); + + // Use the user + searchDatafiles(session, "db/tr", null, null, null, null, null, 10, null, 3); + + // Try a bad user + searchDatafiles(session, "db/fred", null, null, null, null, null, 10, null, 0); + + // Set text and parameters + responseObject = searchDatafiles(session, null, "df2", null, null, parameters, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + expectation.put("name", "df2"); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + // Try sorting and searchAfter + String sort = Json.createObjectBuilder().add("name", "desc").add("date", "asc").build().toString(); + responseObject = searchDatafiles(session, null, null, null, null, null, null, 1, sort, 1); + searchAfter = responseObject.getString("search_after"); + assertNotNull(searchAfter); + expectation.put("name", "df3"); + checkResultsSource(responseObject, Arrays.asList(expectation), false); + + responseObject = searchDatafiles(session, null, null, null, null, null, searchAfter, 1, sort, 1); + searchAfter = responseObject.getString("search_after"); + assertNotNull(searchAfter); + expectation.put("name", "df2"); + checkResultsSource(responseObject, Arrays.asList(expectation), false); + + // Test that changes to the public steps/tables are reflected in returned fields + PublicStep ps = new PublicStep(); + ps.setOrigin("Datafile"); + ps.setField("dataset"); + + ps.setId(wSession.create(ps)); + responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + expectation.put("investigation", "notNull"); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + wSession.delete(ps); + responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + expectation.put("investigation", null); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + wSession.addRule(null, "DatafileFormat", "R"); + responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + expectation.put("text", "df2"); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + wSession.delRule(null, "DatafileFormat", "R"); + responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + expectation.put("text", null); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + // Test searching with someone without authz for the Datafile(s) + ICAT icat = new ICAT(System.getProperty("serverUrl")); + Map credentials = new HashMap<>(); + credentials.put("username", "piOne"); + credentials.put("password", "piOne"); + Session piSession = icat.login("db", credentials); + searchDatafiles(piSession, null, null, null, null, null, null, 10, null, 0); + } + + @Test + public void testSearchDatasets() throws Exception { + Session session = setupLuceneTest(); + DateFormat dft = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + JsonObject responseObject; + String searchAfter; + Map expectation = new HashMap<>(); + expectation.put("text", null); + expectation.put("startDate", "notNull"); + expectation.put("endDate", "notNull"); + expectation.put("investigation", "notNull"); + + // All datasets + searchDatasets(session, null, null, null, null, null, null, 10, null, 5); + + // Use the user + responseObject = searchDatasets(session, "db/tr", null, null, null, null, null, 10, null, 3); + List> expectations = new ArrayList<>(); + expectation.put("name", "ds1"); + expectations.add(new HashMap<>(expectation)); + expectation.put("name", "ds2"); + expectations.add(new HashMap<>(expectation)); + expectation.put("name", "ds3"); + expectations.add(new HashMap<>(expectation)); + checkResultsSource(responseObject, expectations, true); + + // Try a bad user + searchDatasets(session, "db/fred", null, null, null, null, null, 10, null, 0); + + // Try text + responseObject = searchDatasets(session, null, "gamma AND ds3", null, null, null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + // Try parameters + Date lower = dft.parse("2014-05-16T05:09:03+0000"); + Date upper = dft.parse("2014-05-16T05:15:26+0000"); + List parameters = new ArrayList<>(); + Date parameterDate = dft.parse("2014-05-16T16:58:26+0000"); + parameters.add(new ParameterForLucene("colour", "name", "green")); + parameters.add(new ParameterForLucene("birthday", "date", parameterDate, parameterDate)); + parameters.add(new ParameterForLucene("current", "amps", 140, 165)); + + responseObject = searchDatasets(session, null, null, null, null, parameters, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + responseObject = searchDatasets(session, null, "gamma AND ds3", lower, upper, parameters, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + // Try sorting and searchAfter + String sort = Json.createObjectBuilder().add("name", "desc").add("startDate", "asc").build().toString(); + responseObject = searchDatasets(session, null, null, null, null, null, null, 1, sort, 1); + searchAfter = responseObject.getString("search_after"); + assertNotNull(searchAfter); + expectation.put("name", "ds4"); + checkResultsSource(responseObject, Arrays.asList(expectation), false); + responseObject = searchDatasets(session, null, null, null, null, null, searchAfter, 1, sort, 1); + searchAfter = responseObject.getString("search_after"); + assertNotNull(searchAfter); + expectation.put("name", "ds3"); + checkResultsSource(responseObject, Arrays.asList(expectation), false); + + // Test that changes to the public steps/tables are reflected in returned fields + PublicStep ps = new PublicStep(); + ps.setOrigin("Dataset"); + ps.setField("type"); + + ps.setId(wSession.create(ps)); + responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + expectation.put("name", "ds1"); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + wSession.addRule(null, "Sample", "R"); + responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + expectation.put("text", "ds1 calibration calibration alpha Koh-I-Noor diamond"); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + wSession.delete(ps); + responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + expectation.put("text", null); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + wSession.delRule(null, "Sample", "R"); + responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + // Test searching with someone without authz for the Dataset(s) + ICAT icat = new ICAT(System.getProperty("serverUrl")); + Map credentials = new HashMap<>(); + credentials.put("username", "piOne"); + credentials.put("password", "piOne"); + Session piSession = icat.login("db", credentials); + searchDatasets(piSession, null, null, null, null, null, null, 10, null, 0); + } + + @Test + public void testSearchInvestigations() throws Exception { + Session session = setupLuceneTest(); + DateFormat dft = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + JsonObject responseObject; + Map expectation = new HashMap<>(); + expectation.put("name", "expt1"); + expectation.put("startDate", "notNull"); + expectation.put("endDate", "notNull"); + expectation.put("text", null); + + Date lowerOrigin = dft.parse("2011-01-01T00:00:00+0000"); + Date lowerSecond = dft.parse("2011-01-01T00:00:01+0000"); + Date lowerMinute = dft.parse("2011-01-01T00:01:00+0000"); + Date upperOrigin = dft.parse("2011-12-31T23:59:59+0000"); + Date upperSecond = dft.parse("2011-12-31T23:59:58+0000"); + Date upperMinute = dft.parse("2011-12-31T23:58:00+0000"); + List samplesAnd = Arrays.asList("ford AND rust", "koh* AND diamond"); + List samplesPlus = Arrays.asList("ford + rust", "koh + diamond"); + List samplesBad = Arrays.asList("ford AND rust", "kog* AND diamond"); + String textAnd = "title AND one"; + String textTwo = "title AND two"; + String textPlus = "title + one"; + + searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, 3); + + List parameters = new ArrayList<>(); + parameters.add(new ParameterForLucene("colour", "name", "green")); + responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, + samplesAnd, "Professor", null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + // change user + searchInvestigations(session, "db/fred", textAnd, null, null, parameters, null, null, null, 10, null, 0); + + // change text + searchInvestigations(session, "db/tr", textTwo, null, null, parameters, null, null, null, 10, null, 0); + + // Only working to a minute + responseObject = searchInvestigations(session, "db/tr", textAnd, lowerSecond, upperOrigin, parameters, null, + null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperSecond, parameters, null, + null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + searchInvestigations(session, "db/tr", textAnd, lowerMinute, upperOrigin, parameters, null, null, null, + 10, null, 0); + + searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperMinute, parameters, null, null, null, + 10, null, 0); + + // Change parameters + List badParameters = new ArrayList<>(); + badParameters.add(new ParameterForLucene("color", "name", "green")); + searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, badParameters, samplesPlus, null, + null, 10, null, 0); + + // Change samples + searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, samplesBad, null, null, + 10, null, 0); + + // Change userFullName + searchInvestigations(session, "db/tr", textPlus, lowerOrigin, upperOrigin, parameters, samplesAnd, "Doctor", + null, 10, null, 0); + + // TODO currently the only field we can access is name due to how + // Investigation.getDoc() encodes, but this is the same for all the + // investigations so testing sorting is not feasible + + // Test that changes to the public steps/tables are reflected in returned fields + PublicStep ps = new PublicStep(); + ps.setOrigin("Investigation"); + ps.setField("type"); + + ps.setId(wSession.create(ps)); + responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + wSession.addRule(null, "Facility", "R"); + responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + expectation.put("text", "one expt1 Test port facility atype a title in the middle"); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + wSession.delete(ps); + responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + expectation.put("text", null); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + wSession.delRule(null, "Facility", "R"); + responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, 1); + assertFalse(responseObject.keySet().contains("search_after")); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + + // Test searching with someone without authz for the Investigation(s) + ICAT icat = new ICAT(System.getProperty("serverUrl")); + Map credentials = new HashMap<>(); + credentials.put("username", "piOne"); + credentials.put("password", "piOne"); + Session piSession = icat.login("db", credentials); + searchInvestigations(piSession, null, null, null, null, null, null, null, null, 10, null, 0); + } + + @Test + public void testSearchParameterValidation() throws Exception { + Session session = setupLuceneTest(); + List badParameters = new ArrayList<>(); + + badParameters = Arrays.asList(new ParameterForLucene(null, null, null)); + try { + searchInvestigations(session, null, null, null, null, badParameters, null, null, null, 10, null, 0); + fail("BAD_PARAMETER exception not caught"); + } catch (IcatException e) { + assertEquals(IcatExceptionType.BAD_PARAMETER, e.getType()); + assertEquals("name not set in one of parameters", e.getMessage()); + } + + badParameters = Arrays.asList(new ParameterForLucene("color", null, null)); + try { + searchInvestigations(session, null, null, null, null, badParameters, null, null, null, 10, null, 0); + fail("BAD_PARAMETER exception not caught"); + } catch (IcatException e) { + assertEquals(IcatExceptionType.BAD_PARAMETER, e.getType()); + assertEquals("units not set in parameter 'color'", e.getMessage()); + } + + badParameters = Arrays.asList(new ParameterForLucene("color", "string", null)); + try { + searchInvestigations(session, null, null, null, null, badParameters, null, null, null, 10, null, 0); + fail("BAD_PARAMETER exception not caught"); + } catch (IcatException e) { + assertEquals(IcatExceptionType.BAD_PARAMETER, e.getType()); + assertEquals("value not set in parameter 'color'", e.getMessage()); + } + } + private void checkResultFromLuceneSearch(Session session, String val, JsonArray array, String ename, String field) throws IcatException { long n = array.getJsonObject(0).getJsonNumber("id").longValueExact(); @@ -477,6 +956,33 @@ private void checkResultFromLuceneSearch(Session session, String val, JsonArray assertEquals(val, result.getJsonObject(ename).getString(field)); } + private void checkResultsSource(JsonObject responseObject, List> expectations, Boolean scored) { + JsonArray results = responseObject.getJsonArray("results"); + assertEquals(expectations.size(), results.size()); + for (int i = 0; i < expectations.size(); i++) { + JsonObject result = results.getJsonObject(i); + assertTrue(result.keySet().contains("id")); + assertEquals(scored, result.keySet().contains("score")); + + assertTrue(result.keySet().contains("source")); + JsonObject source = result.getJsonObject("source"); + assertTrue(source.keySet().contains("id")); + Map expectation = expectations.get(i); + for (Entry entry: expectation.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (value == null) { + assertFalse("Source " + source.toString() + " should NOT contain " + key, source.keySet().contains(key)); + } else if (value.equals("notNull")) { + assertTrue("Source " + source.toString() + " should contain " + key, source.keySet().contains(key)); + } else { + assertTrue("Source " + source.toString() + " should contain " + key, source.keySet().contains(key)); + assertEquals(value, source.getString(key)); + } + } + } + } + private Session setupLuceneTest() throws Exception { ICAT icat = new ICAT(System.getProperty("serverUrl")); Map credentials = new HashMap<>(); @@ -632,14 +1138,15 @@ public void testSearch() throws Exception { JsonArray array; - JsonObject user = search(session, "SELECT u FROM User u WHERE u.name = 'db/lib'", 1).getJsonObject(0).getJsonObject("User"); + JsonObject user = search(session, "SELECT u FROM User u WHERE u.name = 'db/lib'", 1).getJsonObject(0) + .getJsonObject("User"); assertEquals("Horace", user.getString("givenName")); assertEquals("Worblehat", user.getString("familyName")); assertEquals("Unseen University", user.getString("affiliation")); String query = "SELECT inv FROM Investigation inv JOIN inv.shifts AS s " - + "WHERE s.instrument.pid = 'ig:0815' AND s.comment = 'beamtime' " - + "AND s.startDate <= '2014-01-01 12:00:00' AND s.endDate >= '2014-01-01 12:00:00'"; + + "WHERE s.instrument.pid = 'ig:0815' AND s.comment = 'beamtime' " + + "AND s.startDate <= '2014-01-01 12:00:00' AND s.endDate >= '2014-01-01 12:00:00'"; JsonObject inv = search(session, query, 1).getJsonObject(0).getJsonObject("Investigation"); assertEquals("expt1", inv.getString("name")); assertEquals("zero", inv.getString("visitId")); @@ -1070,6 +1577,37 @@ private JsonArray searchInvestigations(Session session, String user, String text return result; } + private JsonObject searchDatafiles(Session session, String user, String text, Date lower, Date upper, + List parameters, String searchAfter, int limit, String sort, int n) + throws IcatException { + String response = session.searchDatafiles(user, text, lower, upper, parameters, searchAfter, limit, sort); + JsonObject responseObject = Json.createReader(new ByteArrayInputStream(response.getBytes())).readObject(); + JsonArray results = responseObject.getJsonArray("results"); + assertEquals(n, results.size()); + return responseObject; + } + + private JsonObject searchDatasets(Session session, String user, String text, Date lower, Date upper, + List parameters, String searchAfter, int limit, String sort, int n) + throws IcatException { + String response = session.searchDatasets(user, text, lower, upper, parameters, searchAfter, limit, sort); + JsonObject responseObject = Json.createReader(new ByteArrayInputStream(response.getBytes())).readObject(); + JsonArray results = responseObject.getJsonArray("results"); + assertEquals(n, results.size()); + return responseObject; + } + + private JsonObject searchInvestigations(Session session, String user, String text, Date lower, Date upper, + List parameters, List samples, String userFullName, String searchAfter, + int limit, String sort, int n) throws IcatException { + String response = session.searchInvestigations(user, text, lower, upper, parameters, samples, userFullName, + searchAfter, limit, sort); + JsonObject responseObject = Json.createReader(new ByteArrayInputStream(response.getBytes())).readObject(); + JsonArray results = responseObject.getJsonArray("results"); + assertEquals(n, results.size()); + return responseObject; + } + @Test public void testWriteGood() throws Exception { @@ -1095,7 +1633,7 @@ public void testWriteGood() throws Exception { JsonArray array = search(session, "SELECT it.name, it.facility.name FROM InvestigationType it WHERE it.id = " + newInvTypeId, 1) - .getJsonArray(0); + .getJsonArray(0); assertEquals("ztype", array.getString(0)); assertEquals("Test port facility", array.getString(1)); From 3c0d3bc8d398a501d01b027ea0ede5f8f268772e Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 8 Apr 2022 09:26:48 +0100 Subject: [PATCH 13/51] Text fields and related entities #267 --- .../org/icatproject/core/entity/Datafile.java | 35 +- .../core/entity/DatafileFormat.java | 9 + .../core/entity/DatafileParameter.java | 6 +- .../org/icatproject/core/entity/Dataset.java | 59 ++-- .../core/entity/DatasetParameter.java | 6 +- .../icatproject/core/entity/DatasetType.java | 9 + .../core/entity/EntityBaseBean.java | 7 +- .../org/icatproject/core/entity/Facility.java | 9 + .../core/entity/Investigation.java | 72 ++--- .../core/entity/InvestigationParameter.java | 6 +- .../core/entity/InvestigationType.java | 9 + .../core/entity/InvestigationUser.java | 12 +- .../icatproject/core/entity/Parameter.java | 17 +- .../core/entity/ParameterType.java | 10 + .../org/icatproject/core/entity/Sample.java | 30 +- .../icatproject/core/entity/SampleType.java | 24 ++ .../org/icatproject/core/entity/User.java | 12 + .../core/manager/ElasticsearchApi.java | 285 ++++++++-------- .../core/manager/EntityInfoHandler.java | 2 +- .../icatproject/core/manager/LuceneApi.java | 71 +--- .../core/manager/ParameterPOJO.java | 2 +- .../icatproject/core/manager/SearchApi.java | 144 ++++----- .../core/manager/SearchManager.java | 50 +-- .../core/manager/TestElasticsearchApi.java | 76 ++--- .../core/manager/TestEntityInfo.java | 13 +- .../icatproject/core/manager/TestLucene.java | 305 +++++++++++------- .../org/icatproject/integration/TestRS.java | 51 +-- 27 files changed, 690 insertions(+), 641 deletions(-) diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index 6f3eceee..53837e35 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -201,31 +201,26 @@ public void setSourceDatafiles(List sourceDatafiles) { } @Override - public void getDoc(JsonGenerator gen, SearchApi searchApi) { - StringBuilder sb = new StringBuilder(name); + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "name", name); if (description != null) { - sb.append(" " + description); + SearchApi.encodeString(gen, "description", description); } if (doi != null) { - sb.append(" " + doi); + SearchApi.encodeString(gen, "doi", doi); } if (datafileFormat != null) { - sb.append(" " + datafileFormat.getName()); + datafileFormat.getDoc(gen); } - searchApi.encodeTextField(gen, "text", sb.toString()); - searchApi.encodeSortedDocValuesField(gen, "name", name); if (datafileModTime != null) { - searchApi.encodeSortedDocValuesField(gen, "date", datafileModTime); + SearchApi.encodeLong(gen, "date", datafileModTime); } else if (datafileCreateTime != null) { - searchApi.encodeSortedDocValuesField(gen, "date", datafileCreateTime); + SearchApi.encodeLong(gen, "date", datafileCreateTime); } else { - searchApi.encodeSortedDocValuesField(gen, "date", modTime); + SearchApi.encodeLong(gen, "date", modTime); } - searchApi.encodeStringField(gen, "id", id, true); - searchApi.encodeStringField(gen, "dataset", dataset.id); - searchApi.encodeStringField(gen, "investigation", dataset.getInvestigation().id); - - // TODO User and Parameter support for Elasticsearch + SearchApi.encodeString(gen, "id", id); + SearchApi.encodeString(gen, "investigation.id", dataset.getInvestigation().id); } /** @@ -242,16 +237,18 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { public static Map getDocumentFields() throws IcatException { if (documentFields.size() == 0) { EntityInfoHandler eiHandler = EntityInfoHandler.getInstance(); - Relationship[] textRelationships = { + Relationship[] datafileFormatRelationships = { eiHandler.getRelationshipsByName(Datafile.class).get("datafileFormat") }; Relationship[] investigationRelationships = { eiHandler.getRelationshipsByName(Datafile.class).get("dataset") }; - documentFields.put("text", textRelationships); documentFields.put("name", null); + documentFields.put("description", null); + documentFields.put("doi", null); documentFields.put("date", null); documentFields.put("id", null); - documentFields.put("dataset", null); - documentFields.put("investigation", investigationRelationships); + documentFields.put("investigation.id", investigationRelationships); + documentFields.put("datafileFormat.id", null); + documentFields.put("datafileFormat.name", datafileFormatRelationships); } return documentFields; } diff --git a/src/main/java/org/icatproject/core/entity/DatafileFormat.java b/src/main/java/org/icatproject/core/entity/DatafileFormat.java index 71532678..ae09b337 100644 --- a/src/main/java/org/icatproject/core/entity/DatafileFormat.java +++ b/src/main/java/org/icatproject/core/entity/DatafileFormat.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; +import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; @@ -14,6 +15,8 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.icatproject.core.manager.SearchApi; + @Comment("A data file format") @SuppressWarnings("serial") @Entity @@ -95,4 +98,10 @@ public void setVersion(String version) { this.version = version; } + @Override + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "datafileFormat.name", name); + SearchApi.encodeString(gen, "datafileFormat.id", id); + } + } diff --git a/src/main/java/org/icatproject/core/entity/DatafileParameter.java b/src/main/java/org/icatproject/core/entity/DatafileParameter.java index 466adce2..8f25207e 100644 --- a/src/main/java/org/icatproject/core/entity/DatafileParameter.java +++ b/src/main/java/org/icatproject/core/entity/DatafileParameter.java @@ -54,9 +54,9 @@ public void setDatafile(Datafile datafile) { } @Override - public void getDoc(JsonGenerator gen, SearchApi searchApi) { - super.getDoc(gen, searchApi); - searchApi.encodeSortedDocValuesField(gen, "datafile", datafile.id); + public void getDoc(JsonGenerator gen) { + super.getDoc(gen); + SearchApi.encodeString(gen, "datafile.id", datafile.id); } } diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index 9e328dcf..f582203f 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -189,45 +189,31 @@ public void setType(DatasetType type) { } @Override - public void getDoc(JsonGenerator gen, SearchApi searchApi) { - - StringBuilder sb = new StringBuilder(name + " " + type.getName() + " " + type.getName()); // TODO duplicate type.getName() + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "name", name); if (description != null) { - sb.append(" " + description); + SearchApi.encodeString(gen, "description", description); } - if (doi != null) { - sb.append(" " + doi); - } - - if (sample != null) { - sb.append(" " + sample.getName()); - if (sample.getType() != null) { - sb.append(" " + sample.getType().getName()); - } + SearchApi.encodeString(gen, "doi", doi); } - - searchApi.encodeTextField(gen, "text", sb.toString()); - searchApi.encodeSortedDocValuesField(gen, "name", name); - if (startDate != null) { - searchApi.encodeSortedDocValuesField(gen, "startDate", startDate); + SearchApi.encodeLong(gen, "startDate", startDate); } else { - searchApi.encodeSortedDocValuesField(gen, "startDate", createTime); + SearchApi.encodeLong(gen, "startDate", createTime); } - if (endDate != null) { - searchApi.encodeSortedDocValuesField(gen, "endDate", endDate); + SearchApi.encodeLong(gen, "endDate", endDate); } else { - searchApi.encodeSortedDocValuesField(gen, "endDate", modTime); + SearchApi.encodeLong(gen, "endDate", modTime); } - searchApi.encodeStringField(gen, "id", id, true); - - searchApi.encodeSortedDocValuesField(gen, "id", id); + SearchApi.encodeString(gen, "id", id); + SearchApi.encodeString(gen, "investigation.id", investigation.id); - searchApi.encodeStringField(gen, "investigation", investigation.id); - - // TODO User, Parameter and Sample support for Elasticsearch + if (sample != null) { + sample.getDoc(gen, "sample."); + } + type.getDoc(gen); } /** @@ -244,13 +230,24 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { public static Map getDocumentFields() throws IcatException { if (documentFields.size() == 0) { EntityInfoHandler eiHandler = EntityInfoHandler.getInstance(); - Relationship[] textRelationships = { eiHandler.getRelationshipsByName(Dataset.class).get("type"), eiHandler.getRelationshipsByName(Dataset.class).get("sample") }; - documentFields.put("text", textRelationships); + Relationship[] sampleRelationships = { eiHandler.getRelationshipsByName(Dataset.class).get("sample") }; + Relationship[] sampleTypeRelationships = { eiHandler.getRelationshipsByName(Dataset.class).get("sample"), + eiHandler.getRelationshipsByName(Sample.class).get("type") }; + Relationship[] typeRelationships = { eiHandler.getRelationshipsByName(Dataset.class).get("type") }; documentFields.put("name", null); + documentFields.put("description", null); + documentFields.put("doi", null); documentFields.put("startDate", null); documentFields.put("endDate", null); documentFields.put("id", null); - documentFields.put("investigation", null); + documentFields.put("investigation.id", null); + documentFields.put("sample.id", null); + documentFields.put("sample.name", sampleRelationships); + documentFields.put("sample.investigation.id", sampleRelationships); + documentFields.put("sample.type.id", sampleRelationships); + documentFields.put("sample.type.name", sampleTypeRelationships); + documentFields.put("type.id", null); + documentFields.put("type.name", typeRelationships); } return documentFields; } diff --git a/src/main/java/org/icatproject/core/entity/DatasetParameter.java b/src/main/java/org/icatproject/core/entity/DatasetParameter.java index ac77c02b..e9fd22fd 100644 --- a/src/main/java/org/icatproject/core/entity/DatasetParameter.java +++ b/src/main/java/org/icatproject/core/entity/DatasetParameter.java @@ -54,8 +54,8 @@ public void setDataset(Dataset dataset) { } @Override - public void getDoc(JsonGenerator gen, SearchApi searchApi) { - super.getDoc(gen, searchApi); - searchApi.encodeSortedDocValuesField(gen, "dataset", dataset.id); + public void getDoc(JsonGenerator gen) { + super.getDoc(gen); + SearchApi.encodeString(gen, "dataset.id", dataset.id); } } \ No newline at end of file diff --git a/src/main/java/org/icatproject/core/entity/DatasetType.java b/src/main/java/org/icatproject/core/entity/DatasetType.java index 192d0a76..43a16d40 100644 --- a/src/main/java/org/icatproject/core/entity/DatasetType.java +++ b/src/main/java/org/icatproject/core/entity/DatasetType.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; +import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; @@ -14,6 +15,8 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.icatproject.core.manager.SearchApi; + @Comment("A type of data set") @SuppressWarnings("serial") @Entity @@ -71,4 +74,10 @@ public void setName(String name) { this.name = name; } + @Override + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "type.name", name); + SearchApi.encodeString(gen, "type.id", id); + } + } diff --git a/src/main/java/org/icatproject/core/entity/EntityBaseBean.java b/src/main/java/org/icatproject/core/entity/EntityBaseBean.java index 06e917d3..23a343f9 100644 --- a/src/main/java/org/icatproject/core/entity/EntityBaseBean.java +++ b/src/main/java/org/icatproject/core/entity/EntityBaseBean.java @@ -435,8 +435,11 @@ public String toString() { return this.getClass().getSimpleName() + ":" + id; } - /* This should be overridden by classes wishing to index things in a search engine */ - public void getDoc(JsonGenerator gen, SearchApi searchApi) { + /* + * This should be overridden by classes wishing to index things in a search + * engine + */ + public void getDoc(JsonGenerator gen) { } } diff --git a/src/main/java/org/icatproject/core/entity/Facility.java b/src/main/java/org/icatproject/core/entity/Facility.java index fd8831b0..fc75a8cc 100644 --- a/src/main/java/org/icatproject/core/entity/Facility.java +++ b/src/main/java/org/icatproject/core/entity/Facility.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; +import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; @@ -11,6 +12,8 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.icatproject.core.manager.SearchApi; + @Comment("An experimental facility") @SuppressWarnings("serial") @Entity @@ -177,4 +180,10 @@ public void setUrl(String url) { this.url = url; } + @Override + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "facility.name", name); + SearchApi.encodeString(gen, "facility.id", id); + } + } diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index 3239e36c..984b6c7a 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -265,66 +265,32 @@ public void setVisitId(String visitId) { } @Override - public void getDoc(JsonGenerator gen, SearchApi searchApi) { - StringBuilder sb = new StringBuilder(visitId + " " + name + " " + facility.getName() + " " + type.getName()); + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "name", name); + SearchApi.encodeString(gen, "visitId", visitId); + SearchApi.encodeString(gen, "title", title); if (summary != null) { - sb.append(" " + summary); + SearchApi.encodeString(gen, "summary", summary); } if (doi != null) { - sb.append(" " + doi); + SearchApi.encodeString(gen, "doi", doi); } - if (title != null) { - sb.append(" " + title); - } - searchApi.encodeTextField(gen, "text", sb.toString()); - searchApi.encodeSortedDocValuesField(gen, "name", name); if (startDate != null) { - searchApi.encodeSortedDocValuesField(gen, "startDate", startDate); + SearchApi.encodeLong(gen, "startDate", startDate); } else { - searchApi.encodeSortedDocValuesField(gen, "startDate", createTime); + SearchApi.encodeLong(gen, "startDate", createTime); } if (endDate != null) { - searchApi.encodeSortedDocValuesField(gen, "endDate", endDate); + SearchApi.encodeLong(gen, "endDate", endDate); } else { - searchApi.encodeSortedDocValuesField(gen, "endDate", modTime); - } - - investigationUsers.forEach((investigationUser) -> { - searchApi.encodeStringField(gen, "userName", investigationUser.getUser().getName()); - searchApi.encodeTextField(gen, "userFullName", investigationUser.getUser().getFullName()); - }); - - samples.forEach((sample) -> { - // searchApi.encodeSortedSetDocValuesFacetField(gen, "sampleName", - // sample.getName()); - searchApi.encodeTextField(gen, "sampleText", sample.getDocText()); - }); - - for (InvestigationParameter parameter : parameters) { - ParameterType type = parameter.type; - String parameterName = type.getName(); - String parameterUnits = type.getUnits(); - // searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterName", - // parameterName); - searchApi.encodeStringField(gen, "parameterName", parameterName); - searchApi.encodeStringField(gen, "parameterUnits", parameterUnits); - // TODO make all value types facetable... - if (type.getValueType() == ParameterValueType.STRING) { - // searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterStringValue", - // parameter.getStringValue()); - searchApi.encodeStringField(gen, "parameterStringValue", parameter.getStringValue()); - } else if (type.getValueType() == ParameterValueType.DATE_AND_TIME) { - searchApi.encodeStringField(gen, "parameterDateValue", parameter.getDateTimeValue()); - } else if (type.getValueType() == ParameterValueType.NUMERIC) { - searchApi.encodeDoublePoint(gen, "parameterNumericValue", parameter.getNumericValue()); - } + SearchApi.encodeLong(gen, "endDate", modTime); } - searchApi.encodeSortedDocValuesField(gen, "id", id); - - searchApi.encodeStringField(gen, "id", id, true); + SearchApi.encodeString(gen, "id", id); + facility.getDoc(gen); + type.getDoc(gen); } /** @@ -341,13 +307,21 @@ public void getDoc(JsonGenerator gen, SearchApi searchApi) { public static Map getDocumentFields() throws IcatException { if (documentFields.size() == 0) { EntityInfoHandler eiHandler = EntityInfoHandler.getInstance(); - Relationship[] textRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("type"), + Relationship[] typeRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("type") }; + Relationship[] facilityRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("facility") }; - documentFields.put("text", textRelationships); documentFields.put("name", null); + documentFields.put("visitId", null); + documentFields.put("title", null); + documentFields.put("summary", null); + documentFields.put("doi", null); documentFields.put("startDate", null); documentFields.put("endDate", null); documentFields.put("id", null); + documentFields.put("facility.name", facilityRelationships); + documentFields.put("facility.id", null); + documentFields.put("type.name", typeRelationships); + documentFields.put("type.id", null); } return documentFields; } diff --git a/src/main/java/org/icatproject/core/entity/InvestigationParameter.java b/src/main/java/org/icatproject/core/entity/InvestigationParameter.java index f3e632ba..d985237a 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationParameter.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationParameter.java @@ -55,8 +55,8 @@ public void setInvestigation(Investigation investigation) { } @Override - public void getDoc(JsonGenerator gen, SearchApi searchApi) { - super.getDoc(gen, searchApi); - searchApi.encodeSortedDocValuesField(gen, "investigation", investigation.id); + public void getDoc(JsonGenerator gen) { + super.getDoc(gen); + SearchApi.encodeString(gen, "investigation.id", investigation.id); } } \ No newline at end of file diff --git a/src/main/java/org/icatproject/core/entity/InvestigationType.java b/src/main/java/org/icatproject/core/entity/InvestigationType.java index afaee464..f9b5b8f3 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationType.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationType.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; +import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; @@ -14,6 +15,8 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.icatproject.core.manager.SearchApi; + @Comment("A type of investigation") @SuppressWarnings("serial") @Entity @@ -71,4 +74,10 @@ public void setDescription(String description) { this.description = description; } + @Override + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "type.name", name); + SearchApi.encodeString(gen, "type.id", id); + } + } diff --git a/src/main/java/org/icatproject/core/entity/InvestigationUser.java b/src/main/java/org/icatproject/core/entity/InvestigationUser.java index e2b79c0d..fe43019b 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationUser.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationUser.java @@ -38,14 +38,10 @@ public InvestigationUser() { } @Override - public void getDoc(JsonGenerator gen, SearchApi searchApi) { - if (user.getFullName() != null) { - searchApi.encodeTextField(gen, "text", user.getFullName()); - searchApi.encodeTextField(gen, "userFullName", user.getFullName()); - } - searchApi.encodeStringField(gen, "name", user.getName()); - searchApi.encodeStringField(gen, "userName", user.getName()); - searchApi.encodeSortedDocValuesField(gen, "investigation", investigation.id); + public void getDoc(JsonGenerator gen) { + user.getDoc(gen); + SearchApi.encodeString(gen, "investigation.id", investigation.id); + SearchApi.encodeString(gen, "id", id); } public String getRole() { diff --git a/src/main/java/org/icatproject/core/entity/Parameter.java b/src/main/java/org/icatproject/core/entity/Parameter.java index bbb0a8ff..0c7f7b41 100644 --- a/src/main/java/org/icatproject/core/entity/Parameter.java +++ b/src/main/java/org/icatproject/core/entity/Parameter.java @@ -162,21 +162,16 @@ public void postMergeFixup(EntityManager manager, GateKeeper gateKeeper) throws } @Override - public void getDoc(JsonGenerator gen, SearchApi searchApi) { - searchApi.encodeStringField(gen, "name", type.getName()); - searchApi.encodeStringField(gen, "parameterName", type.getName()); - searchApi.encodeStringField(gen, "units", type.getUnits()); - searchApi.encodeStringField(gen, "parameterUnits", type.getUnits()); + public void getDoc(JsonGenerator gen) { if (stringValue != null) { - searchApi.encodeStringField(gen, "stringValue", stringValue); - searchApi.encodeStringField(gen, "parameterStringValue", stringValue); + SearchApi.encodeString(gen, "stringValue", stringValue); } else if (numericValue != null) { - searchApi.encodeDoublePoint(gen, "numericValue", numericValue); - searchApi.encodeDoublePoint(gen, "parameterNumericValue", numericValue); + SearchApi.encodeDouble(gen, "numericValue", numericValue); } else if (dateTimeValue != null) { - searchApi.encodeStringField(gen, "dateTimeValue", dateTimeValue); - searchApi.encodeStringField(gen, "parameterDateValue", dateTimeValue); + SearchApi.encodeLong(gen, "dateTimeValue", dateTimeValue); } + type.getDoc(gen); + SearchApi.encodeString(gen, "id", id); } } diff --git a/src/main/java/org/icatproject/core/entity/ParameterType.java b/src/main/java/org/icatproject/core/entity/ParameterType.java index bd67a1a3..489ec742 100644 --- a/src/main/java/org/icatproject/core/entity/ParameterType.java +++ b/src/main/java/org/icatproject/core/entity/ParameterType.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; +import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; @@ -14,6 +15,8 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.icatproject.core.manager.SearchApi; + @Comment("A parameter type with unique name and units") @SuppressWarnings("serial") @Entity @@ -271,4 +274,11 @@ public void setVerified(boolean verified) { this.verified = verified; } + @Override + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "type.name", name); + SearchApi.encodeString(gen, "type.units", units); + SearchApi.encodeString(gen, "type.id", id); + } + } diff --git a/src/main/java/org/icatproject/core/entity/Sample.java b/src/main/java/org/icatproject/core/entity/Sample.java index 9ac8d437..d0a6b526 100644 --- a/src/main/java/org/icatproject/core/entity/Sample.java +++ b/src/main/java/org/icatproject/core/entity/Sample.java @@ -96,17 +96,31 @@ public void setType(SampleType type) { } @Override - public void getDoc(JsonGenerator gen, SearchApi searchApi) { - searchApi.encodeTextField(gen, "text", getDocText()); - searchApi.encodeTextField(gen, "sampleText", getDocText()); - searchApi.encodeSortedDocValuesField(gen, "investigation", investigation.id); + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "sample.name", name); + SearchApi.encodeString(gen, "id", id); + SearchApi.encodeString(gen, "investigation.id", investigation.id); + if (type != null) { + type.getDoc(gen); + } } - public String getDocText() { - StringBuilder sb = new StringBuilder(name); + /** + * Alternative method for encoding that applies a prefix to potentially + * ambiguous fields: "id" and "investigation.id". In the case of a single + * Dataset Sample, these fields will already be used by the Dataset and so + * cannot be overwritten by the Sample. + * + * @param gen JsonGenerator + * @param prefix String to precede all ambiguous field names. + */ + public void getDoc(JsonGenerator gen, String prefix) { + SearchApi.encodeString(gen, "sample.name", name); + SearchApi.encodeString(gen, prefix + "id", id); + SearchApi.encodeString(gen, prefix + "investigation.id", investigation.id); if (type != null) { - sb.append(" " + type.getName()); + type.getDoc(gen, prefix); } - return sb.toString(); } + } diff --git a/src/main/java/org/icatproject/core/entity/SampleType.java b/src/main/java/org/icatproject/core/entity/SampleType.java index fa402d7a..d4c12c2f 100644 --- a/src/main/java/org/icatproject/core/entity/SampleType.java +++ b/src/main/java/org/icatproject/core/entity/SampleType.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; +import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; @@ -14,6 +15,8 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.icatproject.core.manager.SearchApi; + @Comment("A sample to be used in an investigation") @SuppressWarnings("serial") @Entity @@ -84,4 +87,25 @@ public void setSamples(List samples) { this.samples = samples; } + @Override + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "sample.type.name", name); + SearchApi.encodeString(gen, "type.id", id); + } + + + /** + * Alternative method for encoding that applies a prefix to potentially + * ambiguous fields: "type.id". In the case of a single + * Dataset Sample, this fields will already be used by the Dataset and so + * cannot be overwritten by the Sample. + * + * @param gen JsonGenerator + * @param prefix String to precede all ambiguous field names. + */ + public void getDoc(JsonGenerator gen, String prefix) { + SearchApi.encodeString(gen, "sample.type.name", name); + SearchApi.encodeString(gen, prefix + "type.id", id); + } + } diff --git a/src/main/java/org/icatproject/core/entity/User.java b/src/main/java/org/icatproject/core/entity/User.java index a484997a..6b59731a 100644 --- a/src/main/java/org/icatproject/core/entity/User.java +++ b/src/main/java/org/icatproject/core/entity/User.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; +import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; @@ -11,6 +12,8 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.icatproject.core.manager.SearchApi; + @Comment("A user of the facility") @SuppressWarnings("serial") @Entity @@ -147,4 +150,13 @@ public String toString() { return "User[name=" + name + "]"; } + @Override + public void getDoc(JsonGenerator gen) { + if (fullName != null) { + SearchApi.encodeText(gen, "user.fullName", fullName); + } + SearchApi.encodeString(gen, "user.name", name); + SearchApi.encodeString(gen, "user.id", id); + } + } diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java index 5a294c2e..78e1e21a 100644 --- a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java +++ b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java @@ -146,7 +146,9 @@ private void initMappings() throws IcatException { + "else {ctx._source.parameterNumericValue.addAll(params['parameterNumericValue'])}"))); for (String index : INDEX_PROPERTIES.keySet()) { - client.indices().create(c -> c.index(index).mappings(m -> m.dynamic(DynamicMapping.False).properties(INDEX_PROPERTIES.get(index)))) + client.indices() + .create(c -> c.index(index) + .mappings(m -> m.dynamic(DynamicMapping.False).properties(INDEX_PROPERTIES.get(index)))) .acknowledged(); } // TODO consider both dynamic field names and nested fields @@ -165,29 +167,6 @@ public void clear() throws IcatException { } } - // TODO Ideally want to write these as k:v pairs in an object, not objects in a - // list, but this is to be consistent with Lucene (for now) - - public void encodeSortedDocValuesField(JsonGenerator gen, String name, Long value) { - gen.writeStartObject().write(name, value).writeEnd(); - } - - public void encodeSortedDocValuesField(JsonGenerator gen, String name, String value) { - gen.writeStartObject().write(name, value).writeEnd(); - } - - public void encodeStoredId(JsonGenerator gen, Long id) { - gen.writeStartObject().write("id", Long.toString(id)).writeEnd(); - } - - public void encodeStringField(JsonGenerator gen, String name, Date value) { - String timeString; - synchronized (df) { - timeString = df.format(value); - } - gen.writeStartObject().write(name, timeString).writeEnd(); - } - @Override public void freeSearcher(String uid) throws IcatException { @@ -257,134 +236,144 @@ public List facetSearch(JsonObject facetQuery, int maxResults, i // @Override // public SearchResult getResults(JsonObject query, int maxResults, String sort) - // throws IcatException { - // // TODO sort argument not supported - // try { - // String index; - // if (query.keySet().contains("target")) { - // index = query.getString("target").toLowerCase(); - // } else { - // index = query.getString("_all"); - // } - // OpenPointInTimeResponse pitResponse = client.openPointInTime(p -> p - // .index(index) - // .keepAlive(t -> t.time("1m"))); - // String pit = pitResponse.id(); - // pitMap.put(pit, 0); - // return getResults(pit, query, maxResults); - // } catch (ElasticsearchException | IOException e) { - // throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - // } + // throws IcatException { + // // TODO sort argument not supported + // try { + // String index; + // if (query.keySet().contains("target")) { + // index = query.getString("target").toLowerCase(); + // } else { + // index = query.getString("_all"); + // } + // OpenPointInTimeResponse pitResponse = client.openPointInTime(p -> p + // .index(index) + // .keepAlive(t -> t.time("1m"))); + // String pit = pitResponse.id(); + // pitMap.put(pit, 0); + // return getResults(pit, query, maxResults); + // } catch (ElasticsearchException | IOException e) { + // throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + + // e.getMessage()); + // } // } // @Override // public SearchResult getResults(String uid, JsonObject query, int maxResults) - // throws IcatException { - // try { - // logger.debug("getResults for query: {}", query.toString()); - // Set fields = query.keySet(); - // BoolQuery.Builder builder = new BoolQuery.Builder(); - // for (String field : fields) { - // if (field.equals("text")) { - // String text = query.getString("text"); - // builder.must(m -> m.queryString(q -> q.defaultField("text").query(text))); - // } else if (field.equals("lower")) { - // Long time = decodeTime(query.getString("lower")); - // builder.must(m -> m - // .bool(b -> b - // .should(s -> s - // .range(r -> r - // .field("date") - // .gte(JsonData.of(time)))) - // .should(s -> s - // .range(r -> r - // .field("startDate") - // .gte(JsonData.of(time)))))); - // } else if (field.equals("upper")) { - // Long time = decodeTime(query.getString("upper")); - // builder.must(m -> m - // .bool(b -> b - // .should(s -> s - // .range(r -> r - // .field("date") - // .lte(JsonData.of(time)))) - // .should(s -> s - // .range(r -> r - // .field("endDate") - // .lte(JsonData.of(time)))))); - // } else if (field.equals("user")) { - // String user = query.getString("user"); - // builder.filter(f -> f.match(t -> t - // .field("userName") - // .operator(Operator.And) - // .query(q -> q.stringValue(user)))); - // } else if (field.equals("userFullName")) { - // String userFullName = query.getString("userFullName"); - // builder.filter(f -> f.queryString(q -> q.defaultField("userFullName").query(userFullName))); - // } else if (field.equals("samples")) { - // JsonArray samples = query.getJsonArray("samples"); - // for (int i = 0; i < samples.size(); i++) { - // String sample = samples.getString(i); - // builder.filter( - // f -> f.queryString(q -> q.defaultField("sampleText").query(sample))); - // } - // } else if (field.equals("parameters")) { - // for (JsonValue parameterValue : query.getJsonArray("parameters")) { - // // TODO there are more things to support and consider here... e.g. parameters - // // with a numeric range not a numeric value - // BoolQuery.Builder parameterBuilder = new BoolQuery.Builder(); - // JsonObject parameterObject = (JsonObject) parameterValue; - // String name = parameterObject.getString("name", null); - // String units = parameterObject.getString("units", null); - // String stringValue = parameterObject.getString("stringValue", null); - // Long lowerDate = decodeTime(parameterObject.getString("lowerDateValue", null)); - // Long upperDate = decodeTime(parameterObject.getString("upperDateValue", null)); - // JsonNumber lowerNumeric = parameterObject.getJsonNumber("lowerNumericValue"); - // JsonNumber upperNumeric = parameterObject.getJsonNumber("upperNumericValue"); - // if (name != null) { - // parameterBuilder.must(m -> m.match(a -> a.field("parameterName").operator(Operator.And) - // .query(q -> q.stringValue(name)))); - // } - // if (units != null) { - // parameterBuilder.must(m -> m.match(a -> a.field("parameterUnits").operator(Operator.And) - // .query(q -> q.stringValue(units)))); - // } - // if (stringValue != null) { - // parameterBuilder.must(m -> m.match(a -> a.field("parameterStringValue") - // .operator(Operator.And).query(q -> q.stringValue(stringValue)))); - // } else if (lowerDate != null && upperDate != null) { - // parameterBuilder.must(m -> m.range(r -> r.field("parameterDateValue") - // .gte(JsonData.of(lowerDate)).lte(JsonData.of(upperDate)))); - // } else if (lowerNumeric != null && upperNumeric != null) { - // parameterBuilder.must(m -> m.range( - // r -> r.field("parameterNumericValue").gte(JsonData.of(lowerNumeric.doubleValue())) - // .lte(JsonData.of(upperNumeric.doubleValue())))); - // } - // builder.filter(f -> f.bool(b -> parameterBuilder)); - // } - // // TODO consider support for other fields (would require dynamic fields) - // } - // } - // Integer from = pitMap.get(uid); - // SearchResponse response = client.search(s -> s - // .size(maxResults) - // .pit(p -> p.id(uid).keepAlive(t -> t.time("1m"))) - // .query(q -> q.bool(builder.build())) - // // TODO check the ordering? - // .from(from) - // .sort(o -> o.score(c -> c.order(SortOrder.Desc))) - // .sort(o -> o.field(f -> f.field("id").order(SortOrder.Asc))), ElasticsearchDocument.class); - // SearchResult result = new SearchResult(); - // // result.setUid(uid); - // pitMap.put(uid, from + maxResults); - // List entities = result.getResults(); - // for (Hit hit : response.hits().hits()) { - // entities.add(new ScoredEntityBaseBean(Long.parseLong(hit.id()), hit.score().floatValue(), "")); - // } - // return result; - // } catch (ElasticsearchException | IOException | ParseException e) { - // throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - // } + // throws IcatException { + // try { + // logger.debug("getResults for query: {}", query.toString()); + // Set fields = query.keySet(); + // BoolQuery.Builder builder = new BoolQuery.Builder(); + // for (String field : fields) { + // if (field.equals("text")) { + // String text = query.getString("text"); + // builder.must(m -> m.queryString(q -> q.defaultField("text").query(text))); + // } else if (field.equals("lower")) { + // Long time = decodeTime(query.getString("lower")); + // builder.must(m -> m + // .bool(b -> b + // .should(s -> s + // .range(r -> r + // .field("date") + // .gte(JsonData.of(time)))) + // .should(s -> s + // .range(r -> r + // .field("startDate") + // .gte(JsonData.of(time)))))); + // } else if (field.equals("upper")) { + // Long time = decodeTime(query.getString("upper")); + // builder.must(m -> m + // .bool(b -> b + // .should(s -> s + // .range(r -> r + // .field("date") + // .lte(JsonData.of(time)))) + // .should(s -> s + // .range(r -> r + // .field("endDate") + // .lte(JsonData.of(time)))))); + // } else if (field.equals("user")) { + // String user = query.getString("user"); + // builder.filter(f -> f.match(t -> t + // .field("userName") + // .operator(Operator.And) + // .query(q -> q.stringValue(user)))); + // } else if (field.equals("userFullName")) { + // String userFullName = query.getString("userFullName"); + // builder.filter(f -> f.queryString(q -> + // q.defaultField("userFullName").query(userFullName))); + // } else if (field.equals("samples")) { + // JsonArray samples = query.getJsonArray("samples"); + // for (int i = 0; i < samples.size(); i++) { + // String sample = samples.getString(i); + // builder.filter( + // f -> f.queryString(q -> q.defaultField("sampleText").query(sample))); + // } + // } else if (field.equals("parameters")) { + // for (JsonValue parameterValue : query.getJsonArray("parameters")) { + // // TODO there are more things to support and consider here... e.g. parameters + // // with a numeric range not a numeric value + // BoolQuery.Builder parameterBuilder = new BoolQuery.Builder(); + // JsonObject parameterObject = (JsonObject) parameterValue; + // String name = parameterObject.getString("name", null); + // String units = parameterObject.getString("units", null); + // String stringValue = parameterObject.getString("stringValue", null); + // Long lowerDate = decodeTime(parameterObject.getString("lowerDateValue", + // null)); + // Long upperDate = decodeTime(parameterObject.getString("upperDateValue", + // null)); + // JsonNumber lowerNumeric = parameterObject.getJsonNumber("lowerNumericValue"); + // JsonNumber upperNumeric = parameterObject.getJsonNumber("upperNumericValue"); + // if (name != null) { + // parameterBuilder.must(m -> m.match(a -> + // a.field("parameterName").operator(Operator.And) + // .query(q -> q.stringValue(name)))); + // } + // if (units != null) { + // parameterBuilder.must(m -> m.match(a -> + // a.field("parameterUnits").operator(Operator.And) + // .query(q -> q.stringValue(units)))); + // } + // if (stringValue != null) { + // parameterBuilder.must(m -> m.match(a -> a.field("parameterStringValue") + // .operator(Operator.And).query(q -> q.stringValue(stringValue)))); + // } else if (lowerDate != null && upperDate != null) { + // parameterBuilder.must(m -> m.range(r -> r.field("parameterDateValue") + // .gte(JsonData.of(lowerDate)).lte(JsonData.of(upperDate)))); + // } else if (lowerNumeric != null && upperNumeric != null) { + // parameterBuilder.must(m -> m.range( + // r -> + // r.field("parameterNumericValue").gte(JsonData.of(lowerNumeric.doubleValue())) + // .lte(JsonData.of(upperNumeric.doubleValue())))); + // } + // builder.filter(f -> f.bool(b -> parameterBuilder)); + // } + // // TODO consider support for other fields (would require dynamic fields) + // } + // } + // Integer from = pitMap.get(uid); + // SearchResponse response = client.search(s -> s + // .size(maxResults) + // .pit(p -> p.id(uid).keepAlive(t -> t.time("1m"))) + // .query(q -> q.bool(builder.build())) + // // TODO check the ordering? + // .from(from) + // .sort(o -> o.score(c -> c.order(SortOrder.Desc))) + // .sort(o -> o.field(f -> f.field("id").order(SortOrder.Asc))), + // ElasticsearchDocument.class); + // SearchResult result = new SearchResult(); + // // result.setUid(uid); + // pitMap.put(uid, from + maxResults); + // List entities = result.getResults(); + // for (Hit hit : response.hits().hits()) { + // entities.add(new ScoredEntityBaseBean(Long.parseLong(hit.id()), + // hit.score().floatValue(), "")); + // } + // return result; + // } catch (ElasticsearchException | IOException | ParseException e) { + // throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + + // e.getMessage()); + // } // } @Override diff --git a/src/main/java/org/icatproject/core/manager/EntityInfoHandler.java b/src/main/java/org/icatproject/core/manager/EntityInfoHandler.java index 2e030251..b7d7413f 100644 --- a/src/main/java/org/icatproject/core/manager/EntityInfoHandler.java +++ b/src/main/java/org/icatproject/core/manager/EntityInfoHandler.java @@ -593,7 +593,7 @@ private PrivateEntityInfo buildEi(Class objectClass) t boolean hasSearchDoc = true; try { - objectClass.getDeclaredMethod("getDoc", JsonGenerator.class, SearchApi.class); + objectClass.getDeclaredMethod("getDoc", JsonGenerator.class); } catch (NoSuchMethodException e) { hasSearchDoc = false; } diff --git a/src/main/java/org/icatproject/core/manager/LuceneApi.java b/src/main/java/org/icatproject/core/manager/LuceneApi.java index 6cf11d0e..90b57ace 100644 --- a/src/main/java/org/icatproject/core/manager/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/LuceneApi.java @@ -17,6 +17,7 @@ import javax.json.JsonObject; import javax.json.JsonObjectBuilder; import javax.json.JsonReader; +import javax.json.JsonValue; import javax.json.stream.JsonGenerator; import javax.json.stream.JsonParser; import javax.json.stream.JsonParser.Event; @@ -65,52 +66,6 @@ private String getTargetPath(JsonObject query) throws IcatException { return path; } - // TODO this method of encoding an entity as an array of 3 key objects that - // represent single field each - // is something that should be streamlined, but would require changes to - // icat.lucene - - @Override - public void encodeSortedDocValuesField(JsonGenerator gen, String name, Long value) { - gen.writeStartObject().write("type", "SortedDocValuesField").write("name", name).write("value", value) - .writeEnd(); - } - - @Override - public void encodeSortedDocValuesField(JsonGenerator gen, String name, String value) { - encodeStringField(gen, name, value); // TODO leading to duplications in the _source, do we need both here? - gen.writeStartObject().write("type", "SortedDocValuesField").write("name", name).write("value", value) - .writeEnd(); - } - - @Override - public void encodeDoublePoint(JsonGenerator gen, String name, Double value) { - gen.writeStartObject().write("type", "DoublePoint").write("name", name).write("value", value) - .write("store", true).writeEnd(); - } - - // public void encodeSortedSetDocValuesFacetField(JsonGenerator gen, String name, String value) { - // gen.writeStartObject().write("type", "SortedSetDocValuesFacetField").write("name", name).write("value", value) - // .writeEnd(); - // } - - @Override - public void encodeStringField(JsonGenerator gen, String name, String value) { - gen.writeStartObject().write("type", "StringField").write("name", name).write("value", value).writeEnd(); - } - - @Override - public void encodeStringField(JsonGenerator gen, String name, Long value, Boolean store) { - gen.writeStartObject().write("type", "StringField").write("name", name).write("value", Long.toString(value)).write("store", store).writeEnd(); - } - - @Override - public void encodeTextField(JsonGenerator gen, String name, String value) { - if (value != null) { - gen.writeStartObject().write("type", "TextField").write("name", name).write("value", value).writeEnd(); - } - } - URI server; public LuceneApi(URI server) { @@ -118,7 +73,8 @@ public LuceneApi(URI server) { } public void addNow(String entityName, List ids, EntityManager manager, - Class klass, ExecutorService getBeanDocExecutor) throws IcatException, IOException, URISyntaxException { + Class klass, ExecutorService getBeanDocExecutor) + throws IcatException, IOException, URISyntaxException { URI uri = new URIBuilder(server).setPath(basePath + "/addNow/" + entityName) .build(); @@ -132,8 +88,8 @@ public void addNow(String entityName, List ids, EntityManager manager, for (Long id : ids) { EntityBaseBean bean = (EntityBaseBean) manager.find(klass, id); if (bean != null) { - gen.writeStartArray(); - bean.getDoc(gen, this); + gen.writeStartObject(); + bean.getDoc(gen); gen.writeEnd(); } } @@ -154,7 +110,7 @@ public void addNow(String entityName, List ids, EntityManager manager, } @Override - public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { + public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { JsonObjectBuilder builder = Json.createObjectBuilder(); builder.add("doc", lastBean.getEngineDocId()); builder.add("shardIndex", -1); @@ -167,16 +123,17 @@ public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throw JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); for (String key : object.keySet()) { if (!lastBean.getSource().keySet().contains(key)) { - throw new IcatException(IcatExceptionType.INTERNAL, "Cannot build searchAfter document from source as sorted field " + key + " missing."); + throw new IcatException(IcatExceptionType.INTERNAL, + "Cannot build searchAfter document from source as sorted field " + key + " missing."); } - String value = lastBean.getSource().getString(key); + JsonValue value = lastBean.getSource().get(key); arrayBuilder.add(value); } builder.add("fields", arrayBuilder); } } return builder.build().toString(); - } + } @Override public void clear() throws IcatException { @@ -272,7 +229,8 @@ private List getFacets(URI uri, CloseableHttpClient httpclient, } @Override - public SearchResult getResults(JsonObject query, String searchAfter, int blockSize, String sort, List fields) throws IcatException { + public SearchResult getResults(JsonObject query, String searchAfter, int blockSize, String sort, + List fields) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { String indexPath = getTargetPath(query); URI uri = new URIBuilder(server).setPath(basePath + "/" + indexPath) @@ -310,8 +268,9 @@ private SearchResult getResults(URI uri, CloseableHttpClient httpclient, String Rest.checkStatus(response, IcatExceptionType.INTERNAL); try (JsonReader reader = Json.createReader(response.getEntity().getContent())) { JsonObject responseObject = reader.readObject(); - List resultsArray = responseObject.getJsonArray("results").getValuesAs(JsonObject.class); - for (JsonObject resultObject: resultsArray) { + List resultsArray = responseObject.getJsonArray("results") + .getValuesAs(JsonObject.class); + for (JsonObject resultObject : resultsArray) { int luceneDocId = resultObject.getInt("_id"); Float score = Float.NaN; if (resultObject.keySet().contains("_score")) { diff --git a/src/main/java/org/icatproject/core/manager/ParameterPOJO.java b/src/main/java/org/icatproject/core/manager/ParameterPOJO.java index 52271a7f..bcafe8e8 100644 --- a/src/main/java/org/icatproject/core/manager/ParameterPOJO.java +++ b/src/main/java/org/icatproject/core/manager/ParameterPOJO.java @@ -43,7 +43,7 @@ public String toString() { if (stringValue != null) { sb.append(" stringValue:" + stringValue); } else if (lowerDateValue != null) { - sb.append(" lowerDateValue:" + lowerDateValue + " upperDateValue:" + upperDateValue); + sb.append(" lowerDateValue:" + lowerDateValue.getTime() + " upperDateValue:" + upperDateValue.getTime()); } else if (lowerNumericValue != null) { sb.append(", lowerNumericValue:" + lowerNumericValue + " upperNumericValue:" + upperNumericValue); } diff --git a/src/main/java/org/icatproject/core/manager/SearchApi.java b/src/main/java/org/icatproject/core/manager/SearchApi.java index 8090db03..5237145a 100644 --- a/src/main/java/org/icatproject/core/manager/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/SearchApi.java @@ -113,22 +113,19 @@ protected static String encodeDate(Date dateValue) { } } - // TODO if encoding methods are unified across all APIs, then we can make these - // static - public void encodeDoublePoint(JsonGenerator gen, String name, Double value) { - gen.writeStartObject().write(name, value).writeEnd(); - } - - public void encodeSortedDocValuesField(JsonGenerator gen, String name, Date value) { - encodeSortedDocValuesField(gen, name, encodeDate(value)); - } - - public void encodeSortedDocValuesField(JsonGenerator gen, String name, Long value) { - gen.writeStartObject().write(name, value).writeEnd(); + public static String encodeDeletion(EntityBaseBean bean) { + String entityName = bean.getClass().getSimpleName(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject().writeStartObject("delete"); + gen.write("_index", entityName).write("_id", bean.getId().toString()); + gen.writeEnd().writeEnd(); + } + return baos.toString(); } - public void encodeSortedDocValuesField(JsonGenerator gen, String name, String value) { - encodeStringField(gen, name, value); + public static void encodeDouble(JsonGenerator gen, String name, Double value) { + gen.write(name, value); } // public void encodeSortedSetDocValuesFacetField(JsonGenerator gen, String @@ -136,25 +133,38 @@ public void encodeSortedDocValuesField(JsonGenerator gen, String name, String va // encodeStringField(gen, name, value); // } - public void encodeStringField(JsonGenerator gen, String name, Date value) { - encodeStringField(gen, name, encodeDate(value)); + public static void encodeLong(JsonGenerator gen, String name, Date value) { + gen.write(name, value.getTime()); } - public void encodeStringField(JsonGenerator gen, String name, Long value) { - encodeStringField(gen, name, Long.toString(value)); + public static String encodeOperation(String operation, EntityBaseBean bean) throws IcatException { + Long icatId = bean.getId(); + if (icatId == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, bean.toString() + " had null id"); + } + String entityName = bean.getClass().getSimpleName(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject().writeStartObject(operation); + gen.write("_index", entityName).write("_id", icatId.toString()); + gen.writeStartObject("doc"); + bean.getDoc(gen); + gen.writeEnd().writeEnd().writeEnd(); + } + return baos.toString(); } - public void encodeStringField(JsonGenerator gen, String name, Long value, Boolean store) { - encodeStringField(gen, name, value); + public static void encodeString(JsonGenerator gen, String name, Long value) { + gen.write(name, Long.toString(value)); } - public void encodeStringField(JsonGenerator gen, String name, String value) { - gen.writeStartObject().write(name, value).writeEnd(); + public static void encodeString(JsonGenerator gen, String name, String value) { + gen.write(name, value); } - public void encodeTextField(JsonGenerator gen, String name, String value) { + public static void encodeText(JsonGenerator gen, String name, String value) { if (value != null) { - gen.writeStartObject().write(name, value).writeEnd(); + gen.write(name, value); } } @@ -171,28 +181,31 @@ public void addNow(String entityName, List ids, EntityManager manager, if (bean != null) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); // Document fields are wrapped in an array - bean.getDoc(gen, this); // Fields - gen.writeEnd(); + gen.writeStartObject().writeStartObject("create"); + gen.write("_index", entityName).write("_id", bean.getId().toString()); + gen.writeStartObject("doc"); + bean.getDoc(gen); + gen.writeEnd().writeEnd().writeEnd(); } if (sb.length() != 1) { sb.append(','); } - sb.append("[\"").append(entityName).append("\",null,").append(baos.toString()).append(']'); + sb.append(baos.toString()); } } sb.append("]"); modify(sb.toString()); } - public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { + public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { if (sort != null && !sort.equals("")) { try (JsonReader reader = Json.createReader(new ByteArrayInputStream(sort.getBytes()))) { JsonObject object = reader.readObject(); JsonArrayBuilder builder = Json.createArrayBuilder(); for (String key : object.keySet()) { if (!lastBean.getSource().keySet().contains(key)) { - throw new IcatException(IcatExceptionType.INTERNAL, "Cannot build searchAfter document from source as sorted field " + key + " missing."); + throw new IcatException(IcatExceptionType.INTERNAL, + "Cannot build searchAfter document from source as sorted field " + key + " missing."); } String value = lastBean.getSource().getString(key); builder.add(value); @@ -202,13 +215,14 @@ public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throw } else { JsonArrayBuilder builder = Json.createArrayBuilder(); if (Float.isNaN(lastBean.getScore())) { - throw new IcatException(IcatExceptionType.INTERNAL, "Cannot build searchAfter document from source as score was NaN."); + throw new IcatException(IcatExceptionType.INTERNAL, + "Cannot build searchAfter document from source as score was NaN."); } builder.add(lastBean.getScore()); builder.add(lastBean.getEntityBaseBeanId()); return builder.build().toString(); } - } + } public void clear() throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { @@ -251,7 +265,8 @@ public SearchResult getResults(JsonObject query, int maxResults, String sort) th return getResults(query, null, maxResults, sort, Arrays.asList("id")); } - public SearchResult getResults(JsonObject query, String searchAfter, int blockSize, String sort, List fields) throws IcatException { + public SearchResult getResults(JsonObject query, String searchAfter, int blockSize, String sort, + List fields) throws IcatException { // return getResults(uid.toString(), query, blockSize); // TODO @@ -263,7 +278,7 @@ private SearchResult getResults(String uid, JsonObject query, int maxResults) th String index; Set fields = query.keySet(); if (fields.contains("target")) { - index = query.getString("target").toLowerCase(); + index = query.getString("target"); } else { index = query.getString("_all"); } @@ -370,7 +385,8 @@ private SearchResult getResults(String uid, JsonObject query, int maxResults) th JsonObject jsonObject = jsonReader.readObject(); JsonArray hits = jsonObject.getJsonObject("hits").getJsonArray("hits"); for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { - entities.add(new ScoredEntityBaseBean(hit.getInt("_id"), hit.getJsonNumber("_score").bigDecimalValue().floatValue(), null)); // TODO + entities.add(new ScoredEntityBaseBean(hit.getInt("_id"), + hit.getJsonNumber("_score").bigDecimalValue().floatValue(), null)); // TODO } } return result; @@ -390,56 +406,22 @@ public void unlock(String entityName) throws IcatException { public void modify(String json) throws IcatException { // TODO replace other places with this format // TODO this assumes simple update/create with no relation - // Format should be [{"index": "investigation", "id": "123", "document": {}}, ...] + // Format should be [{"index": "investigation", "id": "123", "document": {}}, + // ...] try (CloseableHttpClient httpclient = HttpClients.createDefault()) { logger.debug("modify: {}", json); StringBuilder sb = new StringBuilder(); JsonReader jsonReader = Json.createReader(new StringReader(json)); JsonArray outerArray = jsonReader.readArray(); for (JsonObject operation : outerArray.getValuesAs(JsonObject.class)) { - String index = operation.getString("index", null); - String id = operation.getString("id", null); - JsonObject document = operation.getJsonObject("document"); - if (index == null) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Cannot modify a document without the target index"); - } - if (document == null) { - // Delete - if (id == null) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Cannot delete a document without an id"); - } - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject().writeStartObject("delete").write("_index", index).write("_id", id).writeEnd().writeEnd(); - } - sb.append(baos.toString()).append("\n"); + if (operation.containsKey("doc")) { + JsonObject document = operation.getJsonObject("doc"); + operation.remove("doc"); + sb.append(operation.toString()).append("\n"); + sb.append(document.toString()).append("\n"); } else { - if (id == null) { - // Create - id = document.getString("id", null); - if (id == null) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Cannot index a document without an id"); - } - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject().writeStartObject("create").write("_index", index).write("_id", id).writeEnd().writeEnd(); - } - sb.append(baos.toString()).append("\n"); - sb.append(document.toString()).append("\n"); - } else { - // Update - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject().writeStartObject("update").write("_index", index).write("_id", id).writeEnd().writeEnd(); - } - sb.append(baos.toString()).append("\n"); - sb.append(document.toString()).append("\n"); - } + sb.append(operation.toString()).append("\n"); } - } URI uri = new URIBuilder(server).setPath(basePath + "/_bulk").build(); HttpPost httpPost = new HttpPost(uri); @@ -479,10 +461,10 @@ public static JsonObject buildQuery(String target, String user, String text, Dat builder.add("text", text); } if (lower != null) { - builder.add("lower", encodeDate(lower)); + builder.add("lower", lower.getTime()); } if (upper != null) { - builder.add("upper", encodeDate(upper)); + builder.add("upper", upper.getTime()); } if (parameters != null && !parameters.isEmpty()) { JsonArrayBuilder parametersBuilder = Json.createArrayBuilder(); @@ -498,10 +480,10 @@ public static JsonObject buildQuery(String target, String user, String text, Dat parameterBuilder.add("stringValue", parameter.stringValue); } if (parameter.lowerDateValue != null) { - parameterBuilder.add("lowerDateValue", encodeDate(parameter.lowerDateValue)); + parameterBuilder.add("lowerDateValue", parameter.lowerDateValue.getTime()); } if (parameter.upperDateValue != null) { - parameterBuilder.add("upperDateValue", encodeDate(parameter.upperDateValue)); + parameterBuilder.add("upperDateValue", parameter.upperDateValue.getTime()); } if (parameter.lowerNumericValue != null) { parameterBuilder.add("lowerNumericValue", parameter.lowerNumericValue); diff --git a/src/main/java/org/icatproject/core/manager/SearchManager.java b/src/main/java/org/icatproject/core/manager/SearchManager.java index 0377404a..43751643 100644 --- a/src/main/java/org/icatproject/core/manager/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/SearchManager.java @@ -1,7 +1,6 @@ package org.icatproject.core.manager; import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileReader; import java.io.FileWriter; @@ -31,9 +30,7 @@ import javax.ejb.EJB; import javax.ejb.Singleton; import javax.ejb.Startup; -import javax.json.Json; import javax.json.JsonObject; -import javax.json.stream.JsonGenerator; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; import javax.persistence.PersistenceUnit; @@ -332,38 +329,17 @@ public static List getPublicSearchFields(GateKeeper gateKeeper, String s } public void addDocument(EntityBaseBean bean) throws IcatException { - String entityName = bean.getClass().getSimpleName(); - if (eiHandler.hasSearchDoc(bean.getClass()) && entitiesToIndex.contains(entityName)) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - bean.getDoc(gen, searchApi); - gen.writeEnd(); - } - enqueue(entityName, baos.toString(), null); + Class klass = bean.getClass(); + if (eiHandler.hasSearchDoc(klass) && entitiesToIndex.contains(klass.getSimpleName())) { + enqueue(SearchApi.encodeOperation("create", bean)); } } - public void enqueue(String entityName, String json, Long id) throws IcatException { - - StringBuilder sb = new StringBuilder(); - sb.append("[\"").append(entityName).append('"'); - if (id != null) { - sb.append(',').append(id); - } else { - sb.append(",null"); - } - if (json != null) { - sb.append(',').append(json); - } else { - sb.append(",null"); - } - sb.append(']'); - + public void enqueue(String json) throws IcatException { synchronized (queueFileLock) { try { FileWriter output = new FileWriter(queueFile, true); - output.write(sb.toString() + "\n"); + output.write(json + "\n"); output.close(); } catch (IOException e) { String msg = "Problems writing to " + queueFile + " " + e.getMessage(); @@ -394,9 +370,7 @@ public void commit() throws IcatException { public void deleteDocument(EntityBaseBean bean) throws IcatException { if (eiHandler.hasSearchDoc(bean.getClass())) { - String entityName = bean.getClass().getSimpleName(); - Long id = bean.getId(); - enqueue(entityName, null, id); + enqueue(SearchApi.encodeDeletion(bean)); } } @@ -548,15 +522,9 @@ public void populate(String entityName, long minid) throws IcatException { } public void updateDocument(EntityBaseBean bean) throws IcatException { - String entityName = bean.getClass().getSimpleName(); - if (eiHandler.hasSearchDoc(bean.getClass()) && entitiesToIndex.contains(entityName)) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - bean.getDoc(gen, searchApi); - gen.writeEnd(); - } - enqueue(entityName, baos.toString(), bean.getId()); + Class klass = bean.getClass(); + if (eiHandler.hasSearchDoc(klass) && entitiesToIndex.contains(klass.getSimpleName())) { + enqueue(SearchApi.encodeOperation("update", bean)); } } diff --git a/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java b/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java index 9269a378..67c78260 100644 --- a/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java @@ -72,11 +72,11 @@ // ByteArrayOutputStream baos = new ByteArrayOutputStream(); // try (JsonGenerator gen = Json.createGenerator(baos)) { // gen.writeStartArray(); -// searchApi.encodeTextField(gen, "text", "Elephants and Aardvarks"); -// searchApi.encodeStringField(gen, "startDate", new Date()); -// searchApi.encodeStringField(gen, "endDate", new Date()); -// searchApi.encodeStoredId(gen, 42L); -// searchApi.encodeStringField(gen, "dataset", 2001L); +// SearchApi.encodeTextField(gen, "text", "Elephants and Aardvarks"); +// SearchApi.encodeStringField(gen, "startDate", new Date()); +// SearchApi.encodeStringField(gen, "endDate", new Date()); +// SearchApi.encodeStoredId(gen, 42L); +// SearchApi.encodeStringField(gen, "dataset", 2001L); // gen.writeEnd(); // } @@ -276,10 +276,10 @@ // gen.write(rel + "Parameter"); // gen.writeNull(); // gen.writeStartArray(); -// searchApi.encodeStringField(gen, "parameterName", "S" + name); -// searchApi.encodeStringField(gen, "parameterUnits", units); -// searchApi.encodeStringField(gen, "parameterStringValue", "v" + i * i); -// searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); +// SearchApi.encodeStringField(gen, "parameterName", "S" + name); +// SearchApi.encodeStringField(gen, "parameterUnits", units); +// SearchApi.encodeStringField(gen, "parameterStringValue", "v" + i * i); +// SearchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); // gen.writeEnd(); // gen.writeEnd(); // System.out.println(rel + " " + i + " '" + "S" + name + "' '" + units + "' 'v" + i * i + "'"); @@ -288,10 +288,10 @@ // gen.write(rel + "Parameter"); // gen.writeNull(); // gen.writeStartArray(); -// searchApi.encodeStringField(gen, "parameterName", "N" + name); -// searchApi.encodeStringField(gen, "parameterUnits", units); -// searchApi.encodeDoublePoint(gen, "parameterNumericValue", new Double(j * j)); -// searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); +// SearchApi.encodeStringField(gen, "parameterName", "N" + name); +// SearchApi.encodeStringField(gen, "parameterUnits", units); +// SearchApi.encodeDoublePoint(gen, "parameterNumericValue", new Double(j * j)); +// SearchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); // gen.writeEnd(); // gen.writeEnd(); // System.out.println(rel + " " + i + " '" + "N" + name + "' '" + units + "' " + new Double(j * j)); @@ -300,10 +300,10 @@ // gen.write(rel + "Parameter"); // gen.writeNull(); // gen.writeStartArray(); -// searchApi.encodeStringField(gen, "parameterName", "D" + name); -// searchApi.encodeStringField(gen, "parameterUnits", units); -// searchApi.encodeStringField(gen, "parameterDateValue", new Date(now + 60000 * k * k)); -// searchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); +// SearchApi.encodeStringField(gen, "parameterName", "D" + name); +// SearchApi.encodeStringField(gen, "parameterUnits", units); +// SearchApi.encodeStringField(gen, "parameterDateValue", new Date(now + 60000 * k * k)); +// SearchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); // gen.writeEnd(); // gen.writeEnd(); // System.out.println( @@ -447,10 +447,10 @@ // gen.writeNull(); // gen.writeStartArray(); -// searchApi.encodeTextField(gen, "userFullName", fn); +// SearchApi.encodeTextField(gen, "userFullName", fn); -// searchApi.encodeStringField(gen, "userName", name); -// searchApi.encodeSortedDocValuesField(gen, "investigation", new Long(i)); +// SearchApi.encodeStringField(gen, "userName", name); +// SearchApi.encodeSortedDocValuesField(gen, "investigation", new Long(i)); // gen.writeEnd(); // gen.writeEnd(); @@ -478,11 +478,11 @@ // gen.write("Investigation"); // gen.writeNull(); // gen.writeStartArray(); -// searchApi.encodeTextField(gen, "text", word); -// searchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); -// searchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); -// searchApi.encodeStoredId(gen, new Long(i)); -// searchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); +// SearchApi.encodeTextField(gen, "text", word); +// SearchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); +// SearchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); +// SearchApi.encodeStoredId(gen, new Long(i)); +// SearchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); // gen.writeEnd(); // gen.writeEnd(); // System.out.println("INVESTIGATION '" + word + "' " + new Date(now + i * 60000) + " " + i); @@ -517,12 +517,12 @@ // gen.write("Dataset"); // gen.writeNull(); // gen.writeStartArray(); -// searchApi.encodeTextField(gen, "text", word); -// searchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); -// searchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); -// searchApi.encodeStoredId(gen, new Long(i)); -// searchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); -// searchApi.encodeStringField(gen, "investigation", new Long(i % NUMINV)); +// SearchApi.encodeTextField(gen, "text", word); +// SearchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); +// SearchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); +// SearchApi.encodeStoredId(gen, new Long(i)); +// SearchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); +// SearchApi.encodeStringField(gen, "investigation", new Long(i % NUMINV)); // gen.writeEnd(); // gen.writeEnd(); // System.out.println("DATASET '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMINV); @@ -554,11 +554,11 @@ // gen.write("Datafile"); // gen.writeNull(); // gen.writeStartArray(); -// searchApi.encodeTextField(gen, "text", word); -// searchApi.encodeStringField(gen, "date", new Date(now + i * 60000)); -// searchApi.encodeStoredId(gen, new Long(i)); -// searchApi.encodeStringField(gen, "dataset", new Long(i % NUMDS)); -// searchApi.encodeStringField(gen, "investigation", new Long((i % NUMDS) % NUMINV)); +// SearchApi.encodeTextField(gen, "text", word); +// SearchApi.encodeStringField(gen, "date", new Date(now + i * 60000)); +// SearchApi.encodeStoredId(gen, new Long(i)); +// SearchApi.encodeStringField(gen, "dataset", new Long(i % NUMDS)); +// SearchApi.encodeStringField(gen, "investigation", new Long((i % NUMDS) % NUMINV)); // gen.writeEnd(); // gen.writeEnd(); // System.out.println("DATAFILE '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMDS); @@ -591,8 +591,8 @@ // gen.write("Sample"); // gen.writeNull(); // gen.writeStartArray(); -// searchApi.encodeTextField(gen, "sampleText", word); -// searchApi.encodeSortedDocValuesField(gen, "investigation", new Long(i % NUMINV)); +// SearchApi.encodeTextField(gen, "sampleText", word); +// SearchApi.encodeSortedDocValuesField(gen, "investigation", new Long(i % NUMINV)); // gen.writeEnd(); // gen.writeEnd(); // System.out.println("SAMPLE '" + word + "' " + i % NUMINV); diff --git a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java index aeda683f..c263a1eb 100644 --- a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java +++ b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java @@ -47,7 +47,9 @@ public void testBadname() throws Exception { @Test public void testHasSearchDoc() throws Exception { Set docdbeans = new HashSet<>(Arrays.asList("Investigation", "Dataset", "Datafile", - "InvestigationParameter", "DatasetParameter", "DatafileParameter", "InvestigationUser", "Sample")); + "InvestigationParameter", "DatasetParameter", "DatafileParameter", "ParameterType", + "InvestigationUser", "User", "Sample", "SampleType", "Facility", "InvestigationType", "DatasetType", + "DatafileFormat")); for (String beanName : EntityInfoHandler.getEntityNamesList()) { @SuppressWarnings("unchecked") Class bean = (Class) Class @@ -79,7 +81,7 @@ public void testExportHeaders() throws Exception { assertEquals( "Job(?:0,application(facility(name:1),name:2,version:3),arguments:4,inputDataCollection(?:5),outputDataCollection(?:6))", eiHandler.getExportHeader(Job.class)); - assertEquals( "Study(?:0,description:1,endDate:2,name:3,pid:4,startDate:5,status:6,user(name:7))", + assertEquals("Study(?:0,description:1,endDate:2,name:3,pid:4,startDate:5,status:6,user(name:7))", eiHandler.getExportHeader(Study.class)); assertEquals( "DatasetParameter(dataset(investigation(facility(name:0),name:1,visitId:2),name:3)," @@ -126,7 +128,7 @@ public void testExportHeaderAll() throws Exception { + "application(facility(name:5),name:6,version:7),arguments:8," + "inputDataCollection(?:9),outputDataCollection(?:10))", eiHandler.getExportHeaderAll(Job.class)); - assertEquals( "Study(?:0,description:1,endDate:2,name:3,pid:4,startDate:5,status:6,user(name:7))", + assertEquals("Study(?:0,description:1,endDate:2,name:3,pid:4,startDate:5,status:6,user(name:7))", eiHandler.getExportHeader(Study.class)); assertEquals( "DatasetParameter(createId:0,createTime:1,modId:2,modTime:3," @@ -181,7 +183,7 @@ public void testFields() throws Exception { testField("dataCollectionDatafiles,dataCollectionDatasets,doi,jobsAsInput,jobsAsOutput,parameters", DataCollection.class); testField("application,arguments,inputDataCollection,outputDataCollection", Job.class); - testField( "description,endDate,name,pid,startDate,status,studyInvestigations,user",Study.class); + testField("description,endDate,name,pid,startDate,status,studyInvestigations,user", Study.class); testField("dataset,dateTimeValue,error,numericValue,rangeBottom,rangeTop,stringValue,type", DatasetParameter.class); testField( @@ -357,7 +359,8 @@ public void stringFields() throws Exception { testSF(Dataset.class, "name 255", "description 255", "location 255", "doi 255"); testSF(Keyword.class, "name 255"); testSF(InvestigationUser.class, "role 255"); - testSF(User.class, "name 255", "fullName 255", "givenName 255", "familyName 255", "affiliation 255", "email 255", "orcidId 255"); + testSF(User.class, "name 255", "fullName 255", "givenName 255", "familyName 255", "affiliation 255", + "email 255", "orcidId 255"); testSF(ParameterType.class, "pid 255", "description 255", "unitsFullName 255", "units 255", "name 255"); testSF(Job.class, "arguments 255"); testSF(Study.class, "name 255", "description 4000", "pid 255"); diff --git a/src/test/java/org/icatproject/core/manager/TestLucene.java b/src/test/java/org/icatproject/core/manager/TestLucene.java index a335978c..3b0d1519 100644 --- a/src/test/java/org/icatproject/core/manager/TestLucene.java +++ b/src/test/java/org/icatproject/core/manager/TestLucene.java @@ -34,11 +34,21 @@ import org.icatproject.core.IcatException; import org.icatproject.core.IcatException.IcatExceptionType; import org.icatproject.core.entity.Datafile; +import org.icatproject.core.entity.DatafileFormat; +import org.icatproject.core.entity.DatafileParameter; import org.icatproject.core.entity.Dataset; +import org.icatproject.core.entity.DatasetParameter; import org.icatproject.core.entity.DatasetType; import org.icatproject.core.entity.Facility; import org.icatproject.core.entity.Investigation; +import org.icatproject.core.entity.InvestigationParameter; import org.icatproject.core.entity.InvestigationType; +import org.icatproject.core.entity.InvestigationUser; +import org.icatproject.core.entity.Parameter; +import org.icatproject.core.entity.ParameterType; +import org.icatproject.core.entity.Sample; +import org.icatproject.core.entity.SampleType; +import org.icatproject.core.entity.User; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -73,103 +83,111 @@ public static void beforeClass() throws Exception { int NUMSAMP = 15; - private class QueueItem { - - private String entityName; - private Long id; - private String json; - - public QueueItem(String entityName, Long id, String json) { - this.entityName = entityName; - this.id = id; - this.json = json; - } - - } - @Test - public void modify() throws IcatException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); + public void modifyDatafile() throws IcatException { Investigation investigation = new Investigation(); investigation.setId(0L); Dataset dataset = new Dataset(); dataset.setId(0L); dataset.setInvestigation(investigation); - Datafile datafile = new Datafile(); - datafile.setName("Elephants and Aardvarks"); - datafile.setDatafileModTime(new Date()); - datafile.setId(new Long(42L)); - datafile.setDataset(dataset); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - datafile.getDoc(gen, luceneApi); - gen.writeEnd(); - } - String elephantJson = baos.toString(); - baos = new ByteArrayOutputStream(); - datafile.setName("Rhinos and Aardvarks"); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - datafile.getDoc(gen, luceneApi); - gen.writeEnd(); - } - String rhinoJson = baos.toString(); + Datafile elephantDatafile = new Datafile(); + elephantDatafile.setName("Elephants and Aardvarks"); + elephantDatafile.setDatafileModTime(new Date()); + elephantDatafile.setId(42L); + elephantDatafile.setDataset(dataset); + + DatafileFormat pdfFormat = new DatafileFormat(); + pdfFormat.setId(0L); + pdfFormat.setName("pdf"); + Datafile rhinoDatafile = new Datafile(); + rhinoDatafile.setName("Rhinos and Aardvarks"); + rhinoDatafile.setDatafileModTime(new Date()); + rhinoDatafile.setId(42L); + rhinoDatafile.setDataset(dataset); + rhinoDatafile.setDatafileFormat(pdfFormat); + + DatafileFormat pngFormat = new DatafileFormat(); + pngFormat.setId(0L); + pngFormat.setName("png"); JsonObject elephantQuery = SearchApi.buildQuery("Datafile", null, "elephant", null, null, null, null, null); JsonObject rhinoQuery = SearchApi.buildQuery("Datafile", null, "rhino", null, null, null, null, null); + JsonObject pdfQuery = SearchApi.buildQuery("Datafile", null, "datafileFormat.name:pdf", null, null, null, null, + null); + JsonObject pngQuery = SearchApi.buildQuery("Datafile", null, "datafileFormat.name:png", null, null, null, null, + null); - Queue queue = new ConcurrentLinkedQueue<>(); - queue.add(new QueueItem("Datafile", null, elephantJson)); + // Original + Queue queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeOperation("create", elephantDatafile)); modifyQueue(queue); checkLsr(luceneApi.getResults(elephantQuery, 5), 42L); checkLsr(luceneApi.getResults(rhinoQuery, 5)); + checkLsr(luceneApi.getResults(pdfQuery, 5)); + checkLsr(luceneApi.getResults(pngQuery, 5)); + + // Change name and add a format + queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeOperation("update", rhinoDatafile)); + modifyQueue(queue); + checkLsr(luceneApi.getResults(elephantQuery, 5)); + checkLsr(luceneApi.getResults(rhinoQuery, 5), 42L); + checkLsr(luceneApi.getResults(pdfQuery, 5), 42L); + checkLsr(luceneApi.getResults(pngQuery, 5)); + // Change just the format queue = new ConcurrentLinkedQueue<>(); - queue.add(new QueueItem("Datafile", 42L, rhinoJson)); + queue.add(SearchApi.encodeOperation("update", pngFormat)); modifyQueue(queue); checkLsr(luceneApi.getResults(elephantQuery, 5)); checkLsr(luceneApi.getResults(rhinoQuery, 5), 42L); + checkLsr(luceneApi.getResults(pdfQuery, 5)); + checkLsr(luceneApi.getResults(pngQuery, 5), 42L); + // Remove the format queue = new ConcurrentLinkedQueue<>(); - queue.add(new QueueItem("Datafile", 42L, null)); - queue.add(new QueueItem("Datafile", 42L, null)); + queue.add(SearchApi.encodeOperation("delete", pngFormat)); + modifyQueue(queue); + checkLsr(luceneApi.getResults(elephantQuery, 5)); + checkLsr(luceneApi.getResults(rhinoQuery, 5), 42L); + checkLsr(luceneApi.getResults(pdfQuery, 5)); + checkLsr(luceneApi.getResults(pngQuery, 5)); + + // Remove the file + queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeDeletion(elephantDatafile)); + queue.add(SearchApi.encodeDeletion(rhinoDatafile)); modifyQueue(queue); checkLsr(luceneApi.getResults(elephantQuery, 5)); checkLsr(luceneApi.getResults(rhinoQuery, 5)); + checkLsr(luceneApi.getResults(pdfQuery, 5)); + checkLsr(luceneApi.getResults(pngQuery, 5)); + // Multiple commands at once queue = new ConcurrentLinkedQueue<>(); - queue.add(new QueueItem("Datafile", null, elephantJson)); - queue.add(new QueueItem("Datafile", 42L, rhinoJson)); - queue.add(new QueueItem("Datafile", 42L, null)); - queue.add(new QueueItem("Datafile", 42L, null)); + queue.add(SearchApi.encodeOperation("create", elephantDatafile)); + queue.add(SearchApi.encodeOperation("update", rhinoDatafile)); + queue.add(SearchApi.encodeDeletion(elephantDatafile)); + queue.add(SearchApi.encodeDeletion(rhinoDatafile)); modifyQueue(queue); checkLsr(luceneApi.getResults(elephantQuery, 5)); checkLsr(luceneApi.getResults(rhinoQuery, 5)); + checkLsr(luceneApi.getResults(pdfQuery, 5)); + checkLsr(luceneApi.getResults(pngQuery, 5)); } - private void modifyQueue(Queue queue) throws IcatException { - Iterator qiter = queue.iterator(); + private void modifyQueue(Queue queue) throws IcatException { + Iterator qiter = queue.iterator(); if (qiter.hasNext()) { StringBuilder sb = new StringBuilder("["); while (qiter.hasNext()) { - QueueItem item = qiter.next(); + String item = qiter.next(); if (sb.length() != 1) { sb.append(','); } - sb.append("[\"").append(item.entityName).append('"'); - if (item.id != null) { - sb.append(',').append(item.id); - } else { - sb.append(",null"); - } - if (item.json != null) { - sb.append(',').append(item.json); - } else { - sb.append(",null"); - } - sb.append(']'); + sb.append(item); qiter.remove(); } sb.append(']'); @@ -205,40 +223,36 @@ private void checkDatafile(ScoredEntityBaseBean datafile) { JsonObject source = datafile.getSource(); assertNotNull(source); Set expectedKeys = new HashSet<>( - Arrays.asList("id", "dataset", "investigation", "name", "text", "date")); + Arrays.asList("id", "investigation.id", "name", "date")); assertEquals(expectedKeys, source.keySet()); assertEquals("0", source.getString("id")); - assertEquals("0", source.getString("dataset")); - assertEquals("0", source.getString("investigation")); + assertEquals("0", source.getString("investigation.id")); assertEquals("DFaaa", source.getString("name")); - assertEquals("DFaaa", source.getString("text")); - assertNotNull(source.getString("date")); + assertNotNull(source.getJsonNumber("date")); } private void checkDataset(ScoredEntityBaseBean dataset) { JsonObject source = dataset.getSource(); assertNotNull(source); Set expectedKeys = new HashSet<>( - Arrays.asList("id", "investigation", "name", "text", "startDate", "endDate")); + Arrays.asList("id", "investigation.id", "name", "startDate", "endDate")); assertEquals(expectedKeys, source.keySet()); assertEquals("0", source.getString("id")); - assertEquals("0", source.getString("investigation")); + assertEquals("0", source.getString("investigation.id")); assertEquals("DSaaa", source.getString("name")); - assertEquals("DSaaa null null", source.getString("text")); - assertNotNull(source.getString("startDate")); - assertNotNull(source.getString("endDate")); + assertNotNull(source.getJsonNumber("startDate")); + assertNotNull(source.getJsonNumber("endDate")); } private void checkInvestigation(ScoredEntityBaseBean investigation) { JsonObject source = investigation.getSource(); assertNotNull(source); - Set expectedKeys = new HashSet<>(Arrays.asList("id", "name", "text", "startDate", "endDate")); + Set expectedKeys = new HashSet<>(Arrays.asList("id", "name", "startDate", "endDate")); assertEquals(expectedKeys, source.keySet()); assertEquals("0", source.getString("id")); assertEquals("a h r", source.getString("name")); - assertEquals("null a h r null null", source.getString("text")); - assertNotNull(source.getString("startDate")); - assertNotNull(source.getString("endDate")); + assertNotNull(source.getJsonNumber("startDate")); + assertNotNull(source.getJsonNumber("endDate")); } private void checkLsr(SearchResult lsr, Long... n) { @@ -288,7 +302,7 @@ public void datafiles() throws Exception { populate(); JsonObject query = SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null); - List fields = Arrays.asList("date", "name", "investigation", "id", "text", "dataset"); + List fields = Arrays.asList("date", "name", "investigation.id", "id"); SearchResult lsr = luceneApi.getResults(query, null, 5, null, fields); String searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -429,7 +443,7 @@ public void datasets() throws Exception { populate(); JsonObject query = SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null); - List fields = Arrays.asList("startDate", "endDate", "name", "investigation", "id", "text"); + List fields = Arrays.asList("startDate", "endDate", "name", "investigation.id", "id"); SearchResult lsr = luceneApi.getResults(query, null, 5, null, fields); String searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -482,7 +496,7 @@ public void datasets() throws Exception { gen.writeEnd(); } sort = baos.toString(); - lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); + lsr = luceneApi.getResults(query, null, 5, sort, fields); checkLsrOrder(lsr, 0L, 26L, 1L, 27L, 2L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -544,31 +558,62 @@ private void fillParms(JsonGenerator gen, int i, String rel) { String name = "nm " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); String units = "u " + letters.substring(k, k + 1) + letters.substring(k, k + 1) + letters.substring(k, k + 1); - gen.writeStartArray(); - luceneApi.encodeStringField(gen, "name", "S" + name); - luceneApi.encodeStringField(gen, "units", units); - luceneApi.encodeStringField(gen, "stringValue", "v" + i * i); - luceneApi.encodeSortedDocValuesField(gen, rel, new Long(i)); + ParameterType dateParameterType = new ParameterType(); + dateParameterType.setId(0L); + dateParameterType.setName("D" + name); + dateParameterType.setUnits(units); + ParameterType numericParameterType = new ParameterType(); + numericParameterType.setId(0L); + numericParameterType.setName("N" + name); + numericParameterType.setUnits(units); + ParameterType stringParameterType = new ParameterType(); + stringParameterType.setId(0L); + stringParameterType.setName("S" + name); + stringParameterType.setUnits(units); + + Parameter parameter; + if (rel.equals("datafile")) { + parameter = new DatafileParameter(); + Datafile datafile = new Datafile(); + datafile.setId(new Long(i)); + ((DatafileParameter) parameter).setDatafile(datafile); + } else if (rel.equals("dataset")) { + parameter = new DatasetParameter(); + Dataset dataset = new Dataset(); + dataset.setId(new Long(i)); + ((DatasetParameter) parameter).setDataset(dataset); + } else if (rel.equals("investigation")) { + parameter = new InvestigationParameter(); + Investigation investigation = new Investigation(); + investigation.setId(new Long(i)); + ((InvestigationParameter) parameter).setInvestigation(investigation); + } else { + fail(rel + " is not valid"); + return; + } + parameter.setId(0L); + + parameter.setType(dateParameterType); + parameter.setDateTimeValue(new Date(now + 60000 * k * k)); + gen.writeStartObject(); + parameter.getDoc(gen); gen.writeEnd(); - System.out.println(rel + " " + i + " '" + "S" + name + "' '" + units + "' 'v" + i * i + "'"); + System.out.println( + rel + " " + i + " '" + "D" + name + "' '" + units + "' '" + new Date(now + 60000 * k * k) + "'"); - gen.writeStartArray(); - luceneApi.encodeStringField(gen, "name", "N" + name); - luceneApi.encodeStringField(gen, "units", units); - luceneApi.encodeDoublePoint(gen, "numericValue", new Double(j * j)); - luceneApi.encodeSortedDocValuesField(gen, rel, new Long(i)); + parameter.setType(numericParameterType); + parameter.setNumericValue(new Double(j * j)); + gen.writeStartObject(); + parameter.getDoc(gen); gen.writeEnd(); System.out.println(rel + " " + i + " '" + "N" + name + "' '" + units + "' " + new Double(j * j)); - gen.writeStartArray(); - luceneApi.encodeStringField(gen, "name", "D" + name); - luceneApi.encodeStringField(gen, "units", units); - luceneApi.encodeStringField(gen, "dateTimeValue", new Date(now + 60000 * k * k)); - luceneApi.encodeSortedDocValuesField(gen, rel, new Long(i)); + parameter.setType(stringParameterType); + parameter.setStringValue("v" + i * i); + gen.writeStartObject(); + parameter.getDoc(gen); gen.writeEnd(); - System.out.println( - rel + " " + i + " '" + "D" + name + "' '" + units + "' '" + new Date(now + 60000 * k * k) + "'"); - + System.out.println(rel + " " + i + " '" + "S" + name + "' '" + units + "' 'v" + i * i + "'"); } @Test @@ -577,7 +622,7 @@ public void investigations() throws Exception { /* Blocked results */ JsonObject query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null); - List fields = Arrays.asList("startDate", "endDate", "name", "id", "text"); + List fields = Arrays.asList("startDate", "endDate", "name", "id"); SearchResult lsr = luceneApi.getResults(query, null, 5, null, fields); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); checkInvestigation(lsr.getResults().get(0)); @@ -746,20 +791,28 @@ private void populate() throws IcatException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { + Long investigationUserId = 0L; gen.writeStartArray(); for (int i = 0; i < NUMINV; i++) { for (int j = 0; j < NUMUSERS; j++) { if (i % (j + 1) == 1) { String fn = "FN " + letters.substring(j, j + 1) + " " + letters.substring(j, j + 1); String name = letters.substring(j, j + 1) + j; - gen.writeStartArray(); - - luceneApi.encodeTextField(gen, "text", fn); - - luceneApi.encodeStringField(gen, "name", name); - luceneApi.encodeSortedDocValuesField(gen, "investigation", new Long(i)); - + User user = new User(); + user.setId(new Long(j)); + user.setName(name); + user.setFullName(fn); + Investigation investigation = new Investigation(); + investigation.setId(new Long(i)); + InvestigationUser investigationUser = new InvestigationUser(); + investigationUser.setId(investigationUserId); + investigationUser.setUser(user); + investigationUser.setInvestigation(investigation); + + gen.writeStartObject(); + investigationUser.getDoc(gen); gen.writeEnd(); + investigationUserId++; System.out.println("'" + fn + "' " + name + " " + i); } } @@ -778,14 +831,22 @@ private void populate() throws IcatException { String word = letters.substring(j, j + 1) + " " + letters.substring(k, k + 1) + " " + letters.substring(l, l + 1); Investigation investigation = new Investigation(); - investigation.setFacility(new Facility()); - investigation.setType(new InvestigationType()); + Facility facility = new Facility(); + facility.setName(""); + facility.setId(0L); + investigation.setFacility(facility); + InvestigationType type = new InvestigationType(); + type.setName(""); + type.setId(0L); + investigation.setType(type); investigation.setName(word); + investigation.setTitle(""); + investigation.setVisitId(""); investigation.setStartDate(new Date(now + i * 60000)); investigation.setEndDate(new Date(now + (i + 1) * 60000)); investigation.setId(new Long(i)); - gen.writeStartArray(); - investigation.getDoc(gen, luceneApi); + gen.writeStartObject(); + investigation.getDoc(gen); gen.writeEnd(); System.out.println("INVESTIGATION '" + word + "' " + new Date(now + i * 60000) + " " + i); } @@ -816,15 +877,18 @@ private void populate() throws IcatException { Investigation investigation = new Investigation(); investigation.setId(new Long(i % NUMINV)); Dataset dataset = new Dataset(); - dataset.setType(new DatasetType()); + DatasetType type = new DatasetType(); + type.setName(""); + type.setId(0L); + dataset.setType(type); dataset.setName(word); dataset.setStartDate(new Date(now + i * 60000)); dataset.setEndDate(new Date(now + (i + 1) * 60000)); dataset.setId(new Long(i)); dataset.setInvestigation(investigation); - gen.writeStartArray(); - dataset.getDoc(gen, luceneApi); + gen.writeStartObject(); + dataset.getDoc(gen); gen.writeEnd(); System.out.println("DATASET '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMINV); } @@ -863,8 +927,8 @@ private void populate() throws IcatException { datafile.setId(new Long(i)); datafile.setDataset(dataset); - gen.writeStartArray(); - datafile.getDoc(gen, luceneApi); + gen.writeStartObject(); + datafile.getDoc(gen); gen.writeEnd(); System.out.println("DATAFILE '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMDS); @@ -892,9 +956,20 @@ private void populate() throws IcatException { int j = i % 26; String word = "SType " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); - gen.writeStartArray(); - luceneApi.encodeTextField(gen, "text", word); - luceneApi.encodeSortedDocValuesField(gen, "investigation", new Long(i % NUMINV)); + + Investigation investigation = new Investigation(); + investigation.setId(new Long(i % NUMINV)); + SampleType sampleType = new SampleType(); + sampleType.setId(0L); + sampleType.setName(""); + Sample sample = new Sample(); + sample.setId(new Long(i)); + sample.setInvestigation(investigation); + sample.setType(sampleType); + sample.setName(word); + + gen.writeStartObject(); + sample.getDoc(gen); gen.writeEnd(); System.out.println("SAMPLE '" + word + "' " + i % NUMINV); } diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index f77584f7..fc62acca 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -630,10 +630,8 @@ public void testSearchDatafiles() throws Exception { JsonObject responseObject; String searchAfter; Map expectation = new HashMap<>(); - expectation.put("investigation", null); - expectation.put("text", null); + expectation.put("investigation.id", null); expectation.put("date", "notNull"); - expectation.put("dataset", "notNull"); List parameters = new ArrayList<>(); parameters.add(new ParameterForLucene("colour", "name", "green")); @@ -675,25 +673,25 @@ public void testSearchDatafiles() throws Exception { ps.setId(wSession.create(ps)); responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, 1); assertFalse(responseObject.keySet().contains("search_after")); - expectation.put("investigation", "notNull"); + expectation.put("investigation.id", "notNull"); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.delete(ps); responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, 1); assertFalse(responseObject.keySet().contains("search_after")); - expectation.put("investigation", null); + expectation.put("investigation.id", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); - wSession.addRule(null, "DatafileFormat", "R"); + wSession.addRule(null, "Dataset", "R"); responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, 1); assertFalse(responseObject.keySet().contains("search_after")); - expectation.put("text", "df2"); + expectation.put("investigation.id", "notNull"); checkResultsSource(responseObject, Arrays.asList(expectation), true); - wSession.delRule(null, "DatafileFormat", "R"); + wSession.delRule(null, "Dataset", "R"); responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, 1); assertFalse(responseObject.keySet().contains("search_after")); - expectation.put("text", null); + expectation.put("investigation.id", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); // Test searching with someone without authz for the Datafile(s) @@ -712,10 +710,12 @@ public void testSearchDatasets() throws Exception { JsonObject responseObject; String searchAfter; Map expectation = new HashMap<>(); - expectation.put("text", null); expectation.put("startDate", "notNull"); expectation.put("endDate", "notNull"); - expectation.put("investigation", "notNull"); + expectation.put("investigation.id", "notNull"); + expectation.put("sample.name", null); + expectation.put("sample.type.name", null); + expectation.put("type.name", null); // All datasets searchDatasets(session, null, null, null, null, null, null, 10, null, 5); @@ -778,23 +778,25 @@ public void testSearchDatasets() throws Exception { responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, 1); assertFalse(responseObject.keySet().contains("search_after")); expectation.put("name", "ds1"); + expectation.put("type.name", "calibration"); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.addRule(null, "Sample", "R"); responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, 1); assertFalse(responseObject.keySet().contains("search_after")); - expectation.put("text", "ds1 calibration calibration alpha Koh-I-Noor diamond"); + expectation.put("sample.name", "Koh-I-Noor"); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.delete(ps); responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, 1); assertFalse(responseObject.keySet().contains("search_after")); - expectation.put("text", null); + expectation.put("type.name", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.delRule(null, "Sample", "R"); responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, 1); assertFalse(responseObject.keySet().contains("search_after")); + expectation.put("sample.name", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); // Test searching with someone without authz for the Dataset(s) @@ -815,7 +817,8 @@ public void testSearchInvestigations() throws Exception { expectation.put("name", "expt1"); expectation.put("startDate", "notNull"); expectation.put("endDate", "notNull"); - expectation.put("text", null); + expectation.put("type.name", null); + expectation.put("facility.name", null); Date lowerOrigin = dft.parse("2011-01-01T00:00:00+0000"); Date lowerSecond = dft.parse("2011-01-01T00:00:01+0000"); @@ -834,6 +837,15 @@ public void testSearchInvestigations() throws Exception { List parameters = new ArrayList<>(); parameters.add(new ParameterForLucene("colour", "name", "green")); + // TODO remove additional checks here + responseObject = searchInvestigations(session, "db/tr", null, lowerOrigin, upperOrigin, null, + null, null, null, 10, null, 1); + responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, null, + null, null, null, 10, null, 1); + responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, + null, null, null, 10, null, 1); + responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, + null, "Professor", null, 10, null, 1); responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, samplesAnd, "Professor", null, 10, null, 1); assertFalse(responseObject.keySet().contains("search_after")); @@ -888,23 +900,25 @@ public void testSearchInvestigations() throws Exception { ps.setId(wSession.create(ps)); responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, 1); assertFalse(responseObject.keySet().contains("search_after")); + expectation.put("type.name", "atype"); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.addRule(null, "Facility", "R"); responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, 1); assertFalse(responseObject.keySet().contains("search_after")); - expectation.put("text", "one expt1 Test port facility atype a title in the middle"); + expectation.put("facility.name", "Test port facility"); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.delete(ps); responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, 1); assertFalse(responseObject.keySet().contains("search_after")); - expectation.put("text", null); + expectation.put("type.name", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.delRule(null, "Facility", "R"); responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, 1); assertFalse(responseObject.keySet().contains("search_after")); + expectation.put("facility.name", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); // Test searching with someone without authz for the Investigation(s) @@ -968,11 +982,12 @@ private void checkResultsSource(JsonObject responseObject, List expectation = expectations.get(i); - for (Entry entry: expectation.entrySet()) { + for (Entry entry : expectation.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); if (value == null) { - assertFalse("Source " + source.toString() + " should NOT contain " + key, source.keySet().contains(key)); + assertFalse("Source " + source.toString() + " should NOT contain " + key, + source.keySet().contains(key)); } else if (value.equals("notNull")) { assertTrue("Source " + source.toString() + " should contain " + key, source.keySet().contains(key)); } else { From 7efdc36771851b9c6bbe4507ea93ed0af1f00bf3 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 14 Apr 2022 00:19:55 +0100 Subject: [PATCH 14/51] Enable Lucene facets #267 --- .../core/manager/ElasticsearchApi.java | 83 +++--- .../core/manager/EntityBeanManager.java | 67 +++-- .../core/manager/FacetDimension.java | 14 +- .../icatproject/core/manager/LuceneApi.java | 12 +- .../icatproject/core/manager/SearchApi.java | 7 +- .../core/manager/SearchManager.java | 18 +- .../core/manager/SearchResult.java | 12 +- .../org/icatproject/exposed/ICATRest.java | 269 +++++++++++------- .../icatproject/core/manager/TestLucene.java | 80 +++++- .../org/icatproject/integration/TestRS.java | 256 ++++++++++++----- 10 files changed, 539 insertions(+), 279 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java index 78e1e21a..83b269b9 100644 --- a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java +++ b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java @@ -189,49 +189,50 @@ public void commit() throws IcatException { } @Override - public List facetSearch(JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException { + public List facetSearch(String target, JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException { // TODO this should be generalised - try { - String index = "_all"; - Set fields = facetQuery.keySet(); - BoolQuery.Builder builder = new BoolQuery.Builder(); - for (String field : fields) { - // Only expecting a target and text field as part of the current facet - // implementation - if (field.equals("target")) { - index = facetQuery.getString("target").toLowerCase(); - } else if (field.equals("text")) { - String text = facetQuery.getString("text"); - builder.must(m -> m.queryString(q -> q.defaultField("text").query(text))); - } - } - String indexFinal = index; - SearchResponse response = client.search(s -> s - .index(indexFinal) - .size(maxResults) - .query(q -> q.bool(builder.build())) - .aggregations("samples", a -> a.terms(t -> t.field("sampleName").size(maxLabels))) - .aggregations("parameters", a -> a.terms(t -> t.field("parameterName").size(maxLabels))), - Object.class); + return null; + // try { + // String index = "_all"; + // Set fields = facetQuery.keySet(); + // BoolQuery.Builder builder = new BoolQuery.Builder(); + // for (String field : fields) { + // // Only expecting a target and text field as part of the current facet + // // implementation + // if (field.equals("target")) { + // index = facetQuery.getString("target").toLowerCase(); + // } else if (field.equals("text")) { + // String text = facetQuery.getString("text"); + // builder.must(m -> m.queryString(q -> q.defaultField("text").query(text))); + // } + // } + // String indexFinal = index; + // SearchResponse response = client.search(s -> s + // .index(indexFinal) + // .size(maxResults) + // .query(q -> q.bool(builder.build())) + // .aggregations("samples", a -> a.terms(t -> t.field("sampleName").size(maxLabels))) + // .aggregations("parameters", a -> a.terms(t -> t.field("parameterName").size(maxLabels))), + // Object.class); - List sampleBuckets = response.aggregations().get("samples").sterms().buckets().array(); - List parameterBuckets = response.aggregations().get("parameters").sterms().buckets() - .array(); - List facetDimensions = new ArrayList<>(); - FacetDimension sampleDimension = new FacetDimension("sampleName"); - FacetDimension parameterDimension = new FacetDimension("parameterName"); - for (StringTermsBucket sampleBucket : sampleBuckets) { - sampleDimension.getFacets().add(new FacetLabel(sampleBucket.key(), sampleBucket.docCount())); - } - for (StringTermsBucket parameterBucket : parameterBuckets) { - parameterDimension.getFacets().add(new FacetLabel(parameterBucket.key(), parameterBucket.docCount())); - } - facetDimensions.add(sampleDimension); - facetDimensions.add(parameterDimension); - return facetDimensions; - } catch (ElasticsearchException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } + // List sampleBuckets = response.aggregations().get("samples").sterms().buckets().array(); + // List parameterBuckets = response.aggregations().get("parameters").sterms().buckets() + // .array(); + // List facetDimensions = new ArrayList<>(); + // FacetDimension sampleDimension = new FacetDimension("sampleName"); + // FacetDimension parameterDimension = new FacetDimension("parameterName"); + // for (StringTermsBucket sampleBucket : sampleBuckets) { + // sampleDimension.getFacets().add(new FacetLabel(sampleBucket.key(), sampleBucket.docCount())); + // } + // for (StringTermsBucket parameterBucket : parameterBuckets) { + // parameterDimension.getFacets().add(new FacetLabel(parameterBucket.key(), parameterBucket.docCount())); + // } + // facetDimensions.add(sampleDimension); + // facetDimensions.add(parameterDimension); + // return facetDimensions; + // } catch (ElasticsearchException | IOException e) { + // throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + // } } // @Override diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index c29d5778..b864bb6d 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; +import java.io.StringReader; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -779,9 +780,9 @@ private void exportTable(String beanName, Set ids, OutputStream output, } } - private ScoredEntityBaseBean filterReadAccess(List results, List allResults, - int maxCount, String userId, EntityManager manager, Class klass) - throws IcatException { + private ScoredEntityBaseBean filterReadAccess(List results, + List allResults, int maxCount, String userId, EntityManager manager, + Class klass) throws IcatException { logger.debug("Got " + allResults.size() + " results from search engine"); for (ScoredEntityBaseBean sr : allResults) { @@ -1423,28 +1424,6 @@ public List freeTextSearch(String userName, JsonObject jo, } while (results.size() < limit); } - // TODO move this to somewhere we can manually call it - // TODO also need to fix it for Elasticsearch (cannot use text fields for - // aggregations...) - // if (results.size() > 0) { - // /* Get facets for the filtered list of IDs */ - // String facetText = ""; - // for (ScoredEntityBaseBean result: results) { - // facetText += " id:" + result.getEntityBaseBeanId(); - // } - // JsonObject facetQuery = Json.createObjectBuilder() - // .add("target", jo.getString("target")) - // .add("text", facetText.substring(1)) - // .build(); - // List facets = searchManager.facetSearch(facetQuery, - // blockSize, 100); // TODO remove hardcode - // for (FacetDimension dimension: facets) { - // logger.debug("Facet dimension: {}", dimension.getDimension()); - // for (FacetLabel facet: dimension.getFacets()) { - // logger.debug("{}: {}", facet.getLabel(), facet.getValue()); - // } - // } - // } if (logRequests.contains("R")) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) { @@ -1461,10 +1440,12 @@ public List freeTextSearch(String userName, JsonObject jo, } public SearchResult freeTextSearchDocs(String userName, JsonObject jo, String searchAfter, int limit, String sort, - EntityManager manager, String ip, Class klass) throws IcatException { + String facets, EntityManager manager, String ip, Class klass) + throws IcatException { long startMillis = log ? System.currentTimeMillis() : 0; List results = new ArrayList<>(); String lastSearchAfter = null; + List dimensions = null; if (searchActive) { SearchResult lastSearchResult = null; List allResults = Collections.emptyList(); @@ -1492,6 +1473,38 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, String se break; } } while (results.size() < limit); + + if (facets != null && !facets.equals("") && results.size() > 0) { + JsonReader reader = Json.createReader(new StringReader(facets)); + List jsonFacets = reader.readArray().getValuesAs(JsonObject.class); + + for (JsonObject jsonFacet : jsonFacets) { + JsonObject facetQuery; + String target = jsonFacet.getString("target"); + if (target.equals(klass.getSimpleName())) { + facetQuery = SearchManager.buildFacetQuery(results, "id", jsonFacet); + } else { + Relationship relationship; + if (target.contains("Parameter")) { + relationship = eiHandler.getRelationshipsByName(klass).get("parameters"); + } else { + // TODO allow more types here, as we decide what to support + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Facet target should be a Parameter, but it was " + target); + } + + if (gateKeeper.allowed(relationship)) { + facetQuery = SearchManager.buildFacetQuery(results, + klass.getSimpleName().toLowerCase() + ".id", jsonFacet); + } else { + logger.debug("Cannot collect facets for {} as Relationship with parent {} is not allowed", + target, klass.getSimpleName()); + continue; + } + } + dimensions = searchManager.facetSearch(target, facetQuery, results.size(), 10); + } + } } if (logRequests.contains("R")) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -1505,7 +1518,7 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, String se transmitter.processMessage("freeTextSearchDocs", ip, baos.toString(), startMillis); } logger.debug("Returning {} results", results.size()); - return new SearchResult(lastSearchAfter, results); + return new SearchResult(lastSearchAfter, results, dimensions); } public List searchGetPopulating() { diff --git a/src/main/java/org/icatproject/core/manager/FacetDimension.java b/src/main/java/org/icatproject/core/manager/FacetDimension.java index 0b6e0b75..bfb85e0b 100644 --- a/src/main/java/org/icatproject/core/manager/FacetDimension.java +++ b/src/main/java/org/icatproject/core/manager/FacetDimension.java @@ -4,15 +4,19 @@ import java.util.List; /** - * Holds information for a single facetable dimension, or field. - * Each dimension will have a list of FacetLabels. + * Holds information for a single faceted dimension, or field. + * Each dimension will have a list of FacetLabels, and to prevent ambiguity is + * associated with the target entity that was faceted. For example, both a + * Dataset and a DatasetParameter might have the "type.name" dimension. */ public class FacetDimension { + private String target; private String dimension; private List facets = new ArrayList<>(); - public FacetDimension(String dimension) { + public FacetDimension(String target, String dimension) { + this.target = target; this.dimension = dimension; } @@ -24,4 +28,8 @@ public String getDimension() { return dimension; } + public String getTarget() { + return target; + } + } diff --git a/src/main/java/org/icatproject/core/manager/LuceneApi.java b/src/main/java/org/icatproject/core/manager/LuceneApi.java index 90b57ace..e9e54191 100644 --- a/src/main/java/org/icatproject/core/manager/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/LuceneApi.java @@ -164,21 +164,19 @@ public void commit() throws IcatException { } @Override - public List facetSearch(JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException { + public List facetSearch(String target, JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - String indexPath = getTargetPath(facetQuery); - URI uri = new URIBuilder(server).setPath(basePath + "/" + indexPath + "/facet") + URI uri = new URIBuilder(server).setPath(basePath + "/" + target + "/facet") .setParameter("maxResults", Integer.toString(maxResults)) .setParameter("maxLabels", Integer.toString(maxLabels)).build(); logger.trace("Making call {}", uri); - return getFacets(uri, httpclient, facetQuery.toString()); - + return getFacets(uri, httpclient, facetQuery.toString(), target); } catch (IOException | URISyntaxException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } } - private List getFacets(URI uri, CloseableHttpClient httpclient, String facetQueryString) + private List getFacets(URI uri, CloseableHttpClient httpclient, String facetQueryString, String target) throws IcatException { logger.debug(facetQueryString); try { @@ -205,7 +203,7 @@ private List getFacets(URI uri, CloseableHttpClient httpclient, if (state == ParserState.None && key != null && key.equals("dimensions")) { state = ParserState.Dimensions; } else if (state == ParserState.Dimensions) { - facetDimensions.add(new FacetDimension(key)); + facetDimensions.add(new FacetDimension(target, key)); state = ParserState.Labels; } } else if (e == (Event.END_OBJECT)) { diff --git a/src/main/java/org/icatproject/core/manager/SearchApi.java b/src/main/java/org/icatproject/core/manager/SearchApi.java index 5237145a..b5a0ac47 100644 --- a/src/main/java/org/icatproject/core/manager/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/SearchApi.java @@ -128,11 +128,6 @@ public static void encodeDouble(JsonGenerator gen, String name, Double value) { gen.write(name, value); } - // public void encodeSortedSetDocValuesFacetField(JsonGenerator gen, String - // name, String value) { - // encodeStringField(gen, name, value); - // } - public static void encodeLong(JsonGenerator gen, String name, Date value) { gen.write(name, value.getTime()); } @@ -254,7 +249,7 @@ public void freeSearcher(String uid) throws IcatException { logger.info("Manually freeing searcher not supported, no request sent"); } - public abstract List facetSearch(JsonObject facetQuery, int maxResults, int maxLabels) + public abstract List facetSearch(String target, JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException; public SearchResult getResults(JsonObject query, int maxResults) throws IcatException { diff --git a/src/main/java/org/icatproject/core/manager/SearchManager.java b/src/main/java/org/icatproject/core/manager/SearchManager.java index 43751643..7f4e7fdd 100644 --- a/src/main/java/org/icatproject/core/manager/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/SearchManager.java @@ -30,7 +30,10 @@ import javax.ejb.EJB; import javax.ejb.Singleton; import javax.ejb.Startup; +import javax.json.Json; +import javax.json.JsonArrayBuilder; import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; import javax.persistence.PersistenceUnit; @@ -374,6 +377,17 @@ public void deleteDocument(EntityBaseBean bean) throws IcatException { } } + public static JsonObject buildFacetQuery(List results, String idField, JsonObject facetJson) { + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + results.forEach(r -> arrayBuilder.add(Long.toString(r.getEntityBaseBeanId()))); + JsonObject terms = Json.createObjectBuilder().add(idField, arrayBuilder.build()).build(); + JsonObjectBuilder objectBuilder = Json.createObjectBuilder().add("query", terms); + if (facetJson.containsKey("dimensions")) { + objectBuilder.add("dimensions", facetJson.getJsonArray("dimensions")); + } + return objectBuilder.build(); + } + private static List buildPublicSearchFields(GateKeeper gateKeeper, Map map) { List fields = new ArrayList<>(); for (Entry entry : map.entrySet()) { @@ -428,9 +442,9 @@ private void exit() { } } - public List facetSearch(JsonObject facetQuery, int maxResults, int maxLabels) + public List facetSearch(String target, JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException { - return searchApi.facetSearch(facetQuery, maxResults, maxLabels); + return searchApi.facetSearch(target, facetQuery, maxResults, maxLabels); } public void freeSearcher(String uid) throws IcatException { diff --git a/src/main/java/org/icatproject/core/manager/SearchResult.java b/src/main/java/org/icatproject/core/manager/SearchResult.java index 09f73498..1c34f0b6 100644 --- a/src/main/java/org/icatproject/core/manager/SearchResult.java +++ b/src/main/java/org/icatproject/core/manager/SearchResult.java @@ -7,12 +7,22 @@ public class SearchResult { private String searchAfter; private List results = new ArrayList<>(); + private List dimensions; public SearchResult() {} - public SearchResult(String searchAfter, List results) { + public SearchResult(String searchAfter, List results, List dimensions) { this.searchAfter = searchAfter; this.results = results; + this.dimensions = dimensions; + } + + public List getDimensions() { + return dimensions; + } + + public void setDimensions(List dimensions) { + this.dimensions = dimensions; } public List getResults() { diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index b9192468..2a87610d 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -76,6 +76,8 @@ import org.icatproject.core.entity.StudyStatus; import org.icatproject.core.manager.EntityBeanManager; import org.icatproject.core.manager.EntityInfoHandler; +import org.icatproject.core.manager.FacetDimension; +import org.icatproject.core.manager.FacetLabel; import org.icatproject.core.manager.GateKeeper; import org.icatproject.core.manager.Porter; import org.icatproject.core.manager.PropertyHandler; @@ -927,7 +929,8 @@ public void logout(@Context HttpServletRequest request, @PathParam("sessionId") } /** - * Perform a free text search against a dedicated (non-DB) search engine component for entity ids. + * Perform a free text search against a dedicated (non-DB) search engine + * component for entity ids. * * @summary Free text id search. * @@ -1111,119 +1114,150 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId } /** - * Perform a free text search against a dedicated (non-DB) search engine component for entire Documents. + * Perform a free text search against a dedicated (non-DB) search engine + * component for entire Documents. * * @summary Free text Document search. * * @param sessionId - * a sessionId of a user which takes the form - * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 + * a sessionId of a user which takes the form + * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 * @param query - * json encoded query object. One of the fields is "target" - * which - * must be "Investigation", "Dataset" or "Datafile". The other - * fields are all optional: - *
- *
user
- *
name of user as in the User table which may include a - * prefix
- *
text
- *
some text occurring somewhere in the entity. This is - * understood by the lucene parser but avoid trying to use fields.
- *
lower
- *
earliest date to search for in the form - * 201509030842 i.e. yyyyMMddHHmm using UTC as - * timezone. In the case of an investigation or data set search - * the date is compared with the start date and in the case of - * a - * data file the date field is used.
- *
upper
- *
latest date to search for in the form - * 201509030842 i.e. yyyyMMddHHmm using UTC as - * timezone. In the case of an investigation or data set search - * the date is compared with the end date and in the case of a - * data file the date field is used.
- *
parameters
- *
this holds a list of json parameter objects all of which - * must match. Parameters have the following fields, all of - * which - * are optional: - *
- *
name
- *
A wildcard search for a parameter with this name. - * Supported wildcards are *, which matches any - * character sequence (including the empty one), and - * ?, which matches any single character. - * \ is the escape character. Note this query can - * be - * slow, as it needs to iterate over many terms. In order to - * prevent extremely slow queries, a name should not start with - * the wildcard *
- *
units
- *
A wildcard search for a parameter with these units. - * Supported wildcards are *, which matches any - * character sequence (including the empty one), and - * ?, which matches any single character. - * \ is the escape character. Note this query can - * be - * slow, as it needs to iterate over many terms. In order to - * prevent extremely slow queries, units should not start with - * the wildcard *
- *
stringValue
- *
A wildcard search for a parameter stringValue. Supported - * wildcards are *, which matches any character - * sequence (including the empty one), and ?, - * which - * matches any single character. \ is the escape - * character. Note this query can be slow, as it needs to - * iterate - * over many terms. In order to prevent extremely slow queries, - * requested stringValues should not start with the wildcard - * *
- *
lowerDateValue and upperDateValue
- *
latest and highest date to search for in the form - * 201509030842 i.e. yyyyMMddHHmm using UTC as - * timezone. This should be used to search on parameters having - * a - * dateValue. If only one bound is set the restriction has not - * effect.
- *
lowerNumericValue and upperNumericValue
- *
This should be used to search on parameters having a - * numericValue. If only one bound is set the restriction has - * not - * effect.
- *
- *
- *
samples
- *
A json array of strings each of which must match text - * found in a sample. This is understood by the lucene parser but avoid trying to use fields. This is - * only respected in the case of an investigation search.
- *
userFullName
- *
Full name of user in the User table which may contain - * titles etc. Matching is done by the lucene parser but avoid trying to use fields. This is - * only respected in the case of an investigation search.
- *
+ * json encoded query object. One of the fields is "target" + * which + * must be "Investigation", "Dataset" or "Datafile". The + * other + * fields are all optional: + *
+ *
user
+ *
name of user as in the User table which may include a + * prefix
+ *
text
+ *
some text occurring somewhere in the entity. This is + * understood by the lucene parser but avoid trying to use fields.
+ *
lower
+ *
earliest date to search for in the form + * 201509030842 i.e. yyyyMMddHHmm using UTC as + * timezone. In the case of an investigation or data set + * search + * the date is compared with the start date and in the case + * of + * a + * data file the date field is used.
+ *
upper
+ *
latest date to search for in the form + * 201509030842 i.e. yyyyMMddHHmm using UTC as + * timezone. In the case of an investigation or data set + * search + * the date is compared with the end date and in the case of + * a + * data file the date field is used.
+ *
parameters
+ *
this holds a list of json parameter objects all of + * which + * must match. Parameters have the following fields, all of + * which + * are optional: + *
+ *
name
+ *
A wildcard search for a parameter with this name. + * Supported wildcards are *, which matches any + * character sequence (including the empty one), and + * ?, which matches any single character. + * \ is the escape character. Note this query + * can + * be + * slow, as it needs to iterate over many terms. In order to + * prevent extremely slow queries, a name should not start + * with + * the wildcard *
+ *
units
+ *
A wildcard search for a parameter with these units. + * Supported wildcards are *, which matches any + * character sequence (including the empty one), and + * ?, which matches any single character. + * \ is the escape character. Note this query + * can + * be + * slow, as it needs to iterate over many terms. In order to + * prevent extremely slow queries, units should not start + * with + * the wildcard *
+ *
stringValue
+ *
A wildcard search for a parameter stringValue. + * Supported + * wildcards are *, which matches any character + * sequence (including the empty one), and ?, + * which + * matches any single character. \ is the escape + * character. Note this query can be slow, as it needs to + * iterate + * over many terms. In order to prevent extremely slow + * queries, + * requested stringValues should not start with the wildcard + * *
+ *
lowerDateValue and upperDateValue
+ *
latest and highest date to search for in the form + * 201509030842 i.e. yyyyMMddHHmm using UTC as + * timezone. This should be used to search on parameters + * having + * a + * dateValue. If only one bound is set the restriction has + * not + * effect.
+ *
lowerNumericValue and upperNumericValue
+ *
This should be used to search on parameters having a + * numericValue. If only one bound is set the restriction has + * not + * effect.
+ *
+ *
+ *
samples
+ *
A json array of strings each of which must match text + * found in a sample. This is understood by the lucene parser but avoid trying to use fields. This is + * only respected in the case of an investigation + * search.
+ *
userFullName
+ *
Full name of user in the User table which may contain + * titles etc. Matching is done by the lucene parser but avoid trying to use fields. This is + * only respected in the case of an investigation + * search.
+ *
* @param searchAfter - * String representing the last returned document of a previous search, so that new results will be from after this document. The representation should be a JSON array, but the nature of the values will depend on the sort applied. + * String representing the last returned document of a + * previous search, so that new results will be from after + * this document. The representation should be a JSON array, + * but the nature of the values will depend on the sort + * applied. * @param sort - * json encoded sort object. Each key should be a field on the - * targeted Document, with a value of "asc" or "desc" to - * specify the order of the results. Multiple pairs can be - * provided, in which case each subsequent sort is used as a - * tiebreaker for the previous one. If no sort is specified, - * then results will be returned in order of relevance to the - * search query, with their search engine id as a tiebreaker. + * json encoded sort object. Each key should be a field on + * the targeted Document, with a value of "asc" or "desc" to + * specify the order of the results. Multiple pairs can be + * provided, in which case each subsequent sort is used as a + * tiebreaker for the previous one. If no sort is specified, + * then results will be returned in order of relevance to the + * search query, with their search engine id as a tiebreaker. * * @param limit - * maximum number of entities to return - * - * @return Set of entity ids, relevance scores and Document source encoded as json. + * maximum number of entities to return + * @param facets + * String representing a JsonArray of JsonObjects. Each + * should define the "target" entity name, and optionally + * another JsonArray of JsonObjects representing specific + * fields to facet. If absent, then all applicable String + * fields will be faceted. These objects must have the + * "dimension" key, and if the field is numeric than the + * "range" key should denote an array of objects with "lower" + * and "upper" values. + * + * @return Set of entity ids, relevance scores and Document source encoded as + * json. * * @throws IcatException * when something is wrong @@ -1232,7 +1266,8 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId @Path("search/documents") @Produces(MediaType.APPLICATION_JSON) public String search(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, - @QueryParam("query") String query, @QueryParam("search_after") String searchAfter, @QueryParam("limit") int limit, @QueryParam("sort") String sort) + @QueryParam("query") String query, @QueryParam("search_after") String searchAfter, + @QueryParam("limit") int limit, @QueryParam("sort") String sort, @QueryParam("facets") String facets) throws IcatException { if (query == null) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "query is not set"); @@ -1260,7 +1295,8 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId && parameter.containsKey("upperDateValue")) || (parameter.containsKey("lowerNumericValue") && parameter.containsKey("upperNumericValue")))) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, "value not set in parameter '" + name + "'"); + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "value not set in parameter '" + name + "'"); } } } @@ -1277,13 +1313,30 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId throw new IcatException(IcatExceptionType.BAD_PARAMETER, "target:" + target + " is not expected"); } logger.debug("Free text search with query: {}", jo.toString()); - result = beanManager.freeTextSearchDocs(userName, jo, searchAfter, limit, sort, manager, request.getRemoteAddr(), klass); + result = beanManager.freeTextSearchDocs(userName, jo, searchAfter, limit, sort, facets, manager, + request.getRemoteAddr(), klass); + JsonGenerator gen = Json.createGenerator(baos); gen.writeStartObject(); + String newSearchAfter = result.getSearchAfter(); if (newSearchAfter != null) { gen.write("search_after", newSearchAfter); } + + List dimensions = result.getDimensions(); + if (dimensions != null && dimensions.size() > 0) { + gen.writeStartObject("dimensions"); + for (FacetDimension dimension : dimensions) { + gen.writeStartObject(dimension.getTarget() + "." + dimension.getDimension()); + for (FacetLabel label : dimension.getFacets()) { + gen.write(label.getLabel(), label.getValue()); + } + gen.writeEnd(); + } + gen.writeEnd(); + } + gen.writeStartArray("results"); for (ScoredEntityBaseBean sb : result.getResults()) { gen.writeStartObject(); diff --git a/src/test/java/org/icatproject/core/manager/TestLucene.java b/src/test/java/org/icatproject/core/manager/TestLucene.java index 3b0d1519..cd7e0c83 100644 --- a/src/test/java/org/icatproject/core/manager/TestLucene.java +++ b/src/test/java/org/icatproject/core/manager/TestLucene.java @@ -21,7 +21,9 @@ import java.util.concurrent.ConcurrentLinkedQueue; import javax.json.Json; +import javax.json.JsonArrayBuilder; import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; import javax.json.stream.JsonGenerator; import javax.ws.rs.core.MediaType; @@ -93,7 +95,7 @@ public void modifyDatafile() throws IcatException { Datafile elephantDatafile = new Datafile(); elephantDatafile.setName("Elephants and Aardvarks"); - elephantDatafile.setDatafileModTime(new Date()); + elephantDatafile.setDatafileModTime(new Date(0L)); elephantDatafile.setId(42L); elephantDatafile.setDataset(dataset); @@ -102,7 +104,7 @@ public void modifyDatafile() throws IcatException { pdfFormat.setName("pdf"); Datafile rhinoDatafile = new Datafile(); rhinoDatafile.setName("Rhinos and Aardvarks"); - rhinoDatafile.setDatafileModTime(new Date()); + rhinoDatafile.setDatafileModTime(new Date(3L)); rhinoDatafile.setId(42L); rhinoDatafile.setDataset(dataset); rhinoDatafile.setDatafileFormat(pdfFormat); @@ -117,6 +119,16 @@ public void modifyDatafile() throws IcatException { null); JsonObject pngQuery = SearchApi.buildQuery("Datafile", null, "datafileFormat.name:png", null, null, null, null, null); + JsonObject queryObject = Json.createObjectBuilder().add("id", Json.createArrayBuilder().add("42")).build(); + JsonObjectBuilder lowRangeBuilder = Json.createObjectBuilder().add("lower", 0L).add("upper", 1L); + JsonObjectBuilder highRangeBuilder = Json.createObjectBuilder().add("lower", 2L).add("upper", 3L); + JsonArrayBuilder rangesBuilder = Json.createArrayBuilder().add(lowRangeBuilder).add(highRangeBuilder); + JsonObjectBuilder dimensionBuilder = Json.createObjectBuilder().add("dimension", "date").add("ranges", + rangesBuilder); + JsonArrayBuilder dimensionsBuilder = Json.createArrayBuilder().add(dimensionBuilder); + JsonObject stringFacetQuery = Json.createObjectBuilder().add("query", queryObject).build(); + JsonObject rangeFacetQuery = Json.createObjectBuilder().add("query", queryObject) + .add("dimensions", dimensionsBuilder).build(); // Original Queue queue = new ConcurrentLinkedQueue<>(); @@ -126,6 +138,19 @@ public void modifyDatafile() throws IcatException { checkLsr(luceneApi.getResults(rhinoQuery, 5)); checkLsr(luceneApi.getResults(pdfQuery, 5)); checkLsr(luceneApi.getResults(pngQuery, 5)); + List facetDimensions = luceneApi.facetSearch("Datafile", stringFacetQuery, 5, 5); + assertEquals(0, facetDimensions.size()); + facetDimensions = luceneApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + FacetDimension facetDimension = facetDimensions.get(0); + assertEquals("date", facetDimension.getDimension()); + assertEquals(2, facetDimension.getFacets().size()); + FacetLabel facetLabel = facetDimension.getFacets().get(0); + assertEquals("0_1", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("2_3", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); // Change name and add a format queue = new ConcurrentLinkedQueue<>(); @@ -135,6 +160,19 @@ public void modifyDatafile() throws IcatException { checkLsr(luceneApi.getResults(rhinoQuery, 5), 42L); checkLsr(luceneApi.getResults(pdfQuery, 5), 42L); checkLsr(luceneApi.getResults(pngQuery, 5)); + facetDimensions = luceneApi.facetSearch("Datafile", stringFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimensions = luceneApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("date", facetDimension.getDimension()); + assertEquals(2, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("0_1", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("2_3", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); // Change just the format queue = new ConcurrentLinkedQueue<>(); @@ -144,6 +182,25 @@ public void modifyDatafile() throws IcatException { checkLsr(luceneApi.getResults(rhinoQuery, 5), 42L); checkLsr(luceneApi.getResults(pdfQuery, 5)); checkLsr(luceneApi.getResults(pngQuery, 5), 42L); + facetDimensions = luceneApi.facetSearch("Datafile", stringFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("datafileFormat.name", facetDimension.getDimension()); + assertEquals(1, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("png", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + facetDimensions = luceneApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("date", facetDimension.getDimension()); + assertEquals(2, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("0_1", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("2_3", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); // Remove the format queue = new ConcurrentLinkedQueue<>(); @@ -153,6 +210,19 @@ public void modifyDatafile() throws IcatException { checkLsr(luceneApi.getResults(rhinoQuery, 5), 42L); checkLsr(luceneApi.getResults(pdfQuery, 5)); checkLsr(luceneApi.getResults(pngQuery, 5)); + facetDimensions = luceneApi.facetSearch("Datafile", stringFacetQuery, 5, 5); + assertEquals(0, facetDimensions.size()); + facetDimensions = luceneApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("date", facetDimension.getDimension()); + assertEquals(2, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("0_1", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("2_3", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); // Remove the file queue = new ConcurrentLinkedQueue<>(); @@ -836,7 +906,7 @@ private void populate() throws IcatException { facility.setId(0L); investigation.setFacility(facility); InvestigationType type = new InvestigationType(); - type.setName(""); + type.setName("test"); type.setId(0L); investigation.setType(type); investigation.setName(word); @@ -878,7 +948,7 @@ private void populate() throws IcatException { investigation.setId(new Long(i % NUMINV)); Dataset dataset = new Dataset(); DatasetType type = new DatasetType(); - type.setName(""); + type.setName("test"); type.setId(0L); dataset.setType(type); dataset.setName(word); @@ -961,7 +1031,7 @@ private void populate() throws IcatException { investigation.setId(new Long(i % NUMINV)); SampleType sampleType = new SampleType(); sampleType.setId(0L); - sampleType.setName(""); + sampleType.setName("test"); Sample sample = new Sample(); sample.setId(new Long(i)); sample.setInvestigation(investigation); diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index fc62acca..6b75b1ea 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -48,6 +48,7 @@ import javax.json.JsonArray; import javax.json.JsonNumber; import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; import javax.json.JsonValue; import javax.json.stream.JsonGenerator; @@ -637,29 +638,29 @@ public void testSearchDatafiles() throws Exception { parameters.add(new ParameterForLucene("colour", "name", "green")); // All data files - searchDatafiles(session, null, null, null, null, null, null, 10, null, 3); + searchDatafiles(session, null, null, null, null, null, null, 10, null, null, 3); // Use the user - searchDatafiles(session, "db/tr", null, null, null, null, null, 10, null, 3); + searchDatafiles(session, "db/tr", null, null, null, null, null, 10, null, null, 3); // Try a bad user - searchDatafiles(session, "db/fred", null, null, null, null, null, 10, null, 0); + searchDatafiles(session, "db/fred", null, null, null, null, null, 10, null, null, 0); // Set text and parameters - responseObject = searchDatafiles(session, null, "df2", null, null, parameters, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchDatafiles(session, null, "df2", null, null, parameters, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); expectation.put("name", "df2"); checkResultsSource(responseObject, Arrays.asList(expectation), true); // Try sorting and searchAfter String sort = Json.createObjectBuilder().add("name", "desc").add("date", "asc").build().toString(); - responseObject = searchDatafiles(session, null, null, null, null, null, null, 1, sort, 1); + responseObject = searchDatafiles(session, null, null, null, null, null, null, 1, sort, null, 1); searchAfter = responseObject.getString("search_after"); assertNotNull(searchAfter); expectation.put("name", "df3"); checkResultsSource(responseObject, Arrays.asList(expectation), false); - responseObject = searchDatafiles(session, null, null, null, null, null, searchAfter, 1, sort, 1); + responseObject = searchDatafiles(session, null, null, null, null, null, searchAfter, 1, sort, null, 1); searchAfter = responseObject.getString("search_after"); assertNotNull(searchAfter); expectation.put("name", "df2"); @@ -671,26 +672,26 @@ public void testSearchDatafiles() throws Exception { ps.setField("dataset"); ps.setId(wSession.create(ps)); - responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); expectation.put("investigation.id", "notNull"); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.delete(ps); - responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); expectation.put("investigation.id", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.addRule(null, "Dataset", "R"); - responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); expectation.put("investigation.id", "notNull"); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.delRule(null, "Dataset", "R"); - responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchDatafiles(session, null, "df2", null, null, null, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); expectation.put("investigation.id", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); @@ -700,7 +701,31 @@ public void testSearchDatafiles() throws Exception { credentials.put("username", "piOne"); credentials.put("password", "piOne"); Session piSession = icat.login("db", credentials); - searchDatafiles(piSession, null, null, null, null, null, null, 10, null, 0); + searchDatafiles(piSession, null, null, null, null, null, null, 10, null, null, 0); + + // Test no facets match on Datafiles + JsonObjectBuilder target = Json.createObjectBuilder().add("target", "Datafile"); + String facets = Json.createArrayBuilder().add(target).build().toString(); + responseObject = searchDatafiles(session, null, null, null, null, null, null, 10, null, facets, 3); + assertFalse(responseObject.containsKey("search_after")); + assertFalse("Did not expect responseObject to contain 'dimensions', but it did", + responseObject.containsKey("dimensions")); + + // Test no facets match on DatafileParameters due to lack of READ access + target = Json.createObjectBuilder().add("target", "DatafileParameter"); + facets = Json.createArrayBuilder().add(target).build().toString(); + responseObject = searchDatafiles(session, null, null, null, null, null, null, 10, null, facets, 3); + assertFalse(responseObject.containsKey("search_after")); + assertFalse("Did not expect responseObject to contain 'dimensions', but it did", + responseObject.containsKey("dimensions")); + + // Test facets match on DatafileParameters + wSession.addRule(null, "DatafileParameter", "R"); + target = Json.createObjectBuilder().add("target", "DatafileParameter"); + facets = Json.createArrayBuilder().add(target).build().toString(); + responseObject = searchDatafiles(session, null, null, null, null, null, null, 10, null, facets, 3); + assertFalse(responseObject.containsKey("search_after")); + checkFacets(responseObject, "DatafileParameter.type.name", Arrays.asList("colour"), Arrays.asList(1L)); } @Test @@ -718,10 +743,10 @@ public void testSearchDatasets() throws Exception { expectation.put("type.name", null); // All datasets - searchDatasets(session, null, null, null, null, null, null, 10, null, 5); + searchDatasets(session, null, null, null, null, null, null, 10, null, null, 5); // Use the user - responseObject = searchDatasets(session, "db/tr", null, null, null, null, null, 10, null, 3); + responseObject = searchDatasets(session, "db/tr", null, null, null, null, null, 10, null, null, 3); List> expectations = new ArrayList<>(); expectation.put("name", "ds1"); expectations.add(new HashMap<>(expectation)); @@ -732,11 +757,11 @@ public void testSearchDatasets() throws Exception { checkResultsSource(responseObject, expectations, true); // Try a bad user - searchDatasets(session, "db/fred", null, null, null, null, null, 10, null, 0); + searchDatasets(session, "db/fred", null, null, null, null, null, 10, null, null, 0); // Try text - responseObject = searchDatasets(session, null, "gamma AND ds3", null, null, null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchDatasets(session, null, "gamma AND ds3", null, null, null, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); checkResultsSource(responseObject, Arrays.asList(expectation), true); // Try parameters @@ -748,22 +773,23 @@ public void testSearchDatasets() throws Exception { parameters.add(new ParameterForLucene("birthday", "date", parameterDate, parameterDate)); parameters.add(new ParameterForLucene("current", "amps", 140, 165)); - responseObject = searchDatasets(session, null, null, null, null, parameters, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchDatasets(session, null, null, null, null, parameters, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); checkResultsSource(responseObject, Arrays.asList(expectation), true); - responseObject = searchDatasets(session, null, "gamma AND ds3", lower, upper, parameters, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchDatasets(session, null, "gamma AND ds3", lower, upper, parameters, null, 10, null, null, + 1); + assertFalse(responseObject.containsKey("search_after")); checkResultsSource(responseObject, Arrays.asList(expectation), true); // Try sorting and searchAfter String sort = Json.createObjectBuilder().add("name", "desc").add("startDate", "asc").build().toString(); - responseObject = searchDatasets(session, null, null, null, null, null, null, 1, sort, 1); + responseObject = searchDatasets(session, null, null, null, null, null, null, 1, sort, null, 1); searchAfter = responseObject.getString("search_after"); assertNotNull(searchAfter); expectation.put("name", "ds4"); checkResultsSource(responseObject, Arrays.asList(expectation), false); - responseObject = searchDatasets(session, null, null, null, null, null, searchAfter, 1, sort, 1); + responseObject = searchDatasets(session, null, null, null, null, null, searchAfter, 1, sort, null, 1); searchAfter = responseObject.getString("search_after"); assertNotNull(searchAfter); expectation.put("name", "ds3"); @@ -775,27 +801,27 @@ public void testSearchDatasets() throws Exception { ps.setField("type"); ps.setId(wSession.create(ps)); - responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); expectation.put("name", "ds1"); expectation.put("type.name", "calibration"); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.addRule(null, "Sample", "R"); - responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); expectation.put("sample.name", "Koh-I-Noor"); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.delete(ps); - responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); expectation.put("type.name", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.delRule(null, "Sample", "R"); - responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchDatasets(session, null, "ds1", null, null, null, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); expectation.put("sample.name", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); @@ -805,7 +831,31 @@ public void testSearchDatasets() throws Exception { credentials.put("username", "piOne"); credentials.put("password", "piOne"); Session piSession = icat.login("db", credentials); - searchDatasets(piSession, null, null, null, null, null, null, 10, null, 0); + searchDatasets(piSession, null, null, null, null, null, null, 10, null, null, 0); + + // Test no facets match on Datasets + JsonObjectBuilder target = Json.createObjectBuilder().add("target", "Dataset"); + String facets = Json.createArrayBuilder().add(target).build().toString(); + responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5); + assertFalse(responseObject.containsKey("search_after")); + checkFacets(responseObject, "Dataset.type.name", Arrays.asList("calibration"), Arrays.asList(5L)); + + // Test no facets match on DatasetParameters due to lack of READ access + target = Json.createObjectBuilder().add("target", "DatasetParameter"); + facets = Json.createArrayBuilder().add(target).build().toString(); + responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5); + assertFalse(responseObject.containsKey("search_after")); + assertFalse("Did not expect responseObject to contain 'dimensions', but it did", + responseObject.containsKey("dimensions")); + + // Test facets match on DatasetParameters + wSession.addRule(null, "DatasetParameter", "R"); + target = Json.createObjectBuilder().add("target", "DatasetParameter"); + facets = Json.createArrayBuilder().add(target).build().toString(); + responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5); + assertFalse(responseObject.containsKey("search_after")); + checkFacets(responseObject, "DatasetParameter.type.name", Arrays.asList("colour", "birthday", "current"), + Arrays.asList(1L, 1L, 1L)); } @Test @@ -833,60 +883,60 @@ public void testSearchInvestigations() throws Exception { String textTwo = "title AND two"; String textPlus = "title + one"; - searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, 3); + searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, null, 3); List parameters = new ArrayList<>(); parameters.add(new ParameterForLucene("colour", "name", "green")); // TODO remove additional checks here responseObject = searchInvestigations(session, "db/tr", null, lowerOrigin, upperOrigin, null, - null, null, null, 10, null, 1); + null, null, null, 10, null, null, 1); responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, null, - null, null, null, 10, null, 1); + null, null, null, 10, null, null, 1); responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, - null, null, null, 10, null, 1); + null, null, null, 10, null, null, 1); responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, - null, "Professor", null, 10, null, 1); + null, "Professor", null, 10, null, null, 1); responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, - samplesAnd, "Professor", null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + samplesAnd, "Professor", null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); checkResultsSource(responseObject, Arrays.asList(expectation), true); // change user - searchInvestigations(session, "db/fred", textAnd, null, null, parameters, null, null, null, 10, null, 0); + searchInvestigations(session, "db/fred", textAnd, null, null, parameters, null, null, null, 10, null, null, 0); // change text - searchInvestigations(session, "db/tr", textTwo, null, null, parameters, null, null, null, 10, null, 0); + searchInvestigations(session, "db/tr", textTwo, null, null, parameters, null, null, null, 10, null, null, 0); // Only working to a minute responseObject = searchInvestigations(session, "db/tr", textAnd, lowerSecond, upperOrigin, parameters, null, - null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + null, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); checkResultsSource(responseObject, Arrays.asList(expectation), true); responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperSecond, parameters, null, - null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + null, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); checkResultsSource(responseObject, Arrays.asList(expectation), true); searchInvestigations(session, "db/tr", textAnd, lowerMinute, upperOrigin, parameters, null, null, null, - 10, null, 0); + 10, null, null, 0); searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperMinute, parameters, null, null, null, - 10, null, 0); + 10, null, null, 0); // Change parameters List badParameters = new ArrayList<>(); badParameters.add(new ParameterForLucene("color", "name", "green")); searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, badParameters, samplesPlus, null, - null, 10, null, 0); + null, 10, null, null, 0); // Change samples searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, samplesBad, null, null, - 10, null, 0); + 10, null, null, 0); // Change userFullName searchInvestigations(session, "db/tr", textPlus, lowerOrigin, upperOrigin, parameters, samplesAnd, "Doctor", - null, 10, null, 0); + null, 10, null, null, 0); // TODO currently the only field we can access is name due to how // Investigation.getDoc() encodes, but this is the same for all the @@ -898,26 +948,30 @@ public void testSearchInvestigations() throws Exception { ps.setField("type"); ps.setId(wSession.create(ps)); - responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, + null, 1); + assertFalse(responseObject.containsKey("search_after")); expectation.put("type.name", "atype"); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.addRule(null, "Facility", "R"); - responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, + null, 1); + assertFalse(responseObject.containsKey("search_after")); expectation.put("facility.name", "Test port facility"); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.delete(ps); - responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, + null, 1); + assertFalse(responseObject.containsKey("search_after")); expectation.put("type.name", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.delRule(null, "Facility", "R"); - responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, 1); - assertFalse(responseObject.keySet().contains("search_after")); + responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, + null, 1); + assertFalse(responseObject.containsKey("search_after")); expectation.put("facility.name", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); @@ -927,7 +981,33 @@ public void testSearchInvestigations() throws Exception { credentials.put("username", "piOne"); credentials.put("password", "piOne"); Session piSession = icat.login("db", credentials); - searchInvestigations(piSession, null, null, null, null, null, null, null, null, 10, null, 0); + searchInvestigations(piSession, null, null, null, null, null, null, null, null, 10, null, null, 0); + + // Test no facets match on Investigations + JsonObjectBuilder target = Json.createObjectBuilder().add("target", "Investigation"); + String facets = Json.createArrayBuilder().add(target).build().toString(); + responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, + 3); + assertFalse(responseObject.containsKey("search_after")); + checkFacets(responseObject, "Investigation.type.name", Arrays.asList("atype"), Arrays.asList(3L)); + + // Test no facets match on InvestigationParameters due to lack of READ access + target = Json.createObjectBuilder().add("target", "InvestigationParameter"); + facets = Json.createArrayBuilder().add(target).build().toString(); + responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, + 3); + assertFalse(responseObject.containsKey("search_after")); + assertFalse("Did not expect responseObject to contain 'dimensions', but it did", + responseObject.containsKey("dimensions")); + + // Test facets match on InvestigationParameters + wSession.addRule(null, "InvestigationParameter", "R"); + target = Json.createObjectBuilder().add("target", "InvestigationParameter"); + facets = Json.createArrayBuilder().add(target).build().toString(); + responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, + 3); + assertFalse(responseObject.containsKey("search_after")); + checkFacets(responseObject, "InvestigationParameter.type.name", Arrays.asList("colour"), Arrays.asList(1L)); } @Test @@ -937,7 +1017,7 @@ public void testSearchParameterValidation() throws Exception { badParameters = Arrays.asList(new ParameterForLucene(null, null, null)); try { - searchInvestigations(session, null, null, null, null, badParameters, null, null, null, 10, null, 0); + searchInvestigations(session, null, null, null, null, badParameters, null, null, null, 10, null, null, 0); fail("BAD_PARAMETER exception not caught"); } catch (IcatException e) { assertEquals(IcatExceptionType.BAD_PARAMETER, e.getType()); @@ -946,7 +1026,7 @@ public void testSearchParameterValidation() throws Exception { badParameters = Arrays.asList(new ParameterForLucene("color", null, null)); try { - searchInvestigations(session, null, null, null, null, badParameters, null, null, null, 10, null, 0); + searchInvestigations(session, null, null, null, null, badParameters, null, null, null, 10, null, null, 0); fail("BAD_PARAMETER exception not caught"); } catch (IcatException e) { assertEquals(IcatExceptionType.BAD_PARAMETER, e.getType()); @@ -955,7 +1035,7 @@ public void testSearchParameterValidation() throws Exception { badParameters = Arrays.asList(new ParameterForLucene("color", "string", null)); try { - searchInvestigations(session, null, null, null, null, badParameters, null, null, null, 10, null, 0); + searchInvestigations(session, null, null, null, null, badParameters, null, null, null, 10, null, null, 0); fail("BAD_PARAMETER exception not caught"); } catch (IcatException e) { assertEquals(IcatExceptionType.BAD_PARAMETER, e.getType()); @@ -963,6 +1043,22 @@ public void testSearchParameterValidation() throws Exception { } } + private void checkFacets(JsonObject responseObject, String dimension, List expectedLabels, + List expectedCounts) { + assertTrue("Expected responseObject to contain 'dimensions', but it did not", + responseObject.containsKey("dimensions")); + JsonObject dimensions = responseObject.getJsonObject("dimensions"); + assertTrue("Expected 'dimensions' to contain " + dimension + " but keys were " + dimensions.keySet(), + dimensions.containsKey(dimension)); + JsonObject labelsObject = dimensions.getJsonObject(dimension); + assertEquals(expectedLabels.size(), labelsObject.size()); + for (int i = 0; i < expectedLabels.size(); i++) { + String expectedLabel = expectedLabels.get(i); + assertTrue(labelsObject.containsKey(expectedLabel)); + assertEquals(expectedCounts.get(i), new Long(labelsObject.getJsonNumber(expectedLabel).longValueExact())); + } + } + private void checkResultFromLuceneSearch(Session session, String val, JsonArray array, String ename, String field) throws IcatException { long n = array.getJsonObject(0).getJsonNumber("id").longValueExact(); @@ -975,23 +1071,23 @@ private void checkResultsSource(JsonObject responseObject, List expectation = expectations.get(i); for (Entry entry : expectation.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); if (value == null) { assertFalse("Source " + source.toString() + " should NOT contain " + key, - source.keySet().contains(key)); + source.containsKey(key)); } else if (value.equals("notNull")) { - assertTrue("Source " + source.toString() + " should contain " + key, source.keySet().contains(key)); + assertTrue("Source " + source.toString() + " should contain " + key, source.containsKey(key)); } else { - assertTrue("Source " + source.toString() + " should contain " + key, source.keySet().contains(key)); + assertTrue("Source " + source.toString() + " should contain " + key, source.containsKey(key)); assertEquals(value, source.getString(key)); } } @@ -1593,9 +1689,10 @@ private JsonArray searchInvestigations(Session session, String user, String text } private JsonObject searchDatafiles(Session session, String user, String text, Date lower, Date upper, - List parameters, String searchAfter, int limit, String sort, int n) + List parameters, String searchAfter, int limit, String sort, String facets, int n) throws IcatException { - String response = session.searchDatafiles(user, text, lower, upper, parameters, searchAfter, limit, sort); + String response = session.searchDatafiles(user, text, lower, upper, parameters, searchAfter, limit, sort, + facets); JsonObject responseObject = Json.createReader(new ByteArrayInputStream(response.getBytes())).readObject(); JsonArray results = responseObject.getJsonArray("results"); assertEquals(n, results.size()); @@ -1603,9 +1700,10 @@ private JsonObject searchDatafiles(Session session, String user, String text, Da } private JsonObject searchDatasets(Session session, String user, String text, Date lower, Date upper, - List parameters, String searchAfter, int limit, String sort, int n) + List parameters, String searchAfter, int limit, String sort, String facets, int n) throws IcatException { - String response = session.searchDatasets(user, text, lower, upper, parameters, searchAfter, limit, sort); + String response = session.searchDatasets(user, text, lower, upper, parameters, searchAfter, limit, sort, + facets); JsonObject responseObject = Json.createReader(new ByteArrayInputStream(response.getBytes())).readObject(); JsonArray results = responseObject.getJsonArray("results"); assertEquals(n, results.size()); @@ -1614,9 +1712,9 @@ private JsonObject searchDatasets(Session session, String user, String text, Dat private JsonObject searchInvestigations(Session session, String user, String text, Date lower, Date upper, List parameters, List samples, String userFullName, String searchAfter, - int limit, String sort, int n) throws IcatException { + int limit, String sort, String facets, int n) throws IcatException { String response = session.searchInvestigations(user, text, lower, upper, parameters, samples, userFullName, - searchAfter, limit, sort); + searchAfter, limit, sort, facets); JsonObject responseObject = Json.createReader(new ByteArrayInputStream(response.getBytes())).readObject(); JsonArray results = responseObject.getJsonArray("results"); assertEquals(n, results.size()); From 5e93b3d540b0ef45ca80f66f1fe909f90878dcef Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 14 Apr 2022 05:02:07 +0100 Subject: [PATCH 15/51] Basic unit conversion tests #267 --- .../icatproject/core/manager/TestLucene.java | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/src/test/java/org/icatproject/core/manager/TestLucene.java b/src/test/java/org/icatproject/core/manager/TestLucene.java index cd7e0c83..6dde9728 100644 --- a/src/test/java/org/icatproject/core/manager/TestLucene.java +++ b/src/test/java/org/icatproject/core/manager/TestLucene.java @@ -21,6 +21,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import javax.json.Json; +import javax.json.JsonArray; import javax.json.JsonArrayBuilder; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; @@ -1052,4 +1053,173 @@ private void populate() throws IcatException { } + @Test + public void unitConversion() throws IcatException { + // Build queries for raw and SI values + // JsonObject queryObject = Json.createObjectBuilder().add("investigation.id", + // Json.createArrayBuilder().add("0")) + // .build(); + JsonObject mKQuery = Json.createObjectBuilder().add("type.units", "mK").build(); + JsonObject celsiusQuery = Json.createObjectBuilder().add("type.units", "celsius").build(); + JsonObject wrongQuery = Json.createObjectBuilder().add("type.units", "wrong").build(); + JsonObject kelvinQuery = Json.createObjectBuilder().add("type.unitsSI", "Kelvin").build(); + JsonObjectBuilder lowRangeBuilder = Json.createObjectBuilder().add("lower", 272.5).add("upper", 273.5); + JsonObjectBuilder midRangeBuilder = Json.createObjectBuilder().add("lower", 272999.5).add("upper", 273000.5); + JsonObjectBuilder highRangeBuilder = Json.createObjectBuilder().add("lower", 273272.5).add("upper", 273273.5); + JsonArray ranges = Json.createArrayBuilder().add(lowRangeBuilder).add(midRangeBuilder).add(highRangeBuilder) + .build(); + JsonObject rawObject = Json.createObjectBuilder().add("dimension", "numericValue").add("ranges", ranges) + .build(); + JsonObject rawFacetQuery = Json.createObjectBuilder().add("query", mKQuery) + .add("dimensions", Json.createArrayBuilder().add(rawObject)).build(); + JsonObject systemObject = Json.createObjectBuilder().add("dimension", "numericValueSI").add("ranges", ranges) + .build(); + JsonObject systemFacetQuery = Json.createObjectBuilder().add("query", kelvinQuery) + .add("dimensions", Json.createArrayBuilder().add(systemObject)).build(); + + // Build entities + Investigation investigation = new Investigation(); + investigation.setId(0L); + investigation.setName("name"); + investigation.setVisitId("visitId"); + investigation.setTitle("title"); + investigation.setCreateTime(new Date()); + investigation.setModTime(new Date()); + Facility facility = new Facility(); + facility.setName("facility"); + facility.setId(0L); + investigation.setFacility(facility); + InvestigationType type = new InvestigationType(); + type.setName("type"); + type.setId(0L); + investigation.setType(type); + + ParameterType numericParameterType = new ParameterType(); + numericParameterType.setId(0L); + numericParameterType.setName("parameter"); + numericParameterType.setUnits("mK"); + InvestigationParameter parameter = new InvestigationParameter(); + parameter.setInvestigation(investigation); + parameter.setType(numericParameterType); + parameter.setNumericValue(273000.); + parameter.setId(0L); + + Queue queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeOperation("create", investigation)); + queue.add(SearchApi.encodeOperation("create", parameter)); + modifyQueue(queue); + + // Assert the raw value is still 273000 (mK) + List facetDimensions = luceneApi.facetSearch("InvestigationParameter", rawFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + FacetDimension facetDimension = facetDimensions.get(0); + assertEquals("numericValue", facetDimension.getDimension()); + assertEquals(3, facetDimension.getFacets().size()); + FacetLabel facetLabel = facetDimension.getFacets().get(0); + assertEquals("272.5_273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("272999.5_273000.5", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(2); + assertEquals("273272.5_273273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + + // Assert the SI value is 273 (K) + facetDimensions = luceneApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("numericValueSI", facetDimension.getDimension()); + assertEquals(3, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("272.5_273.5", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("272999.5_273000.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(2); + assertEquals("273272.5_273273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + + // Change units only to "celsius" + numericParameterType.setUnits("celsius"); + queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeOperation("update", parameter)); + modifyQueue(queue); + rawFacetQuery = Json.createObjectBuilder().add("query", celsiusQuery) + .add("dimensions", Json.createArrayBuilder().add(rawObject)).build(); + + // Assert the raw value is still 273000 (deg C) + facetDimensions = luceneApi.facetSearch("InvestigationParameter", rawFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("numericValue", facetDimension.getDimension()); + assertEquals(3, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("272.5_273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("272999.5_273000.5", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(2); + assertEquals("273272.5_273273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + + // Assert the SI value is 273273.15 (K) + facetDimensions = luceneApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("numericValueSI", facetDimension.getDimension()); + assertEquals(3, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("272.5_273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("272999.5_273000.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(2); + assertEquals("273272.5_273273.5", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + + // Change units to something wrong + numericParameterType.setUnits("wrong"); + queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeOperation("update", parameter)); + modifyQueue(queue); + rawFacetQuery = Json.createObjectBuilder().add("query", wrongQuery) + .add("dimensions", Json.createArrayBuilder().add(rawObject)).build(); + + // Assert the raw value is still 273000 (wrong) + facetDimensions = luceneApi.facetSearch("InvestigationParameter", rawFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("numericValue", facetDimension.getDimension()); + assertEquals(3, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("272.5_273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("272999.5_273000.5", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(2); + assertEquals("273272.5_273273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + + // Assert that the SI value has not been set due to conversion failing + facetDimensions = luceneApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("numericValueSI", facetDimension.getDimension()); + assertEquals(3, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("272.5_273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("272999.5_273000.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(2); + assertEquals("273272.5_273273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + } + } From 7eeb840ba0a84d77c9e097afb1fe3631b47c0342 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Sat, 16 Apr 2022 08:27:30 +0100 Subject: [PATCH 16/51] Add support for Opensearch requests #267 --- pom.xml | 23 +- .../core/entity/DatafileFormat.java | 3 + .../icatproject/core/entity/DatasetType.java | 3 + .../org/icatproject/core/entity/Facility.java | 3 + .../core/entity/InvestigationType.java | 3 + .../core/entity/ParameterType.java | 3 + .../org/icatproject/core/entity/Sample.java | 4 +- .../icatproject/core/entity/SampleType.java | 7 +- .../org/icatproject/core/entity/User.java | 3 + .../core/manager/ElasticsearchApi.java | 5 +- .../core/manager/EntityBeanManager.java | 12 +- .../icatproject/core/manager/LuceneApi.java | 28 +- .../core/manager/PropertyHandler.java | 21 +- .../core/manager/ScoredEntityBaseBean.java | 3 +- .../icatproject/core/manager/SearchApi.java | 931 ++++++++++--- .../core/manager/SearchManager.java | 7 +- .../core/manager/SearchResult.java | 10 +- .../org/icatproject/exposed/ICATRest.java | 12 +- .../icatproject/core/manager/TestLucene.java | 59 +- .../core/manager/TestSearchApi.java | 1147 +++++++++++++++++ .../org/icatproject/integration/TestRS.java | 22 +- src/test/scripts/prepare_test.py | 7 +- 22 files changed, 2076 insertions(+), 240 deletions(-) create mode 100644 src/test/java/org/icatproject/core/manager/TestSearchApi.java diff --git a/pom.xml b/pom.xml index 67a83b78..ccd4c7ad 100644 --- a/pom.xml +++ b/pom.xml @@ -117,7 +117,7 @@ org.icatproject icat.client - 4.11.1 + 4.11.2-SNAPSHOT test @@ -137,12 +137,24 @@ co.elastic.clients elasticsearch-java 8.1.0 - - + + com.fasterxml.jackson.core jackson-databind 2.12.3 - + + + + javax.measure + unit-api + 2.1.3 + + + + tech.units + indriya + 2.1.3 + @@ -237,6 +249,7 @@ ${javax.net.ssl.trustStore} ${luceneUrl} ${elasticsearchUrl} + ${opensearchUrl} false @@ -257,6 +270,7 @@ ${searchEngine} ${luceneUrl} ${elasticsearchUrl} + ${opensearchUrl} @@ -338,6 +352,7 @@ ${searchEngine} ${luceneUrl} ${elasticsearchUrl} + ${opensearchUrl} diff --git a/src/main/java/org/icatproject/core/entity/DatafileFormat.java b/src/main/java/org/icatproject/core/entity/DatafileFormat.java index ae09b337..7021c01a 100644 --- a/src/main/java/org/icatproject/core/entity/DatafileFormat.java +++ b/src/main/java/org/icatproject/core/entity/DatafileFormat.java @@ -2,6 +2,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.json.stream.JsonGenerator; @@ -54,6 +55,8 @@ public void setFacility(Facility facility) { @Column(name = "VERSION", nullable = false) private String version; + public static List docFields = Arrays.asList("datafileFormat.name", "datafileFormat.id"); + /* Needed for JPA */ public DatafileFormat() { } diff --git a/src/main/java/org/icatproject/core/entity/DatasetType.java b/src/main/java/org/icatproject/core/entity/DatasetType.java index 43a16d40..be67ec5a 100644 --- a/src/main/java/org/icatproject/core/entity/DatasetType.java +++ b/src/main/java/org/icatproject/core/entity/DatasetType.java @@ -2,6 +2,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.json.stream.JsonGenerator; @@ -38,6 +39,8 @@ public class DatasetType extends EntityBaseBean implements Serializable { @Column(name = "NAME", nullable = false) private String name; + public static List docFields = Arrays.asList("type.name", "type.id"); + /* Needed for JPA */ public DatasetType() { } diff --git a/src/main/java/org/icatproject/core/entity/Facility.java b/src/main/java/org/icatproject/core/entity/Facility.java index fc75a8cc..f10dadaa 100644 --- a/src/main/java/org/icatproject/core/entity/Facility.java +++ b/src/main/java/org/icatproject/core/entity/Facility.java @@ -2,6 +2,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.json.stream.JsonGenerator; @@ -64,6 +65,8 @@ public class Facility extends EntityBaseBean implements Serializable { @Comment("A URL associated with this facility") private String url; + public static List docFields = Arrays.asList("facility.name", "facility.id"); + /* Needed for JPA */ public Facility() { } diff --git a/src/main/java/org/icatproject/core/entity/InvestigationType.java b/src/main/java/org/icatproject/core/entity/InvestigationType.java index f9b5b8f3..a9703fcd 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationType.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationType.java @@ -2,6 +2,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.json.stream.JsonGenerator; @@ -54,6 +55,8 @@ public void setInvestigations(List investigations) { this.investigations = investigations; } + public static List docFields = Arrays.asList("type.name", "type.id"); + /* Needed for JPA */ public InvestigationType() { } diff --git a/src/main/java/org/icatproject/core/entity/ParameterType.java b/src/main/java/org/icatproject/core/entity/ParameterType.java index 489ec742..f58145a5 100644 --- a/src/main/java/org/icatproject/core/entity/ParameterType.java +++ b/src/main/java/org/icatproject/core/entity/ParameterType.java @@ -2,6 +2,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.json.stream.JsonGenerator; @@ -94,6 +95,8 @@ public class ParameterType extends EntityBaseBean implements Serializable { @Comment("If ordinary users are allowed to create their own parameter types this indicates that this one has been approved") private boolean verified; + public static List docFields = Arrays.asList("type.name", "type.units", "type.unitsSI", "numericValueSI", "type.id"); + /* Needed for JPA */ public ParameterType() { } diff --git a/src/main/java/org/icatproject/core/entity/Sample.java b/src/main/java/org/icatproject/core/entity/Sample.java index d0a6b526..451bc4bf 100644 --- a/src/main/java/org/icatproject/core/entity/Sample.java +++ b/src/main/java/org/icatproject/core/entity/Sample.java @@ -97,7 +97,7 @@ public void setType(SampleType type) { @Override public void getDoc(JsonGenerator gen) { - SearchApi.encodeString(gen, "sample.name", name); + SearchApi.encodeString(gen, "name", name); SearchApi.encodeString(gen, "id", id); SearchApi.encodeString(gen, "investigation.id", investigation.id); if (type != null) { @@ -115,7 +115,7 @@ public void getDoc(JsonGenerator gen) { * @param prefix String to precede all ambiguous field names. */ public void getDoc(JsonGenerator gen, String prefix) { - SearchApi.encodeString(gen, "sample.name", name); + SearchApi.encodeString(gen, prefix + "name", name); SearchApi.encodeString(gen, prefix + "id", id); SearchApi.encodeString(gen, prefix + "investigation.id", investigation.id); if (type != null) { diff --git a/src/main/java/org/icatproject/core/entity/SampleType.java b/src/main/java/org/icatproject/core/entity/SampleType.java index d4c12c2f..2cea752c 100644 --- a/src/main/java/org/icatproject/core/entity/SampleType.java +++ b/src/main/java/org/icatproject/core/entity/SampleType.java @@ -2,6 +2,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.json.stream.JsonGenerator; @@ -43,6 +44,8 @@ public class SampleType extends EntityBaseBean implements Serializable { @OneToMany(cascade = CascadeType.ALL, mappedBy = "type") private List samples = new ArrayList<>(); + public static List docFields = Arrays.asList("sample.type.name", "type.id"); + /* Needed for JPA */ public SampleType() { } @@ -89,7 +92,7 @@ public void setSamples(List samples) { @Override public void getDoc(JsonGenerator gen) { - SearchApi.encodeString(gen, "sample.type.name", name); + SearchApi.encodeString(gen, "type.name", name); SearchApi.encodeString(gen, "type.id", id); } @@ -104,7 +107,7 @@ public void getDoc(JsonGenerator gen) { * @param prefix String to precede all ambiguous field names. */ public void getDoc(JsonGenerator gen, String prefix) { - SearchApi.encodeString(gen, "sample.type.name", name); + SearchApi.encodeString(gen, prefix + "type.name", name); SearchApi.encodeString(gen, prefix + "type.id", id); } diff --git a/src/main/java/org/icatproject/core/entity/User.java b/src/main/java/org/icatproject/core/entity/User.java index 6b59731a..e0ca1367 100644 --- a/src/main/java/org/icatproject/core/entity/User.java +++ b/src/main/java/org/icatproject/core/entity/User.java @@ -2,6 +2,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.json.stream.JsonGenerator; @@ -54,6 +55,8 @@ public class User extends EntityBaseBean implements Serializable { @OneToMany(cascade = CascadeType.ALL, mappedBy = "user") private List studies = new ArrayList(); + public static List docFields = Arrays.asList("user.name", "user.fullName", "user.id"); + public User() { } diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java index 83b269b9..f3dcfb88 100644 --- a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java +++ b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java @@ -106,6 +106,7 @@ public class ElasticsearchApi extends SearchApi { private final Map pitMap = new HashMap<>(); public ElasticsearchApi(List servers) throws IcatException { + super(null); List hosts = new ArrayList(); for (URL server : servers) { hosts.add(new HttpHost(server.getHost(), server.getPort(), server.getProtocol())); @@ -117,7 +118,7 @@ public ElasticsearchApi(List servers) throws IcatException { initMappings(); } - private void initMappings() throws IcatException { + public void initMappings() throws IcatException { try { client.cluster().putSettings(s -> s.persistent("action.auto_create_index", JsonData.of(false))); client.putScript(p -> p.id("update_user").script(s -> s @@ -189,7 +190,7 @@ public void commit() throws IcatException { } @Override - public List facetSearch(String target, JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException { + public List facetSearch(String target, JsonObject facetQuery, Integer maxResults, int maxLabels) throws IcatException { // TODO this should be generalised return null; // try { diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index b864bb6d..0690e002 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -1394,8 +1394,8 @@ public List freeTextSearch(String userName, JsonObject jo, EntityManager manager, String ip, Class klass) throws IcatException { long startMillis = log ? System.currentTimeMillis() : 0; List results = new ArrayList<>(); - String searchAfter = null; - String lastSearchAfter = null; + JsonValue searchAfter = null; + JsonValue lastSearchAfter = null; if (searchActive) { SearchResult lastSearchResult = null; List allResults = Collections.emptyList(); @@ -1415,7 +1415,7 @@ public List freeTextSearch(String userName, JsonObject jo, if (lastSearchAfter == null) { break; // If searchAfter is null, we ran out of results so stop here } - searchAfter = lastSearchAfter.toString(); + searchAfter = lastSearchAfter; } else { // Have stopped early by reaching the limit, so build a searchAfter document lastSearchAfter = searchManager.buildSearchAfter(lastBean, sort); @@ -1439,12 +1439,12 @@ public List freeTextSearch(String userName, JsonObject jo, return results; } - public SearchResult freeTextSearchDocs(String userName, JsonObject jo, String searchAfter, int limit, String sort, + public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue searchAfter, int limit, String sort, String facets, EntityManager manager, String ip, Class klass) throws IcatException { long startMillis = log ? System.currentTimeMillis() : 0; List results = new ArrayList<>(); - String lastSearchAfter = null; + JsonValue lastSearchAfter = null; List dimensions = null; if (searchActive) { SearchResult lastSearchResult = null; @@ -1466,7 +1466,7 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, String se if (lastSearchAfter == null) { break; // If searchAfter is null, we ran out of results so stop here } - searchAfter = lastSearchAfter.toString(); + searchAfter = lastSearchAfter; } else { // Have stopped early by reaching the limit, so build a searchAfter document lastSearchAfter = searchManager.buildSearchAfter(lastBean, sort); diff --git a/src/main/java/org/icatproject/core/manager/LuceneApi.java b/src/main/java/org/icatproject/core/manager/LuceneApi.java index e9e54191..2fd13dbc 100644 --- a/src/main/java/org/icatproject/core/manager/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/LuceneApi.java @@ -13,6 +13,7 @@ import java.util.concurrent.ExecutorService; import javax.json.Json; +import javax.json.JsonArray; import javax.json.JsonArrayBuilder; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; @@ -66,10 +67,8 @@ private String getTargetPath(JsonObject query) throws IcatException { return path; } - URI server; - public LuceneApi(URI server) { - this.server = server; + super(server); } public void addNow(String entityName, List ids, EntityManager manager, @@ -110,7 +109,7 @@ public void addNow(String entityName, List ids, EntityManager manager, } @Override - public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { + public JsonObject buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { JsonObjectBuilder builder = Json.createObjectBuilder(); builder.add("doc", lastBean.getEngineDocId()); builder.add("shardIndex", -1); @@ -132,7 +131,7 @@ public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throw builder.add("fields", arrayBuilder); } } - return builder.build().toString(); + return builder.build(); } @Override @@ -164,7 +163,7 @@ public void commit() throws IcatException { } @Override - public List facetSearch(String target, JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException { + public List facetSearch(String target, JsonObject facetQuery, Integer maxResults, int maxLabels) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URI uri = new URIBuilder(server).setPath(basePath + "/" + target + "/facet") .setParameter("maxResults", Integer.toString(maxResults)) @@ -227,14 +226,18 @@ private List getFacets(URI uri, CloseableHttpClient httpclient, } @Override - public SearchResult getResults(JsonObject query, String searchAfter, int blockSize, String sort, + public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer blockSize, String sort, List fields) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { String indexPath = getTargetPath(query); - URI uri = new URIBuilder(server).setPath(basePath + "/" + indexPath) - .setParameter("search_after", searchAfter) - .setParameter("maxResults", Integer.toString(blockSize)) - .setParameter("sort", sort).build(); + URIBuilder uriBuilder = new URIBuilder(server).setPath(basePath + "/" + indexPath) + .setParameter("maxResults", Integer.toString(blockSize)); + if (searchAfter != null) { + uriBuilder.setParameter("search_after", searchAfter.toString()); + } + if (sort != null) { + uriBuilder.setParameter("sort", sort); + } JsonObjectBuilder objectBuilder = Json.createObjectBuilder(); objectBuilder.add("query", query); if (fields != null && fields.size() > 0) { @@ -243,6 +246,7 @@ public SearchResult getResults(JsonObject query, String searchAfter, int blockSi objectBuilder.add("fields", arrayBuilder.build()); } String queryString = objectBuilder.build().toString(); + URI uri = uriBuilder.build(); logger.trace("Making call {} with queryString {}", uri, queryString); return getResults(uri, httpclient, queryString); @@ -278,7 +282,7 @@ private SearchResult getResults(URI uri, CloseableHttpClient httpclient, String results.add(new ScoredEntityBaseBean(luceneDocId, score, source)); } if (responseObject.containsKey("search_after")) { - lsr.setSearchAfter(responseObject.getJsonObject("search_after").toString()); + lsr.setSearchAfter(responseObject.getJsonObject("search_after")); } } } diff --git a/src/main/java/org/icatproject/core/manager/PropertyHandler.java b/src/main/java/org/icatproject/core/manager/PropertyHandler.java index 81f4a14e..e18cdef9 100644 --- a/src/main/java/org/icatproject/core/manager/PropertyHandler.java +++ b/src/main/java/org/icatproject/core/manager/PropertyHandler.java @@ -225,7 +225,9 @@ public enum CallType { READ, WRITE, SESSION, INFO } - public enum SearchEngine {LUCENE, ELASTICSEARCH} + public enum SearchEngine { + LUCENE, ELASTICSEARCH, OPENSEARCH + } public class ExtendedAuthenticator { @@ -275,16 +277,15 @@ public Set getRootUserNames() { return rootUserNames; } - /** * Configure which entities will be indexed on ingest */ private Set entitiesToIndex = new HashSet(); - + public Set getEntitiesToIndex() { return entitiesToIndex; } - + public int getLifetimeMinutes() { return lifetimeMinutes; } @@ -390,16 +391,17 @@ private void init() { } logger.info("search.entitiesToIndex: {}", entitiesToIndex.toString()); } else { - /* If the property is not specified, we default to all the entities which + /* + * If the property is not specified, we default to all the entities which * currently override the EntityBaseBean.getDoc() method. This should * result in no change to behaviour if the property is not specified. */ - entitiesToIndex.addAll(Arrays.asList("Datafile", "Dataset", "Investigation", "InvestigationUser", + entitiesToIndex.addAll(Arrays.asList("Datafile", "Dataset", "Investigation", "InvestigationUser", "DatafileParameter", "DatasetParameter", "InvestigationParameter", "Sample")); logger.info("search.entitiesToIndex not set. Defaulting to: {}", entitiesToIndex.toString()); } formattedProps.add("search.entitiesToIndex " + entitiesToIndex.toString()); - + /* notification.list */ key = "notification.list"; if (props.has(key)) { @@ -476,13 +478,14 @@ private void init() { } } - if (searchEngine == SearchEngine.LUCENE && searchUrls.size() != 1) { + if ((searchEngine.equals(SearchEngine.LUCENE) || searchEngine.equals(SearchEngine.OPENSEARCH)) + && searchUrls.size() != 1) { String msg = "Exactly one value for search.urls must be provided when using " + searchEngine; throw new IllegalStateException(msg); } else if (searchUrls.size() == 0) { String msg = "At least one value for search.urls must be provided"; throw new IllegalStateException(msg); - } + } formattedProps.add("search.urls" + " " + searchUrls.toString()); logger.info("Using {} as search engine with url(s) {}", searchEngine, searchUrls); diff --git a/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java b/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java index a2741d00..6ac9316c 100644 --- a/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java +++ b/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java @@ -34,7 +34,8 @@ public class ScoredEntityBaseBean { public ScoredEntityBaseBean(int engineDocId, float score, JsonObject source) throws IcatException { if (!source.keySet().contains("id")) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Document source must have 'id' and the entityBaseBeanId as a key-value pair."); + "Document source must have 'id' and the entityBaseBeanId as a key-value pair, but it was " + + source.toString()); } this.engineDocId = engineDocId; this.score = score; diff --git a/src/main/java/org/icatproject/core/manager/SearchApi.java b/src/main/java/org/icatproject/core/manager/SearchApi.java index b5a0ac47..97bf4e76 100644 --- a/src/main/java/org/icatproject/core/manager/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/SearchApi.java @@ -1,6 +1,5 @@ package org.icatproject.core.manager; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.StringReader; @@ -8,9 +7,13 @@ import java.net.URISyntaxException; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TimeZone; import java.util.concurrent.ExecutorService; @@ -23,28 +26,68 @@ import javax.json.JsonObjectBuilder; import javax.json.JsonReader; import javax.json.JsonValue; +import javax.json.JsonValue.ValueType; import javax.json.stream.JsonGenerator; +import javax.measure.IncommensurableException; +import javax.measure.Unit; +import javax.measure.UnitConverter; +import javax.measure.format.MeasurementParseException; import javax.persistence.EntityManager; import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.icatproject.core.IcatException; import org.icatproject.core.IcatException.IcatExceptionType; +import org.icatproject.core.entity.DatafileFormat; +import org.icatproject.core.entity.DatasetType; import org.icatproject.core.entity.EntityBaseBean; +import org.icatproject.core.entity.Facility; +import org.icatproject.core.entity.InvestigationType; +import org.icatproject.core.entity.ParameterType; +import org.icatproject.core.entity.SampleType; +import org.icatproject.core.entity.User; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import tech.units.indriya.format.SimpleUnitFormat; +import tech.units.indriya.unit.Units; + // TODO see what functionality can live here, and possibly convert from abstract to a fully generic API -public abstract class SearchApi { +public class SearchApi { + // TODO this is a duplicate of icat.lucene code (for now...?) + private static class ParentRelationship { + public String parentName; + public String joinName; + public List fields; + + public ParentRelationship(String parentName, String joinName, List fields) { + this.parentName = parentName; + this.joinName = joinName; + this.fields = fields; + } + } + private static final SimpleUnitFormat unitFormat = SimpleUnitFormat.getInstance(); protected static final Logger logger = LoggerFactory.getLogger(SearchApi.class); protected static SimpleDateFormat df; protected static String basePath = ""; - protected static String matchAllQuery; + protected static JsonObject matchAllQuery = Json.createObjectBuilder().add("query", Json.createObjectBuilder() + .add("match_all", Json.createObjectBuilder())).build(); + private static JsonObject indexSettings = Json.createObjectBuilder().add("analysis", Json.createObjectBuilder() + .add("analyzer", Json.createObjectBuilder().add("default", Json.createObjectBuilder() + .add("tokenizer", "lowercase").add("filter", Json.createArrayBuilder().add("porter_stem"))))).build(); + // private static JsonObject mappingsBuilder; + protected static Set indices = new HashSet<>(); + protected static Map scripts = new HashMap<>(); + private static Map> relationships = new HashMap<>(); protected URI server; @@ -52,15 +95,223 @@ public abstract class SearchApi { df = new SimpleDateFormat("yyyyMMddHHmm"); TimeZone tz = TimeZone.getTimeZone("GMT"); df.setTimeZone(tz); + + unitFormat.alias(Units.CELSIUS, "celsius"); // TODO this should be generalised with the units we need + + indices.addAll(Arrays.asList("datafile", "dataset", "investigation")); + + scripts.put("delete_datafileparameter", buildChildrenScript("datafileparameter", false)); + scripts.put("update_datafileparameter", buildChildrenScript("datafileparameter", true)); + scripts.put("delete_datasetparameter", buildChildrenScript("datasetparameter", false)); + scripts.put("update_datasetparameter", buildChildrenScript("datasetparameter", true)); + scripts.put("delete_investigationparameter", buildChildrenScript("investigationparameter", false)); + scripts.put("update_investigationparameter", buildChildrenScript("investigationparameter", true)); + scripts.put("delete_investigationuser", buildChildrenScript("investigationuser", false)); + scripts.put("update_investigationuser", buildChildrenScript("investigationuser", true)); + scripts.put("create_investigationuser", buildCreateScript("investigationuser")); + scripts.put("delete_sample", buildChildrenScript("sample", false)); + scripts.put("update_sample", buildChildrenScript("sample", true)); + scripts.put("delete_datafileparametertype", buildChildrenScript("datafileparameter", ParameterType.docFields, false)); + scripts.put("update_datafileparametertype", buildChildrenScript("datafileparameter", ParameterType.docFields, true)); + scripts.put("delete_datasetparametertype", buildChildrenScript("datasetparameter", ParameterType.docFields, false)); + scripts.put("update_datasetparametertype", buildChildrenScript("datasetparameter", ParameterType.docFields, true)); + scripts.put("delete_investigationparametertype", buildChildrenScript("investigationparameter", ParameterType.docFields, false)); + scripts.put("update_investigationparametertype", buildChildrenScript("investigationparameter", ParameterType.docFields, true)); + scripts.put("delete_user", buildChildrenScript("investigationuser", User.docFields, false)); + scripts.put("update_user", buildChildrenScript("investigationuser", User.docFields, true)); + scripts.put("delete_sampletype", buildChildrenScript("sample", SampleType.docFields, false)); + scripts.put("update_sampletype", buildChildrenScript("sample", SampleType.docFields, true)); + + scripts.put("delete_datafileformat", buildChildScript("datafileformat", DatafileFormat.docFields, false)); + scripts.put("update_datafileformat", buildChildScript("datafileformat", DatafileFormat.docFields, true)); + scripts.put("delete_datasettype", buildChildScript("datasettype", DatasetType.docFields, false)); + scripts.put("update_datasettype", buildChildScript("datasettype", DatasetType.docFields, true)); + scripts.put("delete_investigationtype", buildChildScript("investigationtype", InvestigationType.docFields, false)); + scripts.put("update_investigationtype", buildChildScript("investigationtype", InvestigationType.docFields, true)); + scripts.put("delete_facility", buildChildScript("facility", Facility.docFields, false)); + scripts.put("update_facility", buildChildScript("facility", Facility.docFields, true)); + + relationships.put("datafileparameter", Arrays.asList( + new ParentRelationship("datafile", "datafile", new ArrayList<>()))); + relationships.put("datasetparameter", Arrays.asList( + new ParentRelationship("dataset", "dataset", new ArrayList<>()))); + relationships.put("investigationparameter", Arrays.asList( + new ParentRelationship("investigation", "investigation", new ArrayList<>()))); + relationships.put("investigationuser", Arrays.asList( + new ParentRelationship("investigation", "investigation", new ArrayList<>()))); + relationships.put("sample", Arrays.asList( + new ParentRelationship("investigation", "investigation", new ArrayList<>()))); + + relationships.put("parametertype", Arrays.asList( + new ParentRelationship("investigation", "investigationparameter", ParameterType.docFields), + new ParentRelationship("dataset", "datasetparameter", ParameterType.docFields), + new ParentRelationship("datafile", "datafileparameter", ParameterType.docFields) + )); + relationships.put("user", Arrays.asList( + new ParentRelationship("investigation", "investigationuser", User.docFields))); + relationships.put("sampleType", Arrays.asList( + new ParentRelationship("investigation", "sample", SampleType.docFields))); + + relationships.put("datafileformat", Arrays.asList( + new ParentRelationship("datafile", "datafileFormat", DatafileFormat.docFields))); + relationships.put("datasettype", Arrays.asList( + new ParentRelationship("dataset", "type", DatasetType.docFields))); + relationships.put("investigationtype", Arrays.asList( + new ParentRelationship("investigation", "type", InvestigationType.docFields))); + relationships.put("facility", Arrays.asList( + new ParentRelationship("investigation", "facility", Facility.docFields))); } - static { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject().writeStartObject("query").writeStartObject("match_all") - .writeEnd().writeEnd().writeEnd(); + public SearchApi(URI server) { + this.server = server; + } + + private static String buildCreateScript(String target) { + String source = "ctx._source." + target + " = params.doc"; + JsonObjectBuilder builder = Json.createObjectBuilder().add("lang", "painless").add("source", source); + return Json.createObjectBuilder().add("script", builder).build().toString(); + } + + private static String buildChildrenScript(String target, boolean update) { + String source = "if (ctx._source." + target + " != null) {List ids = new ArrayList(); ctx._source." + target + ".forEach(t -> ids.add(t.id)); if (ids.contains(params.id)) {ctx._source." + target + ".remove(ids.indexOf(params.id))}}"; + if (update) { + source += "if (ctx._source." + target + " != null) {ctx._source." + target + ".addAll(params.doc);} else {ctx._source." + target + " = params.doc;}"; + } + JsonObjectBuilder builder = Json.createObjectBuilder().add("lang", "painless").add("source", source); + return Json.createObjectBuilder().add("script", builder).build().toString(); + } + + private static String buildChildrenScript(String target, List docFields, boolean update) { + String source = "int listIndex; if (ctx._source." + target + " != null) {List ids = new ArrayList(); ctx._source." + target + ".forEach(t -> ids.add(t.id)); if (ids.contains(params.id)) {listIndex = ids.indexOf(params.id)}}"; + String childSource = "ctx._source." + target + ".get(listIndex)"; + for (String field : docFields) { + if (update) { + if (field.equals("numericValueSI")) { + source += "if ("+ childSource + ".numericValue != null && params.containsKey('conversionFactor')) {" + childSource + ".numericValueSI = params.conversionFactor * "+ childSource + ".numericValue;} else {" + childSource + ".remove('numericValueSI');}"; + } else { + source += childSource + "['" + field + "']" + " = params['" + field + "']; "; + } + } else { + source += childSource + ".remove('" + field + "'); "; + } + } + JsonObjectBuilder builder = Json.createObjectBuilder().add("lang", "painless").add("source", source); + return Json.createObjectBuilder().add("script", builder).build().toString(); + } + + private static String buildChildScript(String target, List docFields, boolean update) { + String source = ""; + for (String field : docFields) { + if (update) { + source += "ctx._source['" + field + "'] = params['" + field + "']; "; + } else { + source += "ctx._source.remove('" + field + "'); "; + } + } + JsonObjectBuilder builder = Json.createObjectBuilder().add("lang", "painless").add("source", source); + return Json.createObjectBuilder().add("script", builder).build().toString(); + } + + private static JsonObject buildMappings(String index) { + JsonObject typeLong = Json.createObjectBuilder().add("type", "long").build(); + JsonObjectBuilder propertiesBuilder = Json.createObjectBuilder() + .add("id", typeLong).add("investigationuser", buildNestedMapping("investigation.id", "user.id")); + if (index.equals("investigation")) { + propertiesBuilder.add("type.id", typeLong).add("facility.id", typeLong) + .add("sample", buildNestedMapping("investigation.id", "type.id")) + .add("investigationparameter", buildNestedMapping("investigation.id", "type.id")); + } else if (index.equals("dataset")) { + propertiesBuilder.add("investigation.id", typeLong).add("type.id", typeLong).add("sample.id", typeLong) + .add("datasetparameter", buildNestedMapping("dataset.id", "type.id")); + } else if (index.equals("datafile")) { + propertiesBuilder.add("investigation.id", typeLong).add("datafileFormat.id", typeLong) + .add("datafileparameter", buildNestedMapping("datafile.id", "type.id")); + } + return Json.createObjectBuilder().add("properties", propertiesBuilder).build(); + } + + private static JsonObject buildNestedMapping(String... idFields) { + JsonObject typeLong = Json.createObjectBuilder().add("type", "long").build(); + JsonObjectBuilder propertiesBuilder = Json.createObjectBuilder().add("id", typeLong); + for (String idField : idFields) { + propertiesBuilder.add(idField, typeLong); } - matchAllQuery = baos.toString(); + return Json.createObjectBuilder().add("type", "nested").add("properties", propertiesBuilder).build(); + } + + private static JsonObject buildMatchQuery(String field, String value) { + JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("query", value).add("operator", "and"); + JsonObjectBuilder matchBuilder = Json.createObjectBuilder().add(field + ".keyword", fieldBuilder); + return Json.createObjectBuilder().add("match", matchBuilder).build(); + } + + private static JsonObject buildNestedQuery(String path, JsonObject... queryObjects) { + JsonObject builtQueries = null; + if (queryObjects.length == 0) { + builtQueries = matchAllQuery; + } else if (queryObjects.length == 1) { + builtQueries = queryObjects[0]; + } else { + JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); + for (JsonObject queryObject : queryObjects) { + filterBuilder.add(queryObject); + } + JsonObjectBuilder boolBuilder = Json.createObjectBuilder().add("filter", filterBuilder); + builtQueries = Json.createObjectBuilder().add("bool", boolBuilder).build(); + } + JsonObjectBuilder nestedBuilder = Json.createObjectBuilder().add("path", path).add("query", builtQueries); + return Json.createObjectBuilder().add("nested", nestedBuilder).build(); + } + + private static JsonObject buildStringQuery(String field, String value) { + JsonObjectBuilder queryStringBuilder = Json.createObjectBuilder().add("query", value); + if (field != null) { + queryStringBuilder.add("fields", Json.createArrayBuilder().add(field)); + } + return Json.createObjectBuilder().add("query_string", queryStringBuilder).build(); + } + + private static JsonObject buildTermQuery(String field, String value) { + return Json.createObjectBuilder().add("term", Json.createObjectBuilder().add(field, value)).build(); + } + + private static JsonObject buildTermsQuery(String field, JsonArray values) { + return Json.createObjectBuilder().add("terms", Json.createObjectBuilder().add(field, values)).build(); + } + + private static JsonObject buildLongRangeQuery(String field, Long lowerValue, Long upperValue) { + JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("gte", lowerValue).add("lte", upperValue); + JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); + return Json.createObjectBuilder().add("range", rangeBuilder).build(); + } + + private static JsonObject buildRangeQuery(String field, JsonNumber lowerValue, JsonNumber upperValue) { + JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("gte", lowerValue).add("lte", upperValue); + JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); + return Json.createObjectBuilder().add("range", rangeBuilder).build(); + } + + // TODO (mostly) duplicated code from icat.lucene... + private static Long parseDate(JsonObject jsonObject, String key, int offset, Long defaultValue) throws IcatException { + if (jsonObject.containsKey(key)) { + ValueType valueType = jsonObject.get(key).getValueType(); + switch (valueType) { + case STRING: + String dateString = jsonObject.getString(key); + try { + return decodeTime(dateString) + offset; + } catch (Exception e) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Could not parse date " + dateString + " using expected format yyyyMMddHHmm"); + } + case NUMBER: + return jsonObject.getJsonNumber(key).longValueExact(); + default: + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Dates should be represented by a NUMBER or STRING JsonValue, but got " + valueType); + } + } + return defaultValue; } /** @@ -168,33 +419,27 @@ public void addNow(String entityName, List ids, EntityManager manager, throws IcatException, IOException, URISyntaxException { // getBeanDocExecutor is not used for all implementations, but is // required for the @Override - // TODO Change this string building fake JSON by hand - StringBuilder sb = new StringBuilder(); - sb.append("["); - for (Long id : ids) { - EntityBaseBean bean = (EntityBaseBean) manager.find(klass, id); - if (bean != null) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartArray(); + for (Long id : ids) { + EntityBaseBean bean = (EntityBaseBean) manager.find(klass, id); + if (bean != null) { gen.writeStartObject().writeStartObject("create"); gen.write("_index", entityName).write("_id", bean.getId().toString()); gen.writeStartObject("doc"); bean.getDoc(gen); gen.writeEnd().writeEnd().writeEnd(); } - if (sb.length() != 1) { - sb.append(','); - } - sb.append(baos.toString()); } + gen.writeEnd(); } - sb.append("]"); - modify(sb.toString()); + modify(baos.toString()); } - public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { + public JsonValue buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { if (sort != null && !sort.equals("")) { - try (JsonReader reader = Json.createReader(new ByteArrayInputStream(sort.getBytes()))) { + try (JsonReader reader = Json.createReader(new StringReader(sort))) { JsonObject object = reader.readObject(); JsonArrayBuilder builder = Json.createArrayBuilder(); for (String key : object.keySet()) { @@ -205,7 +450,7 @@ public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throw String value = lastBean.getSource().getString(key); builder.add(value); } - return builder.build().toString(); + return builder.build(); } } else { JsonArrayBuilder builder = Json.createArrayBuilder(); @@ -215,15 +460,15 @@ public String buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throw } builder.add(lastBean.getScore()); builder.add(lastBean.getEntityBaseBeanId()); - return builder.build().toString(); + return builder.build(); } } public void clear() throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/_delete_by_query").build(); + URI uri = new URIBuilder(server).setPath(basePath + "/_all/_delete_by_query").build(); HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(matchAllQuery)); + httpPost.setEntity(new StringEntity(matchAllQuery.toString(), ContentType.APPLICATION_JSON)); try (CloseableHttpResponse response = httpclient.execute(httpPost)) { Rest.checkStatus(response, IcatExceptionType.INTERNAL); } @@ -249,8 +494,105 @@ public void freeSearcher(String uid) throws IcatException { logger.info("Manually freeing searcher not supported, no request sent"); } - public abstract List facetSearch(String target, JsonObject facetQuery, int maxResults, int maxLabels) - throws IcatException; + public List facetSearch(String target, JsonObject facetQuery, Integer maxResults, + int maxLabels) throws IcatException { + List results = new ArrayList<>(); + if (!facetQuery.containsKey("dimensions")) { + return results; + } + String dimensionPrefix = ""; + String index = target.toLowerCase(); + if (relationships.containsKey(index)) { + dimensionPrefix = index + "."; + index = relationships.get(index).get(0).parentName; + } + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URIBuilder builder = new URIBuilder(server).setPath(basePath + "/" + index + "/_search"); + builder.addParameter("size", maxResults.toString()); + URI uri = builder.build(); + logger.trace("Making call {}", uri); + JsonObject queryObject = facetQuery.getJsonObject("query"); + JsonObjectBuilder bodyBuilder = parseQuery(queryObject, null, null, index, queryObject.keySet()); + + JsonObjectBuilder aggsBuilder = Json.createObjectBuilder(); + for (JsonObject dimension : facetQuery.getJsonArray("dimensions").getValuesAs(JsonObject.class)) { + String dimensionString = dimension.getString("dimension"); + if (dimension.containsKey("ranges")) { + aggsBuilder.add(dimensionString, Json.createObjectBuilder().add("range", Json.createObjectBuilder() + .add("field", dimensionPrefix + dimensionString).add("keyed", true).add("ranges", dimension.getJsonArray("ranges")))); + } else { + aggsBuilder.add(dimensionString, Json.createObjectBuilder().add("terms", Json.createObjectBuilder() + .add("field", dimensionPrefix + dimensionString + ".keyword").add("size", maxLabels))); + } + } + if (dimensionPrefix.equals("")) { + bodyBuilder.add("aggs", aggsBuilder); + } else { + bodyBuilder.add("aggs", Json.createObjectBuilder() + .add(dimensionPrefix.substring(0, dimensionPrefix.length() - 1), Json.createObjectBuilder() + .add("nested", Json.createObjectBuilder() + .add("path", dimensionPrefix.substring(0, dimensionPrefix.length() - 1))) + .add("aggs", aggsBuilder))); + } + String body = bodyBuilder.build().toString(); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + logger.trace("Body: {}", body); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + JsonReader jsonReader = Json.createReader(response.getEntity().getContent()); + JsonObject jsonObject = jsonReader.readObject(); + logger.trace("facet response: {}", jsonObject); + JsonObject aggregations = jsonObject.getJsonObject("aggregations"); + + if (!dimensionPrefix.equals("")) { + aggregations = aggregations.getJsonObject(dimensionPrefix.substring(0, dimensionPrefix.length() - 1)); + } + + for (String dimension : aggregations.keySet()) { + if (dimension.equals("doc_count")) { + continue; + } + FacetDimension facetDimension = new FacetDimension(target, dimension); + List facets = facetDimension.getFacets(); + JsonObject aggregation = aggregations.getJsonObject(dimension); + JsonValue bucketsValue = aggregation.get("buckets"); + switch (bucketsValue.getValueType()) { + case ARRAY: + List buckets = aggregation.getJsonArray("buckets").getValuesAs(JsonObject.class); + if (buckets.size() == 0) { + continue; + } + for (JsonObject bucket : buckets) { + long docCount = bucket.getJsonNumber("doc_count").longValueExact(); + facets.add(new FacetLabel(bucket.getString("key"), docCount)); + } + break; + case OBJECT: + JsonObject bucketsObject = aggregation.getJsonObject("buckets"); + Set keySet = bucketsObject.keySet(); + if (keySet.size() == 0) { + continue; + } + for (String key : keySet) { + JsonObject bucket = bucketsObject.getJsonObject(key); + long docCount = bucket.getJsonNumber("doc_count").longValueExact(); + facets.add(new FacetLabel(key, docCount)); + } + break; + default: + String msg = "Excpeted 'buckets' to have ARRAY or OBJECT type, but it was " + + bucketsValue.getValueType(); + throw new IcatException(IcatExceptionType.INTERNAL, msg); + } + results.add(facetDimension); + } + } + return results; + } catch (IOException | URISyntaxException | ParseException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } public SearchResult getResults(JsonObject query, int maxResults) throws IcatException { return getResults(query, null, maxResults, null, Arrays.asList("id")); @@ -260,128 +602,53 @@ public SearchResult getResults(JsonObject query, int maxResults, String sort) th return getResults(query, null, maxResults, sort, Arrays.asList("id")); } - public SearchResult getResults(JsonObject query, String searchAfter, int blockSize, String sort, - List fields) throws IcatException { - - // return getResults(uid.toString(), query, blockSize); - // TODO - return null; - } - - private SearchResult getResults(String uid, JsonObject query, int maxResults) throws IcatException { + public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer blockSize, String sort, + List requestedFields) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { String index; - Set fields = query.keySet(); - if (fields.contains("target")) { - index = query.getString("target"); + Set queryFields = query.keySet(); + if (queryFields.contains("target")) { + index = query.getString("target").toLowerCase(); } else { index = query.getString("_all"); } - URI uri = new URIBuilder(server).setPath(basePath + "/" + index + "/_search").build(); + URIBuilder builder = new URIBuilder(server).setPath(basePath + "/" + index + "/_search"); + StringBuilder sb = new StringBuilder(); + requestedFields.forEach(f -> sb.append(f).append(",")); + builder.addParameter("_source", sb.toString()); + builder.addParameter("size", blockSize.toString()); + URI uri = builder.build(); logger.trace("Making call {}", uri); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - // TODO refactor some of this into re-usable building blocks - gen.writeStartObject().writeStartObject("query").writeStartObject("bool"); - if (fields.contains("text")) { - // TODO consider default field: this would need to be set by index, but the - // default of * makes more sense to me... - String text = query.getString("text"); - gen.writeStartObject("must").writeStartObject("simple_query_string").write("query", text).writeEnd() - .writeEnd(); - } - gen.writeStartArray("filter"); - Long lowerTime = null; - Long upperTime = null; - if (fields.contains("lower")) { - lowerTime = decodeTime(query.getString("lower")); - } - if (fields.contains("upper")) { - upperTime = decodeTime(query.getString("upper")); - } - if (lowerTime != null || upperTime != null) { - gen.writeStartObject().writeStartObject("bool").writeStartArray("should"); - if (lowerTime != null) { - gen.writeStartObject().writeStartObject("range").writeStartObject("date") - .write("gte", lowerTime).writeEnd().writeEnd().writeEnd(); - gen.writeStartObject().writeStartObject("range").writeStartObject("startDate") - .write("gte", lowerTime).writeEnd().writeEnd().writeEnd(); - } - if (upperTime != null) { - gen.writeStartObject().writeStartObject("range").writeStartObject("date") - .write("lte", upperTime).writeEnd().writeEnd().writeEnd(); - gen.writeStartObject().writeStartObject("range").writeStartObject("endDate") - .write("lte", upperTime).writeEnd().writeEnd().writeEnd(); - } - gen.writeEnd().writeEnd().writeEnd(); - } - if (fields.contains("user")) { - String user = query.getString("user"); - gen.writeStartObject().writeStartObject("match").writeStartObject("userName").write("query", user) - .write("operator", "and").writeEnd().writeEnd().writeEnd(); - } - if (fields.contains("userFullName")) { - String userFullName = query.getString("userFullName"); - gen.writeStartObject().writeStartObject("simple_query_string").write("query", userFullName) - .writeStartArray("fields").write("userFullName").writeEnd().writeEnd().writeEnd(); - } - if (fields.contains("samples")) { - JsonArray samples = query.getJsonArray("samples"); - for (int i = 0; i < samples.size(); i++) { - String sample = samples.getString(i); - gen.writeStartObject().writeStartObject("simple_query_string").write("query", sample) - .writeStartArray("fields").write("sampleText").writeEnd().writeEnd().writeEnd(); - } - } - if (fields.contains("parameters")) { - for (JsonValue parameterValue : query.getJsonArray("parameters")) { - JsonObject parameterObject = (JsonObject) parameterValue; - String name = parameterObject.getString("name", null); - String units = parameterObject.getString("units", null); - String stringValue = parameterObject.getString("stringValue", null); - Long lowerDate = decodeTime(parameterObject.getString("lowerDateValue", null)); - Long upperDate = decodeTime(parameterObject.getString("upperDateValue", null)); - JsonNumber lowerNumeric = parameterObject.getJsonNumber("lowerNumericValue"); - JsonNumber upperNumeric = parameterObject.getJsonNumber("upperNumericValue"); - gen.writeStartObject().writeStartObject("bool").writeStartArray("must"); - if (name != null) { - gen.writeStartObject().writeStartObject("match").writeStartObject("parameterName") - .write("query", name).write("operator", "and").writeEnd().writeEnd().writeEnd(); - } - if (units != null) { - gen.writeStartObject().writeStartObject("match").writeStartObject("parameterUnits") - .write("query", units).write("operator", "and").writeEnd().writeEnd().writeEnd(); - } - if (stringValue != null) { - gen.writeStartObject().writeStartObject("match").writeStartObject("parameterStringValue") - .write("query", stringValue).write("operator", "and").writeEnd().writeEnd() - .writeEnd(); - } else if (lowerDate != null && upperDate != null) { - gen.writeStartObject().writeStartObject("range").writeStartObject("parameterDateValue") - .write("gte", lowerDate).write("lte", upperDate).writeEnd().writeEnd().writeEnd(); - } else if (lowerNumeric != null && upperNumeric != null) { - gen.writeStartObject().writeStartObject("range").writeStartObject("parameterNumericValue") - .write("gte", lowerNumeric).write("lte", upperNumeric).writeEnd().writeEnd() - .writeEnd(); - } - gen.writeEnd().writeEnd().writeEnd(); - } - } - gen.writeEnd().writeEnd().writeEnd().writeEnd(); - } - // TODO build returned results + + String body = parseQuery(query, searchAfter, sort, index, queryFields).build().toString(); SearchResult result = new SearchResult(); List entities = result.getResults(); HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(baos.toString())); + httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + logger.trace("Body: {}", body); try (CloseableHttpResponse response = httpclient.execute(httpPost)) { Rest.checkStatus(response, IcatExceptionType.INTERNAL); JsonReader jsonReader = Json.createReader(response.getEntity().getContent()); JsonObject jsonObject = jsonReader.readObject(); JsonArray hits = jsonObject.getJsonObject("hits").getJsonArray("hits"); for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { - entities.add(new ScoredEntityBaseBean(hit.getInt("_id"), - hit.getJsonNumber("_score").bigDecimalValue().floatValue(), null)); // TODO + Float score = Float.NaN; + if (!hit.isNull("_score")) { + score = hit.getJsonNumber("_score").bigDecimalValue().floatValue(); + } + Integer id = new Integer(hit.getString("_id")); + entities.add(new ScoredEntityBaseBean(id, score, hit.getJsonObject("_source"))); + } + if (hits.size() == blockSize) { + JsonObject lastHit = hits.getJsonObject(blockSize - 1); + logger.trace("Building searchAfter from {}", lastHit.toString()); + if (lastHit.containsKey("sort")) { + result.setSearchAfter(lastHit.getJsonArray("sort")); + } else { + ScoredEntityBaseBean lastEntity = entities.get(blockSize - 1); + result.setSearchAfter(Json.createArrayBuilder().add(lastEntity.getScore()) + .add(lastEntity.getEntityBaseBeanId()).build()); + } } } return result; @@ -390,6 +657,161 @@ private SearchResult getResults(String uid, JsonObject query, int maxResults) th } } + private JsonObjectBuilder parseQuery(JsonObject query, JsonValue searchAfter, String sort, String index, + Set queryFields) throws IcatException, ParseException { + JsonObjectBuilder builder = Json.createObjectBuilder(); + if (sort == null) { + builder.add("sort", Json.createArrayBuilder() + .add(Json.createObjectBuilder().add("_score", "desc")) + .add(Json.createObjectBuilder().add("id", "asc")).build()); + } else { + JsonObject sortObject = Json.createReader(new StringReader(sort)).readObject(); + JsonArrayBuilder sortArrayBuilder = Json.createArrayBuilder(); + for (String key : sortObject.keySet()) { + if (key.toLowerCase().contains("date")) { + sortArrayBuilder.add(Json.createObjectBuilder().add(key, sortObject.getString(key))); + } else { + sortArrayBuilder.add(Json.createObjectBuilder() + .add(key + ".keyword", sortObject.getString(key))); + } + } + builder.add("sort", sortArrayBuilder.add(Json.createObjectBuilder().add("id", "asc")).build()); + } + if (searchAfter != null) { + builder.add("search_after", searchAfter); + } + JsonObjectBuilder queryBuilder = Json.createObjectBuilder(); + JsonObjectBuilder boolBuilder = Json.createObjectBuilder(); + JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); + if (queryFields.contains("text")) { + JsonArrayBuilder mustBuilder = Json.createArrayBuilder(); + mustBuilder.add(buildStringQuery(null, query.getString("text"))); + boolBuilder.add("must", mustBuilder); + } + Long lowerTime = parseDate(query, "lower", 0, Long.MIN_VALUE); + Long upperTime = parseDate(query, "upper", 0, Long.MAX_VALUE); + if (lowerTime != Long.MIN_VALUE || upperTime != Long.MAX_VALUE) { + if (index.equals("datafile")) { + // datafile has only one date field + filterBuilder.add(buildLongRangeQuery("date", lowerTime, upperTime)); + } else { + filterBuilder.add(buildLongRangeQuery("startDate", lowerTime, upperTime)); + filterBuilder.add(buildLongRangeQuery("endDate", lowerTime, upperTime)); + } + } + + List investigationUserQueries = new ArrayList<>(); + if (queryFields.contains("user")) { + investigationUserQueries.add(buildMatchQuery("investigationuser.user.name", query.getString("user"))); + } + if (queryFields.contains("userFullName")) { + investigationUserQueries.add(buildStringQuery("investigationuser.user.fullName", query.getString("userFullName"))); + } + if (investigationUserQueries.size() > 0) { + filterBuilder.add(buildNestedQuery("investigationuser", investigationUserQueries.toArray(new JsonObject[0]))); + } + + if (queryFields.contains("samples")) { + JsonArray samples = query.getJsonArray("samples"); + for (int i = 0; i < samples.size(); i++) { + String sample = samples.getString(i); + filterBuilder.add(buildNestedQuery("sample", buildStringQuery("sample.name", sample))); + } + } + if (queryFields.contains("parameters")) { + for (JsonValue parameterValue : query.getJsonArray("parameters")) { + JsonObject parameterObject = (JsonObject) parameterValue; + String name = parameterObject.getString("name", null); + String units = parameterObject.getString("units", null); + String stringValue = parameterObject.getString("stringValue", null); + Long lowerDate = decodeTime(parameterObject.getString("lowerDateValue", null)); + Long upperDate = decodeTime(parameterObject.getString("upperDateValue", null)); + JsonNumber lowerNumeric = parameterObject.getJsonNumber("lowerNumericValue"); + JsonNumber upperNumeric = parameterObject.getJsonNumber("upperNumericValue"); + + String path = index + "parameter"; + List parameterQueries = new ArrayList<>(); + if (name != null) { + parameterQueries.add(buildMatchQuery(path + ".type.name", name)); + } + if (units != null) { + parameterQueries.add(buildMatchQuery(path + ".type.units", units)); + } + if (stringValue != null) { + parameterQueries.add(buildMatchQuery(path + ".stringValue", stringValue)); + } else if (lowerDate != null && upperDate != null) { + parameterQueries.add(buildLongRangeQuery(path + ".dateTimeValue", lowerDate, upperDate)); + } else if (lowerNumeric != null && upperNumeric != null) { + parameterQueries.add(buildRangeQuery(path + ".numericValue", lowerNumeric, upperNumeric)); + } + filterBuilder.add(buildNestedQuery(path, parameterQueries.toArray(new JsonObject[0]))); + } + } + + if (queryFields.contains("id")) { + filterBuilder.add(buildTermsQuery("id", query.getJsonArray("id"))); + } + + JsonArray filterArray = filterBuilder.build(); + if (filterArray.size() > 0) { + boolBuilder.add("filter", filterArray); + } + return builder.add("query", queryBuilder.add("bool", boolBuilder)); + } + + public void initMappings() throws IcatException { + for (String index : indices) { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URI uri = new URIBuilder(server).setPath(basePath + "/" + index).build(); + logger.trace("Making call {}", uri); + HttpHead httpHead = new HttpHead(uri); + try (CloseableHttpResponse response = httpclient.execute(httpHead)) { + int statusCode = response.getStatusLine().getStatusCode(); + // If the index isn't present, we should get 404 and create the index + if (statusCode == 200) { + // If the index already exists (200), do not attempt to create it + continue; + } else if (statusCode != 404) { + // If the code isn't 200 or 404, something has gone wrong + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } + } catch (URISyntaxException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URI uri = new URIBuilder(server).setPath(basePath + "/" + index).build(); + logger.trace("Making call {}", uri); + HttpPut httpPut = new HttpPut(uri); + String body = Json.createObjectBuilder() + .add("settings", indexSettings).add("mappings", buildMappings(index)).build().toString(); + httpPut.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + try (CloseableHttpResponse response = httpclient.execute(httpPut)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } catch (URISyntaxException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + } + + public void initScripts() throws IcatException { + for (String scriptKey : scripts.keySet()) { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URI uri = new URIBuilder(server).setPath(basePath + "/_scripts/" + scriptKey).build(); + logger.trace("Making call {}", uri); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(scripts.get(scriptKey), ContentType.APPLICATION_JSON)); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } catch (URISyntaxException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + } + public void lock(String entityName) throws IcatException { logger.info("Manually locking index not supported, no request sent"); } @@ -399,37 +821,238 @@ public void unlock(String entityName) throws IcatException { } public void modify(String json) throws IcatException { - // TODO replace other places with this format - // TODO this assumes simple update/create with no relation - // Format should be [{"index": "investigation", "id": "123", "document": {}}, - // ...] try (CloseableHttpClient httpclient = HttpClients.createDefault()) { logger.debug("modify: {}", json); + List updatesByQuery = new ArrayList<>(); + Set investigationIds = new HashSet<>(); StringBuilder sb = new StringBuilder(); JsonReader jsonReader = Json.createReader(new StringReader(json)); JsonArray outerArray = jsonReader.readArray(); for (JsonObject operation : outerArray.getValuesAs(JsonObject.class)) { - if (operation.containsKey("doc")) { - JsonObject document = operation.getJsonObject("doc"); - operation.remove("doc"); - sb.append(operation.toString()).append("\n"); - sb.append(document.toString()).append("\n"); + String operationKey; + if (operation.containsKey("create")) { + operationKey = "create"; + } else if (operation.containsKey("update")) { + operationKey = "update"; + } else if (operation.containsKey("delete")) { + operationKey = "delete"; } else { - sb.append(operation.toString()).append("\n"); + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Operation type should be 'create', 'update' or 'delete'"); + } + JsonObject innerOperation = operation.getJsonObject(operationKey); + String index = innerOperation.getString("_index").toLowerCase(); + String id = innerOperation.getString("_id"); + if (relationships.containsKey(index)) { + for (ParentRelationship relation : relationships.get(index)) { + if (operationKey.equals("create") && relation.parentName.equals(relation.joinName)) { + // Don't need it for children of a relative + // Do need it for 0:* relatives, in which case it's just appending to a list of nested objects + JsonObject document = innerOperation.getJsonObject("doc"); + // TODO if the document has type.unit (it's a parameter) then we need to convert units here. Advantage being we (might) have the value + if (document.containsKey("type.units")) { + // Need to rebuild the document... + JsonObjectBuilder rebuilder = Json.createObjectBuilder(); + for (String key : document.keySet()) { + rebuilder.add(key, document.get(key)); + } + String unitString = document.getString("type.units"); + try { + Unit unit = unitFormat.parse(unitString); + Unit systemUnit = unit.getSystemUnit(); + rebuilder.add("type.unitsSI", systemUnit.getName()); + if (document.containsKey("numericValue")) { + double numericValue = document.getJsonNumber("numericValue").doubleValue(); + UnitConverter converter = unit.getConverterToAny(systemUnit); + rebuilder.add("numericValueSI", converter.convert(numericValue)); + } + } catch (IncommensurableException | MeasurementParseException e) { + logger.error("Unable to convert 'type.units' of {} due to {}", unitString, e.getMessage()); + } + document = rebuilder.build(); + } + String parentId = document.getString(relation.parentName + ".id"); + JsonObjectBuilder innerBuilder = Json.createObjectBuilder().add("_id", + parentId).add("_index", relation.parentName); + JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("id", id).add("doc", Json.createArrayBuilder().add(document)); + String scriptId = (index.equals("parametertype")) ? "update_" + relation.parentName + index : "update_" + index; + JsonObjectBuilder scriptBuilder = Json.createObjectBuilder().add("id", scriptId) + .add("params", paramsBuilder); + sb.append(Json.createObjectBuilder().add("update", + innerBuilder).build().toString()).append("\n"); + sb.append(Json.createObjectBuilder() + .add("upsert", Json.createObjectBuilder().add(index, Json.createArrayBuilder().add(document))) + .add("script", scriptBuilder).build().toString()).append("\n"); + } else if (!operationKey.equals("delete")) { + JsonObject document = innerOperation.getJsonObject("doc"); + URI uri = new URIBuilder(server) + .setPath(basePath + "/" + relation.parentName + "/_update_by_query").build(); + HttpPost httpPost = new HttpPost(uri); + JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("id", id); + if (relation.fields.size() == 0) { + // TODO duplicated + if (document.containsKey("type.units")) { + // Need to rebuild the document... + JsonObjectBuilder rebuilder = Json.createObjectBuilder(); + for (String key : document.keySet()) { + rebuilder.add(key, document.get(key)); + } + String unitString = document.getString("type.units"); + try { + Unit unit = unitFormat.parse(unitString); + Unit systemUnit = unit.getSystemUnit(); + rebuilder.add("type.unitsSI", systemUnit.getName()); + if (document.containsKey("numericValue")) { + double numericValue = document.getJsonNumber("numericValue").doubleValue(); + UnitConverter converter = unit.getConverterToAny(systemUnit); + rebuilder.add("numericValueSI", converter.convert(numericValue)); + } + } catch (IncommensurableException | MeasurementParseException e) { + logger.error("Unable to convert 'type.units' of {} due to {}", unitString, e.getMessage()); + } + document = rebuilder.build(); + } + paramsBuilder.add("doc", Json.createArrayBuilder().add(document)); + } else { + UnitConverter converter = null; + for (String field : relation.fields) { + if (field.equals("type.unitsSI")) { + String unitString = document.getString("type.units"); + try { + Unit unit = unitFormat.parse(unitString); + Unit systemUnit = unit.getSystemUnit(); + converter = unit.getConverterToAny(systemUnit); + paramsBuilder.add(field, systemUnit.getName()); + } catch (IncommensurableException | MeasurementParseException e) { + logger.error("Unable to convert 'type.units' of {} due to {}", unitString, e.getMessage()); + } + } else if (field.equals("numericValueSI")) { + if (converter != null) { + // If we convert 1, we then have the necessary factor and can do multiplication by script... + paramsBuilder.add("conversionFactor", converter.convert(1.)); + } + } else { + paramsBuilder.add(field, document.get(field)); + } + } + } + String scriptId = (index.equals("parametertype")) ? "update_" + relation.parentName + index : "update_" + index; + JsonObjectBuilder scriptBuilder = Json.createObjectBuilder().add("id", scriptId) + .add("params", paramsBuilder); + JsonObject queryObject; + String idField = (relation.joinName.equals(relation.parentName)) ? "id" : relation.joinName + ".id"; + if (!Arrays.asList("parametertype", "sampletype", "user").contains(index)) { + queryObject = buildTermQuery(idField, id); + } else { + queryObject = buildNestedQuery(relation.joinName, buildTermQuery(idField, id)); + } + JsonObject bodyJson = Json.createObjectBuilder().add("query", queryObject) + .add("script", scriptBuilder).build(); + logger.trace("update script: {}", bodyJson.toString()); + httpPost.setEntity(new StringEntity(bodyJson.toString(), ContentType.APPLICATION_JSON)); + updatesByQuery.add(httpPost); + } else { + URI uri = new URIBuilder(server) + .setPath(basePath + "/" + relation.parentName + "/_update_by_query").build(); + HttpPost httpPost = new HttpPost(uri); + JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("id", id); + String scriptId = (index.equals("parametertype")) ? "delete_" + relation.parentName + index : "update_" + index; + JsonObjectBuilder scriptBuilder = Json.createObjectBuilder().add("id", scriptId) + .add("params", paramsBuilder); + JsonObject queryObject; + String idField = (relation.joinName.equals(relation.parentName)) ? "id" : relation.joinName + ".id"; + if (!Arrays.asList("parametertype", "sampletype", "user").contains(index)) { + queryObject = buildTermQuery(idField, id); + } else { + queryObject = buildNestedQuery(relation.joinName, buildTermQuery(idField, id)); + } + JsonObject bodyJson = Json.createObjectBuilder().add("query", queryObject) + .add("script", scriptBuilder).build(); + logger.trace("delete script: {}", bodyJson.toString()); + httpPost.setEntity(new StringEntity(bodyJson.toString(), ContentType.APPLICATION_JSON)); + updatesByQuery.add(httpPost); + } + } + } else { + JsonObjectBuilder innerBuilder = Json.createObjectBuilder().add("_id", new Long(id)).add("_index", + index); + if (operationKey.equals("delete")) { + sb.append(Json.createObjectBuilder().add(operationKey, innerBuilder).build().toString()).append("\n"); + } else { + JsonObject document = innerOperation.getJsonObject("doc"); + sb.append(Json.createObjectBuilder().add("update", innerBuilder).build().toString()) + .append("\n"); + sb.append(Json.createObjectBuilder().add("doc", document).add("doc_as_upsert", true).build() + .toString()).append("\n"); + + if (!index.equals("investigation")) { + // TODO Nightmare user lookup + investigationIds.add(document.getString("investigation.id")); + } + } } } - URI uri = new URIBuilder(server).setPath(basePath + "/_bulk").build(); - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(sb.toString())); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); + logger.debug("bulk string: {}", sb.toString()); + if (sb.toString().length() > 0) { + URI uri = new URIBuilder(server).setPath(basePath + "/_bulk").build(); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(sb.toString(), ContentType.APPLICATION_JSON)); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } + if (updatesByQuery.size() > 0) { + commit(); + } + logger.trace("updatesByQuery: {}", updatesByQuery.toString()); + for (HttpPost updateByQuery : updatesByQuery) { + try (CloseableHttpResponse response = httpclient.execute(updateByQuery)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } + if (investigationIds.size() > 0) { + commit(); + } + logger.trace("investigationIds: {}", investigationIds.toString()); + for (String investigationId : investigationIds) { + URI uriGet = new URIBuilder(server).setPath(basePath + "/investigation/_source/" + investigationId) + .build(); + HttpGet httpGet = new HttpGet(uriGet); + try (CloseableHttpResponse responseGet = httpclient.execute(httpGet)) { + if (responseGet.getStatusLine().getStatusCode() == 200) { + // It's possible that the an investigation/investigationUser has not yet been indexed, in which case we cannot update the dataset/file with the user metadata + Rest.checkStatus(responseGet, IcatExceptionType.INTERNAL); + JsonObject responseObject = Json.createReader(responseGet.getEntity().getContent()).readObject(); + logger.trace("GET investigation {} response: {}", investigationId, responseObject); + if (responseObject.containsKey("investigationuser")) { + JsonArray jsonArray = responseObject.getJsonArray("investigationuser"); + for (String index : new String[] {"datafile", "dataset"}) { + URI uri = new URIBuilder(server).setPath(basePath + "/" + index + "/_update_by_query").build(); + HttpPost httpPost = new HttpPost(uri); + JsonObjectBuilder queryBuilder = Json.createObjectBuilder() + .add("term", Json.createObjectBuilder() + .add("investigation.id", investigationId)); + JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("doc", jsonArray); + JsonObject bodyJson = Json.createObjectBuilder() + .add("query", queryBuilder) + .add("script", Json.createObjectBuilder() + .add("id", "create_investigationuser").add("params", paramsBuilder)) + .build(); + httpPost.setEntity(new StringEntity(bodyJson.toString(), ContentType.APPLICATION_JSON)); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } + } + + } + } } } catch (IOException | URISyntaxException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } } - // TODO Remove? /** * Legacy function for building a Query from individual arguments * diff --git a/src/main/java/org/icatproject/core/manager/SearchManager.java b/src/main/java/org/icatproject/core/manager/SearchManager.java index 7f4e7fdd..10256d1b 100644 --- a/src/main/java/org/icatproject/core/manager/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/SearchManager.java @@ -34,6 +34,7 @@ import javax.json.JsonArrayBuilder; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; +import javax.json.JsonValue; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; import javax.persistence.PersistenceUnit; @@ -410,7 +411,7 @@ private static List buildPublicSearchFields(GateKeeper gateKeeper, Map fields) throws IcatException { return searchApi.getResults(jo, searchAfter, blockSize, sort, fields); } @@ -480,6 +481,8 @@ private void init() { searchApi = new LuceneApi(propertyHandler.getSearchUrls().get(0).toURI()); } else if (searchEngine == SearchEngine.ELASTICSEARCH) { searchApi = new ElasticsearchApi(propertyHandler.getSearchUrls()); + } else if (searchEngine == SearchEngine.OPENSEARCH) { + searchApi = new SearchApi(propertyHandler.getSearchUrls().get(0).toURI()); } else { // TODO implement opensearch throw new IcatException(IcatExceptionType.BAD_PARAMETER, diff --git a/src/main/java/org/icatproject/core/manager/SearchResult.java b/src/main/java/org/icatproject/core/manager/SearchResult.java index 1c34f0b6..592e7813 100644 --- a/src/main/java/org/icatproject/core/manager/SearchResult.java +++ b/src/main/java/org/icatproject/core/manager/SearchResult.java @@ -3,15 +3,17 @@ import java.util.ArrayList; import java.util.List; +import javax.json.JsonValue; + public class SearchResult { - private String searchAfter; + private JsonValue searchAfter; private List results = new ArrayList<>(); private List dimensions; public SearchResult() {} - public SearchResult(String searchAfter, List results, List dimensions) { + public SearchResult(JsonValue searchAfter, List results, List dimensions) { this.searchAfter = searchAfter; this.results = results; this.dimensions = dimensions; @@ -29,11 +31,11 @@ public List getResults() { return results; } - public String getSearchAfter() { + public JsonValue getSearchAfter() { return searchAfter; } - public void setSearchAfter(String searchAfter) { + public void setSearchAfter(JsonValue searchAfter) { this.searchAfter = searchAfter; } diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index 2a87610d..5b9c4057 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -1273,8 +1273,14 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId throw new IcatException(IcatExceptionType.BAD_PARAMETER, "query is not set"); } String userName = beanManager.getUserName(sessionId, manager); + JsonValue searchAfterValue = null; + if (searchAfter.length() > 0) { + try (JsonReader jr = Json.createReader(new StringReader(searchAfter))) { + searchAfterValue = jr.read(); + } + } ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonReader jr = Json.createReader(new ByteArrayInputStream(query.getBytes()))) { + try (JsonReader jr = Json.createReader(new StringReader(query))) { JsonObject jo = jr.readObject(); String target = jo.getString("target", null); if (jo.containsKey("parameters")) { @@ -1313,13 +1319,13 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId throw new IcatException(IcatExceptionType.BAD_PARAMETER, "target:" + target + " is not expected"); } logger.debug("Free text search with query: {}", jo.toString()); - result = beanManager.freeTextSearchDocs(userName, jo, searchAfter, limit, sort, facets, manager, + result = beanManager.freeTextSearchDocs(userName, jo, searchAfterValue, limit, sort, facets, manager, request.getRemoteAddr(), klass); JsonGenerator gen = Json.createGenerator(baos); gen.writeStartObject(); - String newSearchAfter = result.getSearchAfter(); + JsonValue newSearchAfter = result.getSearchAfter(); if (newSearchAfter != null) { gen.write("search_after", newSearchAfter); } diff --git a/src/test/java/org/icatproject/core/manager/TestLucene.java b/src/test/java/org/icatproject/core/manager/TestLucene.java index 6dde9728..4442dd6c 100644 --- a/src/test/java/org/icatproject/core/manager/TestLucene.java +++ b/src/test/java/org/icatproject/core/manager/TestLucene.java @@ -25,6 +25,7 @@ import javax.json.JsonArrayBuilder; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; +import javax.json.JsonValue; import javax.json.stream.JsonGenerator; import javax.ws.rs.core.MediaType; @@ -121,15 +122,19 @@ public void modifyDatafile() throws IcatException { JsonObject pngQuery = SearchApi.buildQuery("Datafile", null, "datafileFormat.name:png", null, null, null, null, null); JsonObject queryObject = Json.createObjectBuilder().add("id", Json.createArrayBuilder().add("42")).build(); - JsonObjectBuilder lowRangeBuilder = Json.createObjectBuilder().add("lower", 0L).add("upper", 1L); - JsonObjectBuilder highRangeBuilder = Json.createObjectBuilder().add("lower", 2L).add("upper", 3L); + + JsonObjectBuilder stringDimensionBuilder = Json.createObjectBuilder().add("dimension", "datafileFormat.name"); + JsonArrayBuilder stringDimensionsBuilder = Json.createArrayBuilder().add(stringDimensionBuilder); + JsonObjectBuilder lowRangeBuilder = Json.createObjectBuilder().add("from", 0L).add("to", 2L).add("key", "low"); + JsonObjectBuilder highRangeBuilder = Json.createObjectBuilder().add("from", 2L).add("to", 4L).add("key", "high"); JsonArrayBuilder rangesBuilder = Json.createArrayBuilder().add(lowRangeBuilder).add(highRangeBuilder); - JsonObjectBuilder dimensionBuilder = Json.createObjectBuilder().add("dimension", "date").add("ranges", + JsonObjectBuilder rangedDimensionBuilder = Json.createObjectBuilder().add("dimension", "date").add("ranges", rangesBuilder); - JsonArrayBuilder dimensionsBuilder = Json.createArrayBuilder().add(dimensionBuilder); - JsonObject stringFacetQuery = Json.createObjectBuilder().add("query", queryObject).build(); + JsonArrayBuilder rangedDimensionsBuilder = Json.createArrayBuilder().add(rangedDimensionBuilder); + JsonObject stringFacetQuery = Json.createObjectBuilder().add("query", queryObject) + .add("dimensions", stringDimensionsBuilder).build(); JsonObject rangeFacetQuery = Json.createObjectBuilder().add("query", queryObject) - .add("dimensions", dimensionsBuilder).build(); + .add("dimensions", rangedDimensionsBuilder).build(); // Original Queue queue = new ConcurrentLinkedQueue<>(); @@ -147,10 +152,10 @@ public void modifyDatafile() throws IcatException { assertEquals("date", facetDimension.getDimension()); assertEquals(2, facetDimension.getFacets().size()); FacetLabel facetLabel = facetDimension.getFacets().get(0); - assertEquals("0_1", facetLabel.getLabel()); + assertEquals("low", facetLabel.getLabel()); assertEquals(new Long(1), facetLabel.getValue()); facetLabel = facetDimension.getFacets().get(1); - assertEquals("2_3", facetLabel.getLabel()); + assertEquals("high", facetLabel.getLabel()); assertEquals(new Long(0), facetLabel.getValue()); // Change name and add a format @@ -169,10 +174,10 @@ public void modifyDatafile() throws IcatException { assertEquals("date", facetDimension.getDimension()); assertEquals(2, facetDimension.getFacets().size()); facetLabel = facetDimension.getFacets().get(0); - assertEquals("0_1", facetLabel.getLabel()); + assertEquals("low", facetLabel.getLabel()); assertEquals(new Long(0), facetLabel.getValue()); facetLabel = facetDimension.getFacets().get(1); - assertEquals("2_3", facetLabel.getLabel()); + assertEquals("high", facetLabel.getLabel()); assertEquals(new Long(1), facetLabel.getValue()); // Change just the format @@ -197,10 +202,10 @@ public void modifyDatafile() throws IcatException { assertEquals("date", facetDimension.getDimension()); assertEquals(2, facetDimension.getFacets().size()); facetLabel = facetDimension.getFacets().get(0); - assertEquals("0_1", facetLabel.getLabel()); + assertEquals("low", facetLabel.getLabel()); assertEquals(new Long(0), facetLabel.getValue()); facetLabel = facetDimension.getFacets().get(1); - assertEquals("2_3", facetLabel.getLabel()); + assertEquals("high", facetLabel.getLabel()); assertEquals(new Long(1), facetLabel.getValue()); // Remove the format @@ -219,10 +224,10 @@ public void modifyDatafile() throws IcatException { assertEquals("date", facetDimension.getDimension()); assertEquals(2, facetDimension.getFacets().size()); facetLabel = facetDimension.getFacets().get(0); - assertEquals("0_1", facetLabel.getLabel()); + assertEquals("low", facetLabel.getLabel()); assertEquals(new Long(0), facetLabel.getValue()); facetLabel = facetDimension.getFacets().get(1); - assertEquals("2_3", facetLabel.getLabel()); + assertEquals("high", facetLabel.getLabel()); assertEquals(new Long(1), facetLabel.getValue()); // Remove the file @@ -375,11 +380,11 @@ public void datafiles() throws Exception { JsonObject query = SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null); List fields = Arrays.asList("date", "name", "investigation.id", "id"); SearchResult lsr = luceneApi.getResults(query, null, 5, null, fields); - String searchAfter = lsr.getSearchAfter(); + JsonValue searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); checkDatafile(lsr.getResults().get(0)); - lsr = luceneApi.getResults(query, searchAfter.toString(), 200, null, fields); + lsr = luceneApi.getResults(query, searchAfter, 200, null, fields); assertNull(lsr.getSearchAfter()); assertEquals(95, lsr.getResults().size()); @@ -395,7 +400,7 @@ public void datafiles() throws Exception { checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter.toString(), 5, sort, fields); + lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -412,7 +417,7 @@ public void datafiles() throws Exception { checkLsrOrder(lsr, 99L, 98L, 97L, 96L, 95L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter.toString(), 5, sort, fields); + lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); checkLsrOrder(lsr, 94L, 93L, 92L, 91L, 90L); searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); @@ -516,11 +521,11 @@ public void datasets() throws Exception { JsonObject query = SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null); List fields = Arrays.asList("startDate", "endDate", "name", "investigation.id", "id"); SearchResult lsr = luceneApi.getResults(query, null, 5, null, fields); - String searchAfter = lsr.getSearchAfter(); + JsonValue searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); checkDataset(lsr.getResults().get(0)); - lsr = luceneApi.getResults(query, searchAfter.toString(), 100, null, fields); + lsr = luceneApi.getResults(query, searchAfter, 100, null, fields); assertNull(lsr.getSearchAfter()); checkLsr(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, 25L, 26L, 27L, 28L, 29L); @@ -697,7 +702,7 @@ public void investigations() throws Exception { SearchResult lsr = luceneApi.getResults(query, null, 5, null, fields); checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); checkInvestigation(lsr.getResults().get(0)); - String searchAfter = lsr.getSearchAfter(); + JsonValue searchAfter = lsr.getSearchAfter(); assertNotNull(searchAfter); lsr = luceneApi.getResults(query, searchAfter, 6, null, fields); checkLsr(lsr, 5L, 6L, 7L, 8L, 9L); @@ -1056,16 +1061,13 @@ private void populate() throws IcatException { @Test public void unitConversion() throws IcatException { // Build queries for raw and SI values - // JsonObject queryObject = Json.createObjectBuilder().add("investigation.id", - // Json.createArrayBuilder().add("0")) - // .build(); JsonObject mKQuery = Json.createObjectBuilder().add("type.units", "mK").build(); JsonObject celsiusQuery = Json.createObjectBuilder().add("type.units", "celsius").build(); JsonObject wrongQuery = Json.createObjectBuilder().add("type.units", "wrong").build(); JsonObject kelvinQuery = Json.createObjectBuilder().add("type.unitsSI", "Kelvin").build(); - JsonObjectBuilder lowRangeBuilder = Json.createObjectBuilder().add("lower", 272.5).add("upper", 273.5); - JsonObjectBuilder midRangeBuilder = Json.createObjectBuilder().add("lower", 272999.5).add("upper", 273000.5); - JsonObjectBuilder highRangeBuilder = Json.createObjectBuilder().add("lower", 273272.5).add("upper", 273273.5); + JsonObjectBuilder lowRangeBuilder = Json.createObjectBuilder().add("from", 272.5).add("to", 273.5).add("key", "272.5_273.5"); + JsonObjectBuilder midRangeBuilder = Json.createObjectBuilder().add("from", 272999.5).add("to", 273000.5).add("key", "272999.5_273000.5"); + JsonObjectBuilder highRangeBuilder = Json.createObjectBuilder().add("from", 273272.5).add("to", 273273.5).add("key", "273272.5_273273.5"); JsonArray ranges = Json.createArrayBuilder().add(lowRangeBuilder).add(midRangeBuilder).add(highRangeBuilder) .build(); JsonObject rawObject = Json.createObjectBuilder().add("dimension", "numericValue").add("ranges", ranges) @@ -1184,7 +1186,8 @@ public void unitConversion() throws IcatException { // Change units to something wrong numericParameterType.setUnits("wrong"); queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("update", parameter)); + // queue.add(SearchApi.encodeOperation("update", parameter)); + queue.add(SearchApi.encodeOperation("update", numericParameterType)); modifyQueue(queue); rawFacetQuery = Json.createObjectBuilder().add("query", wrongQuery) .add("dimensions", Json.createArrayBuilder().add(rawObject)).build(); diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java new file mode 100644 index 00000000..071dd244 --- /dev/null +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -0,0 +1,1147 @@ +package org.icatproject.core.manager; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.reflect.Array; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; + +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import javax.json.JsonValue; +import javax.json.stream.JsonGenerator; +import javax.ws.rs.core.MediaType; + +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.icatproject.core.IcatException; +import org.icatproject.core.IcatException.IcatExceptionType; +import org.icatproject.core.entity.Datafile; +import org.icatproject.core.entity.DatafileFormat; +import org.icatproject.core.entity.DatafileParameter; +import org.icatproject.core.entity.Dataset; +import org.icatproject.core.entity.DatasetParameter; +import org.icatproject.core.entity.DatasetType; +import org.icatproject.core.entity.Facility; +import org.icatproject.core.entity.Investigation; +import org.icatproject.core.entity.InvestigationParameter; +import org.icatproject.core.entity.InvestigationType; +import org.icatproject.core.entity.InvestigationUser; +import org.icatproject.core.entity.Parameter; +import org.icatproject.core.entity.ParameterType; +import org.icatproject.core.entity.Sample; +import org.icatproject.core.entity.SampleType; +import org.icatproject.core.entity.User; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestSearchApi { + + static SearchApi searchApi; + private static URI uribase; + final static Logger logger = LoggerFactory.getLogger(TestSearchApi.class); + + @BeforeClass + public static void beforeClass() throws Exception { + String urlString = System.getProperty("opensearchUrl"); + logger.info("Using search service at {}", urlString); + uribase = new URI(urlString); + searchApi = new SearchApi(uribase); + searchApi.initMappings(); + searchApi.initScripts(); + } + + String letters = "abcdefghijklmnopqrstuvwxyz"; + + long now = new Date().getTime(); + + int NUMINV = 10; + + int NUMUSERS = 5; + + int NUMDS = 30; + + int NUMDF = 100; + + int NUMSAMP = 15; + + @Test + public void modifyDatafile() throws IcatException { + Investigation investigation = new Investigation(); + investigation.setId(0L); + Dataset dataset = new Dataset(); + dataset.setId(0L); + dataset.setInvestigation(investigation); + + Datafile elephantDatafile = new Datafile(); + elephantDatafile.setName("Elephants and Aardvarks"); + elephantDatafile.setDatafileModTime(new Date(0L)); + elephantDatafile.setId(42L); + elephantDatafile.setDataset(dataset); + + DatafileFormat pdfFormat = new DatafileFormat(); + pdfFormat.setId(0L); + pdfFormat.setName("pdf"); + Datafile rhinoDatafile = new Datafile(); + rhinoDatafile.setName("Rhinos and Aardvarks"); + rhinoDatafile.setDatafileModTime(new Date(3L)); + rhinoDatafile.setId(42L); + rhinoDatafile.setDataset(dataset); + rhinoDatafile.setDatafileFormat(pdfFormat); + + DatafileFormat pngFormat = new DatafileFormat(); + pngFormat.setId(0L); + pngFormat.setName("png"); + + JsonObject elephantQuery = SearchApi.buildQuery("Datafile", null, "elephant", null, null, null, null, null); + JsonObject rhinoQuery = SearchApi.buildQuery("Datafile", null, "rhino", null, null, null, null, null); + JsonObject pdfQuery = SearchApi.buildQuery("Datafile", null, "datafileFormat.name:pdf", null, null, null, null, + null); + JsonObject pngQuery = SearchApi.buildQuery("Datafile", null, "datafileFormat.name:png", null, null, null, null, + null); + JsonObject queryObject = Json.createObjectBuilder().add("id", Json.createArrayBuilder().add("42")).build(); + + JsonObjectBuilder stringDimensionBuilder = Json.createObjectBuilder().add("dimension", "datafileFormat.name"); + JsonArrayBuilder stringDimensionsBuilder = Json.createArrayBuilder().add(stringDimensionBuilder); + JsonObjectBuilder lowRangeBuilder = Json.createObjectBuilder().add("from", 0L).add("to", 2L).add("key", "low"); + JsonObjectBuilder highRangeBuilder = Json.createObjectBuilder().add("from", 2L).add("to", 4L).add("key", "high"); + JsonArrayBuilder rangesBuilder = Json.createArrayBuilder().add(lowRangeBuilder).add(highRangeBuilder); + JsonObjectBuilder rangedDimensionBuilder = Json.createObjectBuilder().add("dimension", "date").add("ranges", + rangesBuilder); + JsonArrayBuilder rangedDimensionsBuilder = Json.createArrayBuilder().add(rangedDimensionBuilder); + JsonObject stringFacetQuery = Json.createObjectBuilder().add("query", queryObject) + .add("dimensions", stringDimensionsBuilder).build(); + JsonObject rangeFacetQuery = Json.createObjectBuilder().add("query", queryObject) + .add("dimensions", rangedDimensionsBuilder).build(); + + // Original + Queue queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeOperation("create", elephantDatafile)); + modifyQueue(queue); + checkLsr(searchApi.getResults(elephantQuery, 5), 42L); + checkLsr(searchApi.getResults(rhinoQuery, 5)); + checkLsr(searchApi.getResults(pdfQuery, 5)); + checkLsr(searchApi.getResults(pngQuery, 5)); + List facetDimensions = searchApi.facetSearch("Datafile", stringFacetQuery, 5, 5); + assertEquals(0, facetDimensions.size()); + facetDimensions = searchApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + FacetDimension facetDimension = facetDimensions.get(0); + assertEquals("date", facetDimension.getDimension()); + assertEquals(2, facetDimension.getFacets().size()); + FacetLabel facetLabel = facetDimension.getFacets().get(0); + assertEquals("low", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("high", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + + // Change name and add a format + queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeOperation("update", rhinoDatafile)); + modifyQueue(queue); + checkLsr(searchApi.getResults(elephantQuery, 5)); + checkLsr(searchApi.getResults(rhinoQuery, 5), 42L); + checkLsr(searchApi.getResults(pdfQuery, 5), 42L); + checkLsr(searchApi.getResults(pngQuery, 5)); + facetDimensions = searchApi.facetSearch("Datafile", stringFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimensions = searchApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("date", facetDimension.getDimension()); + assertEquals(2, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("low", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("high", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + + // Change just the format + queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeOperation("update", pngFormat)); + modifyQueue(queue); + checkLsr(searchApi.getResults(elephantQuery, 5)); + checkLsr(searchApi.getResults(rhinoQuery, 5), 42L); + checkLsr(searchApi.getResults(pdfQuery, 5)); + checkLsr(searchApi.getResults(pngQuery, 5), 42L); + facetDimensions = searchApi.facetSearch("Datafile", stringFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("datafileFormat.name", facetDimension.getDimension()); + assertEquals(1, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("png", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + facetDimensions = searchApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("date", facetDimension.getDimension()); + assertEquals(2, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("low", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("high", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + + // Remove the format + queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeOperation("delete", pngFormat)); + modifyQueue(queue); + checkLsr(searchApi.getResults(elephantQuery, 5)); + checkLsr(searchApi.getResults(rhinoQuery, 5), 42L); + checkLsr(searchApi.getResults(pdfQuery, 5)); + checkLsr(searchApi.getResults(pngQuery, 5)); + facetDimensions = searchApi.facetSearch("Datafile", stringFacetQuery, 5, 5); + assertEquals(0, facetDimensions.size()); + facetDimensions = searchApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("date", facetDimension.getDimension()); + assertEquals(2, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("low", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("high", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + + // Remove the file + queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeDeletion(elephantDatafile)); + queue.add(SearchApi.encodeDeletion(rhinoDatafile)); + modifyQueue(queue); + checkLsr(searchApi.getResults(elephantQuery, 5)); + checkLsr(searchApi.getResults(rhinoQuery, 5)); + checkLsr(searchApi.getResults(pdfQuery, 5)); + checkLsr(searchApi.getResults(pngQuery, 5)); + + // Multiple commands at once + queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeOperation("create", elephantDatafile)); + queue.add(SearchApi.encodeOperation("update", rhinoDatafile)); + queue.add(SearchApi.encodeDeletion(elephantDatafile)); + queue.add(SearchApi.encodeDeletion(rhinoDatafile)); + modifyQueue(queue); + checkLsr(searchApi.getResults(elephantQuery, 5)); + checkLsr(searchApi.getResults(rhinoQuery, 5)); + checkLsr(searchApi.getResults(pdfQuery, 5)); + checkLsr(searchApi.getResults(pngQuery, 5)); + } + + private void modifyQueue(Queue queue) throws IcatException { + Iterator qiter = queue.iterator(); + if (qiter.hasNext()) { + StringBuilder sb = new StringBuilder("["); + + while (qiter.hasNext()) { + String item = qiter.next(); + if (sb.length() != 1) { + sb.append(','); + } + sb.append(item); + qiter.remove(); + } + sb.append(']'); + logger.debug("XXX " + sb.toString()); + + searchApi.modify(sb.toString()); + searchApi.commit(); + } + } + + private void addDocuments(String entityName, String json) throws IcatException { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URI uri = new URIBuilder(uribase).setPath(SearchApi.basePath + "/addNow/" + entityName).build(); + HttpPost httpPost = new HttpPost(uri); + StringEntity input = new StringEntity(json); + input.setContentType(MediaType.APPLICATION_JSON); + httpPost.setEntity(input); + + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } catch (IOException | URISyntaxException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + @Before + public void before() throws Exception { + searchApi.clear(); + } + + private void checkDatafile(ScoredEntityBaseBean datafile) { + JsonObject source = datafile.getSource(); + assertNotNull(source); + Set expectedKeys = new HashSet<>( + Arrays.asList("id", "investigation.id", "name", "date")); + assertEquals(expectedKeys, source.keySet()); + assertEquals("0", source.getString("id")); + assertEquals("0", source.getString("investigation.id")); + assertEquals("DFaaa", source.getString("name")); + assertNotNull(source.getJsonNumber("date")); + } + + private void checkDataset(ScoredEntityBaseBean dataset) { + JsonObject source = dataset.getSource(); + assertNotNull(source); + Set expectedKeys = new HashSet<>( + Arrays.asList("id", "investigation.id", "name", "startDate", "endDate")); + assertEquals(expectedKeys, source.keySet()); + assertEquals("0", source.getString("id")); + assertEquals("0", source.getString("investigation.id")); + assertEquals("DSaaa", source.getString("name")); + assertNotNull(source.getJsonNumber("startDate")); + assertNotNull(source.getJsonNumber("endDate")); + } + + private void checkInvestigation(ScoredEntityBaseBean investigation) { + JsonObject source = investigation.getSource(); + assertNotNull(source); + Set expectedKeys = new HashSet<>(Arrays.asList("id", "name", "startDate", "endDate")); + assertEquals(expectedKeys, source.keySet()); + assertEquals("0", source.getString("id")); + assertEquals("a h r", source.getString("name")); + assertNotNull(source.getJsonNumber("startDate")); + assertNotNull(source.getJsonNumber("endDate")); + } + + private void checkLsr(SearchResult lsr, Long... n) { + Set wanted = new HashSet<>(Arrays.asList(n)); + Set got = new HashSet<>(); + + for (ScoredEntityBaseBean q : lsr.getResults()) { + got.add(q.getEntityBaseBeanId()); + } + + Set missing = new HashSet<>(wanted); + missing.removeAll(got); + if (!missing.isEmpty()) { + for (Long l : missing) { + logger.error("Entry missing: {}", l); + } + fail("Missing entries"); + } + + missing = new HashSet<>(got); + missing.removeAll(wanted); + if (!missing.isEmpty()) { + for (Long l : missing) { + logger.error("Extra entry: {}", l); + } + fail("Extra entries"); + } + + } + + private void checkLsrOrder(SearchResult lsr, Long... n) { + List results = lsr.getResults(); + if (n.length != results.size()) { + checkLsr(lsr, n); + } + for (int i = 0; i < n.length; i++) { + Long resultId = results.get(i).getEntityBaseBeanId(); + Long expectedId = (Long) Array.get(n, i); + if (resultId != expectedId) { + fail("Expected id " + expectedId + " in position " + i + " but got " + resultId); + } + } + } + + @Test + public void datafiles() throws Exception { + populate(); + + JsonObject query = SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null); + List fields = Arrays.asList("date", "name", "investigation.id", "id"); + SearchResult lsr = searchApi.getResults(query, null, 5, null, fields); + JsonValue searchAfter = lsr.getSearchAfter(); + checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); + checkDatafile(lsr.getResults().get(0)); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + lsr = searchApi.getResults(query, searchAfter, 200, null, fields); + assertNull(lsr.getSearchAfter()); + assertEquals(95, lsr.getResults().size()); + + // Test searchAfter preserves the sorting of original search (asc) + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("date", "asc"); + gen.writeEnd(); + } + String sort = baos.toString(); + lsr = searchApi.getResults(query, null, 5, sort, fields); + checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); + checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + + // Test searchAfter preserves the sorting of original search (desc) + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("date", "desc"); + gen.writeEnd(); + } + sort = baos.toString(); + lsr = searchApi.getResults(query, null, 5, sort, fields); + checkLsrOrder(lsr, 99L, 98L, 97L, 96L, 95L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); + checkLsrOrder(lsr, 94L, 93L, 92L, 91L, 90L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + + // Test tie breaks on fields with identical values (asc) + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("name", "asc"); + gen.writeEnd(); + } + sort = baos.toString(); + lsr = searchApi.getResults(query, null, 5, sort, fields); + checkLsrOrder(lsr, 0L, 26L, 52L, 78L, 1L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("name", "asc"); + gen.write("date", "desc"); + gen.writeEnd(); + } + sort = baos.toString(); + lsr = searchApi.getResults(query, null, 5, sort, fields); + checkLsrOrder(lsr, 78L, 52L, 26L, 0L, 79L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + + // Test tie breaks on fields with identical values (desc) + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("name", "desc"); + gen.writeEnd(); + } + sort = baos.toString(); + lsr = searchApi.getResults(query, null, 5, sort, fields); + checkLsrOrder(lsr, 25L, 51L, 77L, 24L, 50L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("name", "desc"); + gen.write("date", "desc"); + gen.writeEnd(); + } + sort = baos.toString(); + lsr = searchApi.getResults(query, null, 5, sort, fields); + checkLsrOrder(lsr, 77L, 51L, 25L, 76L, 50L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + + query = SearchApi.buildQuery("Datafile", "e4", null, null, null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, 96L); + + query = SearchApi.buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 1L); + + query = SearchApi.buildQuery("Datafile", null, "dfbbb", null, null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 1L, 27L, 53L, 79L); + + query = SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 3L, 4L, 5L, 6L); + + query = SearchApi.buildQuery("Datafile", "b1", "dsddd", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr); + + List pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v25")); + query = SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 5L); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v25")); + query = SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 5L); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, "u sss", null)); + query = SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 13L, 65L); + } + + @Test + public void datasets() throws Exception { + populate(); + + JsonObject query = SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null); + List fields = Arrays.asList("startDate", "endDate", "name", "investigation.id", "id"); + SearchResult lsr = searchApi.getResults(query, null, 5, null, fields); + checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); + checkDataset(lsr.getResults().get(0)); + JsonValue searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + lsr = searchApi.getResults(query, searchAfter, 100, null, fields); + assertNull(lsr.getSearchAfter()); + checkLsr(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, + 25L, 26L, 27L, 28L, 29L); + + // Test searchAfter preserves the sorting of original search (asc) + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("startDate", "asc"); + gen.writeEnd(); + } + String sort = baos.toString(); + lsr = searchApi.getResults(query, 5, sort); + checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); + checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + + // Test searchAfter preserves the sorting of original search (desc) + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("endDate", "desc"); + gen.writeEnd(); + } + sort = baos.toString(); + lsr = searchApi.getResults(query, 5, sort); + checkLsrOrder(lsr, 29L, 28L, 27L, 26L, 25L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); + checkLsrOrder(lsr, 24L, 23L, 22L, 21L, 20L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + + // Test tie breaks on fields with identical values (asc) + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("name", "asc"); + gen.writeEnd(); + } + sort = baos.toString(); + lsr = searchApi.getResults(query, null, 5, sort, fields); + checkLsrOrder(lsr, 0L, 26L, 1L, 27L, 2L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("name", "asc"); + gen.write("endDate", "desc"); + gen.writeEnd(); + } + sort = baos.toString(); + lsr = searchApi.getResults(query, 5, sort); + checkLsrOrder(lsr, 26L, 0L, 27L, 1L, 28L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "e4", null, null, null, null, null, null), 100, + null); + checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L); + + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100, + null); + checkLsr(lsr, 1L); + + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100, + null); + checkLsr(lsr, 1L, 27L); + + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100, null); + checkLsr(lsr, 3L, 4L, 5L); + + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), null, null, null), 100, null); + checkLsr(lsr, 3L); + + List pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v16")); + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, pojos, null, null), 100, + null); + checkLsr(lsr, 4L); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v16")); + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, null), 100, null); + checkLsr(lsr, 4L); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v16")); + lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), + new Date(now + 60000 * 6), pojos, null, null), 100, null); + checkLsr(lsr); + } + + private void fillParms(Queue queue, int i, String rel) throws IcatException { + int j = i % 26; + int k = (i + 5) % 26; + String name = "nm " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); + String units = "u " + letters.substring(k, k + 1) + letters.substring(k, k + 1) + letters.substring(k, k + 1); + + ParameterType dateParameterType = new ParameterType(); + dateParameterType.setId(0L); + dateParameterType.setName("D" + name); + dateParameterType.setUnits(units); + ParameterType numericParameterType = new ParameterType(); + numericParameterType.setId(0L); + numericParameterType.setName("N" + name); + numericParameterType.setUnits(units); + ParameterType stringParameterType = new ParameterType(); + stringParameterType.setId(0L); + stringParameterType.setName("S" + name); + stringParameterType.setUnits(units); + + Parameter parameter; + if (rel.equals("datafile")) { + parameter = new DatafileParameter(); + Datafile datafile = new Datafile(); + datafile.setId(new Long(i)); + ((DatafileParameter) parameter).setDatafile(datafile); + } else if (rel.equals("dataset")) { + parameter = new DatasetParameter(); + Dataset dataset = new Dataset(); + dataset.setId(new Long(i)); + ((DatasetParameter) parameter).setDataset(dataset); + } else if (rel.equals("investigation")) { + parameter = new InvestigationParameter(); + Investigation investigation = new Investigation(); + investigation.setId(new Long(i)); + ((InvestigationParameter) parameter).setInvestigation(investigation); + } else { + fail(rel + " is not valid"); + return; + } + + parameter.setId(new Long(3 * i)); + parameter.setType(dateParameterType); + parameter.setDateTimeValue(new Date(now + 60000 * k * k)); + + queue.add(SearchApi.encodeOperation("create", parameter)); + System.out.println( + rel + " " + i + " '" + "D" + name + "' '" + units + "' '" + new Date(now + 60000 * k * k) + "'"); + + parameter.setId(new Long(3 * i + 1)); + parameter.setType(numericParameterType); + parameter.setNumericValue(new Double(j * j)); + + queue.add(SearchApi.encodeOperation("create", parameter)); + System.out.println(rel + " " + i + " '" + "N" + name + "' '" + units + "' " + new Double(j * j)); + + parameter.setId(new Long(3 * i + 2)); + parameter.setType(stringParameterType); + parameter.setStringValue("v" + i * i); + + queue.add(SearchApi.encodeOperation("create", parameter)); + System.out.println(rel + " " + i + " '" + "S" + name + "' '" + units + "' 'v" + i * i + "'"); + } + + @Test + public void investigations() throws Exception { + populate(); + + /* Blocked results */ + JsonObject query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null); + List fields = Arrays.asList("startDate", "endDate", "name", "id"); + SearchResult lsr = searchApi.getResults(query, null, 5, null, fields); + checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); + checkInvestigation(lsr.getResults().get(0)); + JsonValue searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + lsr = searchApi.getResults(query, searchAfter, 6, null, fields); + checkLsr(lsr, 5L, 6L, 7L, 8L, 9L); + searchAfter = lsr.getSearchAfter(); + assertNull(searchAfter); + + // Test searchAfter preserves the sorting of original search (asc) + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("startDate", "asc"); + gen.writeEnd(); + } + String sort = baos.toString(); + lsr = searchApi.getResults(query, 5, sort); + checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); + checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + + // Test searchAfter preserves the sorting of original search (desc) + baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("endDate", "desc"); + gen.writeEnd(); + } + sort = baos.toString(); + lsr = searchApi.getResults(query, 5, sort); + checkLsrOrder(lsr, 9L, 8L, 7L, 6L, 5L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); + checkLsrOrder(lsr, 4L, 3L, 2L, 1L, 0L); + searchAfter = lsr.getSearchAfter(); + assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + + query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "b"); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); + + query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); + + query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); + + query = SearchApi.buildQuery("Investigation", "b1", null, null, null, null, null, "b"); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); + + query = SearchApi.buildQuery("Investigation", "c1", null, null, null, null, null, "b"); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr); + + query = SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 4L); + + query = SearchApi.buildQuery("Investigation", "b1", "d", null, null, null, null, "b"); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 3L); + + query = SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), + null, null, "b"); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 3L); + + query = SearchApi.buildQuery("Investigation", null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), + null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 3L, 4L, 5L); + + List pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v9")); + query = SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 3L); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v9")); + pojos.add(new ParameterPOJO(null, null, 7, 10)); + pojos.add(new ParameterPOJO(null, null, new Date(now + 60000 * 63), new Date(now + 60000 * 65))); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 3L); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v9")); + query = SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), + pojos, null, "b"); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 3L); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO(null, null, "v9")); + pojos.add(new ParameterPOJO(null, null, "v81")); + query = SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); + query = SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 3L); + + List samples = Arrays.asList("ddd", "nnn"); + query = SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 3L); + + samples = Arrays.asList("ddd", "mmm"); + query = SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr); + + pojos = new ArrayList<>(); + pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); + samples = Arrays.asList("ddd", "nnn"); + query = SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), + pojos, samples, "b"); + lsr = searchApi.getResults(query, 100, null); + checkLsr(lsr, 3L); + } + + /** + * Populate UserGroup, Investigation, InvestigationParameter, + * InvestigationUser, Dataset,DatasetParameter,Datafile, DatafileParameter + * and Sample + */ + private void populate() throws IcatException { + Queue queue = new ConcurrentLinkedQueue<>(); + Long investigationUserId = 0L; + for (int i = 0; i < NUMINV; i++) { + for (int j = 0; j < NUMUSERS; j++) { + if (i % (j + 1) == 1) { + String fn = "FN " + letters.substring(j, j + 1) + " " + letters.substring(j, j + 1); + String name = letters.substring(j, j + 1) + j; + User user = new User(); + user.setId(new Long(j)); + user.setName(name); + user.setFullName(fn); + Investigation investigation = new Investigation(); + investigation.setId(new Long(i)); + InvestigationUser investigationUser = new InvestigationUser(); + investigationUser.setId(investigationUserId); + investigationUser.setUser(user); + investigationUser.setInvestigation(investigation); + + queue.add(SearchApi.encodeOperation("create", investigationUser)); + investigationUserId++; + System.out.println("'" + fn + "' " + name + " " + i); + } + } + } + + for (int i = 0; i < NUMINV; i++) { + int j = i % 26; + int k = (i + 7) % 26; + int l = (i + 17) % 26; + String word = letters.substring(j, j + 1) + " " + letters.substring(k, k + 1) + " " + + letters.substring(l, l + 1); + Investigation investigation = new Investigation(); + Facility facility = new Facility(); + facility.setName(""); + facility.setId(0L); + investigation.setFacility(facility); + InvestigationType type = new InvestigationType(); + type.setName("test"); + type.setId(0L); + investigation.setType(type); + investigation.setName(word); + investigation.setTitle(""); + investigation.setVisitId(""); + investigation.setStartDate(new Date(now + i * 60000)); + investigation.setEndDate(new Date(now + (i + 1) * 60000)); + investigation.setId(new Long(i)); + + queue.add(SearchApi.encodeOperation("create", investigation)); + System.out.println("INVESTIGATION '" + word + "' " + new Date(now + i * 60000) + " " + i); + } + + for (int i = 0; i < NUMINV; i++) { + if (i % 2 == 1) { + fillParms(queue, i, "investigation"); + } + } + + for (int i = 0; i < NUMDS; i++) { + int j = i % 26; + String word = "DS" + letters.substring(j, j + 1) + letters.substring(j, j + 1) + + letters.substring(j, j + 1); + + Investigation investigation = new Investigation(); + investigation.setId(new Long(i % NUMINV)); + Dataset dataset = new Dataset(); + DatasetType type = new DatasetType(); + type.setName("test"); + type.setId(0L); + dataset.setType(type); + dataset.setName(word); + dataset.setStartDate(new Date(now + i * 60000)); + dataset.setEndDate(new Date(now + (i + 1) * 60000)); + dataset.setId(new Long(i)); + dataset.setInvestigation(investigation); + + queue.add(SearchApi.encodeOperation("create", dataset)); + System.out.println("DATASET '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMINV); + } + + for (int i = 0; i < NUMDS; i++) { + if (i % 3 == 1) { + fillParms(queue, i, "dataset"); + } + } + + for (int i = 0; i < NUMDF; i++) { + int j = i % 26; + String word = "DF" + letters.substring(j, j + 1) + letters.substring(j, j + 1) + + letters.substring(j, j + 1); + + Investigation investigation = new Investigation(); + investigation.setId(new Long((i % NUMDS) % NUMINV)); + Dataset dataset = new Dataset(); + dataset.setId(new Long(i % NUMDS)); + dataset.setInvestigation(investigation); + Datafile datafile = new Datafile(); + datafile.setName(word); + datafile.setDatafileModTime(new Date(now + i * 60000)); + datafile.setId(new Long(i)); + datafile.setDataset(dataset); + + queue.add(SearchApi.encodeOperation("create", datafile)); + System.out.println("DATAFILE '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMDS); + + } + + for (int i = 0; i < NUMDF; i++) { + if (i % 4 == 1) { + fillParms(queue, i, "datafile"); + } + } + + for (int i = 0; i < NUMSAMP; i++) { + int j = i % 26; + String word = "SType " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + + letters.substring(j, j + 1); + + Investigation investigation = new Investigation(); + investigation.setId(new Long(i % NUMINV)); + SampleType sampleType = new SampleType(); + sampleType.setId(0L); + sampleType.setName("test"); + Sample sample = new Sample(); + sample.setId(new Long(i)); + sample.setInvestigation(investigation); + sample.setType(sampleType); + sample.setName(word); + + queue.add(SearchApi.encodeOperation("create", sample)); + System.out.println("SAMPLE '" + word + "' " + i % NUMINV); + } + + modifyQueue(queue); + + } + + @Test + public void unitConversion() throws IcatException { + // Build queries for raw and SI values + JsonObject mKQuery = Json.createObjectBuilder().add("type.units", "mK").build(); + JsonObject celsiusQuery = Json.createObjectBuilder().add("type.units", "celsius").build(); + JsonObject wrongQuery = Json.createObjectBuilder().add("type.units", "wrong").build(); + JsonObject kelvinQuery = Json.createObjectBuilder().add("type.unitsSI", "Kelvin").build(); + JsonObjectBuilder lowRangeBuilder = Json.createObjectBuilder().add("from", 272.5).add("to", 273.5).add("key", "272.5_273.5"); + JsonObjectBuilder midRangeBuilder = Json.createObjectBuilder().add("from", 272999.5).add("to", 273000.5).add("key", "272999.5_273000.5"); + JsonObjectBuilder highRangeBuilder = Json.createObjectBuilder().add("from", 273272.5).add("to", 273273.5).add("key", "273272.5_273273.5"); + JsonArray ranges = Json.createArrayBuilder().add(lowRangeBuilder).add(midRangeBuilder).add(highRangeBuilder) + .build(); + JsonObject rawObject = Json.createObjectBuilder().add("dimension", "numericValue").add("ranges", ranges) + .build(); + JsonObject rawFacetQuery = Json.createObjectBuilder().add("query", mKQuery) + .add("dimensions", Json.createArrayBuilder().add(rawObject)).build(); + JsonObject systemObject = Json.createObjectBuilder().add("dimension", "numericValueSI").add("ranges", ranges) + .build(); + JsonObject systemFacetQuery = Json.createObjectBuilder().add("query", kelvinQuery) + .add("dimensions", Json.createArrayBuilder().add(systemObject)).build(); + + // Build entities + Investigation investigation = new Investigation(); + investigation.setId(0L); + investigation.setName("name"); + investigation.setVisitId("visitId"); + investigation.setTitle("title"); + investigation.setCreateTime(new Date()); + investigation.setModTime(new Date()); + Facility facility = new Facility(); + facility.setName("facility"); + facility.setId(0L); + investigation.setFacility(facility); + InvestigationType type = new InvestigationType(); + type.setName("type"); + type.setId(0L); + investigation.setType(type); + + ParameterType numericParameterType = new ParameterType(); + numericParameterType.setId(0L); + numericParameterType.setName("parameter"); + numericParameterType.setUnits("mK"); + InvestigationParameter parameter = new InvestigationParameter(); + parameter.setInvestigation(investigation); + parameter.setType(numericParameterType); + parameter.setNumericValue(273000.); + parameter.setId(0L); + + Queue queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeOperation("create", investigation)); + queue.add(SearchApi.encodeOperation("create", parameter)); + modifyQueue(queue); + + // Assert the raw value is still 273000 (mK) + List facetDimensions = searchApi.facetSearch("InvestigationParameter", rawFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + FacetDimension facetDimension = facetDimensions.get(0); + assertEquals("numericValue", facetDimension.getDimension()); + assertEquals(3, facetDimension.getFacets().size()); + FacetLabel facetLabel = facetDimension.getFacets().get(0); + assertEquals("272.5_273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("272999.5_273000.5", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(2); + assertEquals("273272.5_273273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + + // Assert the SI value is 273 (K) + facetDimensions = searchApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("numericValueSI", facetDimension.getDimension()); + assertEquals(3, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("272.5_273.5", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("272999.5_273000.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(2); + assertEquals("273272.5_273273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + + // Change units only to "celsius" + numericParameterType.setUnits("celsius"); + queue = new ConcurrentLinkedQueue<>(); + queue.add(SearchApi.encodeOperation("update", parameter)); + modifyQueue(queue); + rawFacetQuery = Json.createObjectBuilder().add("query", celsiusQuery) + .add("dimensions", Json.createArrayBuilder().add(rawObject)).build(); + + // Assert the raw value is still 273000 (deg C) + facetDimensions = searchApi.facetSearch("InvestigationParameter", rawFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("numericValue", facetDimension.getDimension()); + assertEquals(3, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("272.5_273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("272999.5_273000.5", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(2); + assertEquals("273272.5_273273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + + // Assert the SI value is 273273.15 (K) + facetDimensions = searchApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("numericValueSI", facetDimension.getDimension()); + assertEquals(3, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("272.5_273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("272999.5_273000.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(2); + assertEquals("273272.5_273273.5", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + + // Change units to something wrong + numericParameterType.setUnits("wrong"); + queue = new ConcurrentLinkedQueue<>(); + // queue.add(SearchApi.encodeOperation("update", parameter)); + queue.add(SearchApi.encodeOperation("update", numericParameterType)); + modifyQueue(queue); + rawFacetQuery = Json.createObjectBuilder().add("query", wrongQuery) + .add("dimensions", Json.createArrayBuilder().add(rawObject)).build(); + + // Assert the raw value is still 273000 (wrong) + facetDimensions = searchApi.facetSearch("InvestigationParameter", rawFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("numericValue", facetDimension.getDimension()); + assertEquals(3, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("272.5_273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("272999.5_273000.5", facetLabel.getLabel()); + assertEquals(new Long(1), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(2); + assertEquals("273272.5_273273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + + // Assert that the SI value has not been set due to conversion failing + facetDimensions = searchApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5); + assertEquals(1, facetDimensions.size()); + facetDimension = facetDimensions.get(0); + assertEquals("numericValueSI", facetDimension.getDimension()); + assertEquals(3, facetDimension.getFacets().size()); + facetLabel = facetDimension.getFacets().get(0); + assertEquals("272.5_273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(1); + assertEquals("272999.5_273000.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + facetLabel = facetDimension.getFacets().get(2); + assertEquals("273272.5_273273.5", facetLabel.getLabel()); + assertEquals(new Long(0), facetLabel.getValue()); + } + +} diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 6b75b1ea..6b278f40 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -93,12 +93,16 @@ private static void clearSearch() String urlString = System.getProperty("luceneUrl"); URI uribase = new URI(urlString); searchApi = new LuceneApi(uribase); + } else if (searchEngine.equals("OPENSEARCH")) { + String urlString = System.getProperty("opensearchUrl"); + URI uribase = new URI(urlString); + searchApi = new SearchApi(uribase); } else if (searchEngine.equals("ELASTICSEARCH")) { String urlString = System.getProperty("elasticsearchUrl"); searchApi = new ElasticsearchApi(Arrays.asList(new URL(urlString))); } else { throw new RuntimeException( - "searchEngine must be one of LUCENE, ELASTICSEARCH, but it was " + searchEngine); + "searchEngine must be one of LUCENE, OPENSEARCH, ELASTICSEARCH, but it was " + searchEngine); } } searchApi.clear(); @@ -629,7 +633,7 @@ public void testLuceneInvestigations() throws Exception { public void testSearchDatafiles() throws Exception { Session session = setupLuceneTest(); JsonObject responseObject; - String searchAfter; + JsonValue searchAfter; Map expectation = new HashMap<>(); expectation.put("investigation.id", null); expectation.put("date", "notNull"); @@ -655,13 +659,13 @@ public void testSearchDatafiles() throws Exception { // Try sorting and searchAfter String sort = Json.createObjectBuilder().add("name", "desc").add("date", "asc").build().toString(); responseObject = searchDatafiles(session, null, null, null, null, null, null, 1, sort, null, 1); - searchAfter = responseObject.getString("search_after"); + searchAfter = responseObject.get("search_after"); assertNotNull(searchAfter); expectation.put("name", "df3"); checkResultsSource(responseObject, Arrays.asList(expectation), false); - responseObject = searchDatafiles(session, null, null, null, null, null, searchAfter, 1, sort, null, 1); - searchAfter = responseObject.getString("search_after"); + responseObject = searchDatafiles(session, null, null, null, null, null, searchAfter.toString(), 1, sort, null, 1); + searchAfter = responseObject.get("search_after"); assertNotNull(searchAfter); expectation.put("name", "df2"); checkResultsSource(responseObject, Arrays.asList(expectation), false); @@ -733,7 +737,7 @@ public void testSearchDatasets() throws Exception { Session session = setupLuceneTest(); DateFormat dft = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); JsonObject responseObject; - String searchAfter; + JsonValue searchAfter; Map expectation = new HashMap<>(); expectation.put("startDate", "notNull"); expectation.put("endDate", "notNull"); @@ -785,12 +789,12 @@ public void testSearchDatasets() throws Exception { // Try sorting and searchAfter String sort = Json.createObjectBuilder().add("name", "desc").add("startDate", "asc").build().toString(); responseObject = searchDatasets(session, null, null, null, null, null, null, 1, sort, null, 1); - searchAfter = responseObject.getString("search_after"); + searchAfter = responseObject.get("search_after"); assertNotNull(searchAfter); expectation.put("name", "ds4"); checkResultsSource(responseObject, Arrays.asList(expectation), false); - responseObject = searchDatasets(session, null, null, null, null, null, searchAfter, 1, sort, null, 1); - searchAfter = responseObject.getString("search_after"); + responseObject = searchDatasets(session, null, null, null, null, null, searchAfter.toString(), 1, sort, null, 1); + searchAfter = responseObject.get("search_after"); assertNotNull(searchAfter); expectation.put("name", "ds3"); checkResultsSource(responseObject, Arrays.asList(expectation), false); diff --git a/src/test/scripts/prepare_test.py b/src/test/scripts/prepare_test.py index 01c2314c..c3fc0043 100644 --- a/src/test/scripts/prepare_test.py +++ b/src/test/scripts/prepare_test.py @@ -8,7 +8,7 @@ from zipfile import ZipFile import subprocess -if len(sys.argv) != 6: +if len(sys.argv) != 7: raise RuntimeError("Wrong number of arguments") containerHome = sys.argv[1] @@ -16,14 +16,17 @@ search_engine = sys.argv[3] lucene_url = sys.argv[4] elasticsearch_url = sys.argv[5] +opensearch_url = sys.argv[6] if search_engine == "LUCENE": search_urls = lucene_url elif search_engine == "ELASTICSEARCH": search_urls = elasticsearch_url +elif search_engine == "OPENSEARCH": + search_urls = opensearch_url else: raise RuntimeError("Search engine %s unrecognised, " % search_engine - + "should be one of LUCENE, ELASTICSEARCH") + + "should be one of LUCENE, ELASTICSEARCH, OPENSEARCH") subst = dict(os.environ) From 9450c4b11435fd33bf8e26801642fa3179fa0df0 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 28 Apr 2022 01:08:46 +0100 Subject: [PATCH 17/51] Integration test fixes and text analysis #267 --- .../icatproject/core/manager/SearchApi.java | 47 +++++++++++------ .../org/icatproject/integration/TestRS.java | 50 +++++++++++-------- 2 files changed, 62 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/SearchApi.java b/src/main/java/org/icatproject/core/manager/SearchApi.java index 97bf4e76..a5ea9c28 100644 --- a/src/main/java/org/icatproject/core/manager/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/SearchApi.java @@ -81,10 +81,20 @@ public ParentRelationship(String parentName, String joinName, List field protected static String basePath = ""; protected static JsonObject matchAllQuery = Json.createObjectBuilder().add("query", Json.createObjectBuilder() .add("match_all", Json.createObjectBuilder())).build(); + // TODO synonym filter in the default_search ONLY private static JsonObject indexSettings = Json.createObjectBuilder().add("analysis", Json.createObjectBuilder() - .add("analyzer", Json.createObjectBuilder().add("default", Json.createObjectBuilder() - .add("tokenizer", "lowercase").add("filter", Json.createArrayBuilder().add("porter_stem"))))).build(); - // private static JsonObject mappingsBuilder; + .add("analyzer", Json.createObjectBuilder() + .add("default", Json.createObjectBuilder() + .add("tokenizer", "classic").add("filter", Json.createArrayBuilder() + .add("possessive_english").add("lowercase").add("porter_stem"))) + .add("default_search", Json.createObjectBuilder() + .add("tokenizer", "classic").add("filter", Json.createArrayBuilder() + .add("possessive_english").add("lowercase").add("porter_stem").add("synonym")))) + .add("filter", Json.createObjectBuilder() + .add("synonym", Json.createObjectBuilder() + .add("type", "synonym").add("synonyms_path", "synonym.txt")) + .add("possessive_english", Json.createObjectBuilder() + .add("type", "stemmer").add("langauge", "possessive_english")))).build(); protected static Set indices = new HashSet<>(); protected static Map scripts = new HashMap<>(); private static Map> relationships = new HashMap<>(); @@ -263,10 +273,14 @@ private static JsonObject buildNestedQuery(String path, JsonObject... queryObjec return Json.createObjectBuilder().add("nested", nestedBuilder).build(); } - private static JsonObject buildStringQuery(String field, String value) { + private static JsonObject buildStringQuery(String value, String... fields) { JsonObjectBuilder queryStringBuilder = Json.createObjectBuilder().add("query", value); - if (field != null) { - queryStringBuilder.add("fields", Json.createArrayBuilder().add(field)); + if (fields.length > 0) { + JsonArrayBuilder fieldsBuilder = Json.createArrayBuilder(); + for (String field : fields) { + fieldsBuilder.add(field); + } + queryStringBuilder.add("fields", fieldsBuilder); } return Json.createObjectBuilder().add("query_string", queryStringBuilder).build(); } @@ -447,9 +461,10 @@ public JsonValue buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) th throw new IcatException(IcatExceptionType.INTERNAL, "Cannot build searchAfter document from source as sorted field " + key + " missing."); } - String value = lastBean.getSource().getString(key); + JsonValue value = lastBean.getSource().get(key); builder.add(value); } + builder.add(lastBean.getEntityBaseBeanId()); return builder.build(); } } else { @@ -632,6 +647,7 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer JsonObject jsonObject = jsonReader.readObject(); JsonArray hits = jsonObject.getJsonObject("hits").getJsonArray("hits"); for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { + logger.trace("Hit: {}", hit.toString()); Float score = Float.NaN; if (!hit.isNull("_score")) { score = hit.getJsonNumber("_score").bigDecimalValue().floatValue(); @@ -660,7 +676,7 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer private JsonObjectBuilder parseQuery(JsonObject query, JsonValue searchAfter, String sort, String index, Set queryFields) throws IcatException, ParseException { JsonObjectBuilder builder = Json.createObjectBuilder(); - if (sort == null) { + if (sort == null || sort.equals("")) { builder.add("sort", Json.createArrayBuilder() .add(Json.createObjectBuilder().add("_score", "desc")) .add(Json.createObjectBuilder().add("id", "asc")).build()); @@ -685,11 +701,11 @@ private JsonObjectBuilder parseQuery(JsonObject query, JsonValue searchAfter, St JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); if (queryFields.contains("text")) { JsonArrayBuilder mustBuilder = Json.createArrayBuilder(); - mustBuilder.add(buildStringQuery(null, query.getString("text"))); + mustBuilder.add(buildStringQuery(query.getString("text"))); boolBuilder.add("must", mustBuilder); } Long lowerTime = parseDate(query, "lower", 0, Long.MIN_VALUE); - Long upperTime = parseDate(query, "upper", 0, Long.MAX_VALUE); + Long upperTime = parseDate(query, "upper", 59999, Long.MAX_VALUE); if (lowerTime != Long.MIN_VALUE || upperTime != Long.MAX_VALUE) { if (index.equals("datafile")) { // datafile has only one date field @@ -705,7 +721,7 @@ private JsonObjectBuilder parseQuery(JsonObject query, JsonValue searchAfter, St investigationUserQueries.add(buildMatchQuery("investigationuser.user.name", query.getString("user"))); } if (queryFields.contains("userFullName")) { - investigationUserQueries.add(buildStringQuery("investigationuser.user.fullName", query.getString("userFullName"))); + investigationUserQueries.add(buildStringQuery(query.getString("userFullName"), "investigationuser.user.fullName")); } if (investigationUserQueries.size() > 0) { filterBuilder.add(buildNestedQuery("investigationuser", investigationUserQueries.toArray(new JsonObject[0]))); @@ -715,7 +731,7 @@ private JsonObjectBuilder parseQuery(JsonObject query, JsonValue searchAfter, St JsonArray samples = query.getJsonArray("samples"); for (int i = 0; i < samples.size(); i++) { String sample = samples.getString(i); - filterBuilder.add(buildNestedQuery("sample", buildStringQuery("sample.name", sample))); + filterBuilder.add(buildNestedQuery("sample", buildStringQuery(sample, "sample.name", "sample.type.name"))); } } if (queryFields.contains("parameters")) { @@ -724,8 +740,8 @@ private JsonObjectBuilder parseQuery(JsonObject query, JsonValue searchAfter, St String name = parameterObject.getString("name", null); String units = parameterObject.getString("units", null); String stringValue = parameterObject.getString("stringValue", null); - Long lowerDate = decodeTime(parameterObject.getString("lowerDateValue", null)); - Long upperDate = decodeTime(parameterObject.getString("upperDateValue", null)); + Long lowerDate = parseDate(parameterObject, "lowerDateValue", 0, null); + Long upperDate = parseDate(parameterObject, "upperDateValue", 59999, null); JsonNumber lowerNumeric = parameterObject.getJsonNumber("lowerNumericValue"); JsonNumber upperNumeric = parameterObject.getJsonNumber("upperNumericValue"); @@ -770,6 +786,7 @@ public void initMappings() throws IcatException { // If the index isn't present, we should get 404 and create the index if (statusCode == 200) { // If the index already exists (200), do not attempt to create it + logger.trace("{} index already exists, continue", index); continue; } else if (statusCode != 404) { // If the code isn't 200 or 404, something has gone wrong @@ -782,10 +799,10 @@ public void initMappings() throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URI uri = new URIBuilder(server).setPath(basePath + "/" + index).build(); - logger.trace("Making call {}", uri); HttpPut httpPut = new HttpPut(uri); String body = Json.createObjectBuilder() .add("settings", indexSettings).add("mappings", buildMappings(index)).build().toString(); + logger.trace("Making call {} with body {}", uri, body); httpPut.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); try (CloseableHttpResponse response = httpclient.execute(httpPut)) { Rest.checkStatus(response, IcatExceptionType.INTERNAL); diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 6b278f40..d2ddaf60 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -552,11 +552,16 @@ public void testLuceneDatasets() throws Exception { // Try parameters List parameters = new ArrayList<>(); - parameters.add(new ParameterForLucene("colour", "name", "green")); - parameters.add(new ParameterForLucene("birthday", "date", dft.parse("2014-05-16T16:58:26+0000"), - dft.parse("2014-05-16T16:58:26+0000"))); - parameters.add(new ParameterForLucene("current", "amps", 140, 165)); - + ParameterForLucene stringParameter = new ParameterForLucene("colour", "name", "green"); + ParameterForLucene dateParameter = new ParameterForLucene("birthday", "date", + dft.parse("2014-05-16T16:58:26+0000"), dft.parse("2014-05-16T16:58:26+0000")); + ParameterForLucene numericParameter = new ParameterForLucene("current", "amps", 140, 165); + array = searchDatasets(session, null, null, null, null, Arrays.asList(stringParameter), 20, 1); + array = searchDatasets(session, null, null, null, null, Arrays.asList(dateParameter), 20, 1); + array = searchDatasets(session, null, null, null, null, Arrays.asList(numericParameter), 20, 1); + parameters.add(stringParameter); + parameters.add(dateParameter); + parameters.add(numericParameter); array = searchDatasets(session, null, null, null, null, parameters, 20, 1); array = searchDatasets(session, null, "gamma AND ds3", dft.parse("2014-05-16T05:09:03+0000"), @@ -716,7 +721,8 @@ public void testSearchDatafiles() throws Exception { responseObject.containsKey("dimensions")); // Test no facets match on DatafileParameters due to lack of READ access - target = Json.createObjectBuilder().add("target", "DatafileParameter"); + target = Json.createObjectBuilder() + .add("target", "DatafileParameter").add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name"))); facets = Json.createArrayBuilder().add(target).build().toString(); responseObject = searchDatafiles(session, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); @@ -725,8 +731,6 @@ public void testSearchDatafiles() throws Exception { // Test facets match on DatafileParameters wSession.addRule(null, "DatafileParameter", "R"); - target = Json.createObjectBuilder().add("target", "DatafileParameter"); - facets = Json.createArrayBuilder().add(target).build().toString(); responseObject = searchDatafiles(session, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); checkFacets(responseObject, "DatafileParameter.type.name", Arrays.asList("colour"), Arrays.asList(1L)); @@ -774,9 +778,16 @@ public void testSearchDatasets() throws Exception { List parameters = new ArrayList<>(); Date parameterDate = dft.parse("2014-05-16T16:58:26+0000"); parameters.add(new ParameterForLucene("colour", "name", "green")); + responseObject = searchDatasets(session, null, null, null, null, parameters, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + parameters.add(new ParameterForLucene("birthday", "date", parameterDate, parameterDate)); - parameters.add(new ParameterForLucene("current", "amps", 140, 165)); + responseObject = searchDatasets(session, null, null, null, null, parameters, null, 10, null, null, 1); + assertFalse(responseObject.containsKey("search_after")); + checkResultsSource(responseObject, Arrays.asList(expectation), true); + parameters.add(new ParameterForLucene("current", "amps", 140, 165)); responseObject = searchDatasets(session, null, null, null, null, parameters, null, 10, null, null, 1); assertFalse(responseObject.containsKey("search_after")); checkResultsSource(responseObject, Arrays.asList(expectation), true); @@ -837,15 +848,17 @@ public void testSearchDatasets() throws Exception { Session piSession = icat.login("db", credentials); searchDatasets(piSession, null, null, null, null, null, null, 10, null, null, 0); - // Test no facets match on Datasets - JsonObjectBuilder target = Json.createObjectBuilder().add("target", "Dataset"); + // Test facets match on Datasets + JsonObjectBuilder target = Json.createObjectBuilder() + .add("target", "Dataset").add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name"))); String facets = Json.createArrayBuilder().add(target).build().toString(); responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5); assertFalse(responseObject.containsKey("search_after")); checkFacets(responseObject, "Dataset.type.name", Arrays.asList("calibration"), Arrays.asList(5L)); // Test no facets match on DatasetParameters due to lack of READ access - target = Json.createObjectBuilder().add("target", "DatasetParameter"); + target = Json.createObjectBuilder() + .add("target", "DatasetParameter").add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name"))); facets = Json.createArrayBuilder().add(target).build().toString(); responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5); assertFalse(responseObject.containsKey("search_after")); @@ -854,8 +867,6 @@ public void testSearchDatasets() throws Exception { // Test facets match on DatasetParameters wSession.addRule(null, "DatasetParameter", "R"); - target = Json.createObjectBuilder().add("target", "DatasetParameter"); - facets = Json.createArrayBuilder().add(target).build().toString(); responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5); assertFalse(responseObject.containsKey("search_after")); checkFacets(responseObject, "DatasetParameter.type.name", Arrays.asList("colour", "birthday", "current"), @@ -891,7 +902,6 @@ public void testSearchInvestigations() throws Exception { List parameters = new ArrayList<>(); parameters.add(new ParameterForLucene("colour", "name", "green")); - // TODO remove additional checks here responseObject = searchInvestigations(session, "db/tr", null, lowerOrigin, upperOrigin, null, null, null, null, 10, null, null, 1); responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, null, @@ -987,8 +997,9 @@ public void testSearchInvestigations() throws Exception { Session piSession = icat.login("db", credentials); searchInvestigations(piSession, null, null, null, null, null, null, null, null, 10, null, null, 0); - // Test no facets match on Investigations - JsonObjectBuilder target = Json.createObjectBuilder().add("target", "Investigation"); + // Test facets match on Investigations + JsonObjectBuilder target = Json.createObjectBuilder() + .add("target", "Investigation").add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name"))); String facets = Json.createArrayBuilder().add(target).build().toString(); responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, 3); @@ -996,7 +1007,8 @@ public void testSearchInvestigations() throws Exception { checkFacets(responseObject, "Investigation.type.name", Arrays.asList("atype"), Arrays.asList(3L)); // Test no facets match on InvestigationParameters due to lack of READ access - target = Json.createObjectBuilder().add("target", "InvestigationParameter"); + target = Json.createObjectBuilder().add("target", "InvestigationParameter") + .add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name"))); facets = Json.createArrayBuilder().add(target).build().toString(); responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, 3); @@ -1006,8 +1018,6 @@ public void testSearchInvestigations() throws Exception { // Test facets match on InvestigationParameters wSession.addRule(null, "InvestigationParameter", "R"); - target = Json.createObjectBuilder().add("target", "InvestigationParameter"); - facets = Json.createArrayBuilder().add(target).build().toString(); responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); From 901e3ad04825d387a5555ca10eaca27e0497de59 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 28 Apr 2022 11:29:37 +0100 Subject: [PATCH 18/51] Refactor SearchApi functions for clarity #267 --- .../core/entity/DatafileFormat.java | 4 +- .../icatproject/core/entity/DatasetType.java | 4 +- .../org/icatproject/core/entity/Facility.java | 4 +- .../core/entity/InvestigationType.java | 4 +- .../core/entity/ParameterType.java | 4 +- .../icatproject/core/entity/SampleType.java | 4 +- .../org/icatproject/core/entity/User.java | 4 +- .../icatproject/core/manager/SearchApi.java | 1038 +++++++++-------- .../core/manager/search/QueryBuilder.java | 87 ++ .../core/manager/TestSearchApi.java | 16 - 10 files changed, 654 insertions(+), 515 deletions(-) create mode 100644 src/main/java/org/icatproject/core/manager/search/QueryBuilder.java diff --git a/src/main/java/org/icatproject/core/entity/DatafileFormat.java b/src/main/java/org/icatproject/core/entity/DatafileFormat.java index 7021c01a..02ccd53e 100644 --- a/src/main/java/org/icatproject/core/entity/DatafileFormat.java +++ b/src/main/java/org/icatproject/core/entity/DatafileFormat.java @@ -3,7 +3,9 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; @@ -55,7 +57,7 @@ public void setFacility(Facility facility) { @Column(name = "VERSION", nullable = false) private String version; - public static List docFields = Arrays.asList("datafileFormat.name", "datafileFormat.id"); + public static Set docFields = new HashSet<>(Arrays.asList("datafileFormat.name", "datafileFormat.id")); /* Needed for JPA */ public DatafileFormat() { diff --git a/src/main/java/org/icatproject/core/entity/DatasetType.java b/src/main/java/org/icatproject/core/entity/DatasetType.java index be67ec5a..23e66d75 100644 --- a/src/main/java/org/icatproject/core/entity/DatasetType.java +++ b/src/main/java/org/icatproject/core/entity/DatasetType.java @@ -3,7 +3,9 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; @@ -39,7 +41,7 @@ public class DatasetType extends EntityBaseBean implements Serializable { @Column(name = "NAME", nullable = false) private String name; - public static List docFields = Arrays.asList("type.name", "type.id"); + public static Set docFields = new HashSet<>(Arrays.asList("type.name", "type.id")); /* Needed for JPA */ public DatasetType() { diff --git a/src/main/java/org/icatproject/core/entity/Facility.java b/src/main/java/org/icatproject/core/entity/Facility.java index f10dadaa..8a1d7dd9 100644 --- a/src/main/java/org/icatproject/core/entity/Facility.java +++ b/src/main/java/org/icatproject/core/entity/Facility.java @@ -3,7 +3,9 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; @@ -65,7 +67,7 @@ public class Facility extends EntityBaseBean implements Serializable { @Comment("A URL associated with this facility") private String url; - public static List docFields = Arrays.asList("facility.name", "facility.id"); + public static Set docFields = new HashSet<>(Arrays.asList("facility.name", "facility.id")); /* Needed for JPA */ public Facility() { diff --git a/src/main/java/org/icatproject/core/entity/InvestigationType.java b/src/main/java/org/icatproject/core/entity/InvestigationType.java index a9703fcd..861a9001 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationType.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationType.java @@ -3,7 +3,9 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; @@ -55,7 +57,7 @@ public void setInvestigations(List investigations) { this.investigations = investigations; } - public static List docFields = Arrays.asList("type.name", "type.id"); + public static Set docFields = new HashSet<>(Arrays.asList("type.name", "type.id")); /* Needed for JPA */ public InvestigationType() { diff --git a/src/main/java/org/icatproject/core/entity/ParameterType.java b/src/main/java/org/icatproject/core/entity/ParameterType.java index f58145a5..abc884c7 100644 --- a/src/main/java/org/icatproject/core/entity/ParameterType.java +++ b/src/main/java/org/icatproject/core/entity/ParameterType.java @@ -3,7 +3,9 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; @@ -95,7 +97,7 @@ public class ParameterType extends EntityBaseBean implements Serializable { @Comment("If ordinary users are allowed to create their own parameter types this indicates that this one has been approved") private boolean verified; - public static List docFields = Arrays.asList("type.name", "type.units", "type.unitsSI", "numericValueSI", "type.id"); + public static Set docFields = new HashSet<>(Arrays.asList("type.name", "type.units", "type.unitsSI", "numericValueSI", "type.id")); /* Needed for JPA */ public ParameterType() { diff --git a/src/main/java/org/icatproject/core/entity/SampleType.java b/src/main/java/org/icatproject/core/entity/SampleType.java index 2cea752c..786929d9 100644 --- a/src/main/java/org/icatproject/core/entity/SampleType.java +++ b/src/main/java/org/icatproject/core/entity/SampleType.java @@ -3,7 +3,9 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; @@ -44,7 +46,7 @@ public class SampleType extends EntityBaseBean implements Serializable { @OneToMany(cascade = CascadeType.ALL, mappedBy = "type") private List samples = new ArrayList<>(); - public static List docFields = Arrays.asList("sample.type.name", "type.id"); + public static Set docFields = new HashSet<>(Arrays.asList("sample.type.name", "type.id")); /* Needed for JPA */ public SampleType() { diff --git a/src/main/java/org/icatproject/core/entity/User.java b/src/main/java/org/icatproject/core/entity/User.java index e0ca1367..03e78088 100644 --- a/src/main/java/org/icatproject/core/entity/User.java +++ b/src/main/java/org/icatproject/core/entity/User.java @@ -3,7 +3,9 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; @@ -55,7 +57,7 @@ public class User extends EntityBaseBean implements Serializable { @OneToMany(cascade = CascadeType.ALL, mappedBy = "user") private List studies = new ArrayList(); - public static List docFields = Arrays.asList("user.name", "user.fullName", "user.id"); + public static Set docFields = new HashSet<>(Arrays.asList("user.name", "user.fullName", "user.id")); public User() { } diff --git a/src/main/java/org/icatproject/core/manager/SearchApi.java b/src/main/java/org/icatproject/core/manager/SearchApi.java index a5ea9c28..52024ee7 100644 --- a/src/main/java/org/icatproject/core/manager/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/SearchApi.java @@ -5,7 +5,6 @@ import java.io.StringReader; import java.net.URI; import java.net.URISyntaxException; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -16,6 +15,7 @@ import java.util.Map; import java.util.Set; import java.util.TimeZone; +import java.util.Map.Entry; import java.util.concurrent.ExecutorService; import javax.json.Json; @@ -34,6 +34,7 @@ import javax.measure.format.MeasurementParseException; import javax.persistence.EntityManager; +import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpHead; @@ -54,6 +55,7 @@ import org.icatproject.core.entity.ParameterType; import org.icatproject.core.entity.SampleType; import org.icatproject.core.entity.User; +import org.icatproject.core.manager.search.QueryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,15 +64,25 @@ // TODO see what functionality can live here, and possibly convert from abstract to a fully generic API public class SearchApi { - // TODO this is a duplicate of icat.lucene code (for now...?) - private static class ParentRelationship { + + private static enum ModificationType { + CREATE, UPDATE, DELETE + }; + + private static enum RelationType { + CHILD, NESTED_CHILD, NESTED_GRANDCHILD + }; + + private static class ParentRelation { + public RelationType relationType; public String parentName; - public String joinName; - public List fields; + public String joinField; + public Set fields; - public ParentRelationship(String parentName, String joinName, List fields) { + public ParentRelation(RelationType relationType, String parentName, String joinField, Set fields) { + this.relationType = relationType; this.parentName = parentName; - this.joinName = joinName; + this.joinField = joinField; this.fields = fields; } } @@ -79,9 +91,6 @@ public ParentRelationship(String parentName, String joinName, List field protected static final Logger logger = LoggerFactory.getLogger(SearchApi.class); protected static SimpleDateFormat df; protected static String basePath = ""; - protected static JsonObject matchAllQuery = Json.createObjectBuilder().add("query", Json.createObjectBuilder() - .add("match_all", Json.createObjectBuilder())).build(); - // TODO synonym filter in the default_search ONLY private static JsonObject indexSettings = Json.createObjectBuilder().add("analysis", Json.createObjectBuilder() .add("analyzer", Json.createObjectBuilder() .add("default", Json.createObjectBuilder() @@ -94,10 +103,10 @@ public ParentRelationship(String parentName, String joinName, List field .add("synonym", Json.createObjectBuilder() .add("type", "synonym").add("synonyms_path", "synonym.txt")) .add("possessive_english", Json.createObjectBuilder() - .add("type", "stemmer").add("langauge", "possessive_english")))).build(); + .add("type", "stemmer").add("langauge", "possessive_english")))) + .build(); protected static Set indices = new HashSet<>(); - protected static Map scripts = new HashMap<>(); - private static Map> relationships = new HashMap<>(); + private static Map> relations = new HashMap<>(); protected URI server; @@ -110,66 +119,51 @@ public ParentRelationship(String parentName, String joinName, List field indices.addAll(Arrays.asList("datafile", "dataset", "investigation")); - scripts.put("delete_datafileparameter", buildChildrenScript("datafileparameter", false)); - scripts.put("update_datafileparameter", buildChildrenScript("datafileparameter", true)); - scripts.put("delete_datasetparameter", buildChildrenScript("datasetparameter", false)); - scripts.put("update_datasetparameter", buildChildrenScript("datasetparameter", true)); - scripts.put("delete_investigationparameter", buildChildrenScript("investigationparameter", false)); - scripts.put("update_investigationparameter", buildChildrenScript("investigationparameter", true)); - scripts.put("delete_investigationuser", buildChildrenScript("investigationuser", false)); - scripts.put("update_investigationuser", buildChildrenScript("investigationuser", true)); - scripts.put("create_investigationuser", buildCreateScript("investigationuser")); - scripts.put("delete_sample", buildChildrenScript("sample", false)); - scripts.put("update_sample", buildChildrenScript("sample", true)); - scripts.put("delete_datafileparametertype", buildChildrenScript("datafileparameter", ParameterType.docFields, false)); - scripts.put("update_datafileparametertype", buildChildrenScript("datafileparameter", ParameterType.docFields, true)); - scripts.put("delete_datasetparametertype", buildChildrenScript("datasetparameter", ParameterType.docFields, false)); - scripts.put("update_datasetparametertype", buildChildrenScript("datasetparameter", ParameterType.docFields, true)); - scripts.put("delete_investigationparametertype", buildChildrenScript("investigationparameter", ParameterType.docFields, false)); - scripts.put("update_investigationparametertype", buildChildrenScript("investigationparameter", ParameterType.docFields, true)); - scripts.put("delete_user", buildChildrenScript("investigationuser", User.docFields, false)); - scripts.put("update_user", buildChildrenScript("investigationuser", User.docFields, true)); - scripts.put("delete_sampletype", buildChildrenScript("sample", SampleType.docFields, false)); - scripts.put("update_sampletype", buildChildrenScript("sample", SampleType.docFields, true)); - - scripts.put("delete_datafileformat", buildChildScript("datafileformat", DatafileFormat.docFields, false)); - scripts.put("update_datafileformat", buildChildScript("datafileformat", DatafileFormat.docFields, true)); - scripts.put("delete_datasettype", buildChildScript("datasettype", DatasetType.docFields, false)); - scripts.put("update_datasettype", buildChildScript("datasettype", DatasetType.docFields, true)); - scripts.put("delete_investigationtype", buildChildScript("investigationtype", InvestigationType.docFields, false)); - scripts.put("update_investigationtype", buildChildScript("investigationtype", InvestigationType.docFields, true)); - scripts.put("delete_facility", buildChildScript("facility", Facility.docFields, false)); - scripts.put("update_facility", buildChildScript("facility", Facility.docFields, true)); - - relationships.put("datafileparameter", Arrays.asList( - new ParentRelationship("datafile", "datafile", new ArrayList<>()))); - relationships.put("datasetparameter", Arrays.asList( - new ParentRelationship("dataset", "dataset", new ArrayList<>()))); - relationships.put("investigationparameter", Arrays.asList( - new ParentRelationship("investigation", "investigation", new ArrayList<>()))); - relationships.put("investigationuser", Arrays.asList( - new ParentRelationship("investigation", "investigation", new ArrayList<>()))); - relationships.put("sample", Arrays.asList( - new ParentRelationship("investigation", "investigation", new ArrayList<>()))); - - relationships.put("parametertype", Arrays.asList( - new ParentRelationship("investigation", "investigationparameter", ParameterType.docFields), - new ParentRelationship("dataset", "datasetparameter", ParameterType.docFields), - new ParentRelationship("datafile", "datafileparameter", ParameterType.docFields) - )); - relationships.put("user", Arrays.asList( - new ParentRelationship("investigation", "investigationuser", User.docFields))); - relationships.put("sampleType", Arrays.asList( - new ParentRelationship("investigation", "sample", SampleType.docFields))); - - relationships.put("datafileformat", Arrays.asList( - new ParentRelationship("datafile", "datafileFormat", DatafileFormat.docFields))); - relationships.put("datasettype", Arrays.asList( - new ParentRelationship("dataset", "type", DatasetType.docFields))); - relationships.put("investigationtype", Arrays.asList( - new ParentRelationship("investigation", "type", InvestigationType.docFields))); - relationships.put("facility", Arrays.asList( - new ParentRelationship("investigation", "facility", Facility.docFields))); + // Non-nested children have a one to one relationship with an indexed entity and + // so do not form an array, and update specific fields by query + relations.put("datafileformat", Arrays.asList( + new ParentRelation(RelationType.CHILD, "datafile", "datafileFormat", DatafileFormat.docFields))); + relations.put("datasettype", Arrays.asList( + new ParentRelation(RelationType.CHILD, "dataset", "type", DatasetType.docFields))); + relations.put("investigationtype", Arrays.asList( + new ParentRelation(RelationType.CHILD, "investigation", "type", InvestigationType.docFields))); + relations.put("facility", Arrays.asList( + new ParentRelation(RelationType.CHILD, "investigation", "facility", Facility.docFields))); + + // Nested children are indexed as an array of objects on their parent entity, + // and know their parent's id (N.B. InvestigationUsers are also mapped to + // Datasets and Datafiles, but using the investigation.id field) + relations.put("datafileparameter", Arrays.asList( + new ParentRelation(RelationType.NESTED_CHILD, "datafile", "datafile", null))); + relations.put("datasetparameter", Arrays.asList( + new ParentRelation(RelationType.NESTED_CHILD, "dataset", "dataset", null))); + relations.put("investigationparameter", Arrays.asList( + new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null))); + relations.put("investigationuser", Arrays.asList( + new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null), + new ParentRelation(RelationType.NESTED_CHILD, "dataset", "investigation", null), + new ParentRelation(RelationType.NESTED_CHILD, "datafile", "investigation", null))); + relations.put("sample", Arrays.asList( + new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null))); + + // Nested grandchildren are entities that are related to one of the nested + // children, but do not have a direct reference to one of the indexed entities, + // and so must be updated by query - they also only affect a subset of the + // nested fields, rather than an entire nested object + relations.put("parametertype", Arrays.asList( + new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "investigationparameter", + ParameterType.docFields), + new ParentRelation(RelationType.NESTED_GRANDCHILD, "dataset", "datasetparameter", + ParameterType.docFields), + new ParentRelation(RelationType.NESTED_GRANDCHILD, "datafile", "datafileparameter", + ParameterType.docFields))); + relations.put("user", Arrays.asList( + new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "investigationuser", + User.docFields), + new ParentRelation(RelationType.NESTED_GRANDCHILD, "dataset", "investigationuser", User.docFields), + new ParentRelation(RelationType.NESTED_GRANDCHILD, "datafile", "investigationuser", User.docFields))); + relations.put("sampleType", Arrays.asList( + new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "sample", SampleType.docFields))); } public SearchApi(URI server) { @@ -182,22 +176,43 @@ private static String buildCreateScript(String target) { return Json.createObjectBuilder().add("script", builder).build().toString(); } - private static String buildChildrenScript(String target, boolean update) { - String source = "if (ctx._source." + target + " != null) {List ids = new ArrayList(); ctx._source." + target + ".forEach(t -> ids.add(t.id)); if (ids.contains(params.id)) {ctx._source." + target + ".remove(ids.indexOf(params.id))}}"; + private static String buildChildScript(Set docFields, boolean update) { + String source = ""; + for (String field : docFields) { + if (update) { + source += "ctx._source['" + field + "'] = params['" + field + "']; "; + } else { + source += "ctx._source.remove('" + field + "'); "; + } + } + JsonObjectBuilder builder = Json.createObjectBuilder().add("lang", "painless").add("source", source); + return Json.createObjectBuilder().add("script", builder).build().toString(); + } + + private static String buildNestedChildScript(String target, boolean update) { + String source = "if (ctx._source." + target + " != null) {List ids = new ArrayList(); ctx._source." + target + + ".forEach(t -> ids.add(t.id)); if (ids.contains(params.id)) {ctx._source." + target + + ".remove(ids.indexOf(params.id))}}"; if (update) { - source += "if (ctx._source." + target + " != null) {ctx._source." + target + ".addAll(params.doc);} else {ctx._source." + target + " = params.doc;}"; + source += "if (ctx._source." + target + " != null) {ctx._source." + target + + ".addAll(params.doc);} else {ctx._source." + target + " = params.doc;}"; } JsonObjectBuilder builder = Json.createObjectBuilder().add("lang", "painless").add("source", source); return Json.createObjectBuilder().add("script", builder).build().toString(); } - private static String buildChildrenScript(String target, List docFields, boolean update) { - String source = "int listIndex; if (ctx._source." + target + " != null) {List ids = new ArrayList(); ctx._source." + target + ".forEach(t -> ids.add(t.id)); if (ids.contains(params.id)) {listIndex = ids.indexOf(params.id)}}"; + private static String buildNestedGrandchildScript(String target, Set docFields, boolean update) { + String source = "int listIndex; if (ctx._source." + target + + " != null) {List ids = new ArrayList(); ctx._source." + target + + ".forEach(t -> ids.add(t.id)); if (ids.contains(params.id)) {listIndex = ids.indexOf(params.id)}}"; String childSource = "ctx._source." + target + ".get(listIndex)"; for (String field : docFields) { if (update) { if (field.equals("numericValueSI")) { - source += "if ("+ childSource + ".numericValue != null && params.containsKey('conversionFactor')) {" + childSource + ".numericValueSI = params.conversionFactor * "+ childSource + ".numericValue;} else {" + childSource + ".remove('numericValueSI');}"; + source += "if (" + childSource + + ".numericValue != null && params.containsKey('conversionFactor')) {" + childSource + + ".numericValueSI = params.conversionFactor * " + childSource + ".numericValue;} else {" + + childSource + ".remove('numericValueSI');}"; } else { source += childSource + "['" + field + "']" + " = params['" + field + "']; "; } @@ -209,34 +224,29 @@ private static String buildChildrenScript(String target, List docFields, return Json.createObjectBuilder().add("script", builder).build().toString(); } - private static String buildChildScript(String target, List docFields, boolean update) { - String source = ""; - for (String field : docFields) { - if (update) { - source += "ctx._source['" + field + "'] = params['" + field + "']; "; - } else { - source += "ctx._source.remove('" + field + "'); "; - } - } - JsonObjectBuilder builder = Json.createObjectBuilder().add("lang", "painless").add("source", source); - return Json.createObjectBuilder().add("script", builder).build().toString(); - } - private static JsonObject buildMappings(String index) { JsonObject typeLong = Json.createObjectBuilder().add("type", "long").build(); JsonObjectBuilder propertiesBuilder = Json.createObjectBuilder() - .add("id", typeLong).add("investigationuser", buildNestedMapping("investigation.id", "user.id")); + .add("id", typeLong) + .add("investigationuser", buildNestedMapping("investigation.id", "user.id")); if (index.equals("investigation")) { - propertiesBuilder.add("type.id", typeLong).add("facility.id", typeLong) + propertiesBuilder + .add("type.id", typeLong) + .add("facility.id", typeLong) .add("sample", buildNestedMapping("investigation.id", "type.id")) .add("investigationparameter", buildNestedMapping("investigation.id", "type.id")); } else if (index.equals("dataset")) { - propertiesBuilder.add("investigation.id", typeLong).add("type.id", typeLong).add("sample.id", typeLong) + propertiesBuilder + .add("investigation.id", typeLong) + .add("type.id", typeLong) + .add("sample.id", typeLong) .add("datasetparameter", buildNestedMapping("dataset.id", "type.id")); } else if (index.equals("datafile")) { - propertiesBuilder.add("investigation.id", typeLong).add("datafileFormat.id", typeLong) + propertiesBuilder + .add("investigation.id", typeLong) + .add("datafileFormat.id", typeLong) .add("datafileparameter", buildNestedMapping("datafile.id", "type.id")); - } + } return Json.createObjectBuilder().add("properties", propertiesBuilder).build(); } @@ -246,67 +256,12 @@ private static JsonObject buildNestedMapping(String... idFields) { for (String idField : idFields) { propertiesBuilder.add(idField, typeLong); } - return Json.createObjectBuilder().add("type", "nested").add("properties", propertiesBuilder).build(); - } - - private static JsonObject buildMatchQuery(String field, String value) { - JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("query", value).add("operator", "and"); - JsonObjectBuilder matchBuilder = Json.createObjectBuilder().add(field + ".keyword", fieldBuilder); - return Json.createObjectBuilder().add("match", matchBuilder).build(); - } - - private static JsonObject buildNestedQuery(String path, JsonObject... queryObjects) { - JsonObject builtQueries = null; - if (queryObjects.length == 0) { - builtQueries = matchAllQuery; - } else if (queryObjects.length == 1) { - builtQueries = queryObjects[0]; - } else { - JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); - for (JsonObject queryObject : queryObjects) { - filterBuilder.add(queryObject); - } - JsonObjectBuilder boolBuilder = Json.createObjectBuilder().add("filter", filterBuilder); - builtQueries = Json.createObjectBuilder().add("bool", boolBuilder).build(); - } - JsonObjectBuilder nestedBuilder = Json.createObjectBuilder().add("path", path).add("query", builtQueries); - return Json.createObjectBuilder().add("nested", nestedBuilder).build(); - } - - private static JsonObject buildStringQuery(String value, String... fields) { - JsonObjectBuilder queryStringBuilder = Json.createObjectBuilder().add("query", value); - if (fields.length > 0) { - JsonArrayBuilder fieldsBuilder = Json.createArrayBuilder(); - for (String field : fields) { - fieldsBuilder.add(field); - } - queryStringBuilder.add("fields", fieldsBuilder); - } - return Json.createObjectBuilder().add("query_string", queryStringBuilder).build(); - } - - private static JsonObject buildTermQuery(String field, String value) { - return Json.createObjectBuilder().add("term", Json.createObjectBuilder().add(field, value)).build(); - } - - private static JsonObject buildTermsQuery(String field, JsonArray values) { - return Json.createObjectBuilder().add("terms", Json.createObjectBuilder().add(field, values)).build(); - } - - private static JsonObject buildLongRangeQuery(String field, Long lowerValue, Long upperValue) { - JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("gte", lowerValue).add("lte", upperValue); - JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); - return Json.createObjectBuilder().add("range", rangeBuilder).build(); - } - - private static JsonObject buildRangeQuery(String field, JsonNumber lowerValue, JsonNumber upperValue) { - JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("gte", lowerValue).add("lte", upperValue); - JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); - return Json.createObjectBuilder().add("range", rangeBuilder).build(); + return Json.createObjectBuilder().add("type", "nested").add("properties", propertiesBuilder).build(); } // TODO (mostly) duplicated code from icat.lucene... - private static Long parseDate(JsonObject jsonObject, String key, int offset, Long defaultValue) throws IcatException { + private static Long parseDate(JsonObject jsonObject, String key, int offset, Long defaultValue) + throws IcatException { if (jsonObject.containsKey(key)) { ValueType valueType = jsonObject.get(key).getValueType(); switch (valueType) { @@ -315,13 +270,13 @@ private static Long parseDate(JsonObject jsonObject, String key, int offset, Lon try { return decodeTime(dateString) + offset; } catch (Exception e) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, + throw new IcatException(IcatExceptionType.BAD_PARAMETER, "Could not parse date " + dateString + " using expected format yyyyMMddHHmm"); } case NUMBER: return jsonObject.getJsonNumber(key).longValueExact(); default: - throw new IcatException(IcatExceptionType.BAD_PARAMETER, + throw new IcatException(IcatExceptionType.BAD_PARAMETER, "Dates should be represented by a NUMBER or STRING JsonValue, but got " + valueType); } } @@ -480,10 +435,13 @@ public JsonValue buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) th } public void clear() throws IcatException { + commit(); try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URI uri = new URIBuilder(server).setPath(basePath + "/_all/_delete_by_query").build(); HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(matchAllQuery.toString(), ContentType.APPLICATION_JSON)); + String body = Json.createObjectBuilder().add("query", QueryBuilder.buildMatchAllQuery()).build().toString(); + httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + logger.trace("Making call {} with body {}", uri, body); try (CloseableHttpResponse response = httpclient.execute(httpPost)) { Rest.checkStatus(response, IcatExceptionType.INTERNAL); } @@ -513,102 +471,115 @@ public List facetSearch(String target, JsonObject facetQuery, In int maxLabels) throws IcatException { List results = new ArrayList<>(); if (!facetQuery.containsKey("dimensions")) { + // If no dimensions were specified, return early return results; } - String dimensionPrefix = ""; + String dimensionPrefix = null; String index = target.toLowerCase(); - if (relationships.containsKey(index)) { - dimensionPrefix = index + "."; - index = relationships.get(index).get(0).parentName; + if (relations.containsKey(index)) { + // If we're attempting to facet a nested entity, use the parent index + dimensionPrefix = index; + index = relations.get(index).get(0).parentName; } try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URIBuilder builder = new URIBuilder(server).setPath(basePath + "/" + index + "/_search"); builder.addParameter("size", maxResults.toString()); URI uri = builder.build(); - logger.trace("Making call {}", uri); + JsonObject queryObject = facetQuery.getJsonObject("query"); - JsonObjectBuilder bodyBuilder = parseQuery(queryObject, null, null, index, queryObject.keySet()); - - JsonObjectBuilder aggsBuilder = Json.createObjectBuilder(); - for (JsonObject dimension : facetQuery.getJsonArray("dimensions").getValuesAs(JsonObject.class)) { - String dimensionString = dimension.getString("dimension"); - if (dimension.containsKey("ranges")) { - aggsBuilder.add(dimensionString, Json.createObjectBuilder().add("range", Json.createObjectBuilder() - .add("field", dimensionPrefix + dimensionString).add("keyed", true).add("ranges", dimension.getJsonArray("ranges")))); - } else { - aggsBuilder.add(dimensionString, Json.createObjectBuilder().add("terms", Json.createObjectBuilder() - .add("field", dimensionPrefix + dimensionString + ".keyword").add("size", maxLabels))); - } - } - if (dimensionPrefix.equals("")) { - bodyBuilder.add("aggs", aggsBuilder); - } else { - bodyBuilder.add("aggs", Json.createObjectBuilder() - .add(dimensionPrefix.substring(0, dimensionPrefix.length() - 1), Json.createObjectBuilder() - .add("nested", Json.createObjectBuilder() - .add("path", dimensionPrefix.substring(0, dimensionPrefix.length() - 1))) - .add("aggs", aggsBuilder))); - } + JsonArray dimensions = facetQuery.getJsonArray("dimensions"); + JsonObjectBuilder bodyBuilder = parseQuery(Json.createObjectBuilder(), queryObject, index); + bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix); String body = bodyBuilder.build().toString(); + HttpPost httpPost = new HttpPost(uri); httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); - logger.trace("Body: {}", body); + logger.trace("Making call {} with body {}", uri, body); try (CloseableHttpResponse response = httpclient.execute(httpPost)) { Rest.checkStatus(response, IcatExceptionType.INTERNAL); JsonReader jsonReader = Json.createReader(response.getEntity().getContent()); JsonObject jsonObject = jsonReader.readObject(); logger.trace("facet response: {}", jsonObject); JsonObject aggregations = jsonObject.getJsonObject("aggregations"); - - if (!dimensionPrefix.equals("")) { - aggregations = aggregations.getJsonObject(dimensionPrefix.substring(0, dimensionPrefix.length() - 1)); + if (dimensionPrefix != null) { + aggregations = aggregations.getJsonObject(dimensionPrefix); } - for (String dimension : aggregations.keySet()) { - if (dimension.equals("doc_count")) { - continue; - } - FacetDimension facetDimension = new FacetDimension(target, dimension); - List facets = facetDimension.getFacets(); - JsonObject aggregation = aggregations.getJsonObject(dimension); - JsonValue bucketsValue = aggregation.get("buckets"); - switch (bucketsValue.getValueType()) { - case ARRAY: - List buckets = aggregation.getJsonArray("buckets").getValuesAs(JsonObject.class); - if (buckets.size() == 0) { - continue; - } - for (JsonObject bucket : buckets) { - long docCount = bucket.getJsonNumber("doc_count").longValueExact(); - facets.add(new FacetLabel(bucket.getString("key"), docCount)); - } - break; - case OBJECT: - JsonObject bucketsObject = aggregation.getJsonObject("buckets"); - Set keySet = bucketsObject.keySet(); - if (keySet.size() == 0) { - continue; - } - for (String key : keySet) { - JsonObject bucket = bucketsObject.getJsonObject(key); - long docCount = bucket.getJsonNumber("doc_count").longValueExact(); - facets.add(new FacetLabel(key, docCount)); - } - break; - default: - String msg = "Excpeted 'buckets' to have ARRAY or OBJECT type, but it was " - + bucketsValue.getValueType(); - throw new IcatException(IcatExceptionType.INTERNAL, msg); - } - results.add(facetDimension); + parseFacetsResponse(results, target, dimension, aggregations); } } return results; - } catch (IOException | URISyntaxException | ParseException e) { + } catch (IOException | URISyntaxException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } } + private JsonObjectBuilder parseFacets(JsonObjectBuilder bodyBuilder, JsonArray dimensions, int maxLabels, + String dimensionPrefix) { + JsonObjectBuilder aggsBuilder = Json.createObjectBuilder(); + for (JsonObject dimensionObject : dimensions.getValuesAs(JsonObject.class)) { + String dimensionString = dimensionObject.getString("dimension"); + String field = dimensionPrefix == null ? dimensionString : dimensionPrefix + "." + dimensionString; + if (dimensionObject.containsKey("ranges")) { + JsonArray ranges = dimensionObject.getJsonArray("ranges"); + aggsBuilder.add(dimensionString, QueryBuilder.buildRangeFacet(field, ranges)); + } else { + aggsBuilder.add(dimensionString, QueryBuilder.buildStringFacet(field, maxLabels)); + } + } + if (dimensionPrefix == null) { + bodyBuilder.add("aggs", aggsBuilder); + } else { + bodyBuilder.add("aggs", Json.createObjectBuilder() + .add(dimensionPrefix, Json.createObjectBuilder() + .add("nested", Json.createObjectBuilder().add("path", dimensionPrefix)) + .add("aggs", aggsBuilder))); + } + return bodyBuilder; + } + + private void parseFacetsResponse(List results, String target, String dimension, + JsonObject aggregations) throws IcatException { + if (dimension.equals("doc_count")) { + // For nested aggregations, there is a doc_count entry at the same level as the + // dimension objects, but we're not interested in this + return; + } + FacetDimension facetDimension = new FacetDimension(target, dimension); + List facets = facetDimension.getFacets(); + JsonObject aggregation = aggregations.getJsonObject(dimension); + JsonValue bucketsValue = aggregation.get("buckets"); + ValueType valueType = bucketsValue.getValueType(); + switch (valueType) { + case ARRAY: + List buckets = ((JsonArray) bucketsValue).getValuesAs(JsonObject.class); + if (buckets.size() == 0) { + return; + } + for (JsonObject bucket : buckets) { + long docCount = bucket.getJsonNumber("doc_count").longValueExact(); + facets.add(new FacetLabel(bucket.getString("key"), docCount)); + } + break; + case OBJECT: + JsonObject bucketsObject = (JsonObject) bucketsValue; + Set keySet = bucketsObject.keySet(); + if (keySet.size() == 0) { + return; + } + for (String key : keySet) { + JsonObject bucket = bucketsObject.getJsonObject(key); + long docCount = bucket.getJsonNumber("doc_count").longValueExact(); + facets.add(new FacetLabel(key, docCount)); + } + break; + default: + String msg = "Expected 'buckets' to have ARRAY or OBJECT type, but it was " + valueType; + throw new IcatException(IcatExceptionType.INTERNAL, msg); + } + results.add(facetDimension); + } + public SearchResult getResults(JsonObject query, int maxResults) throws IcatException { return getResults(query, null, maxResults, null, Arrays.asList("id")); } @@ -620,34 +591,32 @@ public SearchResult getResults(JsonObject query, int maxResults, String sort) th public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer blockSize, String sort, List requestedFields) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - String index; - Set queryFields = query.keySet(); - if (queryFields.contains("target")) { - index = query.getString("target").toLowerCase(); - } else { - index = query.getString("_all"); - } + String index = query.containsKey("target") ? query.getString("target").toLowerCase() : "_all"; URIBuilder builder = new URIBuilder(server).setPath(basePath + "/" + index + "/_search"); StringBuilder sb = new StringBuilder(); requestedFields.forEach(f -> sb.append(f).append(",")); builder.addParameter("_source", sb.toString()); builder.addParameter("size", blockSize.toString()); URI uri = builder.build(); - logger.trace("Making call {}", uri); - String body = parseQuery(query, searchAfter, sort, index, queryFields).build().toString(); + JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); + bodyBuilder = parseSort(bodyBuilder, sort); + bodyBuilder = parseSearchAfter(bodyBuilder, searchAfter); + bodyBuilder = parseQuery(bodyBuilder, query, index); + String body = bodyBuilder.build().toString(); + SearchResult result = new SearchResult(); List entities = result.getResults(); HttpPost httpPost = new HttpPost(uri); httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); - logger.trace("Body: {}", body); + logger.trace("Making call {} with body {}", uri, body); try (CloseableHttpResponse response = httpclient.execute(httpPost)) { Rest.checkStatus(response, IcatExceptionType.INTERNAL); JsonReader jsonReader = Json.createReader(response.getEntity().getContent()); JsonObject jsonObject = jsonReader.readObject(); JsonArray hits = jsonObject.getJsonObject("hits").getJsonArray("hits"); for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { - logger.trace("Hit: {}", hit.toString()); + logger.trace("Hit {}", hit.toString()); Float score = Float.NaN; if (!hit.isNull("_score")) { score = hit.getJsonNumber("_score").bigDecimalValue().floatValue(); @@ -655,29 +624,30 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer Integer id = new Integer(hit.getString("_id")); entities.add(new ScoredEntityBaseBean(id, score, hit.getJsonObject("_source"))); } + + // If we're returning as many results as were asked for, setSearchAfter so + // subsequent searches can continue from the last result if (hits.size() == blockSize) { JsonObject lastHit = hits.getJsonObject(blockSize - 1); - logger.trace("Building searchAfter from {}", lastHit.toString()); if (lastHit.containsKey("sort")) { result.setSearchAfter(lastHit.getJsonArray("sort")); } else { ScoredEntityBaseBean lastEntity = entities.get(blockSize - 1); - result.setSearchAfter(Json.createArrayBuilder().add(lastEntity.getScore()) - .add(lastEntity.getEntityBaseBeanId()).build()); + long id = lastEntity.getEntityBaseBeanId(); + float score = lastEntity.getScore(); + result.setSearchAfter(Json.createArrayBuilder().add(score).add(id).build()); } } } return result; - } catch (IOException | URISyntaxException | ParseException e) { + } catch (IOException | URISyntaxException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } } - private JsonObjectBuilder parseQuery(JsonObject query, JsonValue searchAfter, String sort, String index, - Set queryFields) throws IcatException, ParseException { - JsonObjectBuilder builder = Json.createObjectBuilder(); + private JsonObjectBuilder parseSort(JsonObjectBuilder builder, String sort) { if (sort == null || sort.equals("")) { - builder.add("sort", Json.createArrayBuilder() + return builder.add("sort", Json.createArrayBuilder() .add(Json.createObjectBuilder().add("_score", "desc")) .add(Json.createObjectBuilder().add("id", "asc")).build()); } else { @@ -691,81 +661,103 @@ private JsonObjectBuilder parseQuery(JsonObject query, JsonValue searchAfter, St .add(key + ".keyword", sortObject.getString(key))); } } - builder.add("sort", sortArrayBuilder.add(Json.createObjectBuilder().add("id", "asc")).build()); + return builder.add("sort", sortArrayBuilder.add(Json.createObjectBuilder().add("id", "asc")).build()); } - if (searchAfter != null) { - builder.add("search_after", searchAfter); + } + + private JsonObjectBuilder parseSearchAfter(JsonObjectBuilder builder, JsonValue searchAfter) { + if (searchAfter == null) { + return builder; + } else { + return builder.add("search_after", searchAfter); } + } + + private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query, String index) + throws IcatException { + // In general, we use a boolean query to compound queries on individual fields JsonObjectBuilder queryBuilder = Json.createObjectBuilder(); JsonObjectBuilder boolBuilder = Json.createObjectBuilder(); - JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); - if (queryFields.contains("text")) { + + if (query.containsKey("text")) { + // The free text is the only element we perform scoring on, so "must" occur JsonArrayBuilder mustBuilder = Json.createArrayBuilder(); - mustBuilder.add(buildStringQuery(query.getString("text"))); + mustBuilder.add(QueryBuilder.buildStringQuery(query.getString("text"))); boolBuilder.add("must", mustBuilder); } + + // Non-scored elements are added to the "filter" + JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); + Long lowerTime = parseDate(query, "lower", 0, Long.MIN_VALUE); Long upperTime = parseDate(query, "upper", 59999, Long.MAX_VALUE); if (lowerTime != Long.MIN_VALUE || upperTime != Long.MAX_VALUE) { if (index.equals("datafile")) { // datafile has only one date field - filterBuilder.add(buildLongRangeQuery("date", lowerTime, upperTime)); + filterBuilder.add(QueryBuilder.buildLongRangeQuery("date", lowerTime, upperTime)); } else { - filterBuilder.add(buildLongRangeQuery("startDate", lowerTime, upperTime)); - filterBuilder.add(buildLongRangeQuery("endDate", lowerTime, upperTime)); + filterBuilder.add(QueryBuilder.buildLongRangeQuery("startDate", lowerTime, upperTime)); + filterBuilder.add(QueryBuilder.buildLongRangeQuery("endDate", lowerTime, upperTime)); } } List investigationUserQueries = new ArrayList<>(); - if (queryFields.contains("user")) { - investigationUserQueries.add(buildMatchQuery("investigationuser.user.name", query.getString("user"))); + if (query.containsKey("user")) { + String name = query.getString("user"); + JsonObject nameQuery = QueryBuilder.buildMatchQuery("investigationuser.user.name", name); + investigationUserQueries.add(nameQuery); } - if (queryFields.contains("userFullName")) { - investigationUserQueries.add(buildStringQuery(query.getString("userFullName"), "investigationuser.user.fullName")); + if (query.containsKey("userFullName")) { + String fullName = query.getString("userFullName"); + JsonObject fullNameQuery = QueryBuilder.buildStringQuery(fullName, "investigationuser.user.fullName"); + investigationUserQueries.add(fullNameQuery); } if (investigationUserQueries.size() > 0) { - filterBuilder.add(buildNestedQuery("investigationuser", investigationUserQueries.toArray(new JsonObject[0]))); + JsonObject[] array = investigationUserQueries.toArray(new JsonObject[0]); + filterBuilder.add(QueryBuilder.buildNestedQuery("investigationuser", array)); } - if (queryFields.contains("samples")) { + if (query.containsKey("samples")) { JsonArray samples = query.getJsonArray("samples"); for (int i = 0; i < samples.size(); i++) { String sample = samples.getString(i); - filterBuilder.add(buildNestedQuery("sample", buildStringQuery(sample, "sample.name", "sample.type.name"))); + JsonObject stringQuery = QueryBuilder.buildStringQuery(sample, "sample.name", "sample.type.name"); + filterBuilder.add(QueryBuilder.buildNestedQuery("sample", stringQuery)); } } - if (queryFields.contains("parameters")) { - for (JsonValue parameterValue : query.getJsonArray("parameters")) { - JsonObject parameterObject = (JsonObject) parameterValue; - String name = parameterObject.getString("name", null); - String units = parameterObject.getString("units", null); - String stringValue = parameterObject.getString("stringValue", null); - Long lowerDate = parseDate(parameterObject, "lowerDateValue", 0, null); - Long upperDate = parseDate(parameterObject, "upperDateValue", 59999, null); - JsonNumber lowerNumeric = parameterObject.getJsonNumber("lowerNumericValue"); - JsonNumber upperNumeric = parameterObject.getJsonNumber("upperNumericValue"); + if (query.containsKey("parameters")) { + for (JsonObject parameterObject : query.getJsonArray("parameters").getValuesAs(JsonObject.class)) { String path = index + "parameter"; List parameterQueries = new ArrayList<>(); - if (name != null) { - parameterQueries.add(buildMatchQuery(path + ".type.name", name)); + if (parameterObject.containsKey("name")) { + String name = parameterObject.getString("name"); + parameterQueries.add(QueryBuilder.buildMatchQuery(path + ".type.name", name)); } - if (units != null) { - parameterQueries.add(buildMatchQuery(path + ".type.units", units)); + if (parameterObject.containsKey("units")) { + String units = parameterObject.getString("units"); + parameterQueries.add(QueryBuilder.buildMatchQuery(path + ".type.units", units)); } - if (stringValue != null) { - parameterQueries.add(buildMatchQuery(path + ".stringValue", stringValue)); - } else if (lowerDate != null && upperDate != null) { - parameterQueries.add(buildLongRangeQuery(path + ".dateTimeValue", lowerDate, upperDate)); - } else if (lowerNumeric != null && upperNumeric != null) { - parameterQueries.add(buildRangeQuery(path + ".numericValue", lowerNumeric, upperNumeric)); + if (parameterObject.containsKey("stringValue")) { + String stringValue = parameterObject.getString("stringValue"); + parameterQueries.add(QueryBuilder.buildMatchQuery(path + ".stringValue", stringValue)); + } else if (parameterObject.containsKey("lowerDateValue") + && parameterObject.containsKey("upperDateValue")) { + Long lower = parseDate(parameterObject, "lowerDateValue", 0, Long.MIN_VALUE); + Long upper = parseDate(parameterObject, "upperDateValue", 59999, Long.MAX_VALUE); + parameterQueries.add(QueryBuilder.buildLongRangeQuery(path + ".dateTimeValue", lower, upper)); + } else if (parameterObject.containsKey("lowerNumericValue") + && parameterObject.containsKey("upperNumericValue")) { + JsonNumber lower = parameterObject.getJsonNumber("lowerNumericValue"); + JsonNumber upper = parameterObject.getJsonNumber("upperNumericValue"); + parameterQueries.add(QueryBuilder.buildRangeQuery(path + ".numericValue", lower, upper)); } - filterBuilder.add(buildNestedQuery(path, parameterQueries.toArray(new JsonObject[0]))); + filterBuilder.add(QueryBuilder.buildNestedQuery(path, parameterQueries.toArray(new JsonObject[0]))); } } - if (queryFields.contains("id")) { - filterBuilder.add(buildTermsQuery("id", query.getJsonArray("id"))); + if (query.containsKey("id")) { + filterBuilder.add(QueryBuilder.buildTermsQuery("id", query.getJsonArray("id"))); } JsonArray filterArray = filterBuilder.build(); @@ -800,8 +792,9 @@ public void initMappings() throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URI uri = new URIBuilder(server).setPath(basePath + "/" + index).build(); HttpPut httpPut = new HttpPut(uri); - String body = Json.createObjectBuilder() - .add("settings", indexSettings).add("mappings", buildMappings(index)).build().toString(); + JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); + bodyBuilder.add("settings", indexSettings).add("mappings", buildMappings(index)); + String body = bodyBuilder.build().toString(); logger.trace("Making call {} with body {}", uri, body); httpPut.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); try (CloseableHttpResponse response = httpclient.execute(httpPut)) { @@ -814,18 +807,52 @@ public void initMappings() throws IcatException { } public void initScripts() throws IcatException { - for (String scriptKey : scripts.keySet()) { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/_scripts/" + scriptKey).build(); - logger.trace("Making call {}", uri); - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(scripts.get(scriptKey), ContentType.APPLICATION_JSON)); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } catch (URISyntaxException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + for (Entry> entry : relations.entrySet()) { + String childName = entry.getKey(); + ParentRelation relation = entry.getValue().get(0); + switch (relation.relationType) { + case CHILD: + postScript("update_" + childName, buildChildScript(relation.fields, true)); + postScript("delete_" + childName, buildChildScript(relation.fields, false)); + break; + case NESTED_CHILD: + postScript("create_" + childName, buildCreateScript(childName)); + postScript("update_" + childName, buildNestedChildScript(childName, true)); + postScript("delete_" + childName, buildNestedChildScript(childName, false)); + break; + case NESTED_GRANDCHILD: + if (childName.equals("parametertype")) { + // Special case, as parametertype applies to investigationparameter, + // datasetparameter, datafileparameter + for (String index : indices) { + String updateScript = buildNestedGrandchildScript(index + "parameter", relation.fields, true); + String deleteScript = buildNestedGrandchildScript(index + "parameter", relation.fields, false); + postScript("update_" + index + childName, updateScript); + postScript("delete_" + index + childName, deleteScript); + break; + } + } else { + String updateScript = buildNestedGrandchildScript(relation.joinField, relation.fields, true); + String deleteScript = buildNestedGrandchildScript(relation.joinField, relation.fields, false); + postScript("update_" + childName, updateScript); + postScript("delete_" + childName, deleteScript); + break; + } + } + } + } + + private void postScript(String scriptKey, String body) throws IcatException { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URI uri = new URIBuilder(server).setPath(basePath + "/_scripts/" + scriptKey).build(); + logger.trace("Making call {}", uri); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); } + } catch (URISyntaxException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } } @@ -839,229 +866,70 @@ public void unlock(String entityName) throws IcatException { public void modify(String json) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - logger.debug("modify: {}", json); List updatesByQuery = new ArrayList<>(); Set investigationIds = new HashSet<>(); StringBuilder sb = new StringBuilder(); JsonReader jsonReader = Json.createReader(new StringReader(json)); JsonArray outerArray = jsonReader.readArray(); for (JsonObject operation : outerArray.getValuesAs(JsonObject.class)) { - String operationKey; - if (operation.containsKey("create")) { - operationKey = "create"; - } else if (operation.containsKey("update")) { - operationKey = "update"; - } else if (operation.containsKey("delete")) { - operationKey = "delete"; - } else { + Set operationKeys = operation.keySet(); + if (operationKeys.size() != 1) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Operation type should be 'create', 'update' or 'delete'"); + "Operation should only have one key, but it had " + operationKeys); } - JsonObject innerOperation = operation.getJsonObject(operationKey); + String operationKey = operationKeys.toArray(new String[1])[0]; + ModificationType modificationType = ModificationType.valueOf(operationKey.toUpperCase()); + JsonObject innerOperation = operation.getJsonObject(modificationType.toString().toLowerCase()); String index = innerOperation.getString("_index").toLowerCase(); String id = innerOperation.getString("_id"); - if (relationships.containsKey(index)) { - for (ParentRelationship relation : relationships.get(index)) { - if (operationKey.equals("create") && relation.parentName.equals(relation.joinName)) { - // Don't need it for children of a relative - // Do need it for 0:* relatives, in which case it's just appending to a list of nested objects - JsonObject document = innerOperation.getJsonObject("doc"); - // TODO if the document has type.unit (it's a parameter) then we need to convert units here. Advantage being we (might) have the value - if (document.containsKey("type.units")) { - // Need to rebuild the document... - JsonObjectBuilder rebuilder = Json.createObjectBuilder(); - for (String key : document.keySet()) { - rebuilder.add(key, document.get(key)); - } - String unitString = document.getString("type.units"); - try { - Unit unit = unitFormat.parse(unitString); - Unit systemUnit = unit.getSystemUnit(); - rebuilder.add("type.unitsSI", systemUnit.getName()); - if (document.containsKey("numericValue")) { - double numericValue = document.getJsonNumber("numericValue").doubleValue(); - UnitConverter converter = unit.getConverterToAny(systemUnit); - rebuilder.add("numericValueSI", converter.convert(numericValue)); - } - } catch (IncommensurableException | MeasurementParseException e) { - logger.error("Unable to convert 'type.units' of {} due to {}", unitString, e.getMessage()); - } - document = rebuilder.build(); - } - String parentId = document.getString(relation.parentName + ".id"); - JsonObjectBuilder innerBuilder = Json.createObjectBuilder().add("_id", - parentId).add("_index", relation.parentName); - JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("id", id).add("doc", Json.createArrayBuilder().add(document)); - String scriptId = (index.equals("parametertype")) ? "update_" + relation.parentName + index : "update_" + index; - JsonObjectBuilder scriptBuilder = Json.createObjectBuilder().add("id", scriptId) - .add("params", paramsBuilder); - sb.append(Json.createObjectBuilder().add("update", - innerBuilder).build().toString()).append("\n"); - sb.append(Json.createObjectBuilder() - .add("upsert", Json.createObjectBuilder().add(index, Json.createArrayBuilder().add(document))) - .add("script", scriptBuilder).build().toString()).append("\n"); - } else if (!operationKey.equals("delete")) { - JsonObject document = innerOperation.getJsonObject("doc"); - URI uri = new URIBuilder(server) - .setPath(basePath + "/" + relation.parentName + "/_update_by_query").build(); - HttpPost httpPost = new HttpPost(uri); - JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("id", id); - if (relation.fields.size() == 0) { - // TODO duplicated - if (document.containsKey("type.units")) { - // Need to rebuild the document... - JsonObjectBuilder rebuilder = Json.createObjectBuilder(); - for (String key : document.keySet()) { - rebuilder.add(key, document.get(key)); - } - String unitString = document.getString("type.units"); - try { - Unit unit = unitFormat.parse(unitString); - Unit systemUnit = unit.getSystemUnit(); - rebuilder.add("type.unitsSI", systemUnit.getName()); - if (document.containsKey("numericValue")) { - double numericValue = document.getJsonNumber("numericValue").doubleValue(); - UnitConverter converter = unit.getConverterToAny(systemUnit); - rebuilder.add("numericValueSI", converter.convert(numericValue)); - } - } catch (IncommensurableException | MeasurementParseException e) { - logger.error("Unable to convert 'type.units' of {} due to {}", unitString, e.getMessage()); - } - document = rebuilder.build(); - } - paramsBuilder.add("doc", Json.createArrayBuilder().add(document)); - } else { - UnitConverter converter = null; - for (String field : relation.fields) { - if (field.equals("type.unitsSI")) { - String unitString = document.getString("type.units"); - try { - Unit unit = unitFormat.parse(unitString); - Unit systemUnit = unit.getSystemUnit(); - converter = unit.getConverterToAny(systemUnit); - paramsBuilder.add(field, systemUnit.getName()); - } catch (IncommensurableException | MeasurementParseException e) { - logger.error("Unable to convert 'type.units' of {} due to {}", unitString, e.getMessage()); - } - } else if (field.equals("numericValueSI")) { - if (converter != null) { - // If we convert 1, we then have the necessary factor and can do multiplication by script... - paramsBuilder.add("conversionFactor", converter.convert(1.)); - } - } else { - paramsBuilder.add(field, document.get(field)); - } - } - } - String scriptId = (index.equals("parametertype")) ? "update_" + relation.parentName + index : "update_" + index; - JsonObjectBuilder scriptBuilder = Json.createObjectBuilder().add("id", scriptId) - .add("params", paramsBuilder); - JsonObject queryObject; - String idField = (relation.joinName.equals(relation.parentName)) ? "id" : relation.joinName + ".id"; - if (!Arrays.asList("parametertype", "sampletype", "user").contains(index)) { - queryObject = buildTermQuery(idField, id); - } else { - queryObject = buildNestedQuery(relation.joinName, buildTermQuery(idField, id)); - } - JsonObject bodyJson = Json.createObjectBuilder().add("query", queryObject) - .add("script", scriptBuilder).build(); - logger.trace("update script: {}", bodyJson.toString()); - httpPost.setEntity(new StringEntity(bodyJson.toString(), ContentType.APPLICATION_JSON)); - updatesByQuery.add(httpPost); - } else { - URI uri = new URIBuilder(server) - .setPath(basePath + "/" + relation.parentName + "/_update_by_query").build(); - HttpPost httpPost = new HttpPost(uri); - JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("id", id); - String scriptId = (index.equals("parametertype")) ? "delete_" + relation.parentName + index : "update_" + index; - JsonObjectBuilder scriptBuilder = Json.createObjectBuilder().add("id", scriptId) - .add("params", paramsBuilder); - JsonObject queryObject; - String idField = (relation.joinName.equals(relation.parentName)) ? "id" : relation.joinName + ".id"; - if (!Arrays.asList("parametertype", "sampletype", "user").contains(index)) { - queryObject = buildTermQuery(idField, id); - } else { - queryObject = buildNestedQuery(relation.joinName, buildTermQuery(idField, id)); - } - JsonObject bodyJson = Json.createObjectBuilder().add("query", queryObject) - .add("script", scriptBuilder).build(); - logger.trace("delete script: {}", bodyJson.toString()); - httpPost.setEntity(new StringEntity(bodyJson.toString(), ContentType.APPLICATION_JSON)); - updatesByQuery.add(httpPost); - } + JsonObject document = innerOperation.containsKey("doc") ? innerOperation.getJsonObject("doc") : null; + + if (relations.containsKey(index)) { + // Entities without an index will have one or more parent indices that need to + // be updated with their information + for (ParentRelation relation : relations.get(index)) { + modifyNestedEntity(sb, updatesByQuery, id, index, document, modificationType, relation); } } else { - JsonObjectBuilder innerBuilder = Json.createObjectBuilder().add("_id", new Long(id)).add("_index", - index); - if (operationKey.equals("delete")) { - sb.append(Json.createObjectBuilder().add(operationKey, innerBuilder).build().toString()).append("\n"); - } else { - JsonObject document = innerOperation.getJsonObject("doc"); - sb.append(Json.createObjectBuilder().add("update", innerBuilder).build().toString()) - .append("\n"); - sb.append(Json.createObjectBuilder().add("doc", document).add("doc_as_upsert", true).build() - .toString()).append("\n"); - - if (!index.equals("investigation")) { - // TODO Nightmare user lookup - investigationIds.add(document.getString("investigation.id")); - } - } + // Otherwise we are dealing with an indexed entity + modifyEntity(sb, investigationIds, id, index, document, modificationType); } } - logger.debug("bulk string: {}", sb.toString()); + if (sb.toString().length() > 0) { + // Perform simple bulk modifications URI uri = new URIBuilder(server).setPath(basePath + "/_bulk").build(); HttpPost httpPost = new HttpPost(uri); httpPost.setEntity(new StringEntity(sb.toString(), ContentType.APPLICATION_JSON)); + // logger.trace("Making call {} with body {}", uri, sb.toString()); try (CloseableHttpResponse response = httpclient.execute(httpPost)) { Rest.checkStatus(response, IcatExceptionType.INTERNAL); } } + if (updatesByQuery.size() > 0) { + // Ensure bulk changes are committed before performing updatesByQuery commit(); - } - logger.trace("updatesByQuery: {}", updatesByQuery.toString()); - for (HttpPost updateByQuery : updatesByQuery) { - try (CloseableHttpResponse response = httpclient.execute(updateByQuery)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); + for (HttpPost updateByQuery : updatesByQuery) { + // logger.trace("Making call {} with body {}", + // updateByQuery.getURI(), updateByQuery.getEntity().getContent().toString()); + try (CloseableHttpResponse response = httpclient.execute(updateByQuery)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } } } + if (investigationIds.size() > 0) { + // Ensure bulk changes are committed before checking for InvestigationUsers commit(); - } - logger.trace("investigationIds: {}", investigationIds.toString()); - for (String investigationId : investigationIds) { - URI uriGet = new URIBuilder(server).setPath(basePath + "/investigation/_source/" + investigationId) - .build(); - HttpGet httpGet = new HttpGet(uriGet); - try (CloseableHttpResponse responseGet = httpclient.execute(httpGet)) { - if (responseGet.getStatusLine().getStatusCode() == 200) { - // It's possible that the an investigation/investigationUser has not yet been indexed, in which case we cannot update the dataset/file with the user metadata - Rest.checkStatus(responseGet, IcatExceptionType.INTERNAL); - JsonObject responseObject = Json.createReader(responseGet.getEntity().getContent()).readObject(); - logger.trace("GET investigation {} response: {}", investigationId, responseObject); - if (responseObject.containsKey("investigationuser")) { - JsonArray jsonArray = responseObject.getJsonArray("investigationuser"); - for (String index : new String[] {"datafile", "dataset"}) { - URI uri = new URIBuilder(server).setPath(basePath + "/" + index + "/_update_by_query").build(); - HttpPost httpPost = new HttpPost(uri); - JsonObjectBuilder queryBuilder = Json.createObjectBuilder() - .add("term", Json.createObjectBuilder() - .add("investigation.id", investigationId)); - JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("doc", jsonArray); - JsonObject bodyJson = Json.createObjectBuilder() - .add("query", queryBuilder) - .add("script", Json.createObjectBuilder() - .add("id", "create_investigationuser").add("params", paramsBuilder)) - .build(); - httpPost.setEntity(new StringEntity(bodyJson.toString(), ContentType.APPLICATION_JSON)); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } + for (String investigationId : investigationIds) { + URI uriGet = new URIBuilder(server).setPath(basePath + "/investigation/_source/" + investigationId) + .build(); + HttpGet httpGet = new HttpGet(uriGet); + try (CloseableHttpResponse responseGet = httpclient.execute(httpGet)) { + if (responseGet.getStatusLine().getStatusCode() == 200) { + extractFromInvestigation(httpclient, investigationId, responseGet); } - } } } @@ -1070,6 +938,192 @@ public void modify(String json) throws IcatException { } } + private void extractFromInvestigation(CloseableHttpClient httpclient, String investigationId, + CloseableHttpResponse responseGet) + throws IOException, URISyntaxException, IcatException, ClientProtocolException { + JsonObject responseObject = Json.createReader(responseGet.getEntity().getContent()).readObject(); + if (responseObject.containsKey("investigationuser")) { + JsonArray jsonArray = responseObject.getJsonArray("investigationuser"); + for (String index : new String[] { "datafile", "dataset" }) { + URI uri = new URIBuilder(server).setPath(basePath + "/" + index + "/_update_by_query").build(); + HttpPost httpPost = new HttpPost(uri); + JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("doc", jsonArray); + JsonObjectBuilder scriptBuilder = Json.createObjectBuilder(); + scriptBuilder.add("id", "create_investigationuser").add("params", paramsBuilder); + JsonObject queryObject = QueryBuilder.buildTermQuery("investigation.id", investigationId); + JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); + String body = bodyBuilder.add("query", queryObject).add("script", scriptBuilder).build().toString(); + httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + // logger.trace("Making call {} with body {}", uri, body); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } + } + } + + private void modifyNestedEntity(StringBuilder sb, List updatesByQuery, String id, String index, + JsonObject document, ModificationType modificationType, ParentRelation relation) + throws URISyntaxException { + + switch (modificationType) { + case CREATE: + if (relation.parentName.equals(relation.joinField)) { + // If the target parent is the same as the joining field, we're appending the + // nested child to a list of objects which can be sent as a bulk update request + document = convertUnits(document); + createNestedEntity(sb, id, index, document, relation); + } else if (index.equals("sampletype")) { + // Otherwise, in most cases we don't need to update, as User and ParameterType + // cannot be null on their parent InvestigationUser or InvestigationParameter + // when that parent is created so the information is captured. However, since + // SampleType can be null upon creation of a Sample, need to account for the + // creation of a SampleType at a later date. + updateNestedEntityByQuery(updatesByQuery, id, index, document, relation, true); + } + break; + case UPDATE: + updateNestedEntityByQuery(updatesByQuery, id, index, document, relation, true); + break; + case DELETE: + updateNestedEntityByQuery(updatesByQuery, id, index, document, relation, false); + break; + } + } + + private static void createNestedEntity(StringBuilder sb, String id, String index, JsonObject document, + ParentRelation relation) { + + String parentId = document.getString(relation.parentName + ".id"); + JsonObjectBuilder innerBuilder = Json.createObjectBuilder() + .add("_id", parentId).add("_index", relation.parentName); + // For nested 0:* relationships, wrap single documents in an array + JsonArray docArray = Json.createArrayBuilder().add(document).build(); + JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("id", id).add("doc", docArray); + // ParameterType is a special case where script needs to include the parentName + String scriptId = index.equals("parametertype") ? "update_" + relation.parentName + index : "update_" + index; + JsonObjectBuilder scriptBuilder = Json.createObjectBuilder().add("id", scriptId).add("params", paramsBuilder); + JsonObjectBuilder upsertBuilder = Json.createObjectBuilder().add(index, docArray); + JsonObjectBuilder payloadBuilder = Json.createObjectBuilder() + .add("upsert", upsertBuilder).add("script", scriptBuilder); + sb.append(Json.createObjectBuilder().add("update", innerBuilder).build().toString()).append("\n"); + sb.append(payloadBuilder.build().toString()).append("\n"); + } + + private void updateNestedEntityByQuery(List updatesByQuery, String id, String index, JsonObject document, + ParentRelation relation, boolean update) throws URISyntaxException { + String path = basePath + "/" + relation.parentName + "/_update_by_query"; + URI uri = new URIBuilder(server).setPath(path).build(); + HttpPost httpPost = new HttpPost(uri); + + String scriptId = update ? "update_" : "delete_"; + scriptId += index.equals("parametertype") ? relation.parentName + index : index; + JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("id", id); + if (update) { + if (relation.fields == null) { + // Update affects all of the nested fields, so can add the entire document + document = convertUnits(document); + paramsBuilder.add("doc", Json.createArrayBuilder().add(document)); + } else { + // Need to update individual nested fields + paramsBuilder = convertUnits(paramsBuilder, document, relation.fields); + } + } + JsonObjectBuilder scriptBuilder = Json.createObjectBuilder().add("id", scriptId).add("params", paramsBuilder); + JsonObject queryObject; + String idField = relation.joinField.equals(relation.parentName) ? "id" : relation.joinField + ".id"; + if (!Arrays.asList("parametertype", "sampletype", "user").contains(index)) { // TODO generalise? + queryObject = QueryBuilder.buildTermQuery(idField, id); + } else { + queryObject = QueryBuilder.buildNestedQuery(relation.joinField, QueryBuilder.buildTermQuery(idField, id)); + } + JsonObject bodyJson = Json.createObjectBuilder().add("query", queryObject).add("script", scriptBuilder).build(); + logger.trace("updateByQuery script: {}", bodyJson.toString()); + httpPost.setEntity(new StringEntity(bodyJson.toString(), ContentType.APPLICATION_JSON)); + updatesByQuery.add(httpPost); + } + + private static JsonObject convertUnits(JsonObject document) { + if (!document.containsKey("type.units")) { + return document; + } + // Need to rebuild the document... + JsonObjectBuilder rebuilder = Json.createObjectBuilder(); + for (String key : document.keySet()) { + rebuilder.add(key, document.get(key)); + } + String unitString = document.getString("type.units"); + try { + Unit unit = unitFormat.parse(unitString); + Unit systemUnit = unit.getSystemUnit(); + rebuilder.add("type.unitsSI", systemUnit.getName()); + if (document.containsKey("numericValue")) { + double numericValue = document.getJsonNumber("numericValue").doubleValue(); + UnitConverter converter = unit.getConverterToAny(systemUnit); + rebuilder.add("numericValueSI", converter.convert(numericValue)); + } + } catch (IncommensurableException | MeasurementParseException e) { + logger.error("Unable to convert 'type.units' of {} due to {}", unitString, + e.getMessage()); + } + document = rebuilder.build(); + return document; + } + + private static JsonObjectBuilder convertUnits(JsonObjectBuilder paramsBuilder, JsonObject document, + Set fields) { + UnitConverter converter = null; + for (String field : fields) { + if (field.equals("type.unitsSI")) { + String unitString = document.getString("type.units"); + try { + Unit unit = unitFormat.parse(unitString); + Unit systemUnit = unit.getSystemUnit(); + converter = unit.getConverterToAny(systemUnit); + paramsBuilder.add(field, systemUnit.getName()); + } catch (IncommensurableException | MeasurementParseException e) { + logger.error("Unable to convert 'type.units' of {} due to {}", unitString, + e.getMessage()); + } + } else if (field.equals("numericValueSI")) { + if (converter != null) { + // If we convert 1, we then have the necessary factor and can do + // multiplication by script... + paramsBuilder.add("conversionFactor", converter.convert(1.)); + } + } else { + paramsBuilder.add(field, document.get(field)); + } + } + return paramsBuilder; + } + + private static void modifyEntity(StringBuilder sb, Set investigationIds, String id, String index, + JsonObject document, ModificationType modificationType) { + + JsonObject targetObject = Json.createObjectBuilder().add("_id", new Long(id)).add("_index", index).build(); + JsonObject update = Json.createObjectBuilder().add("update", targetObject).build(); + JsonObject docAsUpsert; + switch (modificationType) { + case CREATE: + docAsUpsert = Json.createObjectBuilder().add("doc", document).add("doc_as_upsert", true).build(); + sb.append(update.toString()).append("\n").append(docAsUpsert.toString()).append("\n"); + if (!index.equals("investigation")) { + // In principle a Dataset/Datafile could be created after InvestigationUser + // entities are attached to an Investigation, so need to check for those + investigationIds.add(document.getString("investigation.id")); + } + break; + case UPDATE: + docAsUpsert = Json.createObjectBuilder().add("doc", document).add("doc_as_upsert", true).build(); + sb.append(update.toString()).append("\n").append(docAsUpsert.toString()).append("\n"); + break; + case DELETE: + sb.append(Json.createObjectBuilder().add("delete", targetObject).build().toString()).append("\n"); + break; + } + } + /** * Legacy function for building a Query from individual arguments * diff --git a/src/main/java/org/icatproject/core/manager/search/QueryBuilder.java b/src/main/java/org/icatproject/core/manager/search/QueryBuilder.java new file mode 100644 index 00000000..c6531357 --- /dev/null +++ b/src/main/java/org/icatproject/core/manager/search/QueryBuilder.java @@ -0,0 +1,87 @@ +package org.icatproject.core.manager.search; + +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonArrayBuilder; +import javax.json.JsonNumber; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; + +public class QueryBuilder { + + private static JsonObject matchAllQuery = Json.createObjectBuilder().add("match_all", Json.createObjectBuilder()) + .build(); + + public static JsonObject buildMatchAllQuery() { + return matchAllQuery; + } + + public static JsonObject buildMatchQuery(String field, String value) { + JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("query", value).add("operator", "and"); + JsonObjectBuilder matchBuilder = Json.createObjectBuilder().add(field + ".keyword", fieldBuilder); + return Json.createObjectBuilder().add("match", matchBuilder).build(); + } + + public static JsonObject buildNestedQuery(String path, JsonObject... queryObjects) { + JsonObject builtQueries = null; + if (queryObjects.length == 0) { + builtQueries = matchAllQuery; + } else if (queryObjects.length == 1) { + builtQueries = queryObjects[0]; + } else { + JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); + for (JsonObject queryObject : queryObjects) { + filterBuilder.add(queryObject); + } + JsonObjectBuilder boolBuilder = Json.createObjectBuilder().add("filter", filterBuilder); + builtQueries = Json.createObjectBuilder().add("bool", boolBuilder).build(); + } + JsonObjectBuilder nestedBuilder = Json.createObjectBuilder().add("path", path).add("query", builtQueries); + return Json.createObjectBuilder().add("nested", nestedBuilder).build(); + } + + public static JsonObject buildStringQuery(String value, String... fields) { + JsonObjectBuilder queryStringBuilder = Json.createObjectBuilder().add("query", value); + if (fields.length > 0) { + JsonArrayBuilder fieldsBuilder = Json.createArrayBuilder(); + for (String field : fields) { + fieldsBuilder.add(field); + } + queryStringBuilder.add("fields", fieldsBuilder); + } + return Json.createObjectBuilder().add("query_string", queryStringBuilder).build(); + } + + public static JsonObject buildTermQuery(String field, String value) { + return Json.createObjectBuilder().add("term", Json.createObjectBuilder().add(field, value)).build(); + } + + public static JsonObject buildTermsQuery(String field, JsonArray values) { + return Json.createObjectBuilder().add("terms", Json.createObjectBuilder().add(field, values)).build(); + } + + public static JsonObject buildLongRangeQuery(String field, Long lowerValue, Long upperValue) { + JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("gte", lowerValue).add("lte", upperValue); + JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); + return Json.createObjectBuilder().add("range", rangeBuilder).build(); + } + + public static JsonObject buildRangeQuery(String field, JsonNumber lowerValue, JsonNumber upperValue) { + JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("gte", lowerValue).add("lte", upperValue); + JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); + return Json.createObjectBuilder().add("range", rangeBuilder).build(); + } + + public static JsonObject buildRangeFacet(String field, JsonArray ranges) { + JsonObjectBuilder rangeBuilder = Json.createObjectBuilder(); + rangeBuilder.add("field", field).add("keyed", true).add("ranges", ranges); + return Json.createObjectBuilder().add("range", rangeBuilder).build(); + } + + public static JsonObject buildStringFacet(String field, int maxLabels) { + JsonObjectBuilder termsBuilder = Json.createObjectBuilder(); + termsBuilder.add("field", field + ".keyword").add("size", maxLabels); + return Json.createObjectBuilder().add("terms", termsBuilder).build(); + } + +} diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 071dd244..7fee9136 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -277,22 +277,6 @@ private void modifyQueue(Queue queue) throws IcatException { } } - private void addDocuments(String entityName, String json) throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(uribase).setPath(SearchApi.basePath + "/addNow/" + entityName).build(); - HttpPost httpPost = new HttpPost(uri); - StringEntity input = new StringEntity(json); - input.setContentType(MediaType.APPLICATION_JSON); - httpPost.setEntity(input); - - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } catch (IOException | URISyntaxException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - @Before public void before() throws Exception { searchApi.clear(); From 3aa8d3a71ec6b6e087cbe733ea577787ea1da514 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Sat, 30 Apr 2022 02:06:38 +0100 Subject: [PATCH 19/51] SearchApi and unit conversion refactors #267 --- pom.xml | 28 +- .../org/icatproject/core/entity/Datafile.java | 2 +- .../core/entity/DatafileFormat.java | 2 +- .../core/entity/DatafileParameter.java | 2 +- .../org/icatproject/core/entity/Dataset.java | 2 +- .../core/entity/DatasetParameter.java | 2 +- .../icatproject/core/entity/DatasetType.java | 2 +- .../core/entity/EntityBaseBean.java | 4 +- .../org/icatproject/core/entity/Facility.java | 2 +- .../core/entity/Investigation.java | 2 +- .../core/entity/InvestigationParameter.java | 2 +- .../core/entity/InvestigationType.java | 2 +- .../core/entity/InvestigationUser.java | 2 +- .../icatproject/core/entity/Parameter.java | 2 +- .../core/entity/ParameterType.java | 2 +- .../org/icatproject/core/entity/Sample.java | 2 +- .../icatproject/core/entity/SampleType.java | 2 +- .../org/icatproject/core/entity/User.java | 2 +- .../core/manager/ElasticsearchApi.java | 558 ------- .../core/manager/ElasticsearchDocument.java | 273 ---- .../core/manager/EntityBeanManager.java | 4 + .../icatproject/core/manager/LuceneApi.java | 344 ---- .../core/manager/PropertyHandler.java | 16 +- .../org/icatproject/core/manager/Rest.java | 2 +- .../manager/{ => search}/FacetDimension.java | 10 +- .../core/manager/{ => search}/FacetLabel.java | 2 +- .../core/manager/search/LuceneApi.java | 204 +++ .../OpensearchApi.java} | 607 ++----- .../manager/{ => search}/ParameterPOJO.java | 16 +- .../core/manager/search/QueryBuilder.java | 4 + .../{ => search}/ScoredEntityBaseBean.java | 2 +- .../core/manager/search/SearchApi.java | 328 ++++ .../manager/{ => search}/SearchManager.java | 18 +- .../manager/{ => search}/SearchResult.java | 2 +- .../core/manager/search/URIParameter.java | 0 .../org/icatproject/exposed/ICATRest.java | 8 +- src/main/resources/run.properties | 1 + .../core/manager/TestElasticsearchApi.java | 608 ------- .../icatproject/core/manager/TestLucene.java | 1228 -------------- .../core/manager/TestSearchApi.java | 1409 ++++++++--------- .../org/icatproject/integration/TestRS.java | 334 ++-- src/test/scripts/prepare_test.py | 9 +- 42 files changed, 1559 insertions(+), 4492 deletions(-) delete mode 100644 src/main/java/org/icatproject/core/manager/ElasticsearchApi.java delete mode 100644 src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java delete mode 100644 src/main/java/org/icatproject/core/manager/LuceneApi.java rename src/main/java/org/icatproject/core/manager/{ => search}/FacetDimension.java (75%) rename src/main/java/org/icatproject/core/manager/{ => search}/FacetLabel.java (90%) create mode 100644 src/main/java/org/icatproject/core/manager/search/LuceneApi.java rename src/main/java/org/icatproject/core/manager/{SearchApi.java => search/OpensearchApi.java} (62%) rename src/main/java/org/icatproject/core/manager/{ => search}/ParameterPOJO.java (83%) rename src/main/java/org/icatproject/core/manager/{ => search}/ScoredEntityBaseBean.java (97%) create mode 100644 src/main/java/org/icatproject/core/manager/search/SearchApi.java rename src/main/java/org/icatproject/core/manager/{ => search}/SearchManager.java (97%) rename src/main/java/org/icatproject/core/manager/{ => search}/SearchResult.java (95%) create mode 100644 src/main/java/org/icatproject/core/manager/search/URIParameter.java delete mode 100644 src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java delete mode 100644 src/test/java/org/icatproject/core/manager/TestLucene.java diff --git a/pom.xml b/pom.xml index ccd4c7ad..057721ee 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ org.icatproject icat.utils - 4.16.1 + 4.16.2-SNAPSHOT @@ -133,29 +133,6 @@ 4.3.4 - - co.elastic.clients - elasticsearch-java - 8.1.0 - - - com.fasterxml.jackson.core - jackson-databind - 2.12.3 - - - - javax.measure - unit-api - 2.1.3 - - - - tech.units - indriya - 2.1.3 - - @@ -248,7 +225,6 @@ ${javax.net.ssl.trustStore} ${luceneUrl} - ${elasticsearchUrl} ${opensearchUrl} false @@ -269,7 +245,6 @@ ${serverUrl} ${searchEngine} ${luceneUrl} - ${elasticsearchUrl} ${opensearchUrl} @@ -351,7 +326,6 @@ ${serverUrl} ${searchEngine} ${luceneUrl} - ${elasticsearchUrl} ${opensearchUrl} diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index 53837e35..3905dc87 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -24,8 +24,8 @@ import org.icatproject.core.IcatException; import org.icatproject.core.manager.EntityInfoHandler; -import org.icatproject.core.manager.SearchApi; import org.icatproject.core.manager.EntityInfoHandler.Relationship; +import org.icatproject.core.manager.search.SearchApi; @Comment("A data file") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/entity/DatafileFormat.java b/src/main/java/org/icatproject/core/entity/DatafileFormat.java index 02ccd53e..fb2e530d 100644 --- a/src/main/java/org/icatproject/core/entity/DatafileFormat.java +++ b/src/main/java/org/icatproject/core/entity/DatafileFormat.java @@ -18,7 +18,7 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; -import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.search.SearchApi; @Comment("A data file format") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/entity/DatafileParameter.java b/src/main/java/org/icatproject/core/entity/DatafileParameter.java index 8f25207e..9328ceaa 100644 --- a/src/main/java/org/icatproject/core/entity/DatafileParameter.java +++ b/src/main/java/org/icatproject/core/entity/DatafileParameter.java @@ -13,8 +13,8 @@ import org.icatproject.core.IcatException; import org.icatproject.core.manager.EntityBeanManager.PersistMode; +import org.icatproject.core.manager.search.SearchApi; import org.icatproject.core.manager.GateKeeper; -import org.icatproject.core.manager.SearchApi; @Comment("A parameter associated with a data file") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index f582203f..7ebb8980 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -23,8 +23,8 @@ import org.icatproject.core.IcatException; import org.icatproject.core.manager.EntityInfoHandler; -import org.icatproject.core.manager.SearchApi; import org.icatproject.core.manager.EntityInfoHandler.Relationship; +import org.icatproject.core.manager.search.SearchApi; @Comment("A collection of data files and part of an investigation") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/entity/DatasetParameter.java b/src/main/java/org/icatproject/core/entity/DatasetParameter.java index e9fd22fd..f4bdcac0 100644 --- a/src/main/java/org/icatproject/core/entity/DatasetParameter.java +++ b/src/main/java/org/icatproject/core/entity/DatasetParameter.java @@ -13,8 +13,8 @@ import org.icatproject.core.IcatException; import org.icatproject.core.manager.EntityBeanManager.PersistMode; +import org.icatproject.core.manager.search.SearchApi; import org.icatproject.core.manager.GateKeeper; -import org.icatproject.core.manager.SearchApi; @Comment("A parameter associated with a data set") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/entity/DatasetType.java b/src/main/java/org/icatproject/core/entity/DatasetType.java index 23e66d75..656bd439 100644 --- a/src/main/java/org/icatproject/core/entity/DatasetType.java +++ b/src/main/java/org/icatproject/core/entity/DatasetType.java @@ -18,7 +18,7 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; -import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.search.SearchApi; @Comment("A type of data set") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/entity/EntityBaseBean.java b/src/main/java/org/icatproject/core/entity/EntityBaseBean.java index 23a343f9..cb7b68c9 100644 --- a/src/main/java/org/icatproject/core/entity/EntityBaseBean.java +++ b/src/main/java/org/icatproject/core/entity/EntityBaseBean.java @@ -30,9 +30,9 @@ import org.icatproject.core.manager.EntityBeanManager.PersistMode; import org.icatproject.core.manager.EntityInfoHandler; import org.icatproject.core.manager.EntityInfoHandler.Relationship; +import org.icatproject.core.manager.search.SearchApi; +import org.icatproject.core.manager.search.SearchManager; import org.icatproject.core.manager.GateKeeper; -import org.icatproject.core.manager.SearchApi; -import org.icatproject.core.manager.SearchManager; import org.icatproject.core.parser.IncludeClause.Step; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/icatproject/core/entity/Facility.java b/src/main/java/org/icatproject/core/entity/Facility.java index 8a1d7dd9..259c3c0b 100644 --- a/src/main/java/org/icatproject/core/entity/Facility.java +++ b/src/main/java/org/icatproject/core/entity/Facility.java @@ -15,7 +15,7 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; -import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.search.SearchApi; @Comment("An experimental facility") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index 984b6c7a..50ccd474 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -22,8 +22,8 @@ import org.icatproject.core.IcatException; import org.icatproject.core.manager.EntityInfoHandler; -import org.icatproject.core.manager.SearchApi; import org.icatproject.core.manager.EntityInfoHandler.Relationship; +import org.icatproject.core.manager.search.SearchApi; @Comment("An investigation or experiment") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/entity/InvestigationParameter.java b/src/main/java/org/icatproject/core/entity/InvestigationParameter.java index d985237a..e2153fec 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationParameter.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationParameter.java @@ -13,8 +13,8 @@ import org.icatproject.core.IcatException; import org.icatproject.core.manager.EntityBeanManager.PersistMode; +import org.icatproject.core.manager.search.SearchApi; import org.icatproject.core.manager.GateKeeper; -import org.icatproject.core.manager.SearchApi; @Comment("A parameter associated with an investigation") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/entity/InvestigationType.java b/src/main/java/org/icatproject/core/entity/InvestigationType.java index 861a9001..9166ac3a 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationType.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationType.java @@ -18,7 +18,7 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; -import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.search.SearchApi; @Comment("A type of investigation") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/entity/InvestigationUser.java b/src/main/java/org/icatproject/core/entity/InvestigationUser.java index fe43019b..55c203b0 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationUser.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationUser.java @@ -10,7 +10,7 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; -import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.search.SearchApi; @Comment("Many to many relationship between investigation and user. It is expected that this will show the association of " + "individual users with an investigation which might be derived from the proposal. It may also be used as the " diff --git a/src/main/java/org/icatproject/core/entity/Parameter.java b/src/main/java/org/icatproject/core/entity/Parameter.java index 0c7f7b41..feb877ab 100644 --- a/src/main/java/org/icatproject/core/entity/Parameter.java +++ b/src/main/java/org/icatproject/core/entity/Parameter.java @@ -16,8 +16,8 @@ import org.icatproject.core.IcatException; import org.icatproject.core.manager.EntityBeanManager.PersistMode; +import org.icatproject.core.manager.search.SearchApi; import org.icatproject.core.manager.GateKeeper; -import org.icatproject.core.manager.SearchApi; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/icatproject/core/entity/ParameterType.java b/src/main/java/org/icatproject/core/entity/ParameterType.java index abc884c7..9fe8ae96 100644 --- a/src/main/java/org/icatproject/core/entity/ParameterType.java +++ b/src/main/java/org/icatproject/core/entity/ParameterType.java @@ -18,7 +18,7 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; -import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.search.SearchApi; @Comment("A parameter type with unique name and units") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/entity/Sample.java b/src/main/java/org/icatproject/core/entity/Sample.java index 451bc4bf..3d6a816f 100644 --- a/src/main/java/org/icatproject/core/entity/Sample.java +++ b/src/main/java/org/icatproject/core/entity/Sample.java @@ -15,7 +15,7 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; -import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.search.SearchApi; @Comment("A sample to be used in an investigation") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/entity/SampleType.java b/src/main/java/org/icatproject/core/entity/SampleType.java index 786929d9..a8033086 100644 --- a/src/main/java/org/icatproject/core/entity/SampleType.java +++ b/src/main/java/org/icatproject/core/entity/SampleType.java @@ -18,7 +18,7 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; -import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.search.SearchApi; @Comment("A sample to be used in an investigation") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/entity/User.java b/src/main/java/org/icatproject/core/entity/User.java index 03e78088..8337ce72 100644 --- a/src/main/java/org/icatproject/core/entity/User.java +++ b/src/main/java/org/icatproject/core/entity/User.java @@ -15,7 +15,7 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; -import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.search.SearchApi; @Comment("A user of the facility") @SuppressWarnings("serial") diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java b/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java deleted file mode 100644 index f3dcfb88..00000000 --- a/src/main/java/org/icatproject/core/manager/ElasticsearchApi.java +++ /dev/null @@ -1,558 +0,0 @@ -package org.icatproject.core.manager; - -import java.io.IOException; -import java.io.StringReader; -import java.net.URL; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import javax.json.Json; -import javax.json.JsonArray; -import javax.json.JsonNumber; -import javax.json.JsonObject; -import javax.json.JsonReader; -import javax.json.JsonValue; -import javax.json.stream.JsonGenerator; - -import org.apache.http.HttpHost; -import org.elasticsearch.client.RestClient; -import org.icatproject.core.IcatException; -import org.icatproject.core.IcatException.IcatExceptionType; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.elasticsearch._types.ElasticsearchException; -import co.elastic.clients.elasticsearch._types.SortOrder; -import co.elastic.clients.elasticsearch._types.aggregations.StringTermsBucket; -import co.elastic.clients.elasticsearch._types.mapping.DynamicMapping; -import co.elastic.clients.elasticsearch._types.mapping.Property; -import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; -import co.elastic.clients.elasticsearch._types.query_dsl.Operator; -import co.elastic.clients.elasticsearch.core.BulkResponse; -import co.elastic.clients.elasticsearch.core.GetResponse; -import co.elastic.clients.elasticsearch.core.OpenPointInTimeResponse; -import co.elastic.clients.elasticsearch.core.SearchResponse; -import co.elastic.clients.elasticsearch.core.UpdateByQueryRequest; -import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; -import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; -import co.elastic.clients.elasticsearch.core.search.Hit; -import co.elastic.clients.json.JsonData; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; - -public class ElasticsearchApi extends SearchApi { - - private static ElasticsearchClient client; - private final static Map> INDEX_PROPERTIES = new HashMap<>(); - private final static Map TARGET_MAP = new HashMap<>(); - private final static Map UPDATE_BY_QUERY_MAP = new HashMap<>(); - - static { - // Add mappings from related entities to the searchable entity they should be - // flattened to - TARGET_MAP.put("sample", "investigation"); - TARGET_MAP.put("investigationparameter", "investigation"); - TARGET_MAP.put("datasetparameter", "dataset"); - TARGET_MAP.put("datafileparameter", "datafile"); - TARGET_MAP.put("investigationuser", "investigation"); - - // Child entities that should also update by query to other supported entities, - // not just their direct parent - UPDATE_BY_QUERY_MAP.put("investigationuser", "investigation"); - - // Mapping properties that are common to all TARGETS - Map commonProperties = new HashMap<>(); - // commonProperties.put("id", new Property.Builder().text(t -> t).build()); - commonProperties.put("id", new Property.Builder().long_(t -> t).build()); - commonProperties.put("text", new Property.Builder().text(t -> t).build()); - commonProperties.put("userName", new Property.Builder().text(t -> t).build()); - commonProperties.put("userFullName", new Property.Builder().text(t -> t).build()); - commonProperties.put("parameterName", new Property.Builder().text(t -> t).build()); - commonProperties.put("parameterUnits", new Property.Builder().text(t -> t).build()); - commonProperties.put("parameterStringValue", new Property.Builder().text(t -> t).build()); - commonProperties.put("parameterDateValue", new Property.Builder().date(d -> d).build()); - commonProperties.put("parameterNumericValue", new Property.Builder().double_(d -> d).build()); - - // Datafile - Map datafileProperties = new HashMap<>(); - datafileProperties.put("date", new Property.Builder().date(d -> d).build()); - datafileProperties.put("dataset", new Property.Builder().text(t -> t).build()); - datafileProperties.put("investigation", new Property.Builder().text(t -> t).build()); - INDEX_PROPERTIES.put("datafile", datafileProperties); - - // Dataset - Map datasetProperties = new HashMap<>(); - datasetProperties.put("startDate", new Property.Builder().date(d -> d).build()); - datasetProperties.put("endDate", new Property.Builder().date(d -> d).build()); - datasetProperties.put("investigation", new Property.Builder().text(t -> t).build()); - INDEX_PROPERTIES.put("dataset", datasetProperties); - - // Investigation - Map investigationProperties = new HashMap<>(); - investigationProperties.put("startDate", new Property.Builder().date(d -> d).build()); - investigationProperties.put("endDate", new Property.Builder().date(d -> d).build()); - investigationProperties.put("sampleName", new Property.Builder().text(t -> t).build()); - investigationProperties.put("sampleText", new Property.Builder().text(t -> t).build()); - INDEX_PROPERTIES.put("investigation", investigationProperties); - } - - // Maps Elasticsearch Points In Time (PIT) to the number of results to skip - // for successive searching - private final Map pitMap = new HashMap<>(); - - public ElasticsearchApi(List servers) throws IcatException { - super(null); - List hosts = new ArrayList(); - for (URL server : servers) { - hosts.add(new HttpHost(server.getHost(), server.getPort(), server.getProtocol())); - } - RestClient restClient = RestClient.builder(hosts.toArray(new HttpHost[1])).build(); - ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper()); - new JacksonJsonpMapper(); - client = new ElasticsearchClient(transport); - initMappings(); - } - - public void initMappings() throws IcatException { - try { - client.cluster().putSettings(s -> s.persistent("action.auto_create_index", JsonData.of(false))); - client.putScript(p -> p.id("update_user").script(s -> s - .lang("painless") - .source("if (ctx._source.userName == null) {ctx._source.userName = params['userName']}" - + "else {ctx._source.userName.addAll(params['userName'])}" - + "if (ctx._source.userFullName == null) {ctx._source.userFullName = params['userFullName']}" - + "else {ctx._source.userFullName.addAll(params['userFullName'])}"))); - client.putScript(p -> p.id("update_sample").script(s -> s - .lang("painless") - .source("if (ctx._source.sampleName == null) {ctx._source.sampleName = params['sampleName']}" - + "else {ctx._source.sampleName.addAll(params['sampleName'])}" - + "if (ctx._source.sampleText == null) {ctx._source.sampleText = params['sampleText']}" - + "else {ctx._source.sampleText.addAll(params['sampleText'])}"))); - client.putScript(p -> p.id("update_parameter").script(s -> s - .lang("painless") - .source("if (ctx._source.parameterName == null) {ctx._source.parameterName = params['parameterName']}" - + "else {ctx._source.parameterName.addAll(params['parameterName'])}" - + "if (ctx._source.parameterUnits == null) {ctx._source.parameterUnits = params['parameterUnits']}" - + "else {ctx._source.parameterUnits.addAll(params['parameterUnits'])}" - + "if (ctx._source.parameterStringValue == null) {ctx._source.parameterStringValue = params['parameterStringValue']}" - + "else {ctx._source.parameterStringValue.addAll(params['parameterStringValue'])}" - + "if (ctx._source.parameterDateValue == null) {ctx._source.parameterDateValue = params['parameterDateValue']}" - + "else {ctx._source.parameterDateValue.addAll(params['parameterDateValue'])}" - + "if (ctx._source.parameterNumericValue == null) {ctx._source.parameterNumericValue = params['parameterNumericValue']}" - + "else {ctx._source.parameterNumericValue.addAll(params['parameterNumericValue'])}"))); - - for (String index : INDEX_PROPERTIES.keySet()) { - client.indices() - .create(c -> c.index(index) - .mappings(m -> m.dynamic(DynamicMapping.False).properties(INDEX_PROPERTIES.get(index)))) - .acknowledged(); - } - // TODO consider both dynamic field names and nested fields - } catch (ElasticsearchException | IOException e) { - logger.warn("Unable to initialise mappings due to error {}, {}", e.getClass(), e.getMessage()); - } - } - - @Override - public void clear() throws IcatException { - try { - commit(); - client.deleteByQuery(d -> d.index("_all").query(q -> q.matchAll(m -> m))); - } catch (ElasticsearchException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - @Override - public void freeSearcher(String uid) - throws IcatException { - try { - pitMap.remove(uid); - client.closePointInTime(p -> p.id(uid)); - } catch (ElasticsearchException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - @Override - public void commit() throws IcatException { - try { - logger.debug("Manual commit of Elastic search called, refreshing indices"); - client.indices().refresh(); - } catch (ElasticsearchException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - @Override - public List facetSearch(String target, JsonObject facetQuery, Integer maxResults, int maxLabels) throws IcatException { - // TODO this should be generalised - return null; - // try { - // String index = "_all"; - // Set fields = facetQuery.keySet(); - // BoolQuery.Builder builder = new BoolQuery.Builder(); - // for (String field : fields) { - // // Only expecting a target and text field as part of the current facet - // // implementation - // if (field.equals("target")) { - // index = facetQuery.getString("target").toLowerCase(); - // } else if (field.equals("text")) { - // String text = facetQuery.getString("text"); - // builder.must(m -> m.queryString(q -> q.defaultField("text").query(text))); - // } - // } - // String indexFinal = index; - // SearchResponse response = client.search(s -> s - // .index(indexFinal) - // .size(maxResults) - // .query(q -> q.bool(builder.build())) - // .aggregations("samples", a -> a.terms(t -> t.field("sampleName").size(maxLabels))) - // .aggregations("parameters", a -> a.terms(t -> t.field("parameterName").size(maxLabels))), - // Object.class); - - // List sampleBuckets = response.aggregations().get("samples").sterms().buckets().array(); - // List parameterBuckets = response.aggregations().get("parameters").sterms().buckets() - // .array(); - // List facetDimensions = new ArrayList<>(); - // FacetDimension sampleDimension = new FacetDimension("sampleName"); - // FacetDimension parameterDimension = new FacetDimension("parameterName"); - // for (StringTermsBucket sampleBucket : sampleBuckets) { - // sampleDimension.getFacets().add(new FacetLabel(sampleBucket.key(), sampleBucket.docCount())); - // } - // for (StringTermsBucket parameterBucket : parameterBuckets) { - // parameterDimension.getFacets().add(new FacetLabel(parameterBucket.key(), parameterBucket.docCount())); - // } - // facetDimensions.add(sampleDimension); - // facetDimensions.add(parameterDimension); - // return facetDimensions; - // } catch (ElasticsearchException | IOException e) { - // throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - // } - } - - // @Override - // public SearchResult getResults(JsonObject query, int maxResults, String sort) - // throws IcatException { - // // TODO sort argument not supported - // try { - // String index; - // if (query.keySet().contains("target")) { - // index = query.getString("target").toLowerCase(); - // } else { - // index = query.getString("_all"); - // } - // OpenPointInTimeResponse pitResponse = client.openPointInTime(p -> p - // .index(index) - // .keepAlive(t -> t.time("1m"))); - // String pit = pitResponse.id(); - // pitMap.put(pit, 0); - // return getResults(pit, query, maxResults); - // } catch (ElasticsearchException | IOException e) { - // throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + - // e.getMessage()); - // } - // } - - // @Override - // public SearchResult getResults(String uid, JsonObject query, int maxResults) - // throws IcatException { - // try { - // logger.debug("getResults for query: {}", query.toString()); - // Set fields = query.keySet(); - // BoolQuery.Builder builder = new BoolQuery.Builder(); - // for (String field : fields) { - // if (field.equals("text")) { - // String text = query.getString("text"); - // builder.must(m -> m.queryString(q -> q.defaultField("text").query(text))); - // } else if (field.equals("lower")) { - // Long time = decodeTime(query.getString("lower")); - // builder.must(m -> m - // .bool(b -> b - // .should(s -> s - // .range(r -> r - // .field("date") - // .gte(JsonData.of(time)))) - // .should(s -> s - // .range(r -> r - // .field("startDate") - // .gte(JsonData.of(time)))))); - // } else if (field.equals("upper")) { - // Long time = decodeTime(query.getString("upper")); - // builder.must(m -> m - // .bool(b -> b - // .should(s -> s - // .range(r -> r - // .field("date") - // .lte(JsonData.of(time)))) - // .should(s -> s - // .range(r -> r - // .field("endDate") - // .lte(JsonData.of(time)))))); - // } else if (field.equals("user")) { - // String user = query.getString("user"); - // builder.filter(f -> f.match(t -> t - // .field("userName") - // .operator(Operator.And) - // .query(q -> q.stringValue(user)))); - // } else if (field.equals("userFullName")) { - // String userFullName = query.getString("userFullName"); - // builder.filter(f -> f.queryString(q -> - // q.defaultField("userFullName").query(userFullName))); - // } else if (field.equals("samples")) { - // JsonArray samples = query.getJsonArray("samples"); - // for (int i = 0; i < samples.size(); i++) { - // String sample = samples.getString(i); - // builder.filter( - // f -> f.queryString(q -> q.defaultField("sampleText").query(sample))); - // } - // } else if (field.equals("parameters")) { - // for (JsonValue parameterValue : query.getJsonArray("parameters")) { - // // TODO there are more things to support and consider here... e.g. parameters - // // with a numeric range not a numeric value - // BoolQuery.Builder parameterBuilder = new BoolQuery.Builder(); - // JsonObject parameterObject = (JsonObject) parameterValue; - // String name = parameterObject.getString("name", null); - // String units = parameterObject.getString("units", null); - // String stringValue = parameterObject.getString("stringValue", null); - // Long lowerDate = decodeTime(parameterObject.getString("lowerDateValue", - // null)); - // Long upperDate = decodeTime(parameterObject.getString("upperDateValue", - // null)); - // JsonNumber lowerNumeric = parameterObject.getJsonNumber("lowerNumericValue"); - // JsonNumber upperNumeric = parameterObject.getJsonNumber("upperNumericValue"); - // if (name != null) { - // parameterBuilder.must(m -> m.match(a -> - // a.field("parameterName").operator(Operator.And) - // .query(q -> q.stringValue(name)))); - // } - // if (units != null) { - // parameterBuilder.must(m -> m.match(a -> - // a.field("parameterUnits").operator(Operator.And) - // .query(q -> q.stringValue(units)))); - // } - // if (stringValue != null) { - // parameterBuilder.must(m -> m.match(a -> a.field("parameterStringValue") - // .operator(Operator.And).query(q -> q.stringValue(stringValue)))); - // } else if (lowerDate != null && upperDate != null) { - // parameterBuilder.must(m -> m.range(r -> r.field("parameterDateValue") - // .gte(JsonData.of(lowerDate)).lte(JsonData.of(upperDate)))); - // } else if (lowerNumeric != null && upperNumeric != null) { - // parameterBuilder.must(m -> m.range( - // r -> - // r.field("parameterNumericValue").gte(JsonData.of(lowerNumeric.doubleValue())) - // .lte(JsonData.of(upperNumeric.doubleValue())))); - // } - // builder.filter(f -> f.bool(b -> parameterBuilder)); - // } - // // TODO consider support for other fields (would require dynamic fields) - // } - // } - // Integer from = pitMap.get(uid); - // SearchResponse response = client.search(s -> s - // .size(maxResults) - // .pit(p -> p.id(uid).keepAlive(t -> t.time("1m"))) - // .query(q -> q.bool(builder.build())) - // // TODO check the ordering? - // .from(from) - // .sort(o -> o.score(c -> c.order(SortOrder.Desc))) - // .sort(o -> o.field(f -> f.field("id").order(SortOrder.Asc))), - // ElasticsearchDocument.class); - // SearchResult result = new SearchResult(); - // // result.setUid(uid); - // pitMap.put(uid, from + maxResults); - // List entities = result.getResults(); - // for (Hit hit : response.hits().hits()) { - // entities.add(new ScoredEntityBaseBean(Long.parseLong(hit.id()), - // hit.score().floatValue(), "")); - // } - // return result; - // } catch (ElasticsearchException | IOException | ParseException e) { - // throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + - // e.getMessage()); - // } - // } - - @Override - public void modify(String json) throws IcatException { - // Format should be [[, , ], ...] - logger.debug("modify: {}", json); - JsonReader jsonReader = Json.createReader(new StringReader(json)); - JsonArray outerArray = jsonReader.readArray(); - List operations = new ArrayList<>(); - List updateByQueryRequests = new ArrayList<>(); - Map investigationsMap = new HashMap<>(); - try { - for (JsonArray innerArray : outerArray.getValuesAs(JsonArray.class)) { - // Index should always be present, and be recognised - if (innerArray.isNull(0)) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Cannot modify a document without the target index"); - } - String index = innerArray.getString(0).toLowerCase(); - if (!INDEX_PROPERTIES.keySet().contains(index)) { - if (UPDATE_BY_QUERY_MAP.containsKey(index)) { - String parentIndex = TARGET_MAP.get(index); - if (!innerArray.isNull(2)) { - // Both creating and updates are handled by the index operation - // ElasticsearchDocument document = buildDocument(innerArray.getJsonArray(2), - // index, parentIndex); - logger.trace("{}, {}, {}", innerArray.getJsonArray(2).toString(), index, parentIndex); - ElasticsearchDocument document = new ElasticsearchDocument(innerArray.getJsonArray(2), - index, parentIndex); - logger.trace(document.toString()); - // String documentId = document.getId(); - Long documentId = document.getId(); - if (documentId == null) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Cannot index a document without an id"); - } - // TODO generalise, currently this assumes we have a user - ArrayList indices = new ArrayList<>(INDEX_PROPERTIES.keySet()); - indices.remove(parentIndex); - logger.debug("Adding update by query with: {}, {}, {}, {}", parentIndex, documentId, - document.getUserName(), document.getUserFullName()); - updateByQueryRequests.add(new UpdateByQueryRequest.Builder() - .index(indices) - .query(q -> q.term( - t -> t.field(parentIndex).value(v -> v.stringValue(documentId.toString())))) - .script(s -> s.stored(i -> i - .id("update_user") - .params("userName", JsonData.of(document.getUserName())) - .params("userFullName", JsonData.of(document.getUserFullName()))))); - } - } - if (TARGET_MAP.containsKey(index)) { - String parentIndex = TARGET_MAP.get(index); - if (innerArray.isNull(2)) { - // TODO we need to delete all the fields on the parent entitity that start with - // the sample name, this should be possible I think...? - // But we have can't because we provide the child ID, but never index this - // Either here, or for Lucene - logger.warn( - "Cannot delete document for related entity {}, instead update parent document {}", - index, parentIndex); - } else { - // Both creating and updates are handled by the index operation - ElasticsearchDocument document = new ElasticsearchDocument(innerArray.getJsonArray(2), - index, parentIndex); - Long documentId = document.getId(); - if (documentId == null) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Cannot index a document without an id"); - } - String scriptId = "update_"; - if (index.equals("investigationuser")) { - scriptId += "user"; - } else if (index.equals("investigationparameter")) { - scriptId += "parameter"; - } else if (index.equals("datasetparameter")) { - scriptId += "parameter"; - } else if (index.equals("datafileparameter")) { - scriptId += "parameter"; - } else if (index.equals("sample")) { - scriptId += "sample"; - } else { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Cannot map target {} to a parent index"); - } - String scriptIdFinal = scriptId; - operations.add(new BulkOperation.Builder().update(c -> c - .index(parentIndex) - .id(documentId.toString()) - .action(a -> a.upsert(document).script(s -> s.stored(t -> t - .id(scriptIdFinal) - .params("userName", JsonData.of(document.getUserName())) - .params("userFullName", JsonData.of(document.getUserFullName())) - .params("sampleName", JsonData.of(document.getSampleName())) - .params("sampleText", JsonData.of(document.getSampleText())) - .params("parameterName", JsonData.of(document.getParameterName())) - .params("parameterUnits", JsonData.of(document.getParameterUnits())) - .params("parameterStringValue", - JsonData.of(document.getParameterStringValue())) - .params("parameterDateValue", JsonData.of(document.getParameterDateValue())) - .params("parameterNumericValue", - JsonData.of(document.getParameterNumericValue())))))) - .build()); - } - } else { - logger.warn("Cannot index document for unsupported index {}", index); - continue; - } - } else { - if (innerArray.isNull(2)) { - // If the representation is null, delete the document with provided id - if (innerArray.isNull(1)) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Cannot modify document when both the id and object representing its fields are null"); - } - String id = String.valueOf(innerArray.getInt(1)); - operations.add(new BulkOperation.Builder().delete(c -> c.index(index).id(id)).build()); - } else { - ElasticsearchDocument document = new ElasticsearchDocument(innerArray.getJsonArray(2)); - // Get information on user, which might be on the parent investigation - String investigationId = document.getInvestigation(); - logger.debug("Looking for investigations with id: {} for index: {}", investigationId, index); - if (investigationId != null) { - ElasticsearchDocument investigation = investigationsMap.get(investigationId); - if (investigation == null) { - GetResponse getResponse = client.get( - g -> g.index("investigation").id(investigationId), ElasticsearchDocument.class); - if (getResponse.found()) { - investigation = getResponse.source(); - } else { - investigation = new ElasticsearchDocument(); - } - investigationsMap.put(investigationId, investigation); - } - document.getUserName().addAll(investigation.getUserName()); - document.getUserFullName().addAll(investigation.getUserFullName()); - } - - // TODO REVERT? - // Both creating and updates are handled by the index operation - String id; - if (innerArray.isNull(1)) { - // If we weren't given an id, try and get one from the document - // Avoid using a generated id, as this prevents us updating the document later - Long documentId = document.getId(); - if (documentId == null) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Cannot index a document without an id"); - } - id = documentId.toString(); - } else { - id = String.valueOf(innerArray.getInt(1)); - } - operations.add(new BulkOperation.Builder().update(c -> c - .index(index) - .id(id) - .action(a -> a.doc(document).docAsUpsert(true))).build()); - } - } - } - BulkResponse bulkResponse = client.bulk(c -> c.operations(operations)); - if (bulkResponse.errors()) { - // Throw an Exception for the first error we had in the list of operations - for (BulkResponseItem responseItem : bulkResponse.items()) { - if (responseItem.error() != null) { - throw new IcatException(IcatExceptionType.INTERNAL, responseItem.error().reason()); - } - } - } - // TODO this isn't bulked - a single failure will not invalidate the rest... - for (UpdateByQueryRequest.Builder request : updateByQueryRequests) { - commit(); - client.updateByQuery(request.build()); - } - } catch (ElasticsearchException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - -} diff --git a/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java b/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java deleted file mode 100644 index 3ae9c81e..00000000 --- a/src/main/java/org/icatproject/core/manager/ElasticsearchDocument.java +++ /dev/null @@ -1,273 +0,0 @@ -package org.icatproject.core.manager; - -import java.text.ParseException; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map.Entry; - -import javax.json.JsonArray; -import javax.json.JsonObject; -import javax.json.JsonValue; -import javax.json.JsonValue.ValueType; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; - -import org.icatproject.core.IcatException; -import org.icatproject.core.IcatException.IcatExceptionType; - -/** - * This class is required in order to map to and from JSON for Elasticsearch - * client functions - */ -@JsonInclude(Include.NON_EMPTY) -public class ElasticsearchDocument { - - private Long id; - private String investigation; - private String dataset; - private String text; - private Date date; - private Date startDate; - private Date endDate; - private List userName = new ArrayList<>(); - private List userFullName = new ArrayList<>(); - private List sampleName = new ArrayList<>(); - private List sampleText = new ArrayList<>(); - private List parameterName = new ArrayList<>(); - private List parameterUnits = new ArrayList<>(); - private List parameterStringValue = new ArrayList<>(); - private List parameterDateValue = new ArrayList<>(); - private List parameterNumericValue = new ArrayList<>(); - - public ElasticsearchDocument() { - } - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public ElasticsearchDocument(JsonArray jsonArray) throws IcatException { - try { - for (JsonValue fieldValue : jsonArray) { - JsonObject fieldObject = (JsonObject) fieldValue; - for (Entry fieldEntry : fieldObject.entrySet()) { - // TODO this is hideous, replace with something more dynamic? or at least a - // switch? - if (fieldEntry.getKey().equals("id")) { - if (fieldEntry.getValue().getValueType().equals(ValueType.STRING)) { - id = Long.valueOf(fieldObject.getString("id")); - } else if (fieldEntry.getValue().getValueType().equals(ValueType.NUMBER)) { - id = fieldObject.getJsonNumber("id").longValueExact(); - } - } else if (fieldEntry.getKey().equals("investigation")) { - if (fieldEntry.getValue().getValueType().equals(ValueType.STRING)) { - investigation = fieldObject.getString("investigation"); - } else if (fieldEntry.getValue().getValueType().equals(ValueType.NUMBER)) { - investigation = String.valueOf(fieldObject.getInt("investigation")); - } - } else if (fieldEntry.getKey().equals("dataset")) { - if (fieldEntry.getValue().getValueType().equals(ValueType.STRING)) { - dataset = fieldObject.getString("dataset"); - } else if (fieldEntry.getValue().getValueType().equals(ValueType.NUMBER)) { - dataset = String.valueOf(fieldObject.getInt("dataset")); - } - } else if (fieldEntry.getKey().equals("text")) { - text = fieldObject.getString("text"); - } else if (fieldEntry.getKey().equals("date")) { - date = SearchApi.decodeDate(fieldObject.getString("date")); - } else if (fieldEntry.getKey().equals("startDate")) { - startDate = SearchApi.decodeDate(fieldObject.getString("startDate")); - } else if (fieldEntry.getKey().equals("endDate")) { - endDate = SearchApi.decodeDate(fieldObject.getString("endDate")); - } else if (fieldEntry.getKey().equals("user.name")) { - userName.add(fieldObject.getString("user.name")); - } else if (fieldEntry.getKey().equals("user.fullName")) { - userFullName.add(fieldObject.getString("user.fullName")); - } else if (fieldEntry.getKey().equals("sample.name")) { - sampleName.add(fieldObject.getString("sample.name")); - } else if (fieldEntry.getKey().equals("sample.text")) { - sampleText.add(fieldObject.getString("sample.text")); - } else if (fieldEntry.getKey().equals("parameter.name")) { - parameterName.add(fieldObject.getString("parameter.name")); - } else if (fieldEntry.getKey().equals("parameter.units")) { - parameterUnits.add(fieldObject.getString("parameter.units")); - } else if (fieldEntry.getKey().equals("parameter.stringValue")) { - parameterStringValue.add(fieldObject.getString("parameter.stringValue")); - } else if (fieldEntry.getKey().equals("parameter.dateValue")) { - parameterDateValue.add(SearchApi.decodeDate(fieldObject.getString("parameter.dateValue"))); - } else if (fieldEntry.getKey().equals("parameter.numericValue")) { - parameterNumericValue - .add(fieldObject.getJsonNumber("parameter.numericValue").doubleValue()); - } - } - } - } catch (ParseException e) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, e.getClass() + " " + e.getMessage()); - } - } - - public ElasticsearchDocument(JsonArray jsonArray, String index, String parentIndex) throws IcatException { - try { - for (JsonValue fieldValue : jsonArray) { - JsonObject fieldObject = (JsonObject) fieldValue; - for (Entry fieldEntry : fieldObject.entrySet()) { - if (fieldEntry.getKey().equals(parentIndex)) { - if (fieldEntry.getValue().getValueType().equals(ValueType.STRING)) { - id = Long.valueOf(fieldObject.getString(parentIndex)); - } else if (fieldEntry.getValue().getValueType().equals(ValueType.NUMBER)) { - id = fieldObject.getJsonNumber(parentIndex).longValueExact(); - } - } else if (fieldEntry.getKey().equals("userName")) { - userName.add(fieldObject.getString("userName")); - } else if (fieldEntry.getKey().equals("userFullName")) { - userFullName.add(fieldObject.getString("userFullName")); - } else if (fieldEntry.getKey().equals("sampleName")) { - sampleName.add(fieldObject.getString("sampleName")); - } else if (fieldEntry.getKey().equals("sampleText")) { - sampleText.add(fieldObject.getString("sampleText")); - } else if (fieldEntry.getKey().equals("parameterName")) { - parameterName.add(fieldObject.getString("parameterName")); - } else if (fieldEntry.getKey().equals("parameterUnits")) { - parameterUnits.add(fieldObject.getString("parameterUnits")); - } else if (fieldEntry.getKey().equals("parameterStringValue")) { - parameterStringValue.add(fieldObject.getString("parameterStringValue")); - } else if (fieldEntry.getKey().equals("parameterDateValue")) { - parameterDateValue.add(SearchApi.decodeDate(fieldObject.getString("parameterDateValue"))); - } else if (fieldEntry.getKey().equals("parameterNumericValue")) { - parameterNumericValue - .add(fieldObject.getJsonNumber("parameterNumericValue").doubleValue()); - } - } - } - } catch (ParseException e) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, e.getClass() + " " + e.getMessage()); - } - } - - public String getDataset() { - return dataset; - } - - public void setDataset(String dataset) { - this.dataset = dataset; - } - - public String getInvestigation() { - return investigation; - } - - public void setInvestigation(String investigation) { - this.investigation = investigation; - } - - public List getParameterNumericValue() { - return parameterNumericValue; - } - - public void setParameterNumericValue(List parameterNumericValue) { - this.parameterNumericValue = parameterNumericValue; - } - - public List getParameterDateValue() { - return parameterDateValue; - } - - public void setParameterDateValue(List parameterDateValue) { - this.parameterDateValue = parameterDateValue; - } - - public List getParameterStringValue() { - return parameterStringValue; - } - - public void setParameterStringValue(List parameterStringValue) { - this.parameterStringValue = parameterStringValue; - } - - public List getParameterUnits() { - return parameterUnits; - } - - public void setParameterUnits(List parameterUnits) { - this.parameterUnits = parameterUnits; - } - - public List getParameterName() { - return parameterName; - } - - public void setParameterName(List parameterName) { - this.parameterName = parameterName; - } - - public List getSampleText() { - return sampleText; - } - - public void setSampleText(List sampleText) { - this.sampleText = sampleText; - } - - public List getSampleName() { - return sampleName; - } - - public void setSampleName(List sampleName) { - this.sampleName = sampleName; - } - - public List getUserFullName() { - return userFullName; - } - - public void setUserFullName(List userFullName) { - this.userFullName = userFullName; - } - - public List getUserName() { - return userName; - } - - public void setUserName(List userName) { - this.userName = userName; - } - - public Date getEndDate() { - return endDate; - } - - public void setEndDate(Date endDate) { - this.endDate = endDate; - } - - public Date getStartDate() { - return startDate; - } - - public void setStartDate(Date startDate) { - this.startDate = startDate; - } - - public Date getDate() { - return date; - } - - public void setDate(Date date) { - this.date = date; - } - - public String getText() { - return text; - } - - public void setText(String text) { - this.text = text; - } - -} diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 0690e002..ab07ba8a 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -68,6 +68,10 @@ import org.icatproject.core.manager.EntityInfoHandler.Relationship; import org.icatproject.core.manager.PropertyHandler.CallType; import org.icatproject.core.manager.PropertyHandler.Operation; +import org.icatproject.core.manager.search.FacetDimension; +import org.icatproject.core.manager.search.ScoredEntityBaseBean; +import org.icatproject.core.manager.search.SearchManager; +import org.icatproject.core.manager.search.SearchResult; import org.icatproject.core.oldparser.OldGetQuery; import org.icatproject.core.oldparser.OldInput; import org.icatproject.core.oldparser.OldLexerException; diff --git a/src/main/java/org/icatproject/core/manager/LuceneApi.java b/src/main/java/org/icatproject/core/manager/LuceneApi.java deleted file mode 100644 index 2fd13dbc..00000000 --- a/src/main/java/org/icatproject/core/manager/LuceneApi.java +++ /dev/null @@ -1,344 +0,0 @@ -package org.icatproject.core.manager; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutorService; - -import javax.json.Json; -import javax.json.JsonArray; -import javax.json.JsonArrayBuilder; -import javax.json.JsonObject; -import javax.json.JsonObjectBuilder; -import javax.json.JsonReader; -import javax.json.JsonValue; -import javax.json.stream.JsonGenerator; -import javax.json.stream.JsonParser; -import javax.json.stream.JsonParser.Event; -import javax.persistence.EntityManager; -import javax.ws.rs.core.MediaType; - -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.icatproject.core.IcatException; -import org.icatproject.core.IcatException.IcatExceptionType; -import org.icatproject.core.entity.EntityBaseBean; - -public class LuceneApi extends SearchApi { - private enum ParserState { - None, Results, Dimensions, Labels - } - - static String basePath = "/icat.lucene"; - - /* - * Serves as a record of target entities where search is supported, and the - * relevant path to the search endpoint - */ - private static Map targetPaths = new HashMap<>(); - static { - targetPaths.put("Investigation", "investigations"); - targetPaths.put("Dataset", "datasets"); - targetPaths.put("Datafile", "datafiles"); - } - - private String getTargetPath(JsonObject query) throws IcatException { - if (!query.containsKey("target")) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "'target' must be present in query for LuceneApi, but it was " + query.toString()); - } - String path = targetPaths.get(query.getString("target")); - if (path == null) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "'target' must be one of " + targetPaths.keySet() + ", but it was " + query.toString()); - } - return path; - } - - public LuceneApi(URI server) { - super(server); - } - - public void addNow(String entityName, List ids, EntityManager manager, - Class klass, ExecutorService getBeanDocExecutor) - throws IcatException, IOException, URISyntaxException { - URI uri = new URIBuilder(server).setPath(basePath + "/addNow/" + entityName) - .build(); - - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - HttpPost httpPost = new HttpPost(uri); - PipedOutputStream beanDocs = new PipedOutputStream(); - httpPost.setEntity(new InputStreamEntity(new PipedInputStream(beanDocs))); - getBeanDocExecutor.submit(() -> { - try (JsonGenerator gen = Json.createGenerator(beanDocs)) { - gen.writeStartArray(); - for (Long id : ids) { - EntityBaseBean bean = (EntityBaseBean) manager.find(klass, id); - if (bean != null) { - gen.writeStartObject(); - bean.getDoc(gen); - gen.writeEnd(); - } - } - gen.writeEnd(); - return null; - } catch (Exception e) { - logger.error("About to throw internal exception because of", e); - throw new IcatException(IcatExceptionType.INTERNAL, e.getMessage()); - } finally { - manager.close(); - } - }); - - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } - } - - @Override - public JsonObject buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { - JsonObjectBuilder builder = Json.createObjectBuilder(); - builder.add("doc", lastBean.getEngineDocId()); - builder.add("shardIndex", -1); - if (!Float.isNaN(lastBean.getScore())) { - builder.add("score", lastBean.getScore()); - } - if (sort != null && !sort.equals("")) { - try (JsonReader reader = Json.createReader(new ByteArrayInputStream(sort.getBytes()))) { - JsonObject object = reader.readObject(); - JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); - for (String key : object.keySet()) { - if (!lastBean.getSource().keySet().contains(key)) { - throw new IcatException(IcatExceptionType.INTERNAL, - "Cannot build searchAfter document from source as sorted field " + key + " missing."); - } - JsonValue value = lastBean.getSource().get(key); - arrayBuilder.add(value); - } - builder.add("fields", arrayBuilder); - } - } - return builder.build(); - } - - @Override - public void clear() throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/clear").build(); - HttpPost httpPost = new HttpPost(uri); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } catch (IOException | URISyntaxException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - - } - - @Override - public void commit() throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/commit").build(); - logger.trace("Making call {}", uri); - HttpPost httpPost = new HttpPost(uri); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } catch (URISyntaxException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - @Override - public List facetSearch(String target, JsonObject facetQuery, Integer maxResults, int maxLabels) throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/" + target + "/facet") - .setParameter("maxResults", Integer.toString(maxResults)) - .setParameter("maxLabels", Integer.toString(maxLabels)).build(); - logger.trace("Making call {}", uri); - return getFacets(uri, httpclient, facetQuery.toString(), target); - } catch (IOException | URISyntaxException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - private List getFacets(URI uri, CloseableHttpClient httpclient, String facetQueryString, String target) - throws IcatException { - logger.debug(facetQueryString); - try { - StringEntity input = new StringEntity(facetQueryString); - input.setContentType(MediaType.APPLICATION_JSON); - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(input); - - List facetDimensions = new ArrayList<>(); - ParserState state = ParserState.None; - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - try (JsonParser p = Json.createParser(response.getEntity().getContent())) { - String key = null; - while (p.hasNext()) { - // Get next event in the stream - Event e = p.next(); - if (e.equals(Event.KEY_NAME)) { - // The key name will indicate the content to expect next, and is expected to be - // one of - // "dimensions", a specific dimension, or a label within that dimension. - key = p.getString(); - } else if (e == Event.START_OBJECT) { - if (state == ParserState.None && key != null && key.equals("dimensions")) { - state = ParserState.Dimensions; - } else if (state == ParserState.Dimensions) { - facetDimensions.add(new FacetDimension(target, key)); - state = ParserState.Labels; - } - } else if (e == (Event.END_OBJECT)) { - if (state == ParserState.Labels) { - // We may have multiple dimensions, so change state so we can read the next one - state = ParserState.Dimensions; - } else if (state == ParserState.Dimensions) { - state = ParserState.None; - } - } else if (state == ParserState.Labels) { - FacetDimension currentFacets = facetDimensions.get(facetDimensions.size() - 1); - currentFacets.getFacets().add(new FacetLabel(key, p.getLong())); - } - } - } - } - return facetDimensions; - } catch (IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - @Override - public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer blockSize, String sort, - List fields) throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - String indexPath = getTargetPath(query); - URIBuilder uriBuilder = new URIBuilder(server).setPath(basePath + "/" + indexPath) - .setParameter("maxResults", Integer.toString(blockSize)); - if (searchAfter != null) { - uriBuilder.setParameter("search_after", searchAfter.toString()); - } - if (sort != null) { - uriBuilder.setParameter("sort", sort); - } - JsonObjectBuilder objectBuilder = Json.createObjectBuilder(); - objectBuilder.add("query", query); - if (fields != null && fields.size() > 0) { - JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); - fields.forEach((field) -> arrayBuilder.add(field)); - objectBuilder.add("fields", arrayBuilder.build()); - } - String queryString = objectBuilder.build().toString(); - URI uri = uriBuilder.build(); - logger.trace("Making call {} with queryString {}", uri, queryString); - return getResults(uri, httpclient, queryString); - - } catch (IOException | URISyntaxException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - private SearchResult getResults(URI uri, CloseableHttpClient httpclient, String queryString) - throws IcatException { - logger.debug(queryString); - try { - StringEntity input = new StringEntity(queryString); - input.setContentType(MediaType.APPLICATION_JSON); - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(input); - - SearchResult lsr = new SearchResult(); - List results = lsr.getResults(); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - try (JsonReader reader = Json.createReader(response.getEntity().getContent())) { - JsonObject responseObject = reader.readObject(); - List resultsArray = responseObject.getJsonArray("results") - .getValuesAs(JsonObject.class); - for (JsonObject resultObject : resultsArray) { - int luceneDocId = resultObject.getInt("_id"); - Float score = Float.NaN; - if (resultObject.keySet().contains("_score")) { - score = resultObject.getJsonNumber("_score").bigDecimalValue().floatValue(); - } - JsonObject source = resultObject.getJsonObject("_source"); - results.add(new ScoredEntityBaseBean(luceneDocId, score, source)); - } - if (responseObject.containsKey("search_after")) { - lsr.setSearchAfter(responseObject.getJsonObject("search_after")); - } - } - } - return lsr; - } catch (IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - @Override - public void lock(String entityName) throws IcatException { - try { - URI uri = new URIBuilder(server).setPath(basePath + "/lock/" + entityName).build(); - logger.trace("Making call {}", uri); - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - HttpPost httpPost = new HttpPost(uri); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } - } catch (URISyntaxException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - @Override - public void unlock(String entityName) throws IcatException { - try { - URI uri = new URIBuilder(server).setPath(basePath + "/unlock/" + entityName).build(); - logger.trace("Making call {}", uri); - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - HttpPost httpPost = new HttpPost(uri); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } - } catch (URISyntaxException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - @Override - public void modify(String json) throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/modify").build(); - HttpPost httpPost = new HttpPost(uri); - StringEntity input = new StringEntity(json); - input.setContentType(MediaType.APPLICATION_JSON); - httpPost.setEntity(input); - - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } catch (IOException | URISyntaxException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - -} diff --git a/src/main/java/org/icatproject/core/manager/PropertyHandler.java b/src/main/java/org/icatproject/core/manager/PropertyHandler.java index e18cdef9..276a28d6 100644 --- a/src/main/java/org/icatproject/core/manager/PropertyHandler.java +++ b/src/main/java/org/icatproject/core/manager/PropertyHandler.java @@ -308,6 +308,7 @@ public int getLifetimeMinutes() { private int searchPopulateBlockSize; private Path searchDirectory; private long searchBacklogHandlerIntervalMillis; + private String unitAliasOptions; private Map cluster = new HashMap<>(); private long searchEnqueuedRequestIntervalMillis; @@ -478,13 +479,15 @@ private void init() { } } - if ((searchEngine.equals(SearchEngine.LUCENE) || searchEngine.equals(SearchEngine.OPENSEARCH)) - && searchUrls.size() != 1) { + // In principle, clustered engines like OPENSEARCH or ELASTICSEARCH should + // support multiple urls for the nodes in the cluster, however this is not yet + // implemented + if (searchUrls.size() != 1) { String msg = "Exactly one value for search.urls must be provided when using " + searchEngine; throw new IllegalStateException(msg); - } else if (searchUrls.size() == 0) { - String msg = "At least one value for search.urls must be provided"; - throw new IllegalStateException(msg); + // } else if (searchUrls.size() == 0) { + // String msg = "At least one value for search.urls must be provided"; + // throw new IllegalStateException(msg); } formattedProps.add("search.urls" + " " + searchUrls.toString()); logger.info("Using {} as search engine with url(s) {}", searchEngine, searchUrls); @@ -511,6 +514,9 @@ private void init() { logger.info("'search.engine' entry not present so no free text search available"); } + + unitAliasOptions = props.getString("units", ""); + /* * maxEntities, importCacheSize, exportCacheSize, maxIdsInQuery, key */ diff --git a/src/main/java/org/icatproject/core/manager/Rest.java b/src/main/java/org/icatproject/core/manager/Rest.java index c3b0fcfe..9aff5f48 100644 --- a/src/main/java/org/icatproject/core/manager/Rest.java +++ b/src/main/java/org/icatproject/core/manager/Rest.java @@ -18,7 +18,7 @@ public class Rest { - static void checkStatus(HttpResponse response, IcatExceptionType et) throws IcatException { + public static void checkStatus(HttpResponse response, IcatExceptionType et) throws IcatException { StatusLine status = response.getStatusLine(); if (status == null) { throw new IcatException(IcatExceptionType.INTERNAL, "Status line in response is empty"); diff --git a/src/main/java/org/icatproject/core/manager/FacetDimension.java b/src/main/java/org/icatproject/core/manager/search/FacetDimension.java similarity index 75% rename from src/main/java/org/icatproject/core/manager/FacetDimension.java rename to src/main/java/org/icatproject/core/manager/search/FacetDimension.java index bfb85e0b..870d89b4 100644 --- a/src/main/java/org/icatproject/core/manager/FacetDimension.java +++ b/src/main/java/org/icatproject/core/manager/search/FacetDimension.java @@ -1,4 +1,4 @@ -package org.icatproject.core.manager; +package org.icatproject.core.manager.search; import java.util.ArrayList; import java.util.List; @@ -20,6 +20,14 @@ public FacetDimension(String target, String dimension) { this.dimension = dimension; } + public FacetDimension(String target, String dimension, FacetLabel... labels) { + this.target = target; + this.dimension = dimension; + for (FacetLabel label : labels) { + facets.add(label); + } + } + public List getFacets() { return facets; } diff --git a/src/main/java/org/icatproject/core/manager/FacetLabel.java b/src/main/java/org/icatproject/core/manager/search/FacetLabel.java similarity index 90% rename from src/main/java/org/icatproject/core/manager/FacetLabel.java rename to src/main/java/org/icatproject/core/manager/search/FacetLabel.java index 60b8e389..4a85a5d9 100644 --- a/src/main/java/org/icatproject/core/manager/FacetLabel.java +++ b/src/main/java/org/icatproject/core/manager/search/FacetLabel.java @@ -1,4 +1,4 @@ -package org.icatproject.core.manager; +package org.icatproject.core.manager.search; /** * Holds information for a single label value pair. diff --git a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java new file mode 100644 index 00000000..a422abad --- /dev/null +++ b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java @@ -0,0 +1,204 @@ +package org.icatproject.core.manager.search; + +import java.io.IOException; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; + +import javax.json.Json; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import javax.json.JsonValue; +import javax.json.stream.JsonGenerator; +import javax.persistence.EntityManager; + +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.icatproject.core.IcatException; +import org.icatproject.core.IcatException.IcatExceptionType; +import org.icatproject.core.entity.EntityBaseBean; +import org.icatproject.core.manager.Rest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LuceneApi extends SearchApi { + + public String basePath = "/icat.lucene"; + private static final Logger logger = LoggerFactory.getLogger(LuceneApi.class); + + private static String getTargetPath(JsonObject query) throws IcatException { + if (!query.containsKey("target")) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "'target' must be present in query for LuceneApi, but it was " + query.toString()); + } + String path = query.getString("target").toLowerCase(); + if (!indices.contains(path)) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "'target' must be one of " + indices + ", but it was " + path); + } + return path; + } + + @Override + public JsonObject buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { + // As icat.lucene always requires the Lucene id, shardIndex and score + // irrespective of the sort, override the default implementation + JsonObjectBuilder builder = Json.createObjectBuilder(); + builder.add("doc", lastBean.getEngineDocId()); + builder.add("shardIndex", -1); + float score = lastBean.getScore(); + if (!Float.isNaN(score)) { + builder.add("score", score); + } + JsonArrayBuilder arrayBuilder; + if (sort == null || sort.equals("")) { + arrayBuilder = Json.createArrayBuilder().add(score); + } else { + arrayBuilder = searchAfterArrayBuilder(lastBean, sort); + } + builder.add("fields", arrayBuilder.add(lastBean.getEntityBaseBeanId())); + return builder.build(); + } + + public LuceneApi(URI server) { + super(server); + } + + @Override + public void addNow(String entityName, List ids, EntityManager manager, + Class klass, ExecutorService getBeanDocExecutor) + throws IcatException, IOException, URISyntaxException { + URI uri = new URIBuilder(server).setPath(basePath + "/addNow/" + entityName).build(); + + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + HttpPost httpPost = new HttpPost(uri); + PipedOutputStream beanDocs = new PipedOutputStream(); + httpPost.setEntity(new InputStreamEntity(new PipedInputStream(beanDocs))); + getBeanDocExecutor.submit(() -> { + try (JsonGenerator gen = Json.createGenerator(beanDocs)) { + gen.writeStartArray(); + for (Long id : ids) { + EntityBaseBean bean = (EntityBaseBean) manager.find(klass, id); + if (bean != null) { + gen.writeStartObject(); + bean.getDoc(gen); + gen.writeEnd(); + } + } + gen.writeEnd(); + return null; + } catch (Exception e) { + logger.error("About to throw internal exception because of", e); + throw new IcatException(IcatExceptionType.INTERNAL, e.getMessage()); + } finally { + manager.close(); + } + }); + + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } + } + + @Override + public void clear() throws IcatException { + post(basePath + "/clear"); + } + + @Override + public void commit() throws IcatException { + post(basePath + "/commit"); + } + + @Override + public List facetSearch(String target, JsonObject facetQuery, Integer maxResults, Integer maxLabels) + throws IcatException { + String path = basePath + "/" + target + "/facet"; + + Map parameterMap = new HashMap<>(); + parameterMap.put("maxResults", maxResults.toString()); + parameterMap.put("maxLabels", maxLabels.toString()); + + JsonObject postResponse = postResponse(path, facetQuery.toString(), parameterMap); + + List results = new ArrayList<>(); + JsonObject aggregations = postResponse.getJsonObject("aggregations"); + for (String dimension : aggregations.keySet()) { + parseFacetsResponse(results, target, dimension, aggregations); + } + return results; + } + + @Override + public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer blockSize, String sort, + List fields) throws IcatException { + String indexPath = getTargetPath(query); + + Map parameterMap = new HashMap<>(); + parameterMap.put("maxResults", blockSize.toString()); + if (searchAfter != null) { + parameterMap.put("search_after", searchAfter.toString()); + } + if (sort != null) { + parameterMap.put("sort", sort); + } + + JsonObjectBuilder objectBuilder = Json.createObjectBuilder(); + objectBuilder.add("query", query); + if (fields != null && fields.size() > 0) { + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + fields.forEach((field) -> arrayBuilder.add(field)); + objectBuilder.add("fields", arrayBuilder.build()); + } + String queryString = objectBuilder.build().toString(); + + JsonObject postResponse = postResponse(basePath + "/" + indexPath, queryString, parameterMap); + SearchResult lsr = new SearchResult(); + List results = lsr.getResults(); + List resultsArray = postResponse.getJsonArray("results").getValuesAs(JsonObject.class); + for (JsonObject resultObject : resultsArray) { + int luceneDocId = resultObject.getInt("_id"); + Float score = Float.NaN; + if (resultObject.keySet().contains("_score")) { + score = resultObject.getJsonNumber("_score").bigDecimalValue().floatValue(); + } + JsonObject source = resultObject.getJsonObject("_source"); + ScoredEntityBaseBean result = new ScoredEntityBaseBean(luceneDocId, score, source); + results.add(result); + logger.trace("Result id {} with score {}", result.getEntityBaseBeanId(), score); + } + if (postResponse.containsKey("search_after")) { + lsr.setSearchAfter(postResponse.getJsonObject("search_after")); + } + + return lsr; + } + + @Override + public void lock(String entityName) throws IcatException { + post(basePath + "/lock/" + entityName); + } + + @Override + public void unlock(String entityName) throws IcatException { + post(basePath + "/unlock/" + entityName); + } + + @Override + public void modify(String json) throws IcatException { + post(basePath + "/modify", json); + } + +} diff --git a/src/main/java/org/icatproject/core/manager/SearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java similarity index 62% rename from src/main/java/org/icatproject/core/manager/SearchApi.java rename to src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index 52024ee7..e1dd83ee 100644 --- a/src/main/java/org/icatproject/core/manager/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -1,22 +1,19 @@ -package org.icatproject.core.manager; +package org.icatproject.core.manager.search; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.StringReader; import java.net.URI; import java.net.URISyntaxException; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.TimeZone; import java.util.Map.Entry; import java.util.concurrent.ExecutorService; +import java.util.Set; import javax.json.Json; import javax.json.JsonArray; @@ -28,10 +25,6 @@ import javax.json.JsonValue; import javax.json.JsonValue.ValueType; import javax.json.stream.JsonGenerator; -import javax.measure.IncommensurableException; -import javax.measure.Unit; -import javax.measure.UnitConverter; -import javax.measure.format.MeasurementParseException; import javax.persistence.EntityManager; import org.apache.http.client.ClientProtocolException; @@ -55,15 +48,13 @@ import org.icatproject.core.entity.ParameterType; import org.icatproject.core.entity.SampleType; import org.icatproject.core.entity.User; -import org.icatproject.core.manager.search.QueryBuilder; +import org.icatproject.core.manager.Rest; +import org.icatproject.utils.IcatUnits; +import org.icatproject.utils.IcatUnits.SystemValue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import tech.units.indriya.format.SimpleUnitFormat; -import tech.units.indriya.unit.Units; - -// TODO see what functionality can live here, and possibly convert from abstract to a fully generic API -public class SearchApi { +public class OpensearchApi extends SearchApi { private static enum ModificationType { CREATE, UPDATE, DELETE @@ -87,10 +78,8 @@ public ParentRelation(RelationType relationType, String parentName, String joinF } } - private static final SimpleUnitFormat unitFormat = SimpleUnitFormat.getInstance(); - protected static final Logger logger = LoggerFactory.getLogger(SearchApi.class); - protected static SimpleDateFormat df; - protected static String basePath = ""; + private IcatUnits icatUnits; + protected static final Logger logger = LoggerFactory.getLogger(OpensearchApi.class); private static JsonObject indexSettings = Json.createObjectBuilder().add("analysis", Json.createObjectBuilder() .add("analyzer", Json.createObjectBuilder() .add("default", Json.createObjectBuilder() @@ -105,20 +94,9 @@ public ParentRelation(RelationType relationType, String parentName, String joinF .add("possessive_english", Json.createObjectBuilder() .add("type", "stemmer").add("langauge", "possessive_english")))) .build(); - protected static Set indices = new HashSet<>(); private static Map> relations = new HashMap<>(); - protected URI server; - static { - df = new SimpleDateFormat("yyyyMMddHHmm"); - TimeZone tz = TimeZone.getTimeZone("GMT"); - df.setTimeZone(tz); - - unitFormat.alias(Units.CELSIUS, "celsius"); // TODO this should be generalised with the units we need - - indices.addAll(Arrays.asList("datafile", "dataset", "investigation")); - // Non-nested children have a one to one relationship with an indexed entity and // so do not form an array, and update specific fields by query relations.put("datafileformat", Arrays.asList( @@ -166,8 +144,14 @@ public ParentRelation(RelationType relationType, String parentName, String joinF new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "sample", SampleType.docFields))); } - public SearchApi(URI server) { - this.server = server; + public OpensearchApi(URI server) { + super(server); + icatUnits = new IcatUnits(); + } + + public OpensearchApi(URI server, String unitAliasOptions) { + super(server); + icatUnits = new IcatUnits(unitAliasOptions); } private static String buildCreateScript(String target) { @@ -259,7 +243,6 @@ private static JsonObject buildNestedMapping(String... idFields) { return Json.createObjectBuilder().add("type", "nested").add("properties", propertiesBuilder).build(); } - // TODO (mostly) duplicated code from icat.lucene... private static Long parseDate(JsonObject jsonObject, String key, int offset, Long defaultValue) throws IcatException { if (jsonObject.containsKey(key)) { @@ -283,110 +266,11 @@ private static Long parseDate(JsonObject jsonObject, String key, int offset, Lon return defaultValue; } - /** - * Converts String into Date object. - * - * @param value String representing a Date in the format "yyyyMMddHHmm". - * @return Date object, or null if value was null. - * @throws java.text.ParseException - */ - protected static Date decodeDate(String value) throws java.text.ParseException { - if (value == null) { - return null; - } else { - synchronized (df) { - return df.parse(value); - } - } - } - - /** - * Converts String into number of ms since epoch. - * - * @param value String representing a Date in the format "yyyyMMddHHmm". - * @return Number of ms since epoch, or null if value was null - * @throws java.text.ParseException - */ - protected static Long decodeTime(String value) throws java.text.ParseException { - if (value == null) { - return null; - } else { - synchronized (df) { - return df.parse(value).getTime(); - } - } - } - - /** - * Converts Date object into String format. - * - * @param dateValue Date object to be converted. - * @return String representing a Date in the format "yyyyMMddHHmm". - */ - protected static String encodeDate(Date dateValue) { - if (dateValue == null) { - return null; - } else { - synchronized (df) { - return df.format(dateValue); - } - } - } - - public static String encodeDeletion(EntityBaseBean bean) { - String entityName = bean.getClass().getSimpleName(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject().writeStartObject("delete"); - gen.write("_index", entityName).write("_id", bean.getId().toString()); - gen.writeEnd().writeEnd(); - } - return baos.toString(); - } - - public static void encodeDouble(JsonGenerator gen, String name, Double value) { - gen.write(name, value); - } - - public static void encodeLong(JsonGenerator gen, String name, Date value) { - gen.write(name, value.getTime()); - } - - public static String encodeOperation(String operation, EntityBaseBean bean) throws IcatException { - Long icatId = bean.getId(); - if (icatId == null) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, bean.toString() + " had null id"); - } - String entityName = bean.getClass().getSimpleName(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject().writeStartObject(operation); - gen.write("_index", entityName).write("_id", icatId.toString()); - gen.writeStartObject("doc"); - bean.getDoc(gen); - gen.writeEnd().writeEnd().writeEnd(); - } - return baos.toString(); - } - - public static void encodeString(JsonGenerator gen, String name, Long value) { - gen.write(name, Long.toString(value)); - } - - public static void encodeString(JsonGenerator gen, String name, String value) { - gen.write(name, value); - } - - public static void encodeText(JsonGenerator gen, String name, String value) { - if (value != null) { - gen.write(name, value); - } - } - + @Override public void addNow(String entityName, List ids, EntityManager manager, Class klass, ExecutorService getBeanDocExecutor) throws IcatException, IOException, URISyntaxException { - // getBeanDocExecutor is not used for all implementations, but is + // getBeanDocExecutor is not used for this implementation, but is // required for the @Override ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { @@ -406,69 +290,21 @@ public void addNow(String entityName, List ids, EntityManager manager, modify(baos.toString()); } - public JsonValue buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { - if (sort != null && !sort.equals("")) { - try (JsonReader reader = Json.createReader(new StringReader(sort))) { - JsonObject object = reader.readObject(); - JsonArrayBuilder builder = Json.createArrayBuilder(); - for (String key : object.keySet()) { - if (!lastBean.getSource().keySet().contains(key)) { - throw new IcatException(IcatExceptionType.INTERNAL, - "Cannot build searchAfter document from source as sorted field " + key + " missing."); - } - JsonValue value = lastBean.getSource().get(key); - builder.add(value); - } - builder.add(lastBean.getEntityBaseBeanId()); - return builder.build(); - } - } else { - JsonArrayBuilder builder = Json.createArrayBuilder(); - if (Float.isNaN(lastBean.getScore())) { - throw new IcatException(IcatExceptionType.INTERNAL, - "Cannot build searchAfter document from source as score was NaN."); - } - builder.add(lastBean.getScore()); - builder.add(lastBean.getEntityBaseBeanId()); - return builder.build(); - } - } - + @Override public void clear() throws IcatException { commit(); - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/_all/_delete_by_query").build(); - HttpPost httpPost = new HttpPost(uri); - String body = Json.createObjectBuilder().add("query", QueryBuilder.buildMatchAllQuery()).build().toString(); - httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); - logger.trace("Making call {} with body {}", uri, body); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } catch (IOException | URISyntaxException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } + String body = QueryBuilder.addQuery(QueryBuilder.buildMatchAllQuery()).build().toString(); + post("/_all/_delete_by_query", body); } + @Override public void commit() throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/_refresh").build(); - logger.trace("Making call {}", uri); - HttpPost httpPost = new HttpPost(uri); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } catch (URISyntaxException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - public void freeSearcher(String uid) throws IcatException { - logger.info("Manually freeing searcher not supported, no request sent"); + post("/_refresh"); } + @Override public List facetSearch(String target, JsonObject facetQuery, Integer maxResults, - int maxLabels) throws IcatException { + Integer maxLabels) throws IcatException { List results = new ArrayList<>(); if (!facetQuery.containsKey("dimensions")) { // If no dimensions were specified, return early @@ -481,37 +317,26 @@ public List facetSearch(String target, JsonObject facetQuery, In dimensionPrefix = index; index = relations.get(index).get(0).parentName; } - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URIBuilder builder = new URIBuilder(server).setPath(basePath + "/" + index + "/_search"); - builder.addParameter("size", maxResults.toString()); - URI uri = builder.build(); - - JsonObject queryObject = facetQuery.getJsonObject("query"); - JsonArray dimensions = facetQuery.getJsonArray("dimensions"); - JsonObjectBuilder bodyBuilder = parseQuery(Json.createObjectBuilder(), queryObject, index); - bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix); - String body = bodyBuilder.build().toString(); - - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); - logger.trace("Making call {} with body {}", uri, body); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - JsonReader jsonReader = Json.createReader(response.getEntity().getContent()); - JsonObject jsonObject = jsonReader.readObject(); - logger.trace("facet response: {}", jsonObject); - JsonObject aggregations = jsonObject.getJsonObject("aggregations"); - if (dimensionPrefix != null) { - aggregations = aggregations.getJsonObject(dimensionPrefix); - } - for (String dimension : aggregations.keySet()) { - parseFacetsResponse(results, target, dimension, aggregations); - } - } - return results; - } catch (IOException | URISyntaxException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + + JsonObject queryObject = facetQuery.getJsonObject("query"); + JsonArray dimensions = facetQuery.getJsonArray("dimensions"); + JsonObjectBuilder bodyBuilder = parseQuery(Json.createObjectBuilder(), queryObject, index); + bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix); + String body = bodyBuilder.build().toString(); + + Map parameterMap = new HashMap<>(); + parameterMap.put("size", maxResults.toString()); + + JsonObject postResponse = postResponse("/" + index + "/_search", body, parameterMap); + + JsonObject aggregations = postResponse.getJsonObject("aggregations"); + if (dimensionPrefix != null) { + aggregations = aggregations.getJsonObject(dimensionPrefix); + } + for (String dimension : aggregations.keySet()) { + parseFacetsResponse(results, target, dimension, aggregations); } + return results; } private JsonObjectBuilder parseFacets(JsonObjectBuilder bodyBuilder, JsonArray dimensions, int maxLabels, @@ -538,111 +363,52 @@ private JsonObjectBuilder parseFacets(JsonObjectBuilder bodyBuilder, JsonArray d return bodyBuilder; } - private void parseFacetsResponse(List results, String target, String dimension, - JsonObject aggregations) throws IcatException { - if (dimension.equals("doc_count")) { - // For nested aggregations, there is a doc_count entry at the same level as the - // dimension objects, but we're not interested in this - return; - } - FacetDimension facetDimension = new FacetDimension(target, dimension); - List facets = facetDimension.getFacets(); - JsonObject aggregation = aggregations.getJsonObject(dimension); - JsonValue bucketsValue = aggregation.get("buckets"); - ValueType valueType = bucketsValue.getValueType(); - switch (valueType) { - case ARRAY: - List buckets = ((JsonArray) bucketsValue).getValuesAs(JsonObject.class); - if (buckets.size() == 0) { - return; - } - for (JsonObject bucket : buckets) { - long docCount = bucket.getJsonNumber("doc_count").longValueExact(); - facets.add(new FacetLabel(bucket.getString("key"), docCount)); - } - break; - case OBJECT: - JsonObject bucketsObject = (JsonObject) bucketsValue; - Set keySet = bucketsObject.keySet(); - if (keySet.size() == 0) { - return; - } - for (String key : keySet) { - JsonObject bucket = bucketsObject.getJsonObject(key); - long docCount = bucket.getJsonNumber("doc_count").longValueExact(); - facets.add(new FacetLabel(key, docCount)); - } - break; - default: - String msg = "Expected 'buckets' to have ARRAY or OBJECT type, but it was " + valueType; - throw new IcatException(IcatExceptionType.INTERNAL, msg); - } - results.add(facetDimension); - } - - public SearchResult getResults(JsonObject query, int maxResults) throws IcatException { - return getResults(query, null, maxResults, null, Arrays.asList("id")); - } - - public SearchResult getResults(JsonObject query, int maxResults, String sort) throws IcatException { - return getResults(query, null, maxResults, sort, Arrays.asList("id")); - } - + @Override public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer blockSize, String sort, List requestedFields) throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - String index = query.containsKey("target") ? query.getString("target").toLowerCase() : "_all"; - URIBuilder builder = new URIBuilder(server).setPath(basePath + "/" + index + "/_search"); - StringBuilder sb = new StringBuilder(); - requestedFields.forEach(f -> sb.append(f).append(",")); - builder.addParameter("_source", sb.toString()); - builder.addParameter("size", blockSize.toString()); - URI uri = builder.build(); - - JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); - bodyBuilder = parseSort(bodyBuilder, sort); - bodyBuilder = parseSearchAfter(bodyBuilder, searchAfter); - bodyBuilder = parseQuery(bodyBuilder, query, index); - String body = bodyBuilder.build().toString(); - - SearchResult result = new SearchResult(); - List entities = result.getResults(); - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); - logger.trace("Making call {} with body {}", uri, body); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - JsonReader jsonReader = Json.createReader(response.getEntity().getContent()); - JsonObject jsonObject = jsonReader.readObject(); - JsonArray hits = jsonObject.getJsonObject("hits").getJsonArray("hits"); - for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { - logger.trace("Hit {}", hit.toString()); - Float score = Float.NaN; - if (!hit.isNull("_score")) { - score = hit.getJsonNumber("_score").bigDecimalValue().floatValue(); - } - Integer id = new Integer(hit.getString("_id")); - entities.add(new ScoredEntityBaseBean(id, score, hit.getJsonObject("_source"))); - } - - // If we're returning as many results as were asked for, setSearchAfter so - // subsequent searches can continue from the last result - if (hits.size() == blockSize) { - JsonObject lastHit = hits.getJsonObject(blockSize - 1); - if (lastHit.containsKey("sort")) { - result.setSearchAfter(lastHit.getJsonArray("sort")); - } else { - ScoredEntityBaseBean lastEntity = entities.get(blockSize - 1); - long id = lastEntity.getEntityBaseBeanId(); - float score = lastEntity.getScore(); - result.setSearchAfter(Json.createArrayBuilder().add(score).add(id).build()); - } - } + String index = query.containsKey("target") ? query.getString("target").toLowerCase() : "_all"; + + JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); + bodyBuilder = parseSort(bodyBuilder, sort); + bodyBuilder = parseSearchAfter(bodyBuilder, searchAfter); + bodyBuilder = parseQuery(bodyBuilder, query, index); + String body = bodyBuilder.build().toString(); + + Map parameterMap = new HashMap<>(); + StringBuilder sb = new StringBuilder(); + requestedFields.forEach(f -> sb.append(f).append(",")); + parameterMap.put("_source", sb.toString()); + parameterMap.put("size", blockSize.toString()); + + JsonObject postResponse = postResponse("/" + index + "/_search", body, parameterMap); + + SearchResult result = new SearchResult(); + List entities = result.getResults(); + JsonArray hits = postResponse.getJsonObject("hits").getJsonArray("hits"); + for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { + Float score = Float.NaN; + if (!hit.isNull("_score")) { + score = hit.getJsonNumber("_score").bigDecimalValue().floatValue(); + } + Integer id = new Integer(hit.getString("_id")); + entities.add(new ScoredEntityBaseBean(id, score, hit.getJsonObject("_source"))); + } + + // If we're returning as many results as were asked for, setSearchAfter so + // subsequent searches can continue from the last result + if (hits.size() == blockSize) { + JsonObject lastHit = hits.getJsonObject(blockSize - 1); + if (lastHit.containsKey("sort")) { + result.setSearchAfter(lastHit.getJsonArray("sort")); + } else { + ScoredEntityBaseBean lastEntity = entities.get(blockSize - 1); + long id = lastEntity.getEntityBaseBeanId(); + float score = lastEntity.getScore(); + result.setSearchAfter(Json.createArrayBuilder().add(score).add(id).build()); } - return result; - } catch (IOException | URISyntaxException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } + + return result; } private JsonObjectBuilder parseSort(JsonObjectBuilder builder, String sort) { @@ -770,7 +536,7 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query public void initMappings() throws IcatException { for (String index : indices) { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/" + index).build(); + URI uri = new URIBuilder(server).setPath("/" + index).build(); logger.trace("Making call {}", uri); HttpHead httpHead = new HttpHead(uri); try (CloseableHttpResponse response = httpclient.execute(httpHead)) { @@ -790,7 +556,7 @@ public void initMappings() throws IcatException { } try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/" + index).build(); + URI uri = new URIBuilder(server).setPath("/" + index).build(); HttpPut httpPut = new HttpPut(uri); JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); bodyBuilder.add("settings", indexSettings).add("mappings", buildMappings(index)); @@ -812,58 +578,36 @@ public void initScripts() throws IcatException { ParentRelation relation = entry.getValue().get(0); switch (relation.relationType) { case CHILD: - postScript("update_" + childName, buildChildScript(relation.fields, true)); - postScript("delete_" + childName, buildChildScript(relation.fields, false)); + post("/_scripts/update_" + childName, buildChildScript(relation.fields, true)); + post("/_scripts/delete_" + childName, buildChildScript(relation.fields, false)); break; case NESTED_CHILD: - postScript("create_" + childName, buildCreateScript(childName)); - postScript("update_" + childName, buildNestedChildScript(childName, true)); - postScript("delete_" + childName, buildNestedChildScript(childName, false)); + post("/_scripts/create_" + childName, buildCreateScript(childName)); + post("/_scripts/update_" + childName, buildNestedChildScript(childName, true)); + post("/_scripts/delete" + childName, buildNestedChildScript(childName, false)); break; case NESTED_GRANDCHILD: if (childName.equals("parametertype")) { // Special case, as parametertype applies to investigationparameter, // datasetparameter, datafileparameter for (String index : indices) { - String updateScript = buildNestedGrandchildScript(index + "parameter", relation.fields, true); - String deleteScript = buildNestedGrandchildScript(index + "parameter", relation.fields, false); - postScript("update_" + index + childName, updateScript); - postScript("delete_" + index + childName, deleteScript); - break; + String target = index + "parameter"; + String updateScript = buildNestedGrandchildScript(target, relation.fields, true); + String deleteScript = buildNestedGrandchildScript(target, relation.fields, false); + post("/_scripts/update_" + index + childName, updateScript); + post("/_scripts/delete_" + index + childName, deleteScript); } } else { String updateScript = buildNestedGrandchildScript(relation.joinField, relation.fields, true); String deleteScript = buildNestedGrandchildScript(relation.joinField, relation.fields, false); - postScript("update_" + childName, updateScript); - postScript("delete_" + childName, deleteScript); - break; + post("/_scripts/update_" + childName, updateScript); + post("/_scripts/delete_" + childName, deleteScript); } + break; } } } - private void postScript(String scriptKey, String body) throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(basePath + "/_scripts/" + scriptKey).build(); - logger.trace("Making call {}", uri); - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } catch (URISyntaxException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - public void lock(String entityName) throws IcatException { - logger.info("Manually locking index not supported, no request sent"); - } - - public void unlock(String entityName) throws IcatException { - logger.info("Manually unlocking index not supported, no request sent"); - } - public void modify(String json) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { List updatesByQuery = new ArrayList<>(); @@ -898,7 +642,7 @@ public void modify(String json) throws IcatException { if (sb.toString().length() > 0) { // Perform simple bulk modifications - URI uri = new URIBuilder(server).setPath(basePath + "/_bulk").build(); + URI uri = new URIBuilder(server).setPath("/_bulk").build(); HttpPost httpPost = new HttpPost(uri); httpPost.setEntity(new StringEntity(sb.toString(), ContentType.APPLICATION_JSON)); // logger.trace("Making call {} with body {}", uri, sb.toString()); @@ -911,8 +655,6 @@ public void modify(String json) throws IcatException { // Ensure bulk changes are committed before performing updatesByQuery commit(); for (HttpPost updateByQuery : updatesByQuery) { - // logger.trace("Making call {} with body {}", - // updateByQuery.getURI(), updateByQuery.getEntity().getContent().toString()); try (CloseableHttpResponse response = httpclient.execute(updateByQuery)) { Rest.checkStatus(response, IcatExceptionType.INTERNAL); } @@ -923,7 +665,7 @@ public void modify(String json) throws IcatException { // Ensure bulk changes are committed before checking for InvestigationUsers commit(); for (String investigationId : investigationIds) { - URI uriGet = new URIBuilder(server).setPath(basePath + "/investigation/_source/" + investigationId) + URI uriGet = new URIBuilder(server).setPath("/investigation/_source/" + investigationId) .build(); HttpGet httpGet = new HttpGet(uriGet); try (CloseableHttpResponse responseGet = httpclient.execute(httpGet)) { @@ -945,7 +687,7 @@ private void extractFromInvestigation(CloseableHttpClient httpclient, String inv if (responseObject.containsKey("investigationuser")) { JsonArray jsonArray = responseObject.getJsonArray("investigationuser"); for (String index : new String[] { "datafile", "dataset" }) { - URI uri = new URIBuilder(server).setPath(basePath + "/" + index + "/_update_by_query").build(); + URI uri = new URIBuilder(server).setPath("/" + index + "/_update_by_query").build(); HttpPost httpPost = new HttpPost(uri); JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("doc", jsonArray); JsonObjectBuilder scriptBuilder = Json.createObjectBuilder(); @@ -971,7 +713,7 @@ private void modifyNestedEntity(StringBuilder sb, List updatesByQuery, if (relation.parentName.equals(relation.joinField)) { // If the target parent is the same as the joining field, we're appending the // nested child to a list of objects which can be sent as a bulk update request - document = convertUnits(document); + document = convertDocumentUnits(document); createNestedEntity(sb, id, index, document, relation); } else if (index.equals("sampletype")) { // Otherwise, in most cases we don't need to update, as User and ParameterType @@ -1012,7 +754,7 @@ private static void createNestedEntity(StringBuilder sb, String id, String index private void updateNestedEntityByQuery(List updatesByQuery, String id, String index, JsonObject document, ParentRelation relation, boolean update) throws URISyntaxException { - String path = basePath + "/" + relation.parentName + "/_update_by_query"; + String path = "/" + relation.parentName + "/_update_by_query"; URI uri = new URIBuilder(server).setPath(path).build(); HttpPost httpPost = new HttpPost(uri); @@ -1022,28 +764,40 @@ private void updateNestedEntityByQuery(List updatesByQuery, String id, if (update) { if (relation.fields == null) { // Update affects all of the nested fields, so can add the entire document - document = convertUnits(document); + document = convertDocumentUnits(document); paramsBuilder.add("doc", Json.createArrayBuilder().add(document)); } else { // Need to update individual nested fields - paramsBuilder = convertUnits(paramsBuilder, document, relation.fields); + paramsBuilder = convertScriptUnits(paramsBuilder, document, relation.fields); } } JsonObjectBuilder scriptBuilder = Json.createObjectBuilder().add("id", scriptId).add("params", paramsBuilder); JsonObject queryObject; String idField = relation.joinField.equals(relation.parentName) ? "id" : relation.joinField + ".id"; - if (!Arrays.asList("parametertype", "sampletype", "user").contains(index)) { // TODO generalise? - queryObject = QueryBuilder.buildTermQuery(idField, id); - } else { + if (relation.relationType.equals(RelationType.NESTED_GRANDCHILD)) { queryObject = QueryBuilder.buildNestedQuery(relation.joinField, QueryBuilder.buildTermQuery(idField, id)); + } else { + queryObject = QueryBuilder.buildTermQuery(idField, id); } JsonObject bodyJson = Json.createObjectBuilder().add("query", queryObject).add("script", scriptBuilder).build(); - logger.trace("updateByQuery script: {}", bodyJson.toString()); + // logger.trace("updateByQuery script: {}", bodyJson.toString()); httpPost.setEntity(new StringEntity(bodyJson.toString(), ContentType.APPLICATION_JSON)); updatesByQuery.add(httpPost); } - private static JsonObject convertUnits(JsonObject document) { + private void convertUnits(JsonObject document, JsonObjectBuilder rebuilder, String valueString, + Double numericValue) { + String unitString = document.getString("type.units"); + SystemValue systemValue = icatUnits.new SystemValue(numericValue, unitString); + if (systemValue.units != null) { + rebuilder.add("type.unitsSI", systemValue.units); + } + if (systemValue.value != null) { + rebuilder.add(valueString, systemValue.value); + } + } + + private JsonObject convertDocumentUnits(JsonObject document) { if (!document.containsKey("type.units")) { return document; } @@ -1052,45 +806,21 @@ private static JsonObject convertUnits(JsonObject document) { for (String key : document.keySet()) { rebuilder.add(key, document.get(key)); } - String unitString = document.getString("type.units"); - try { - Unit unit = unitFormat.parse(unitString); - Unit systemUnit = unit.getSystemUnit(); - rebuilder.add("type.unitsSI", systemUnit.getName()); - if (document.containsKey("numericValue")) { - double numericValue = document.getJsonNumber("numericValue").doubleValue(); - UnitConverter converter = unit.getConverterToAny(systemUnit); - rebuilder.add("numericValueSI", converter.convert(numericValue)); - } - } catch (IncommensurableException | MeasurementParseException e) { - logger.error("Unable to convert 'type.units' of {} due to {}", unitString, - e.getMessage()); - } + Double numericValue = document.containsKey("numericValue") + ? document.getJsonNumber("numericValue").doubleValue() + : null; + convertUnits(document, rebuilder, "numericValueSI", numericValue); document = rebuilder.build(); return document; } - private static JsonObjectBuilder convertUnits(JsonObjectBuilder paramsBuilder, JsonObject document, + private JsonObjectBuilder convertScriptUnits(JsonObjectBuilder paramsBuilder, JsonObject document, Set fields) { - UnitConverter converter = null; for (String field : fields) { if (field.equals("type.unitsSI")) { - String unitString = document.getString("type.units"); - try { - Unit unit = unitFormat.parse(unitString); - Unit systemUnit = unit.getSystemUnit(); - converter = unit.getConverterToAny(systemUnit); - paramsBuilder.add(field, systemUnit.getName()); - } catch (IncommensurableException | MeasurementParseException e) { - logger.error("Unable to convert 'type.units' of {} due to {}", unitString, - e.getMessage()); - } + convertUnits(document, paramsBuilder, "conversionFactor", 1.); } else if (field.equals("numericValueSI")) { - if (converter != null) { - // If we convert 1, we then have the necessary factor and can do - // multiplication by script... - paramsBuilder.add("conversionFactor", converter.convert(1.)); - } + continue; } else { paramsBuilder.add(field, document.get(field)); } @@ -1123,77 +853,4 @@ private static void modifyEntity(StringBuilder sb, Set investigationIds, break; } } - - /** - * Legacy function for building a Query from individual arguments - * - * @param target - * @param user - * @param text - * @param lower - * @param upper - * @param parameters - * @param samples - * @param userFullName - * @return - */ - public static JsonObject buildQuery(String target, String user, String text, Date lower, Date upper, - List parameters, List samples, String userFullName) { - JsonObjectBuilder builder = Json.createObjectBuilder(); - if (target != null) { - builder.add("target", target); - } - if (user != null) { - builder.add("user", user); - } - if (text != null) { - builder.add("text", text); - } - if (lower != null) { - builder.add("lower", lower.getTime()); - } - if (upper != null) { - builder.add("upper", upper.getTime()); - } - if (parameters != null && !parameters.isEmpty()) { - JsonArrayBuilder parametersBuilder = Json.createArrayBuilder(); - for (ParameterPOJO parameter : parameters) { - JsonObjectBuilder parameterBuilder = Json.createObjectBuilder(); - if (parameter.name != null) { - parameterBuilder.add("name", parameter.name); - } - if (parameter.units != null) { - parameterBuilder.add("units", parameter.units); - } - if (parameter.stringValue != null) { - parameterBuilder.add("stringValue", parameter.stringValue); - } - if (parameter.lowerDateValue != null) { - parameterBuilder.add("lowerDateValue", parameter.lowerDateValue.getTime()); - } - if (parameter.upperDateValue != null) { - parameterBuilder.add("upperDateValue", parameter.upperDateValue.getTime()); - } - if (parameter.lowerNumericValue != null) { - parameterBuilder.add("lowerNumericValue", parameter.lowerNumericValue); - } - if (parameter.upperNumericValue != null) { - parameterBuilder.add("upperNumericValue", parameter.upperNumericValue); - } - parametersBuilder.add(parameterBuilder); - } - builder.add("parameters", parametersBuilder); - } - if (samples != null && !samples.isEmpty()) { - JsonArrayBuilder samplesBuilder = Json.createArrayBuilder(); - for (String sample : samples) { - samplesBuilder.add(sample); - } - builder.add("samples", samplesBuilder); - } - if (userFullName != null) { - builder.add("userFullName", userFullName); - } - return builder.build(); - } } diff --git a/src/main/java/org/icatproject/core/manager/ParameterPOJO.java b/src/main/java/org/icatproject/core/manager/search/ParameterPOJO.java similarity index 83% rename from src/main/java/org/icatproject/core/manager/ParameterPOJO.java rename to src/main/java/org/icatproject/core/manager/search/ParameterPOJO.java index bcafe8e8..c5d948dd 100644 --- a/src/main/java/org/icatproject/core/manager/ParameterPOJO.java +++ b/src/main/java/org/icatproject/core/manager/search/ParameterPOJO.java @@ -1,4 +1,4 @@ -package org.icatproject.core.manager; +package org.icatproject.core.manager.search; import java.io.Serializable; import java.util.Date; @@ -6,13 +6,13 @@ @SuppressWarnings("serial") public class ParameterPOJO implements Serializable { - String name; - String units; - String stringValue; - Date lowerDateValue; - Date upperDateValue; - Double lowerNumericValue; - Double upperNumericValue; + public String name; + public String units; + public String stringValue; + public Date lowerDateValue; + public Date upperDateValue; + public Double lowerNumericValue; + public Double upperNumericValue; public ParameterPOJO(String name, String units, String stringValue) { this.name = name; diff --git a/src/main/java/org/icatproject/core/manager/search/QueryBuilder.java b/src/main/java/org/icatproject/core/manager/search/QueryBuilder.java index c6531357..56460ef7 100644 --- a/src/main/java/org/icatproject/core/manager/search/QueryBuilder.java +++ b/src/main/java/org/icatproject/core/manager/search/QueryBuilder.java @@ -12,6 +12,10 @@ public class QueryBuilder { private static JsonObject matchAllQuery = Json.createObjectBuilder().add("match_all", Json.createObjectBuilder()) .build(); + public static JsonObjectBuilder addQuery(JsonObject query) { + return Json.createObjectBuilder().add("query", query); + } + public static JsonObject buildMatchAllQuery() { return matchAllQuery; } diff --git a/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java b/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java similarity index 97% rename from src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java rename to src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java index 6ac9316c..1fec7421 100644 --- a/src/main/java/org/icatproject/core/manager/ScoredEntityBaseBean.java +++ b/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java @@ -1,4 +1,4 @@ -package org.icatproject.core.manager; +package org.icatproject.core.manager.search; import javax.json.JsonObject; diff --git a/src/main/java/org/icatproject/core/manager/search/SearchApi.java b/src/main/java/org/icatproject/core/manager/search/SearchApi.java new file mode 100644 index 00000000..d418e64f --- /dev/null +++ b/src/main/java/org/icatproject/core/manager/search/SearchApi.java @@ -0,0 +1,328 @@ +package org.icatproject.core.manager.search; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.StringReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.Map.Entry; +import java.util.concurrent.ExecutorService; + +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObject; +import javax.json.JsonReader; +import javax.json.JsonValue; +import javax.json.JsonValue.ValueType; +import javax.json.stream.JsonGenerator; +import javax.persistence.EntityManager; + +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.icatproject.core.IcatException; +import org.icatproject.core.IcatException.IcatExceptionType; +import org.icatproject.core.entity.EntityBaseBean; +import org.icatproject.core.manager.Rest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class SearchApi { + + protected static final Logger logger = LoggerFactory.getLogger(SearchApi.class); + protected static SimpleDateFormat df; + protected static final Set indices = new HashSet<>(Arrays.asList("datafile", "dataset", "investigation")); + + protected URI server; + + static { + df = new SimpleDateFormat("yyyyMMddHHmm"); + TimeZone tz = TimeZone.getTimeZone("GMT"); + df.setTimeZone(tz); + } + + public SearchApi(URI server) { + this.server = server; + } + + /** + * Converts String into Date object. + * + * @param value String representing a Date in the format "yyyyMMddHHmm". + * @return Date object, or null if value was null. + * @throws java.text.ParseException + */ + protected static Date decodeDate(String value) throws java.text.ParseException { + if (value == null) { + return null; + } else { + synchronized (df) { + return df.parse(value); + } + } + } + + /** + * Converts String into number of ms since epoch. + * + * @param value String representing a Date in the format "yyyyMMddHHmm". + * @return Number of ms since epoch, or null if value was null + * @throws java.text.ParseException + */ + protected static Long decodeTime(String value) throws java.text.ParseException { + if (value == null) { + return null; + } else { + synchronized (df) { + return df.parse(value).getTime(); + } + } + } + + /** + * Converts Date object into String format. + * + * @param dateValue Date object to be converted. + * @return String representing a Date in the format "yyyyMMddHHmm". + */ + protected static String encodeDate(Date dateValue) { + if (dateValue == null) { + return null; + } else { + synchronized (df) { + return df.format(dateValue); + } + } + } + + public static String encodeDeletion(EntityBaseBean bean) { + String entityName = bean.getClass().getSimpleName(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject().writeStartObject("delete"); + gen.write("_index", entityName).write("_id", bean.getId().toString()); + gen.writeEnd().writeEnd(); + } + return baos.toString(); + } + + public static void encodeDouble(JsonGenerator gen, String name, Double value) { + gen.write(name, value); + } + + public static void encodeLong(JsonGenerator gen, String name, Date value) { + gen.write(name, value.getTime()); + } + + public static String encodeOperation(String operation, EntityBaseBean bean) throws IcatException { + Long icatId = bean.getId(); + if (icatId == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, bean.toString() + " had null id"); + } + String entityName = bean.getClass().getSimpleName(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject().writeStartObject(operation); + gen.write("_index", entityName).write("_id", icatId.toString()); + gen.writeStartObject("doc"); + bean.getDoc(gen); + gen.writeEnd().writeEnd().writeEnd(); + } + return baos.toString(); + } + + public static void encodeString(JsonGenerator gen, String name, Long value) { + gen.write(name, Long.toString(value)); + } + + public static void encodeString(JsonGenerator gen, String name, String value) { + gen.write(name, value); + } + + public static void encodeText(JsonGenerator gen, String name, String value) { + if (value != null) { + gen.write(name, value); + } + } + + /** + * Builds a Json representation of the final search result based on the sort + * criteria used. This allows future searches to efficiently "search after" this + * result. + * + * @param lastBean The last ScoredEntityBaseBean of the current search results. + * @param sort String representing a JsonObject of sort criteria. + * @return JsonValue representing the lastBean to allow future searches to + * search after it. + * @throws IcatException If the score of the lastBean is NaN, or one of the sort + * fields is not present in the source of the lastBean. + */ + public JsonValue buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { + JsonArrayBuilder arrayBuilder; + if (sort != null && !sort.equals("")) { + arrayBuilder = searchAfterArrayBuilder(lastBean, sort); + } else { + arrayBuilder = Json.createArrayBuilder(); + if (Float.isNaN(lastBean.getScore())) { + throw new IcatException(IcatExceptionType.INTERNAL, + "Cannot build searchAfter document from source as score was NaN."); + } + arrayBuilder.add(lastBean.getScore()); + } + arrayBuilder.add(lastBean.getEntityBaseBeanId()); + return arrayBuilder.build(); + } + + protected static JsonArrayBuilder searchAfterArrayBuilder(ScoredEntityBaseBean lastBean, String sort) + throws IcatException { + try (JsonReader reader = Json.createReader(new StringReader(sort))) { + JsonObject object = reader.readObject(); + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + for (String key : object.keySet()) { + if (!lastBean.getSource().containsKey(key)) { + throw new IcatException(IcatExceptionType.INTERNAL, + "Cannot build searchAfter document from source as sorted field " + key + " missing."); + } + JsonValue value = lastBean.getSource().get(key); + arrayBuilder.add(value); + } + return arrayBuilder; + } + } + + protected static void parseFacetsResponse(List results, String target, String dimension, + JsonObject aggregations) throws IcatException { + if (dimension.equals("doc_count")) { + // For nested aggregations, there is a doc_count entry at the same level as the + // dimension objects, but we're not interested in this + return; + } + FacetDimension facetDimension = new FacetDimension(target, dimension); + List facets = facetDimension.getFacets(); + + JsonObject aggregation = aggregations.getJsonObject(dimension); + JsonValue bucketsValue = aggregation.get("buckets"); + ValueType valueType = bucketsValue.getValueType(); + switch (valueType) { + case ARRAY: + List buckets = ((JsonArray) bucketsValue).getValuesAs(JsonObject.class); + if (buckets.size() == 0) { + return; + } + for (JsonObject bucket : buckets) { + long docCount = bucket.getJsonNumber("doc_count").longValueExact(); + facets.add(new FacetLabel(bucket.getString("key"), docCount)); + } + break; + case OBJECT: + JsonObject bucketsObject = (JsonObject) bucketsValue; + Set keySet = bucketsObject.keySet(); + if (keySet.size() == 0) { + return; + } + for (String key : keySet) { + JsonObject bucket = bucketsObject.getJsonObject(key); + long docCount = bucket.getJsonNumber("doc_count").longValueExact(); + facets.add(new FacetLabel(key, docCount)); + } + break; + default: + String msg = "Expected 'buckets' to have ARRAY or OBJECT type, but it was " + valueType; + throw new IcatException(IcatExceptionType.INTERNAL, msg); + } + results.add(facetDimension); + } + + public abstract void addNow(String entityName, List ids, EntityManager manager, + Class klass, ExecutorService getBeanDocExecutor) + throws IcatException, IOException, URISyntaxException; + + public abstract void clear() throws IcatException; + + public abstract void commit() throws IcatException; + + public abstract List facetSearch(String target, JsonObject facetQuery, Integer maxResults, + Integer maxLabels) throws IcatException; + + public SearchResult getResults(JsonObject query, int maxResults) throws IcatException { + return getResults(query, null, maxResults, null, Arrays.asList("id")); + } + + public SearchResult getResults(JsonObject query, int maxResults, String sort) throws IcatException { + return getResults(query, null, maxResults, sort, Arrays.asList("id")); + } + + public abstract SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer blockSize, String sort, + List requestedFields) throws IcatException; + + public void lock(String entityName) throws IcatException { + logger.info("Manually locking index not supported, no request sent"); + } + + public void unlock(String entityName) throws IcatException { + logger.info("Manually unlocking index not supported, no request sent"); + } + + public abstract void modify(String json) throws IcatException; + + protected void post(String path) throws IcatException { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URI uri = new URIBuilder(server).setPath(path).build(); + HttpPost httpPost = new HttpPost(uri); + logger.trace("Making call {}", uri); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } catch (URISyntaxException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + protected void post(String path, String body) throws IcatException { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URI uri = new URIBuilder(server).setPath(path).build(); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + logger.trace("Making call {} with body {}", uri, body); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } catch (URISyntaxException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + protected JsonObject postResponse(String path, String body, Map parameterMap) throws IcatException { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URIBuilder builder = new URIBuilder(server).setPath(path); + for (Entry entry : parameterMap.entrySet()) { + builder.addParameter(entry.getKey(), entry.getValue()); + } + URI uri = builder.build(); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + logger.trace("Making call {} with body {}", uri, body); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + JsonReader jsonReader = Json.createReader(response.getEntity().getContent()); + return jsonReader.readObject(); + } + } catch (URISyntaxException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + +} diff --git a/src/main/java/org/icatproject/core/manager/SearchManager.java b/src/main/java/org/icatproject/core/manager/search/SearchManager.java similarity index 97% rename from src/main/java/org/icatproject/core/manager/SearchManager.java rename to src/main/java/org/icatproject/core/manager/search/SearchManager.java index 10256d1b..9ec0a658 100644 --- a/src/main/java/org/icatproject/core/manager/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchManager.java @@ -1,4 +1,4 @@ -package org.icatproject.core.manager; +package org.icatproject.core.manager.search; import java.io.BufferedReader; import java.io.File; @@ -46,6 +46,9 @@ import org.icatproject.core.entity.Dataset; import org.icatproject.core.entity.EntityBaseBean; import org.icatproject.core.entity.Investigation; +import org.icatproject.core.manager.EntityInfoHandler; +import org.icatproject.core.manager.GateKeeper; +import org.icatproject.core.manager.PropertyHandler; import org.icatproject.core.manager.EntityInfoHandler.Relationship; import org.icatproject.core.manager.PropertyHandler.SearchEngine; import org.slf4j.Logger; @@ -448,10 +451,6 @@ public List facetSearch(String target, JsonObject facetQuery, in return searchApi.facetSearch(target, facetQuery, maxResults, maxLabels); } - public void freeSearcher(String uid) throws IcatException { - searchApi.freeSearcher(uid); - } - public List getPopulating() { List result = new ArrayList<>(); for (Entry e : populateMap.entrySet()) { @@ -479,14 +478,11 @@ private void init() { try { if (searchEngine == SearchEngine.LUCENE) { searchApi = new LuceneApi(propertyHandler.getSearchUrls().get(0).toURI()); - } else if (searchEngine == SearchEngine.ELASTICSEARCH) { - searchApi = new ElasticsearchApi(propertyHandler.getSearchUrls()); - } else if (searchEngine == SearchEngine.OPENSEARCH) { - searchApi = new SearchApi(propertyHandler.getSearchUrls().get(0).toURI()); + } else if (searchEngine == SearchEngine.ELASTICSEARCH || searchEngine == SearchEngine.OPENSEARCH) { + searchApi = new OpensearchApi(propertyHandler.getSearchUrls().get(0).toURI()); } else { - // TODO implement opensearch throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Search engine {} not supported, must be one of LUCENE, ELASTICSEARCH"); + "Search engine {} not supported, must be one of " + SearchEngine.values()); } populateBlockSize = propertyHandler.getSearchPopulateBlockSize(); diff --git a/src/main/java/org/icatproject/core/manager/SearchResult.java b/src/main/java/org/icatproject/core/manager/search/SearchResult.java similarity index 95% rename from src/main/java/org/icatproject/core/manager/SearchResult.java rename to src/main/java/org/icatproject/core/manager/search/SearchResult.java index 592e7813..0bbe063c 100644 --- a/src/main/java/org/icatproject/core/manager/SearchResult.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchResult.java @@ -1,4 +1,4 @@ -package org.icatproject.core.manager; +package org.icatproject.core.manager.search; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/org/icatproject/core/manager/search/URIParameter.java b/src/main/java/org/icatproject/core/manager/search/URIParameter.java new file mode 100644 index 00000000..e69de29b diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index 5b9c4057..4603712e 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -76,14 +76,14 @@ import org.icatproject.core.entity.StudyStatus; import org.icatproject.core.manager.EntityBeanManager; import org.icatproject.core.manager.EntityInfoHandler; -import org.icatproject.core.manager.FacetDimension; -import org.icatproject.core.manager.FacetLabel; import org.icatproject.core.manager.GateKeeper; import org.icatproject.core.manager.Porter; import org.icatproject.core.manager.PropertyHandler; import org.icatproject.core.manager.PropertyHandler.ExtendedAuthenticator; -import org.icatproject.core.manager.ScoredEntityBaseBean; -import org.icatproject.core.manager.SearchResult; +import org.icatproject.core.manager.search.FacetDimension; +import org.icatproject.core.manager.search.FacetLabel; +import org.icatproject.core.manager.search.ScoredEntityBaseBean; +import org.icatproject.core.manager.search.SearchResult; import org.icatproject.utils.ContainerGetter.ContainerType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/resources/run.properties b/src/main/resources/run.properties index 8a8c7734..4cf67f9c 100644 --- a/src/main/resources/run.properties +++ b/src/main/resources/run.properties @@ -25,6 +25,7 @@ search.enqueuedRequestIntervalSeconds = 3 # Configure this option to prevent certain entities being indexed # For example, remove Datafile and DatafileParameter !search.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample +units = \u2103: celsius degC, K: kelvin !cluster = https://smfisher:8181 diff --git a/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java b/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java deleted file mode 100644 index 67c78260..00000000 --- a/src/test/java/org/icatproject/core/manager/TestElasticsearchApi.java +++ /dev/null @@ -1,608 +0,0 @@ -// package org.icatproject.core.manager; - -// import static org.junit.Assert.assertEquals; -// import static org.junit.Assert.fail; - -// import java.io.ByteArrayOutputStream; -// import java.net.URL; -// import java.util.ArrayList; -// import java.util.Arrays; -// import java.util.Date; -// import java.util.HashSet; -// import java.util.Iterator; -// import java.util.List; -// import java.util.Queue; -// import java.util.Set; -// import java.util.concurrent.ConcurrentLinkedQueue; - -// import javax.json.Json; -// import javax.json.stream.JsonGenerator; - -// import org.icatproject.core.IcatException; -// import org.junit.Before; -// import org.junit.BeforeClass; -// import org.junit.Test; -// import org.slf4j.Logger; -// import org.slf4j.LoggerFactory; - -// public class TestElasticsearchApi { - -// static ElasticsearchApi searchApi; -// final static Logger logger = LoggerFactory.getLogger(TestElasticsearchApi.class); - -// @BeforeClass -// public static void beforeClass() throws Exception { -// String urlString = System.getProperty("elasticsearchUrl"); -// logger.info("Using Elasticsearch service at {}", urlString); -// searchApi = new ElasticsearchApi(Arrays.asList(new URL(urlString))); -// } - -// String letters = "abcdefghijklmnopqrstuvwxyz"; - -// long now = new Date().getTime(); - -// int NUMINV = 10; - -// int NUMUSERS = 5; - -// int NUMDS = 30; - -// int NUMDF = 100; - -// int NUMSAMP = 15; - -// private class QueueItem { - -// private String entityName; -// private Long id; -// private String json; - -// public QueueItem(String entityName, Long id, String json) { -// this.entityName = entityName; -// this.id = id; -// this.json = json; -// } - -// } - -// @Test -// public void modify() throws IcatException { -// Queue queue = new ConcurrentLinkedQueue<>(); - -// ByteArrayOutputStream baos = new ByteArrayOutputStream(); -// try (JsonGenerator gen = Json.createGenerator(baos)) { -// gen.writeStartArray(); -// SearchApi.encodeTextField(gen, "text", "Elephants and Aardvarks"); -// SearchApi.encodeStringField(gen, "startDate", new Date()); -// SearchApi.encodeStringField(gen, "endDate", new Date()); -// SearchApi.encodeStoredId(gen, 42L); -// SearchApi.encodeStringField(gen, "dataset", 2001L); -// gen.writeEnd(); -// } - -// String json = baos.toString(); -// // Create -// queue.add(new QueueItem("Datafile", null, json)); -// // Update -// queue.add(new QueueItem("Datafile", 42L, json)); -// // Delete -// queue.add(new QueueItem("Datafile", 42L, null)); -// queue.add(new QueueItem("Datafile", 42L, null)); - -// Iterator qiter = queue.iterator(); -// if (qiter.hasNext()) { -// StringBuilder sb = new StringBuilder("["); -// while (qiter.hasNext()) { -// QueueItem item = qiter.next(); -// if (sb.length() != 1) { -// sb.append(','); -// } -// sb.append("[\"").append(item.entityName).append('"'); -// if (item.id != null) { -// sb.append(',').append(item.id); -// } else { -// sb.append(",null"); -// } -// if (item.json != null) { -// sb.append(',').append(item.json); -// } else { -// sb.append(",null"); -// } -// sb.append(']'); -// qiter.remove(); -// } -// sb.append(']'); -// searchApi.modify(sb.toString()); -// } -// } - -// @Before -// public void before() throws Exception { -// searchApi.clear(); -// } - -// private void checkLsr(SearchResult lsr, Long... n) { -// Set wanted = new HashSet<>(Arrays.asList(n)); -// Set got = new HashSet<>(); - -// for (ScoredEntityBaseBean q : lsr.getResults()) { -// got.add(q.getEntityBaseBeanId()); -// } - -// Set missing = new HashSet<>(wanted); -// missing.removeAll(got); -// if (!missing.isEmpty()) { -// for (Long l : missing) { -// logger.error("Entry missing: {}", l); -// } -// fail("Missing entries"); -// } - -// missing = new HashSet<>(got); -// missing.removeAll(wanted); -// if (!missing.isEmpty()) { -// for (Long l : missing) { -// logger.error("Extra entry: {}", l); -// } -// fail("Extra entries"); -// } - -// } - -// @Test -// public void datafiles() throws Exception { -// populate(); - -// SearchResult lsr = searchApi -// .getResults(SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), 5); -// String uid = lsr.getUid(); - -// checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); -// System.out.println(uid); -// lsr = searchApi.getResults(uid, SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null), -// 200); -// // assertTrue(lsr.getUid() == null); -// assertEquals(95, lsr.getResults().size()); -// searchApi.freeSearcher(uid); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", "e4", null, null, null, null, null, null), 100); -// checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, 96L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null), 100); -// checkLsr(lsr, 1L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, "dfbbb", null, null, null, null, null), 100); -// checkLsr(lsr, 1L, 27L, 53L, 79L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), -// new Date(now + 60000 * 6), null, null, null), 100); -// checkLsr(lsr, 3L, 4L, 5L, 6L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", "b1", "dsddd", new Date(now + 60000 * 3), -// new Date(now + 60000 * 6), null, null, null), 100); -// checkLsr(lsr); -// searchApi.freeSearcher(lsr.getUid()); - -// List pojos = new ArrayList<>(); -// pojos.add(new ParameterPOJO(null, null, "v25")); -// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), -// new Date(now + 60000 * 6), pojos, null, null), 100); -// checkLsr(lsr, 5L); -// searchApi.freeSearcher(lsr.getUid()); - -// pojos = new ArrayList<>(); -// pojos.add(new ParameterPOJO(null, null, "v25")); -// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100); -// checkLsr(lsr, 5L); -// searchApi.freeSearcher(lsr.getUid()); - -// pojos = new ArrayList<>(); -// pojos.add(new ParameterPOJO(null, "u sss", null)); -// lsr = searchApi.getResults(SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null), 100); -// checkLsr(lsr, 13L, 65L); -// searchApi.freeSearcher(lsr.getUid()); -// } - -// @Test -// public void datasets() throws Exception { -// populate(); -// SearchResult lsr = searchApi -// .getResults(SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 5); - -// String uid = lsr.getUid(); -// checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); -// System.out.println(uid); -// lsr = searchApi.getResults(uid, SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null), 100); -// // assertTrue(lsr.getUid() == null); -// checkLsr(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, -// 25L, 26L, 27L, 28L, 29L); -// searchApi.freeSearcher(uid); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "e4", null, null, null, null, null, null), 100); -// checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100); -// checkLsr(lsr, 1L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100); -// checkLsr(lsr, 1L, 27L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), -// new Date(now + 60000 * 6), null, null, null), 100); -// checkLsr(lsr, 3L, 4L, 5L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), -// new Date(now + 60000 * 6), null, null, null), 100); -// checkLsr(lsr, 3L); -// searchApi.freeSearcher(lsr.getUid()); - -// List pojos = new ArrayList<>(); -// pojos.add(new ParameterPOJO(null, null, "v16")); -// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, pojos, null, null), 100); -// checkLsr(lsr, 4L); -// searchApi.freeSearcher(lsr.getUid()); - -// pojos = new ArrayList<>(); -// pojos.add(new ParameterPOJO(null, null, "v16")); -// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), -// new Date(now + 60000 * 6), pojos, null, null), 100); -// checkLsr(lsr, 4L); -// searchApi.freeSearcher(lsr.getUid()); - -// pojos = new ArrayList<>(); -// pojos.add(new ParameterPOJO(null, null, "v16")); -// lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), -// new Date(now + 60000 * 6), pojos, null, null), 100); -// checkLsr(lsr); -// searchApi.freeSearcher(lsr.getUid()); - -// } - -// private void fillParameters(JsonGenerator gen, int i, String rel) { -// int j = i % 26; -// int k = (i + 5) % 26; -// String name = "nm " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); -// String units = "u " + letters.substring(k, k + 1) + letters.substring(k, k + 1) + letters.substring(k, k + 1); - -// gen.writeStartArray(); -// gen.write(rel + "Parameter"); -// gen.writeNull(); -// gen.writeStartArray(); -// SearchApi.encodeStringField(gen, "parameterName", "S" + name); -// SearchApi.encodeStringField(gen, "parameterUnits", units); -// SearchApi.encodeStringField(gen, "parameterStringValue", "v" + i * i); -// SearchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); -// gen.writeEnd(); -// gen.writeEnd(); -// System.out.println(rel + " " + i + " '" + "S" + name + "' '" + units + "' 'v" + i * i + "'"); - -// gen.writeStartArray(); -// gen.write(rel + "Parameter"); -// gen.writeNull(); -// gen.writeStartArray(); -// SearchApi.encodeStringField(gen, "parameterName", "N" + name); -// SearchApi.encodeStringField(gen, "parameterUnits", units); -// SearchApi.encodeDoublePoint(gen, "parameterNumericValue", new Double(j * j)); -// SearchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); -// gen.writeEnd(); -// gen.writeEnd(); -// System.out.println(rel + " " + i + " '" + "N" + name + "' '" + units + "' " + new Double(j * j)); - -// gen.writeStartArray(); -// gen.write(rel + "Parameter"); -// gen.writeNull(); -// gen.writeStartArray(); -// SearchApi.encodeStringField(gen, "parameterName", "D" + name); -// SearchApi.encodeStringField(gen, "parameterUnits", units); -// SearchApi.encodeStringField(gen, "parameterDateValue", new Date(now + 60000 * k * k)); -// SearchApi.encodeSortedDocValuesField(gen, rel, new Long(i)); -// gen.writeEnd(); -// gen.writeEnd(); -// System.out.println( -// rel + " " + i + " '" + "D" + name + "' '" + units + "' '" + new Date(now + 60000 * k * k) + "'"); - -// } - -// @Test -// public void investigations() throws Exception { -// populate(); - -// /* Blocked results */ -// SearchResult lsr = searchApi.getResults( -// SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), -// 5); -// String uid = lsr.getUid(); -// checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); -// System.out.println(uid); -// lsr = searchApi.getResults(uid, SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), -// 6); -// // assertTrue(lsr.getUid() == null); -// checkLsr(lsr, 5L, 6L, 7L, 8L, 9L); -// searchApi.freeSearcher(uid); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "b"), 100); -// checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"), -// 100); -// checkLsr(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults( -// SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""), -// 100); -// checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", null, null, null, null, null, "b"), 100); -// checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "c1", null, null, null, null, null, "b"), 100); -// checkLsr(lsr); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null), -// 100); -// checkLsr(lsr, 4L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", null, null, null, null, "b"), 100); -// checkLsr(lsr, 3L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), -// new Date(now + 60000 * 6), null, null, "b"), 100); -// checkLsr(lsr, 3L); -// searchApi.freeSearcher(lsr.getUid()); - -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, new Date(now + 60000 * 3), -// new Date(now + 60000 * 6), null, null, null), 100); -// checkLsr(lsr, 3L, 4L, 5L); -// searchApi.freeSearcher(lsr.getUid()); - -// List pojos = new ArrayList<>(); -// pojos.add(new ParameterPOJO(null, null, "v9")); -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), -// 100); -// checkLsr(lsr, 3L); -// searchApi.freeSearcher(lsr.getUid()); - -// pojos = new ArrayList<>(); -// pojos.add(new ParameterPOJO(null, null, "v9")); -// pojos.add(new ParameterPOJO(null, null, 7, 10)); -// pojos.add(new ParameterPOJO(null, null, new Date(now + 60000 * 63), new Date(now + 60000 * 65))); -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), -// 100); -// checkLsr(lsr, 3L); -// searchApi.freeSearcher(lsr.getUid()); - -// pojos = new ArrayList<>(); -// pojos.add(new ParameterPOJO(null, null, "v9")); -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), -// new Date(now + 60000 * 6), pojos, null, "b"), 100); -// checkLsr(lsr, 3L); -// searchApi.freeSearcher(lsr.getUid()); - -// pojos = new ArrayList<>(); -// pojos.add(new ParameterPOJO(null, null, "v9")); -// pojos.add(new ParameterPOJO(null, null, "v81")); -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), -// 100); -// checkLsr(lsr); -// searchApi.freeSearcher(lsr.getUid()); - -// pojos = new ArrayList<>(); -// pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null), -// 100); -// checkLsr(lsr, 3L); -// searchApi.freeSearcher(lsr.getUid()); - -// List samples = Arrays.asList("ddd", "nnn"); -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), -// 100); -// checkLsr(lsr, 3L); -// searchApi.freeSearcher(lsr.getUid()); - -// samples = Arrays.asList("ddd", "mmm"); -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null), -// 100); -// checkLsr(lsr); -// searchApi.freeSearcher(lsr.getUid()); - -// pojos = new ArrayList<>(); -// pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); -// samples = Arrays.asList("ddd", "nnn"); -// lsr = searchApi.getResults(SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), -// new Date(now + 60000 * 6), pojos, samples, "b"), 100); -// checkLsr(lsr, 3L); -// searchApi.freeSearcher(lsr.getUid()); -// } - -// /** -// * Populate Investigation, Dataset, Datafile -// */ -// private void populate() throws IcatException { - -// ByteArrayOutputStream baos = new ByteArrayOutputStream(); -// try (JsonGenerator gen = Json.createGenerator(baos)) { -// gen.writeStartArray(); -// for (int i = 0; i < NUMINV; i++) { -// for (int j = 0; j < NUMUSERS; j++) { -// if (i % (j + 1) == 1) { -// String fn = "FN " + letters.substring(j, j + 1) + " " + letters.substring(j, j + 1); -// String name = letters.substring(j, j + 1) + j; -// gen.writeStartArray(); -// gen.write("InvestigationUser"); -// gen.writeNull(); -// gen.writeStartArray(); - -// SearchApi.encodeTextField(gen, "userFullName", fn); - -// SearchApi.encodeStringField(gen, "userName", name); -// SearchApi.encodeSortedDocValuesField(gen, "investigation", new Long(i)); - -// gen.writeEnd(); -// gen.writeEnd(); -// System.out.println("'" + fn + "' " + name + " " + i); -// } -// } -// } -// gen.writeEnd(); -// } -// searchApi.modify(baos.toString()); -// logger.debug("IUs added:"); -// searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), 100); // TODO -// // RM - -// baos = new ByteArrayOutputStream(); -// try (JsonGenerator gen = Json.createGenerator(baos)) { -// gen.writeStartArray(); -// for (int i = 0; i < NUMINV; i++) { -// int j = i % 26; -// int k = (i + 7) % 26; -// int l = (i + 17) % 26; -// String word = letters.substring(j, j + 1) + " " + letters.substring(k, k + 1) + " " -// + letters.substring(l, l + 1); -// gen.writeStartArray(); -// gen.write("Investigation"); -// gen.writeNull(); -// gen.writeStartArray(); -// SearchApi.encodeTextField(gen, "text", word); -// SearchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); -// SearchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); -// SearchApi.encodeStoredId(gen, new Long(i)); -// SearchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); -// gen.writeEnd(); -// gen.writeEnd(); -// System.out.println("INVESTIGATION '" + word + "' " + new Date(now + i * 60000) + " " + i); -// } -// gen.writeEnd(); -// } -// searchApi.modify(baos.toString()); -// logger.debug("Is added:"); -// searchApi.getResults(SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null), 100); // TODO -// // RM - -// baos = new ByteArrayOutputStream(); -// try (JsonGenerator gen = Json.createGenerator(baos)) { -// gen.writeStartArray(); -// for (int i = 0; i < NUMINV; i++) { -// if (i % 2 == 1) { -// fillParameters(gen, i, "investigation"); -// } -// } -// gen.writeEnd(); -// } -// searchApi.modify(baos.toString()); - -// baos = new ByteArrayOutputStream(); -// try (JsonGenerator gen = Json.createGenerator(baos)) { -// gen.writeStartArray(); -// for (int i = 0; i < NUMDS; i++) { -// int j = i % 26; -// String word = "DS" + letters.substring(j, j + 1) + letters.substring(j, j + 1) -// + letters.substring(j, j + 1); -// gen.writeStartArray(); -// gen.write("Dataset"); -// gen.writeNull(); -// gen.writeStartArray(); -// SearchApi.encodeTextField(gen, "text", word); -// SearchApi.encodeStringField(gen, "startDate", new Date(now + i * 60000)); -// SearchApi.encodeStringField(gen, "endDate", new Date(now + (i + 1) * 60000)); -// SearchApi.encodeStoredId(gen, new Long(i)); -// SearchApi.encodeSortedDocValuesField(gen, "id", new Long(i)); -// SearchApi.encodeStringField(gen, "investigation", new Long(i % NUMINV)); -// gen.writeEnd(); -// gen.writeEnd(); -// System.out.println("DATASET '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMINV); -// } -// gen.writeEnd(); -// } -// searchApi.modify(baos.toString()); - -// baos = new ByteArrayOutputStream(); -// try (JsonGenerator gen = Json.createGenerator(baos)) { -// gen.writeStartArray(); -// for (int i = 0; i < NUMDS; i++) { -// if (i % 3 == 1) { -// fillParameters(gen, i, "dataset"); -// } -// } -// gen.writeEnd(); -// } -// searchApi.modify(baos.toString()); - -// baos = new ByteArrayOutputStream(); -// try (JsonGenerator gen = Json.createGenerator(baos)) { -// gen.writeStartArray(); -// for (int i = 0; i < NUMDF; i++) { -// int j = i % 26; -// String word = "DF" + letters.substring(j, j + 1) + letters.substring(j, j + 1) -// + letters.substring(j, j + 1); -// gen.writeStartArray(); -// gen.write("Datafile"); -// gen.writeNull(); -// gen.writeStartArray(); -// SearchApi.encodeTextField(gen, "text", word); -// SearchApi.encodeStringField(gen, "date", new Date(now + i * 60000)); -// SearchApi.encodeStoredId(gen, new Long(i)); -// SearchApi.encodeStringField(gen, "dataset", new Long(i % NUMDS)); -// SearchApi.encodeStringField(gen, "investigation", new Long((i % NUMDS) % NUMINV)); -// gen.writeEnd(); -// gen.writeEnd(); -// System.out.println("DATAFILE '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMDS); - -// } -// gen.writeEnd(); -// } -// searchApi.modify(baos.toString()); - -// baos = new ByteArrayOutputStream(); -// try (JsonGenerator gen = Json.createGenerator(baos)) { -// gen.writeStartArray(); -// for (int i = 0; i < NUMDF; i++) { -// if (i % 4 == 1) { -// fillParameters(gen, i, "datafile"); -// } -// } -// gen.writeEnd(); -// } -// searchApi.modify(baos.toString()); - -// baos = new ByteArrayOutputStream(); -// try (JsonGenerator gen = Json.createGenerator(baos)) { -// gen.writeStartArray(); -// for (int i = 0; i < NUMSAMP; i++) { -// int j = i % 26; -// String word = "SType " + letters.substring(j, j + 1) + letters.substring(j, j + 1) -// + letters.substring(j, j + 1); -// gen.writeStartArray(); -// gen.write("Sample"); -// gen.writeNull(); -// gen.writeStartArray(); -// SearchApi.encodeTextField(gen, "sampleText", word); -// SearchApi.encodeSortedDocValuesField(gen, "investigation", new Long(i % NUMINV)); -// gen.writeEnd(); -// gen.writeEnd(); -// System.out.println("SAMPLE '" + word + "' " + i % NUMINV); -// } -// gen.writeEnd(); - -// } -// searchApi.modify(baos.toString()); - -// searchApi.commit(); - -// } -// } diff --git a/src/test/java/org/icatproject/core/manager/TestLucene.java b/src/test/java/org/icatproject/core/manager/TestLucene.java deleted file mode 100644 index 4442dd6c..00000000 --- a/src/test/java/org/icatproject/core/manager/TestLucene.java +++ /dev/null @@ -1,1228 +0,0 @@ -package org.icatproject.core.manager; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.lang.reflect.Array; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.ConcurrentLinkedQueue; - -import javax.json.Json; -import javax.json.JsonArray; -import javax.json.JsonArrayBuilder; -import javax.json.JsonObject; -import javax.json.JsonObjectBuilder; -import javax.json.JsonValue; -import javax.json.stream.JsonGenerator; -import javax.ws.rs.core.MediaType; - -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.icatproject.core.IcatException; -import org.icatproject.core.IcatException.IcatExceptionType; -import org.icatproject.core.entity.Datafile; -import org.icatproject.core.entity.DatafileFormat; -import org.icatproject.core.entity.DatafileParameter; -import org.icatproject.core.entity.Dataset; -import org.icatproject.core.entity.DatasetParameter; -import org.icatproject.core.entity.DatasetType; -import org.icatproject.core.entity.Facility; -import org.icatproject.core.entity.Investigation; -import org.icatproject.core.entity.InvestigationParameter; -import org.icatproject.core.entity.InvestigationType; -import org.icatproject.core.entity.InvestigationUser; -import org.icatproject.core.entity.Parameter; -import org.icatproject.core.entity.ParameterType; -import org.icatproject.core.entity.Sample; -import org.icatproject.core.entity.SampleType; -import org.icatproject.core.entity.User; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TestLucene { - - static LuceneApi luceneApi; - private static URI uribase; - final static Logger logger = LoggerFactory.getLogger(TestLucene.class); - - @BeforeClass - public static void beforeClass() throws Exception { - String urlString = System.getProperty("luceneUrl"); - logger.info("Using lucene service at {}", urlString); - uribase = new URI(urlString); - luceneApi = new LuceneApi(uribase); - } - - String letters = "abcdefghijklmnopqrstuvwxyz"; - - long now = new Date().getTime(); - - int NUMINV = 10; - - int NUMUSERS = 5; - - int NUMDS = 30; - - int NUMDF = 100; - - int NUMSAMP = 15; - - @Test - public void modifyDatafile() throws IcatException { - Investigation investigation = new Investigation(); - investigation.setId(0L); - Dataset dataset = new Dataset(); - dataset.setId(0L); - dataset.setInvestigation(investigation); - - Datafile elephantDatafile = new Datafile(); - elephantDatafile.setName("Elephants and Aardvarks"); - elephantDatafile.setDatafileModTime(new Date(0L)); - elephantDatafile.setId(42L); - elephantDatafile.setDataset(dataset); - - DatafileFormat pdfFormat = new DatafileFormat(); - pdfFormat.setId(0L); - pdfFormat.setName("pdf"); - Datafile rhinoDatafile = new Datafile(); - rhinoDatafile.setName("Rhinos and Aardvarks"); - rhinoDatafile.setDatafileModTime(new Date(3L)); - rhinoDatafile.setId(42L); - rhinoDatafile.setDataset(dataset); - rhinoDatafile.setDatafileFormat(pdfFormat); - - DatafileFormat pngFormat = new DatafileFormat(); - pngFormat.setId(0L); - pngFormat.setName("png"); - - JsonObject elephantQuery = SearchApi.buildQuery("Datafile", null, "elephant", null, null, null, null, null); - JsonObject rhinoQuery = SearchApi.buildQuery("Datafile", null, "rhino", null, null, null, null, null); - JsonObject pdfQuery = SearchApi.buildQuery("Datafile", null, "datafileFormat.name:pdf", null, null, null, null, - null); - JsonObject pngQuery = SearchApi.buildQuery("Datafile", null, "datafileFormat.name:png", null, null, null, null, - null); - JsonObject queryObject = Json.createObjectBuilder().add("id", Json.createArrayBuilder().add("42")).build(); - - JsonObjectBuilder stringDimensionBuilder = Json.createObjectBuilder().add("dimension", "datafileFormat.name"); - JsonArrayBuilder stringDimensionsBuilder = Json.createArrayBuilder().add(stringDimensionBuilder); - JsonObjectBuilder lowRangeBuilder = Json.createObjectBuilder().add("from", 0L).add("to", 2L).add("key", "low"); - JsonObjectBuilder highRangeBuilder = Json.createObjectBuilder().add("from", 2L).add("to", 4L).add("key", "high"); - JsonArrayBuilder rangesBuilder = Json.createArrayBuilder().add(lowRangeBuilder).add(highRangeBuilder); - JsonObjectBuilder rangedDimensionBuilder = Json.createObjectBuilder().add("dimension", "date").add("ranges", - rangesBuilder); - JsonArrayBuilder rangedDimensionsBuilder = Json.createArrayBuilder().add(rangedDimensionBuilder); - JsonObject stringFacetQuery = Json.createObjectBuilder().add("query", queryObject) - .add("dimensions", stringDimensionsBuilder).build(); - JsonObject rangeFacetQuery = Json.createObjectBuilder().add("query", queryObject) - .add("dimensions", rangedDimensionsBuilder).build(); - - // Original - Queue queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("create", elephantDatafile)); - modifyQueue(queue); - checkLsr(luceneApi.getResults(elephantQuery, 5), 42L); - checkLsr(luceneApi.getResults(rhinoQuery, 5)); - checkLsr(luceneApi.getResults(pdfQuery, 5)); - checkLsr(luceneApi.getResults(pngQuery, 5)); - List facetDimensions = luceneApi.facetSearch("Datafile", stringFacetQuery, 5, 5); - assertEquals(0, facetDimensions.size()); - facetDimensions = luceneApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - FacetDimension facetDimension = facetDimensions.get(0); - assertEquals("date", facetDimension.getDimension()); - assertEquals(2, facetDimension.getFacets().size()); - FacetLabel facetLabel = facetDimension.getFacets().get(0); - assertEquals("low", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("high", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - - // Change name and add a format - queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("update", rhinoDatafile)); - modifyQueue(queue); - checkLsr(luceneApi.getResults(elephantQuery, 5)); - checkLsr(luceneApi.getResults(rhinoQuery, 5), 42L); - checkLsr(luceneApi.getResults(pdfQuery, 5), 42L); - checkLsr(luceneApi.getResults(pngQuery, 5)); - facetDimensions = luceneApi.facetSearch("Datafile", stringFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimensions = luceneApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("date", facetDimension.getDimension()); - assertEquals(2, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("low", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("high", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - - // Change just the format - queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("update", pngFormat)); - modifyQueue(queue); - checkLsr(luceneApi.getResults(elephantQuery, 5)); - checkLsr(luceneApi.getResults(rhinoQuery, 5), 42L); - checkLsr(luceneApi.getResults(pdfQuery, 5)); - checkLsr(luceneApi.getResults(pngQuery, 5), 42L); - facetDimensions = luceneApi.facetSearch("Datafile", stringFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("datafileFormat.name", facetDimension.getDimension()); - assertEquals(1, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("png", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - facetDimensions = luceneApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("date", facetDimension.getDimension()); - assertEquals(2, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("low", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("high", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - - // Remove the format - queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("delete", pngFormat)); - modifyQueue(queue); - checkLsr(luceneApi.getResults(elephantQuery, 5)); - checkLsr(luceneApi.getResults(rhinoQuery, 5), 42L); - checkLsr(luceneApi.getResults(pdfQuery, 5)); - checkLsr(luceneApi.getResults(pngQuery, 5)); - facetDimensions = luceneApi.facetSearch("Datafile", stringFacetQuery, 5, 5); - assertEquals(0, facetDimensions.size()); - facetDimensions = luceneApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("date", facetDimension.getDimension()); - assertEquals(2, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("low", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("high", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - - // Remove the file - queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeDeletion(elephantDatafile)); - queue.add(SearchApi.encodeDeletion(rhinoDatafile)); - modifyQueue(queue); - checkLsr(luceneApi.getResults(elephantQuery, 5)); - checkLsr(luceneApi.getResults(rhinoQuery, 5)); - checkLsr(luceneApi.getResults(pdfQuery, 5)); - checkLsr(luceneApi.getResults(pngQuery, 5)); - - // Multiple commands at once - queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("create", elephantDatafile)); - queue.add(SearchApi.encodeOperation("update", rhinoDatafile)); - queue.add(SearchApi.encodeDeletion(elephantDatafile)); - queue.add(SearchApi.encodeDeletion(rhinoDatafile)); - modifyQueue(queue); - checkLsr(luceneApi.getResults(elephantQuery, 5)); - checkLsr(luceneApi.getResults(rhinoQuery, 5)); - checkLsr(luceneApi.getResults(pdfQuery, 5)); - checkLsr(luceneApi.getResults(pngQuery, 5)); - } - - private void modifyQueue(Queue queue) throws IcatException { - Iterator qiter = queue.iterator(); - if (qiter.hasNext()) { - StringBuilder sb = new StringBuilder("["); - - while (qiter.hasNext()) { - String item = qiter.next(); - if (sb.length() != 1) { - sb.append(','); - } - sb.append(item); - qiter.remove(); - } - sb.append(']'); - logger.debug("XXX " + sb.toString()); - - luceneApi.modify(sb.toString()); - luceneApi.commit(); - } - } - - private void addDocuments(String entityName, String json) throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(uribase).setPath(LuceneApi.basePath + "/addNow/" + entityName).build(); - HttpPost httpPost = new HttpPost(uri); - StringEntity input = new StringEntity(json); - input.setContentType(MediaType.APPLICATION_JSON); - httpPost.setEntity(input); - - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } catch (IOException | URISyntaxException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } - } - - @Before - public void before() throws Exception { - luceneApi.clear(); - } - - private void checkDatafile(ScoredEntityBaseBean datafile) { - JsonObject source = datafile.getSource(); - assertNotNull(source); - Set expectedKeys = new HashSet<>( - Arrays.asList("id", "investigation.id", "name", "date")); - assertEquals(expectedKeys, source.keySet()); - assertEquals("0", source.getString("id")); - assertEquals("0", source.getString("investigation.id")); - assertEquals("DFaaa", source.getString("name")); - assertNotNull(source.getJsonNumber("date")); - } - - private void checkDataset(ScoredEntityBaseBean dataset) { - JsonObject source = dataset.getSource(); - assertNotNull(source); - Set expectedKeys = new HashSet<>( - Arrays.asList("id", "investigation.id", "name", "startDate", "endDate")); - assertEquals(expectedKeys, source.keySet()); - assertEquals("0", source.getString("id")); - assertEquals("0", source.getString("investigation.id")); - assertEquals("DSaaa", source.getString("name")); - assertNotNull(source.getJsonNumber("startDate")); - assertNotNull(source.getJsonNumber("endDate")); - } - - private void checkInvestigation(ScoredEntityBaseBean investigation) { - JsonObject source = investigation.getSource(); - assertNotNull(source); - Set expectedKeys = new HashSet<>(Arrays.asList("id", "name", "startDate", "endDate")); - assertEquals(expectedKeys, source.keySet()); - assertEquals("0", source.getString("id")); - assertEquals("a h r", source.getString("name")); - assertNotNull(source.getJsonNumber("startDate")); - assertNotNull(source.getJsonNumber("endDate")); - } - - private void checkLsr(SearchResult lsr, Long... n) { - Set wanted = new HashSet<>(Arrays.asList(n)); - Set got = new HashSet<>(); - - for (ScoredEntityBaseBean q : lsr.getResults()) { - got.add(q.getEntityBaseBeanId()); - } - - Set missing = new HashSet<>(wanted); - missing.removeAll(got); - if (!missing.isEmpty()) { - for (Long l : missing) { - logger.error("Entry missing: {}", l); - } - fail("Missing entries"); - } - - missing = new HashSet<>(got); - missing.removeAll(wanted); - if (!missing.isEmpty()) { - for (Long l : missing) { - logger.error("Extra entry: {}", l); - } - fail("Extra entries"); - } - - } - - private void checkLsrOrder(SearchResult lsr, Long... n) { - List results = lsr.getResults(); - if (n.length != results.size()) { - checkLsr(lsr, n); - } - for (int i = 0; i < n.length; i++) { - Long resultId = results.get(i).getEntityBaseBeanId(); - Long expectedId = (Long) Array.get(n, i); - if (resultId != expectedId) { - fail("Expected id " + expectedId + " in position " + i + " but got " + resultId); - } - } - } - - @Test - public void datafiles() throws Exception { - populate(); - - JsonObject query = SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null); - List fields = Arrays.asList("date", "name", "investigation.id", "id"); - SearchResult lsr = luceneApi.getResults(query, null, 5, null, fields); - JsonValue searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); - checkDatafile(lsr.getResults().get(0)); - lsr = luceneApi.getResults(query, searchAfter, 200, null, fields); - assertNull(lsr.getSearchAfter()); - assertEquals(95, lsr.getResults().size()); - - // Test searchAfter preserves the sorting of original search (asc) - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("date", "asc"); - gen.writeEnd(); - } - String sort = baos.toString(); - lsr = luceneApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); - checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - - // Test searchAfter preserves the sorting of original search (desc) - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("date", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); - lsr = luceneApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 99L, 98L, 97L, 96L, 95L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); - checkLsrOrder(lsr, 94L, 93L, 92L, 91L, 90L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - - // Test tie breaks on fields with identical values (asc) - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("name", "asc"); - gen.writeEnd(); - } - sort = baos.toString(); - lsr = luceneApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 0L, 26L, 52L, 78L, 1L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("name", "asc"); - gen.write("date", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); - lsr = luceneApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 78L, 52L, 26L, 0L, 79L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - - // Test tie breaks on fields with identical values (desc) - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("name", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); - lsr = luceneApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 25L, 51L, 77L, 24L, 50L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("name", "desc"); - gen.write("date", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); - lsr = luceneApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 77L, 51L, 25L, 76L, 50L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - - query = SearchApi.buildQuery("Datafile", "e4", null, null, null, null, null, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, 96L); - - query = SearchApi.buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 1L); - - query = SearchApi.buildQuery("Datafile", null, "dfbbb", null, null, null, null, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 1L, 27L, 53L, 79L); - - query = SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 3L, 4L, 5L, 6L); - - query = SearchApi.buildQuery("Datafile", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr); - - List pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v25")); - query = SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 5L); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v25")); - query = SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 5L); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, "u sss", null)); - query = SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 13L, 65L); - } - - @Test - public void datasets() throws Exception { - populate(); - - JsonObject query = SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null); - List fields = Arrays.asList("startDate", "endDate", "name", "investigation.id", "id"); - SearchResult lsr = luceneApi.getResults(query, null, 5, null, fields); - JsonValue searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); - checkDataset(lsr.getResults().get(0)); - lsr = luceneApi.getResults(query, searchAfter, 100, null, fields); - assertNull(lsr.getSearchAfter()); - checkLsr(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, - 25L, 26L, 27L, 28L, 29L); - - // Test searchAfter preserves the sorting of original search (asc) - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("startDate", "asc"); - gen.writeEnd(); - } - String sort = baos.toString(); - lsr = luceneApi.getResults(query, 5, sort); - checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); - checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - - // Test searchAfter preserves the sorting of original search (desc) - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("endDate", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); - lsr = luceneApi.getResults(query, 5, sort); - checkLsrOrder(lsr, 29L, 28L, 27L, 26L, 25L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); - checkLsrOrder(lsr, 24L, 23L, 22L, 21L, 20L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - - // Test tie breaks on fields with identical values (asc) - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("name", "asc"); - gen.writeEnd(); - } - sort = baos.toString(); - lsr = luceneApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 0L, 26L, 1L, 27L, 2L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("name", "asc"); - gen.write("endDate", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); - lsr = luceneApi.getResults(query, 5, sort); - checkLsrOrder(lsr, 26L, 0L, 27L, 1L, 28L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "e4", null, null, null, null, null, null), 100, - null); - checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L); - - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100, - null); - checkLsr(lsr, 1L); - - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100, - null); - checkLsr(lsr, 1L, 27L); - - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100, null); - checkLsr(lsr, 3L, 4L, 5L); - - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100, null); - checkLsr(lsr, 3L); - - List pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, pojos, null, null), 100, - null); - checkLsr(lsr, 4L); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null), 100, null); - checkLsr(lsr, 4L); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = luceneApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null), 100, null); - checkLsr(lsr); - } - - private void fillParms(JsonGenerator gen, int i, String rel) { - int j = i % 26; - int k = (i + 5) % 26; - String name = "nm " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); - String units = "u " + letters.substring(k, k + 1) + letters.substring(k, k + 1) + letters.substring(k, k + 1); - - ParameterType dateParameterType = new ParameterType(); - dateParameterType.setId(0L); - dateParameterType.setName("D" + name); - dateParameterType.setUnits(units); - ParameterType numericParameterType = new ParameterType(); - numericParameterType.setId(0L); - numericParameterType.setName("N" + name); - numericParameterType.setUnits(units); - ParameterType stringParameterType = new ParameterType(); - stringParameterType.setId(0L); - stringParameterType.setName("S" + name); - stringParameterType.setUnits(units); - - Parameter parameter; - if (rel.equals("datafile")) { - parameter = new DatafileParameter(); - Datafile datafile = new Datafile(); - datafile.setId(new Long(i)); - ((DatafileParameter) parameter).setDatafile(datafile); - } else if (rel.equals("dataset")) { - parameter = new DatasetParameter(); - Dataset dataset = new Dataset(); - dataset.setId(new Long(i)); - ((DatasetParameter) parameter).setDataset(dataset); - } else if (rel.equals("investigation")) { - parameter = new InvestigationParameter(); - Investigation investigation = new Investigation(); - investigation.setId(new Long(i)); - ((InvestigationParameter) parameter).setInvestigation(investigation); - } else { - fail(rel + " is not valid"); - return; - } - parameter.setId(0L); - - parameter.setType(dateParameterType); - parameter.setDateTimeValue(new Date(now + 60000 * k * k)); - gen.writeStartObject(); - parameter.getDoc(gen); - gen.writeEnd(); - System.out.println( - rel + " " + i + " '" + "D" + name + "' '" + units + "' '" + new Date(now + 60000 * k * k) + "'"); - - parameter.setType(numericParameterType); - parameter.setNumericValue(new Double(j * j)); - gen.writeStartObject(); - parameter.getDoc(gen); - gen.writeEnd(); - System.out.println(rel + " " + i + " '" + "N" + name + "' '" + units + "' " + new Double(j * j)); - - parameter.setType(stringParameterType); - parameter.setStringValue("v" + i * i); - gen.writeStartObject(); - parameter.getDoc(gen); - gen.writeEnd(); - System.out.println(rel + " " + i + " '" + "S" + name + "' '" + units + "' 'v" + i * i + "'"); - } - - @Test - public void investigations() throws Exception { - populate(); - - /* Blocked results */ - JsonObject query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null); - List fields = Arrays.asList("startDate", "endDate", "name", "id"); - SearchResult lsr = luceneApi.getResults(query, null, 5, null, fields); - checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); - checkInvestigation(lsr.getResults().get(0)); - JsonValue searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter, 6, null, fields); - checkLsr(lsr, 5L, 6L, 7L, 8L, 9L); - searchAfter = lsr.getSearchAfter(); - assertNull(searchAfter); - - // Test searchAfter preserves the sorting of original search (asc) - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("startDate", "asc"); - gen.writeEnd(); - } - String sort = baos.toString(); - lsr = luceneApi.getResults(query, 5, sort); - checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); - checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - - // Test searchAfter preserves the sorting of original search (desc) - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("endDate", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); - lsr = luceneApi.getResults(query, 5, sort); - checkLsrOrder(lsr, 9L, 8L, 7L, 6L, 5L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - lsr = luceneApi.getResults(query, searchAfter, 5, sort, fields); - checkLsrOrder(lsr, 4L, 3L, 2L, 1L, 0L); - searchAfter = lsr.getSearchAfter(); - assertNotNull(searchAfter); - - query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "b"); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); - - query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); - - query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); - - query = SearchApi.buildQuery("Investigation", "b1", null, null, null, null, null, "b"); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); - - query = SearchApi.buildQuery("Investigation", "c1", null, null, null, null, null, "b"); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr); - - query = SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 4L); - - query = SearchApi.buildQuery("Investigation", "b1", "d", null, null, null, null, "b"); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 3L); - - query = SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), - null, null, "b"); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 3L); - - query = SearchApi.buildQuery("Investigation", null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), - null, null, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 3L, 4L, 5L); - - List pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v9")); - query = SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 3L); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v9")); - pojos.add(new ParameterPOJO(null, null, 7, 10)); - pojos.add(new ParameterPOJO(null, null, new Date(now + 60000 * 63), new Date(now + 60000 * 65))); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 3L); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v9")); - query = SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), - pojos, null, "b"); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 3L); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO(null, null, "v9")); - pojos.add(new ParameterPOJO(null, null, "v81")); - query = SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); - query = SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 3L); - - List samples = Arrays.asList("ddd", "nnn"); - query = SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 3L); - - samples = Arrays.asList("ddd", "mmm"); - query = SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr); - - pojos = new ArrayList<>(); - pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); - samples = Arrays.asList("ddd", "nnn"); - query = SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), - pojos, samples, "b"); - lsr = luceneApi.getResults(query, 100, null); - checkLsr(lsr, 3L); - } - - @Test - public void locking() throws IcatException { - - try { - luceneApi.unlock("Dataset"); - fail(); - } catch (IcatException e) { - assertEquals("Lucene is not currently locked for Dataset", e.getMessage()); - } - luceneApi.lock("Dataset"); - try { - luceneApi.lock("Dataset"); - fail(); - } catch (IcatException e) { - assertEquals("Lucene already locked for Dataset", e.getMessage()); - } - luceneApi.unlock("Dataset"); - try { - luceneApi.unlock("Dataset"); - fail(); - } catch (IcatException e) { - assertEquals("Lucene is not currently locked for Dataset", e.getMessage()); - } - } - - /** - * Populate UserGroup, Investigation, InvestigationParameter, - * InvestigationUser, Dataset,DatasetParameter,Datafile, DatafileParameter - * and Sample - */ - private void populate() throws IcatException { - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - Long investigationUserId = 0L; - gen.writeStartArray(); - for (int i = 0; i < NUMINV; i++) { - for (int j = 0; j < NUMUSERS; j++) { - if (i % (j + 1) == 1) { - String fn = "FN " + letters.substring(j, j + 1) + " " + letters.substring(j, j + 1); - String name = letters.substring(j, j + 1) + j; - User user = new User(); - user.setId(new Long(j)); - user.setName(name); - user.setFullName(fn); - Investigation investigation = new Investigation(); - investigation.setId(new Long(i)); - InvestigationUser investigationUser = new InvestigationUser(); - investigationUser.setId(investigationUserId); - investigationUser.setUser(user); - investigationUser.setInvestigation(investigation); - - gen.writeStartObject(); - investigationUser.getDoc(gen); - gen.writeEnd(); - investigationUserId++; - System.out.println("'" + fn + "' " + name + " " + i); - } - } - } - gen.writeEnd(); - } - addDocuments("InvestigationUser", baos.toString()); - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMINV; i++) { - int j = i % 26; - int k = (i + 7) % 26; - int l = (i + 17) % 26; - String word = letters.substring(j, j + 1) + " " + letters.substring(k, k + 1) + " " - + letters.substring(l, l + 1); - Investigation investigation = new Investigation(); - Facility facility = new Facility(); - facility.setName(""); - facility.setId(0L); - investigation.setFacility(facility); - InvestigationType type = new InvestigationType(); - type.setName("test"); - type.setId(0L); - investigation.setType(type); - investigation.setName(word); - investigation.setTitle(""); - investigation.setVisitId(""); - investigation.setStartDate(new Date(now + i * 60000)); - investigation.setEndDate(new Date(now + (i + 1) * 60000)); - investigation.setId(new Long(i)); - gen.writeStartObject(); - investigation.getDoc(gen); - gen.writeEnd(); - System.out.println("INVESTIGATION '" + word + "' " + new Date(now + i * 60000) + " " + i); - } - gen.writeEnd(); - } - addDocuments("Investigation", baos.toString()); - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMINV; i++) { - if (i % 2 == 1) { - fillParms(gen, i, "investigation"); - } - } - gen.writeEnd(); - } - addDocuments("InvestigationParameter", baos.toString()); - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMDS; i++) { - int j = i % 26; - String word = "DS" + letters.substring(j, j + 1) + letters.substring(j, j + 1) - + letters.substring(j, j + 1); - - Investigation investigation = new Investigation(); - investigation.setId(new Long(i % NUMINV)); - Dataset dataset = new Dataset(); - DatasetType type = new DatasetType(); - type.setName("test"); - type.setId(0L); - dataset.setType(type); - dataset.setName(word); - dataset.setStartDate(new Date(now + i * 60000)); - dataset.setEndDate(new Date(now + (i + 1) * 60000)); - dataset.setId(new Long(i)); - dataset.setInvestigation(investigation); - - gen.writeStartObject(); - dataset.getDoc(gen); - gen.writeEnd(); - System.out.println("DATASET '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMINV); - } - gen.writeEnd(); - } - addDocuments("Dataset", baos.toString()); - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMDS; i++) { - if (i % 3 == 1) { - fillParms(gen, i, "dataset"); - } - } - gen.writeEnd(); - } - addDocuments("DatasetParameter", baos.toString()); - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMDF; i++) { - int j = i % 26; - String word = "DF" + letters.substring(j, j + 1) + letters.substring(j, j + 1) - + letters.substring(j, j + 1); - - Investigation investigation = new Investigation(); - investigation.setId(new Long((i % NUMDS) % NUMINV)); - Dataset dataset = new Dataset(); - dataset.setId(new Long(i % NUMDS)); - dataset.setInvestigation(investigation); - Datafile datafile = new Datafile(); - datafile.setName(word); - datafile.setDatafileModTime(new Date(now + i * 60000)); - datafile.setId(new Long(i)); - datafile.setDataset(dataset); - - gen.writeStartObject(); - datafile.getDoc(gen); - gen.writeEnd(); - System.out.println("DATAFILE '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMDS); - - } - gen.writeEnd(); - } - addDocuments("Datafile", baos.toString()); - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMDF; i++) { - if (i % 4 == 1) { - fillParms(gen, i, "datafile"); - } - } - gen.writeEnd(); - } - addDocuments("DatafileParameter", baos.toString()); - - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartArray(); - for (int i = 0; i < NUMSAMP; i++) { - int j = i % 26; - String word = "SType " + letters.substring(j, j + 1) + letters.substring(j, j + 1) - + letters.substring(j, j + 1); - - Investigation investigation = new Investigation(); - investigation.setId(new Long(i % NUMINV)); - SampleType sampleType = new SampleType(); - sampleType.setId(0L); - sampleType.setName("test"); - Sample sample = new Sample(); - sample.setId(new Long(i)); - sample.setInvestigation(investigation); - sample.setType(sampleType); - sample.setName(word); - - gen.writeStartObject(); - sample.getDoc(gen); - gen.writeEnd(); - System.out.println("SAMPLE '" + word + "' " + i % NUMINV); - } - gen.writeEnd(); - - } - addDocuments("Sample", baos.toString()); - - luceneApi.commit(); - - } - - @Test - public void unitConversion() throws IcatException { - // Build queries for raw and SI values - JsonObject mKQuery = Json.createObjectBuilder().add("type.units", "mK").build(); - JsonObject celsiusQuery = Json.createObjectBuilder().add("type.units", "celsius").build(); - JsonObject wrongQuery = Json.createObjectBuilder().add("type.units", "wrong").build(); - JsonObject kelvinQuery = Json.createObjectBuilder().add("type.unitsSI", "Kelvin").build(); - JsonObjectBuilder lowRangeBuilder = Json.createObjectBuilder().add("from", 272.5).add("to", 273.5).add("key", "272.5_273.5"); - JsonObjectBuilder midRangeBuilder = Json.createObjectBuilder().add("from", 272999.5).add("to", 273000.5).add("key", "272999.5_273000.5"); - JsonObjectBuilder highRangeBuilder = Json.createObjectBuilder().add("from", 273272.5).add("to", 273273.5).add("key", "273272.5_273273.5"); - JsonArray ranges = Json.createArrayBuilder().add(lowRangeBuilder).add(midRangeBuilder).add(highRangeBuilder) - .build(); - JsonObject rawObject = Json.createObjectBuilder().add("dimension", "numericValue").add("ranges", ranges) - .build(); - JsonObject rawFacetQuery = Json.createObjectBuilder().add("query", mKQuery) - .add("dimensions", Json.createArrayBuilder().add(rawObject)).build(); - JsonObject systemObject = Json.createObjectBuilder().add("dimension", "numericValueSI").add("ranges", ranges) - .build(); - JsonObject systemFacetQuery = Json.createObjectBuilder().add("query", kelvinQuery) - .add("dimensions", Json.createArrayBuilder().add(systemObject)).build(); - - // Build entities - Investigation investigation = new Investigation(); - investigation.setId(0L); - investigation.setName("name"); - investigation.setVisitId("visitId"); - investigation.setTitle("title"); - investigation.setCreateTime(new Date()); - investigation.setModTime(new Date()); - Facility facility = new Facility(); - facility.setName("facility"); - facility.setId(0L); - investigation.setFacility(facility); - InvestigationType type = new InvestigationType(); - type.setName("type"); - type.setId(0L); - investigation.setType(type); - - ParameterType numericParameterType = new ParameterType(); - numericParameterType.setId(0L); - numericParameterType.setName("parameter"); - numericParameterType.setUnits("mK"); - InvestigationParameter parameter = new InvestigationParameter(); - parameter.setInvestigation(investigation); - parameter.setType(numericParameterType); - parameter.setNumericValue(273000.); - parameter.setId(0L); - - Queue queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("create", investigation)); - queue.add(SearchApi.encodeOperation("create", parameter)); - modifyQueue(queue); - - // Assert the raw value is still 273000 (mK) - List facetDimensions = luceneApi.facetSearch("InvestigationParameter", rawFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - FacetDimension facetDimension = facetDimensions.get(0); - assertEquals("numericValue", facetDimension.getDimension()); - assertEquals(3, facetDimension.getFacets().size()); - FacetLabel facetLabel = facetDimension.getFacets().get(0); - assertEquals("272.5_273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("272999.5_273000.5", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(2); - assertEquals("273272.5_273273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - - // Assert the SI value is 273 (K) - facetDimensions = luceneApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("numericValueSI", facetDimension.getDimension()); - assertEquals(3, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("272.5_273.5", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("272999.5_273000.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(2); - assertEquals("273272.5_273273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - - // Change units only to "celsius" - numericParameterType.setUnits("celsius"); - queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("update", parameter)); - modifyQueue(queue); - rawFacetQuery = Json.createObjectBuilder().add("query", celsiusQuery) - .add("dimensions", Json.createArrayBuilder().add(rawObject)).build(); - - // Assert the raw value is still 273000 (deg C) - facetDimensions = luceneApi.facetSearch("InvestigationParameter", rawFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("numericValue", facetDimension.getDimension()); - assertEquals(3, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("272.5_273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("272999.5_273000.5", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(2); - assertEquals("273272.5_273273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - - // Assert the SI value is 273273.15 (K) - facetDimensions = luceneApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("numericValueSI", facetDimension.getDimension()); - assertEquals(3, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("272.5_273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("272999.5_273000.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(2); - assertEquals("273272.5_273273.5", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - - // Change units to something wrong - numericParameterType.setUnits("wrong"); - queue = new ConcurrentLinkedQueue<>(); - // queue.add(SearchApi.encodeOperation("update", parameter)); - queue.add(SearchApi.encodeOperation("update", numericParameterType)); - modifyQueue(queue); - rawFacetQuery = Json.createObjectBuilder().add("query", wrongQuery) - .add("dimensions", Json.createArrayBuilder().add(rawObject)).build(); - - // Assert the raw value is still 273000 (wrong) - facetDimensions = luceneApi.facetSearch("InvestigationParameter", rawFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("numericValue", facetDimension.getDimension()); - assertEquals(3, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("272.5_273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("272999.5_273000.5", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(2); - assertEquals("273272.5_273273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - - // Assert that the SI value has not been set due to conversion failing - facetDimensions = luceneApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("numericValueSI", facetDimension.getDimension()); - assertEquals(3, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("272.5_273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("272999.5_273000.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(2); - assertEquals("273272.5_273273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - } - -} diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 7fee9136..88f2a5e4 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -5,8 +5,6 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.lang.reflect.Array; import java.net.URI; import java.net.URISyntaxException; @@ -14,35 +12,23 @@ import java.util.Arrays; import java.util.Date; import java.util.HashSet; -import java.util.Iterator; import java.util.List; -import java.util.Queue; import java.util.Set; -import java.util.concurrent.ConcurrentLinkedQueue; import javax.json.Json; -import javax.json.JsonArray; import javax.json.JsonArrayBuilder; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; import javax.json.JsonValue; -import javax.json.stream.JsonGenerator; -import javax.ws.rs.core.MediaType; - -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; + import org.icatproject.core.IcatException; -import org.icatproject.core.IcatException.IcatExceptionType; import org.icatproject.core.entity.Datafile; import org.icatproject.core.entity.DatafileFormat; import org.icatproject.core.entity.DatafileParameter; import org.icatproject.core.entity.Dataset; import org.icatproject.core.entity.DatasetParameter; import org.icatproject.core.entity.DatasetType; +import org.icatproject.core.entity.EntityBaseBean; import org.icatproject.core.entity.Facility; import org.icatproject.core.entity.Investigation; import org.icatproject.core.entity.InvestigationParameter; @@ -53,233 +39,145 @@ import org.icatproject.core.entity.Sample; import org.icatproject.core.entity.SampleType; import org.icatproject.core.entity.User; +import org.icatproject.core.manager.search.FacetDimension; +import org.icatproject.core.manager.search.FacetLabel; +import org.icatproject.core.manager.search.LuceneApi; +import org.icatproject.core.manager.search.OpensearchApi; +import org.icatproject.core.manager.search.ParameterPOJO; +import org.icatproject.core.manager.search.ScoredEntityBaseBean; +import org.icatproject.core.manager.search.SearchApi; +import org.icatproject.core.manager.search.SearchResult; import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@RunWith(Parameterized.class) public class TestSearchApi { - static SearchApi searchApi; - private static URI uribase; + private static final String SEARCH_AFTER_NOT_NULL = "Expected searchAfter to be set, but it was null"; + final static Logger logger = LoggerFactory.getLogger(TestSearchApi.class); - @BeforeClass - public static void beforeClass() throws Exception { - String urlString = System.getProperty("opensearchUrl"); - logger.info("Using search service at {}", urlString); - uribase = new URI(urlString); - searchApi = new SearchApi(uribase); - searchApi.initMappings(); - searchApi.initScripts(); - } + @Parameterized.Parameters + public static Iterable data() throws URISyntaxException { + String luceneUrl = System.getProperty("luceneUrl"); + logger.info("Using Lucene service at {}", luceneUrl); + URI luceneUri = new URI(luceneUrl); - String letters = "abcdefghijklmnopqrstuvwxyz"; + String opensearchUrl = System.getProperty("opensearchUrl"); + logger.info("Using Opensearch/Elasticsearch service at {}", opensearchUrl); + URI opensearchUri = new URI(opensearchUrl); - long now = new Date().getTime(); + return Arrays.asList(new LuceneApi(luceneUri), new OpensearchApi(opensearchUri, "\u2103: celsius")); + } - int NUMINV = 10; + @Parameterized.Parameter + public SearchApi searchApi; + String letters = "abcdefghijklmnopqrstuvwxyz"; + Date date = new Date(); + long now = date.getTime(); + int NUMINV = 10; int NUMUSERS = 5; - int NUMDS = 30; - int NUMDF = 100; - int NUMSAMP = 15; - @Test - public void modifyDatafile() throws IcatException { - Investigation investigation = new Investigation(); - investigation.setId(0L); - Dataset dataset = new Dataset(); - dataset.setId(0L); - dataset.setInvestigation(investigation); - - Datafile elephantDatafile = new Datafile(); - elephantDatafile.setName("Elephants and Aardvarks"); - elephantDatafile.setDatafileModTime(new Date(0L)); - elephantDatafile.setId(42L); - elephantDatafile.setDataset(dataset); - - DatafileFormat pdfFormat = new DatafileFormat(); - pdfFormat.setId(0L); - pdfFormat.setName("pdf"); - Datafile rhinoDatafile = new Datafile(); - rhinoDatafile.setName("Rhinos and Aardvarks"); - rhinoDatafile.setDatafileModTime(new Date(3L)); - rhinoDatafile.setId(42L); - rhinoDatafile.setDataset(dataset); - rhinoDatafile.setDatafileFormat(pdfFormat); - - DatafileFormat pngFormat = new DatafileFormat(); - pngFormat.setId(0L); - pngFormat.setName("png"); - - JsonObject elephantQuery = SearchApi.buildQuery("Datafile", null, "elephant", null, null, null, null, null); - JsonObject rhinoQuery = SearchApi.buildQuery("Datafile", null, "rhino", null, null, null, null, null); - JsonObject pdfQuery = SearchApi.buildQuery("Datafile", null, "datafileFormat.name:pdf", null, null, null, null, - null); - JsonObject pngQuery = SearchApi.buildQuery("Datafile", null, "datafileFormat.name:png", null, null, null, null, - null); - JsonObject queryObject = Json.createObjectBuilder().add("id", Json.createArrayBuilder().add("42")).build(); - - JsonObjectBuilder stringDimensionBuilder = Json.createObjectBuilder().add("dimension", "datafileFormat.name"); - JsonArrayBuilder stringDimensionsBuilder = Json.createArrayBuilder().add(stringDimensionBuilder); - JsonObjectBuilder lowRangeBuilder = Json.createObjectBuilder().add("from", 0L).add("to", 2L).add("key", "low"); - JsonObjectBuilder highRangeBuilder = Json.createObjectBuilder().add("from", 2L).add("to", 4L).add("key", "high"); - JsonArrayBuilder rangesBuilder = Json.createArrayBuilder().add(lowRangeBuilder).add(highRangeBuilder); - JsonObjectBuilder rangedDimensionBuilder = Json.createObjectBuilder().add("dimension", "date").add("ranges", - rangesBuilder); - JsonArrayBuilder rangedDimensionsBuilder = Json.createArrayBuilder().add(rangedDimensionBuilder); - JsonObject stringFacetQuery = Json.createObjectBuilder().add("query", queryObject) - .add("dimensions", stringDimensionsBuilder).build(); - JsonObject rangeFacetQuery = Json.createObjectBuilder().add("query", queryObject) - .add("dimensions", rangedDimensionsBuilder).build(); - - // Original - Queue queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("create", elephantDatafile)); - modifyQueue(queue); - checkLsr(searchApi.getResults(elephantQuery, 5), 42L); - checkLsr(searchApi.getResults(rhinoQuery, 5)); - checkLsr(searchApi.getResults(pdfQuery, 5)); - checkLsr(searchApi.getResults(pngQuery, 5)); - List facetDimensions = searchApi.facetSearch("Datafile", stringFacetQuery, 5, 5); - assertEquals(0, facetDimensions.size()); - facetDimensions = searchApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - FacetDimension facetDimension = facetDimensions.get(0); - assertEquals("date", facetDimension.getDimension()); - assertEquals(2, facetDimension.getFacets().size()); - FacetLabel facetLabel = facetDimension.getFacets().get(0); - assertEquals("low", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("high", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - - // Change name and add a format - queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("update", rhinoDatafile)); - modifyQueue(queue); - checkLsr(searchApi.getResults(elephantQuery, 5)); - checkLsr(searchApi.getResults(rhinoQuery, 5), 42L); - checkLsr(searchApi.getResults(pdfQuery, 5), 42L); - checkLsr(searchApi.getResults(pngQuery, 5)); - facetDimensions = searchApi.facetSearch("Datafile", stringFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimensions = searchApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("date", facetDimension.getDimension()); - assertEquals(2, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("low", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("high", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - - // Change just the format - queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("update", pngFormat)); - modifyQueue(queue); - checkLsr(searchApi.getResults(elephantQuery, 5)); - checkLsr(searchApi.getResults(rhinoQuery, 5), 42L); - checkLsr(searchApi.getResults(pdfQuery, 5)); - checkLsr(searchApi.getResults(pngQuery, 5), 42L); - facetDimensions = searchApi.facetSearch("Datafile", stringFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("datafileFormat.name", facetDimension.getDimension()); - assertEquals(1, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("png", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - facetDimensions = searchApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("date", facetDimension.getDimension()); - assertEquals(2, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("low", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("high", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - - // Remove the format - queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("delete", pngFormat)); - modifyQueue(queue); - checkLsr(searchApi.getResults(elephantQuery, 5)); - checkLsr(searchApi.getResults(rhinoQuery, 5), 42L); - checkLsr(searchApi.getResults(pdfQuery, 5)); - checkLsr(searchApi.getResults(pngQuery, 5)); - facetDimensions = searchApi.facetSearch("Datafile", stringFacetQuery, 5, 5); - assertEquals(0, facetDimensions.size()); - facetDimensions = searchApi.facetSearch("Datafile", rangeFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("date", facetDimension.getDimension()); - assertEquals(2, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("low", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("high", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - - // Remove the file - queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeDeletion(elephantDatafile)); - queue.add(SearchApi.encodeDeletion(rhinoDatafile)); - modifyQueue(queue); - checkLsr(searchApi.getResults(elephantQuery, 5)); - checkLsr(searchApi.getResults(rhinoQuery, 5)); - checkLsr(searchApi.getResults(pdfQuery, 5)); - checkLsr(searchApi.getResults(pngQuery, 5)); + /** + * Utility function for building a Query from individual arguments + */ + public static JsonObject buildQuery(String target, String user, String text, Date lower, Date upper, + List parameters, List samples, String userFullName) { + JsonObjectBuilder builder = Json.createObjectBuilder(); + if (target != null) { + builder.add("target", target); + } + if (user != null) { + builder.add("user", user); + } + if (text != null) { + builder.add("text", text); + } + if (lower != null) { + builder.add("lower", lower.getTime()); + } + if (upper != null) { + builder.add("upper", upper.getTime()); + } + if (parameters != null && !parameters.isEmpty()) { + JsonArrayBuilder parametersBuilder = Json.createArrayBuilder(); + for (ParameterPOJO parameter : parameters) { + JsonObjectBuilder parameterBuilder = Json.createObjectBuilder(); + if (parameter.name != null) { + parameterBuilder.add("name", parameter.name); + } + if (parameter.units != null) { + parameterBuilder.add("units", parameter.units); + } + if (parameter.stringValue != null) { + parameterBuilder.add("stringValue", parameter.stringValue); + } + if (parameter.lowerDateValue != null) { + parameterBuilder.add("lowerDateValue", parameter.lowerDateValue.getTime()); + } + if (parameter.upperDateValue != null) { + parameterBuilder.add("upperDateValue", parameter.upperDateValue.getTime()); + } + if (parameter.lowerNumericValue != null) { + parameterBuilder.add("lowerNumericValue", parameter.lowerNumericValue); + } + if (parameter.upperNumericValue != null) { + parameterBuilder.add("upperNumericValue", parameter.upperNumericValue); + } + parametersBuilder.add(parameterBuilder); + } + builder.add("parameters", parametersBuilder); + } + if (samples != null && !samples.isEmpty()) { + JsonArrayBuilder samplesBuilder = Json.createArrayBuilder(); + for (String sample : samples) { + samplesBuilder.add(sample); + } + builder.add("samples", samplesBuilder); + } + if (userFullName != null) { + builder.add("userFullName", userFullName); + } + return builder.build(); + } - // Multiple commands at once - queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("create", elephantDatafile)); - queue.add(SearchApi.encodeOperation("update", rhinoDatafile)); - queue.add(SearchApi.encodeDeletion(elephantDatafile)); - queue.add(SearchApi.encodeDeletion(rhinoDatafile)); - modifyQueue(queue); - checkLsr(searchApi.getResults(elephantQuery, 5)); - checkLsr(searchApi.getResults(rhinoQuery, 5)); - checkLsr(searchApi.getResults(pdfQuery, 5)); - checkLsr(searchApi.getResults(pngQuery, 5)); + private static JsonObject buildFacetIdQuery(String id) { + return Json.createObjectBuilder().add("id", Json.createArrayBuilder().add(id)).build(); } - private void modifyQueue(Queue queue) throws IcatException { - Iterator qiter = queue.iterator(); - if (qiter.hasNext()) { - StringBuilder sb = new StringBuilder("["); + private static JsonObject buildFacetRangeObject(String key, double from, double to) { + return Json.createObjectBuilder().add("from", from).add("to", to).add("key", key).build(); + } - while (qiter.hasNext()) { - String item = qiter.next(); - if (sb.length() != 1) { - sb.append(','); - } - sb.append(item); - qiter.remove(); - } - sb.append(']'); - logger.debug("XXX " + sb.toString()); + private static JsonObject buildFacetRangeObject(String key, long from, long to) { + return Json.createObjectBuilder().add("from", from).add("to", to).add("key", key).build(); + } - searchApi.modify(sb.toString()); - searchApi.commit(); + private static JsonObject buildFacetRangeRequest(JsonObject queryObject, String dimension, + JsonObject... rangeObjects) { + JsonArrayBuilder rangesBuilder = Json.createArrayBuilder(); + for (JsonObject rangeObject : rangeObjects) { + rangesBuilder.add(rangeObject); } + JsonObjectBuilder rangedDimensionBuilder = Json.createObjectBuilder().add("dimension", dimension).add("ranges", + rangesBuilder); + JsonArrayBuilder rangedDimensionsBuilder = Json.createArrayBuilder().add(rangedDimensionBuilder); + return Json.createObjectBuilder().add("query", queryObject).add("dimensions", rangedDimensionsBuilder).build(); } - @Before - public void before() throws Exception { - searchApi.clear(); + private static JsonObject buildFacetStringRequest(String id, String dimension) { + JsonObject idQuery = buildFacetIdQuery(id); + JsonObjectBuilder stringDimensionBuilder = Json.createObjectBuilder().add("dimension", dimension); + JsonArrayBuilder stringDimensionsBuilder = Json.createArrayBuilder().add(stringDimensionBuilder); + return Json.createObjectBuilder().add("query", idQuery).add("dimensions", stringDimensionsBuilder).build(); } private void checkDatafile(ScoredEntityBaseBean datafile) { @@ -307,6 +205,24 @@ private void checkDataset(ScoredEntityBaseBean dataset) { assertNotNull(source.getJsonNumber("endDate")); } + private void checkFacets(List facetDimensions, FacetDimension... dimensions) { + assertEquals(dimensions.length, facetDimensions.size()); + for (int i = 0; i < dimensions.length; i++) { + FacetDimension expectedFacet = dimensions[i]; + FacetDimension actualFacet = facetDimensions.get(i); + assertEquals(expectedFacet.getDimension(), actualFacet.getDimension()); + List expectedLabels = expectedFacet.getFacets(); + List actualLabels = actualFacet.getFacets(); + assertEquals(expectedLabels.size(), actualLabels.size()); + for (int j = 0; j < expectedLabels.size(); j++) { + FacetLabel expectedLabel = expectedLabels.get(j); + FacetLabel actualLabel = actualLabels.get(j); + assertEquals(expectedLabel.getLabel(), actualLabel.getLabel()); + assertEquals(expectedLabel.getValue(), actualLabel.getValue()); + } + } + } + private void checkInvestigation(ScoredEntityBaseBean investigation) { JsonObject source = investigation.getSource(); assertNotNull(source); @@ -318,7 +234,7 @@ private void checkInvestigation(ScoredEntityBaseBean investigation) { assertNotNull(source.getJsonNumber("endDate")); } - private void checkLsr(SearchResult lsr, Long... n) { + private void checkResults(SearchResult lsr, Long... n) { Set wanted = new HashSet<>(Arrays.asList(n)); Set got = new HashSet<>(); @@ -346,10 +262,10 @@ private void checkLsr(SearchResult lsr, Long... n) { } - private void checkLsrOrder(SearchResult lsr, Long... n) { + private void checkOrder(SearchResult lsr, Long... n) { List results = lsr.getResults(); if (n.length != results.size()) { - checkLsr(lsr, n); + checkResults(lsr, n); } for (int i = 0; i < n.length; i++) { Long resultId = results.get(i).getEntityBaseBeanId(); @@ -360,772 +276,745 @@ private void checkLsrOrder(SearchResult lsr, Long... n) { } } + private Datafile datafile(long id, String name, Date date, Dataset dataset) { + Datafile datafile = new Datafile(); + datafile.setId(id); + datafile.setName(name); + datafile.setDatafileModTime(date); + datafile.setDataset(dataset); + return datafile; + } + + private DatafileFormat datafileFormat(long id, String name) { + DatafileFormat datafileFormat = new DatafileFormat(); + datafileFormat.setId(id); + datafileFormat.setName(name); + return datafileFormat; + } + + private Dataset dataset(long id, String name, Date startDate, Date endDate, Investigation investigation) { + DatasetType type = new DatasetType(); + type.setName("type"); + type.setId(0L); + Dataset dataset = new Dataset(); + dataset.setId(id); + dataset.setName(name); + dataset.setCreateTime(startDate); + dataset.setModTime(endDate); + dataset.setType(type); + dataset.setInvestigation(investigation); + return dataset; + } + + private Investigation investigation(long id, String name, Date startDate, Date endDate) { + InvestigationType type = new InvestigationType(); + type.setName("type"); + type.setId(0L); + Facility facility = new Facility(); + facility.setName("facility"); + facility.setId(0L); + Investigation investigation = new Investigation(); + investigation.setId(id); + investigation.setName(name); + investigation.setVisitId("visitId"); + investigation.setTitle("title"); + investigation.setCreateTime(startDate); + investigation.setModTime(endDate); + investigation.setFacility(facility); + investigation.setType(type); + return investigation; + } + + private InvestigationUser investigationUser(long id, long userId, String name, String fullName, + Investigation investigation) { + User user = new User(); + user.setName(name); + user.setFullName(fullName); + user.setId(userId); + InvestigationUser investigationUser = new InvestigationUser(); + investigationUser.setId(id); + investigationUser.setInvestigation(investigation); + investigationUser.setUser(user); + return investigationUser; + } + + private Parameter parameter(long id, Date value, ParameterType parameterType, EntityBaseBean parent) { + Parameter parameter = parameter(id, parameterType, parent); + parameter.setDateTimeValue(value); + return parameter; + } + + private Parameter parameter(long id, String value, ParameterType parameterType, EntityBaseBean parent) { + Parameter parameter = parameter(id, parameterType, parent); + parameter.setStringValue(value); + return parameter; + } + + private Parameter parameter(long id, double value, ParameterType parameterType, EntityBaseBean parent) { + Parameter parameter = parameter(id, parameterType, parent); + parameter.setNumericValue(value); + return parameter; + } + + private Parameter parameter(long id, ParameterType parameterType, EntityBaseBean parent) { + Parameter parameter; + if (parent instanceof Datafile) { + parameter = new DatafileParameter(); + ((DatafileParameter) parameter).setDatafile((Datafile) parent); + } else if (parent instanceof Dataset) { + parameter = new DatasetParameter(); + ((DatasetParameter) parameter).setDataset((Dataset) parent); + } else if (parent instanceof Investigation) { + parameter = new InvestigationParameter(); + ((InvestigationParameter) parameter).setInvestigation((Investigation) parent); + } else { + fail(parent.getClass().getSimpleName() + " is not valid"); + return null; + } + parameter.setType(parameterType); + parameter.setId(id); + return parameter; + } + + private ParameterType parameterType(long id, String name, String units) { + ParameterType parameterType = new ParameterType(); + parameterType.setId(id); + parameterType.setName(name); + parameterType.setUnits(units); + return parameterType; + } + + private Sample sample(long id, String name, Investigation investigation) { + SampleType sampleType = new SampleType(); + sampleType.setId(0L); + sampleType.setName("test"); + Sample sample = new Sample(); + sample.setId(id); + sample.setName(name); + sample.setInvestigation(investigation); + return sample; + } + + private void modify(String... operations) throws IcatException { + StringBuilder sb = new StringBuilder("["); + for (String operation : operations) { + if (sb.length() != 1) { + sb.append(','); + } + sb.append(operation); + } + sb.append(']'); + searchApi.modify(sb.toString()); + searchApi.commit(); + } + + private void populateParameters(List queue, int i, EntityBaseBean parent) throws IcatException { + int j = i % 26; + int k = (i + 5) % 26; + String name = "nm " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); + String units = "u " + letters.substring(k, k + 1) + letters.substring(k, k + 1) + letters.substring(k, k + 1); + ParameterType dateParameterType = parameterType(0, "D" + name, units); + ParameterType numericParameterType = parameterType(0, "N" + name, units); + ParameterType stringParameterType = parameterType(0, "S" + name, units); + Parameter dateParameter = parameter(3 * i, new Date(now + 60000 * k * k), dateParameterType, parent); + Parameter numericParameter = parameter(3 * i + 1, new Double(j * j), numericParameterType, parent); + Parameter stringParameter = parameter(3 * i + 2, "v" + i * i, stringParameterType, parent); + queue.add(SearchApi.encodeOperation("create", dateParameter)); + queue.add(SearchApi.encodeOperation("create", numericParameter)); + queue.add(SearchApi.encodeOperation("create", stringParameter)); + } + + /** + * Populate UserGroup, Investigation, InvestigationParameter, + * InvestigationUser, Dataset,DatasetParameter,Datafile, DatafileParameter + * and Sample + */ + private void populate() throws IcatException { + List queue = new ArrayList<>(); + Long investigationUserId = 0L; + + for (int investigationId = 0; investigationId < NUMINV; investigationId++) { + String word = word(investigationId % 26, (investigationId + 7) % 26, (investigationId + 17) % 26); + Date startDate = new Date(now + investigationId * 60000); + Date endDate = new Date(now + (investigationId + 1) * 60000); + Investigation investigation = investigation(investigationId, word, startDate, endDate); + queue.add(SearchApi.encodeOperation("create", investigation)); + + for (int userId = 0; userId < NUMUSERS; userId++) { + if (investigationId % (userId + 1) == 1) { + String fullName = "FN " + letters.substring(userId, userId + 1) + " " + + letters.substring(userId, userId + 1); + String name = letters.substring(userId, userId + 1) + userId; + InvestigationUser investigationUser = investigationUser(investigationUserId, userId, name, fullName, + investigation); + queue.add(SearchApi.encodeOperation("create", investigationUser)); + investigationUserId++; + } + } + + if (investigationId % 2 == 1) { + populateParameters(queue, investigationId, investigation); + } + + for (int sampleBatch = 0; sampleBatch * NUMINV < NUMSAMP; sampleBatch++) { + int sampleId = sampleBatch * NUMINV + investigationId; + if (sampleId >= NUMSAMP) { + break; + } + word = word("SType ", sampleId % 26); + Sample sample = sample(sampleId, word, investigation); + queue.add(SearchApi.encodeOperation("create", sample)); + } + + for (int datasetBatch = 0; datasetBatch * NUMINV < NUMDS; datasetBatch++) { + int datasetId = datasetBatch * NUMINV + investigationId; + if (datasetId >= NUMDS) { + break; + } + startDate = new Date(now + datasetId * 60000); + endDate = new Date(now + (datasetId + 1) * 60000); + word = word("DS", datasetId % 26); + Dataset dataset = dataset(datasetId, word, startDate, endDate, investigation); + queue.add(SearchApi.encodeOperation("create", dataset)); + + if (datasetId % 3 == 1) { + populateParameters(queue, datasetId, dataset); + } + + for (int datafileBatch = 0; datafileBatch * NUMDS < NUMDF; datafileBatch++) { + int datafileId = datafileBatch * NUMDS + datasetId; + if (datafileId >= NUMDF) { + break; + } + word = word("DF", datafileId % 26); + Datafile datafile = datafile(datafileId, word, new Date(now + datafileId * 60000), dataset); + queue.add(SearchApi.encodeOperation("create", datafile)); + + if (datafileId % 4 == 1) { + populateParameters(queue, datafileId, datafile); + } + } + } + } + + modify(queue.toArray(new String[0])); + } + + private String word(int j, int k, int l) { + String jString = letters.substring(j, j + 1); + String kString = letters.substring(k, k + 1); + String lString = letters.substring(l, l + 1); + return jString + " " + kString + " " + lString; + } + + private String word(String prefix, int j) { + String jString = letters.substring(j, j + 1); + return prefix + jString + jString + jString; + } + + @Before + public void before() throws Exception { + searchApi.clear(); + } + @Test public void datafiles() throws Exception { populate(); + JsonObjectBuilder sortBuilder = Json.createObjectBuilder(); + String sort; - JsonObject query = SearchApi.buildQuery("Datafile", null, null, null, null, null, null, null); + // Test size and searchAfter + JsonObject query = buildQuery("Datafile", null, null, null, null, null, null, null); List fields = Arrays.asList("date", "name", "investigation.id", "id"); SearchResult lsr = searchApi.getResults(query, null, 5, null, fields); JsonValue searchAfter = lsr.getSearchAfter(); - checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L); checkDatafile(lsr.getResults().get(0)); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); + lsr = searchApi.getResults(query, searchAfter, 200, null, fields); assertNull(lsr.getSearchAfter()); assertEquals(95, lsr.getResults().size()); // Test searchAfter preserves the sorting of original search (asc) - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("date", "asc"); - gen.writeEnd(); - } - String sort = baos.toString(); + sort = sortBuilder.add("date", "asc").build().toString(); lsr = searchApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); + checkOrder(lsr, 0L, 1L, 2L, 3L, 4L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); - checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); + checkOrder(lsr, 5L, 6L, 7L, 8L, 9L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); // Test searchAfter preserves the sorting of original search (desc) - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("date", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); + sort = sortBuilder.add("date", "desc").build().toString(); lsr = searchApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 99L, 98L, 97L, 96L, 95L); + checkOrder(lsr, 99L, 98L, 97L, 96L, 95L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); - checkLsrOrder(lsr, 94L, 93L, 92L, 91L, 90L); + checkOrder(lsr, 94L, 93L, 92L, 91L, 90L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); // Test tie breaks on fields with identical values (asc) - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("name", "asc"); - gen.writeEnd(); - } - sort = baos.toString(); + sort = sortBuilder.add("name", "asc").build().toString(); lsr = searchApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 0L, 26L, 52L, 78L, 1L); + checkOrder(lsr, 0L, 26L, 52L, 78L, 1L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("name", "asc"); - gen.write("date", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); + sort = sortBuilder.add("name", "asc").add("date", "desc").build().toString(); lsr = searchApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 78L, 52L, 26L, 0L, 79L); + checkOrder(lsr, 78L, 52L, 26L, 0L, 79L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); // Test tie breaks on fields with identical values (desc) - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("name", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); + sort = sortBuilder.add("name", "desc").build().toString(); lsr = searchApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 25L, 51L, 77L, 24L, 50L); + checkOrder(lsr, 25L, 51L, 77L, 24L, 50L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("name", "desc"); - gen.write("date", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); + sort = sortBuilder.add("name", "desc").add("date", "desc").build().toString(); lsr = searchApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 77L, 51L, 25L, 76L, 50L); + checkOrder(lsr, 77L, 51L, 25L, 76L, 50L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - query = SearchApi.buildQuery("Datafile", "e4", null, null, null, null, null, null); + query = buildQuery("Datafile", "e4", null, null, null, null, null, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, 96L); + checkResults(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, + 96L); - query = SearchApi.buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null); + query = buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 1L); + checkResults(lsr, 1L); - query = SearchApi.buildQuery("Datafile", null, "dfbbb", null, null, null, null, null); + query = buildQuery("Datafile", null, "dfbbb", null, null, null, null, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 1L, 27L, 53L, 79L); + checkResults(lsr, 1L, 27L, 53L, 79L); - query = SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), + query = buildQuery("Datafile", null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, null, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 3L, 4L, 5L, 6L); + checkResults(lsr, 3L, 4L, 5L, 6L); - query = SearchApi.buildQuery("Datafile", "b1", "dsddd", new Date(now + 60000 * 3), + query = buildQuery("Datafile", "b1", "dsddd", new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, null, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr); + checkResults(lsr); List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v25")); - query = SearchApi.buildQuery("Datafile", null, null, new Date(now + 60000 * 3), + query = buildQuery("Datafile", null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), pojos, null, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 5L); + checkResults(lsr, 5L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v25")); - query = SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null); + query = buildQuery("Datafile", null, null, null, null, pojos, null, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 5L); + checkResults(lsr, 5L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, "u sss", null)); - query = SearchApi.buildQuery("Datafile", null, null, null, null, pojos, null, null); + query = buildQuery("Datafile", null, null, null, null, pojos, null, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 13L, 65L); + checkResults(lsr, 13L, 65L); } @Test public void datasets() throws Exception { populate(); + JsonObjectBuilder sortBuilder = Json.createObjectBuilder(); + String sort; - JsonObject query = SearchApi.buildQuery("Dataset", null, null, null, null, null, null, null); + JsonObject query = buildQuery("Dataset", null, null, null, null, null, null, null); List fields = Arrays.asList("startDate", "endDate", "name", "investigation.id", "id"); SearchResult lsr = searchApi.getResults(query, null, 5, null, fields); - checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L); checkDataset(lsr.getResults().get(0)); JsonValue searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); lsr = searchApi.getResults(query, searchAfter, 100, null, fields); assertNull(lsr.getSearchAfter()); - checkLsr(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, + checkResults(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, 25L, 26L, 27L, 28L, 29L); // Test searchAfter preserves the sorting of original search (asc) - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("startDate", "asc"); - gen.writeEnd(); - } - String sort = baos.toString(); + sort = sortBuilder.add("startDate", "asc").build().toString(); lsr = searchApi.getResults(query, 5, sort); - checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); + checkOrder(lsr, 0L, 1L, 2L, 3L, 4L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); - checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); + checkOrder(lsr, 5L, 6L, 7L, 8L, 9L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); // Test searchAfter preserves the sorting of original search (desc) - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("endDate", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); + sort = sortBuilder.add("endDate", "desc").build().toString(); lsr = searchApi.getResults(query, 5, sort); - checkLsrOrder(lsr, 29L, 28L, 27L, 26L, 25L); + checkOrder(lsr, 29L, 28L, 27L, 26L, 25L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); - checkLsrOrder(lsr, 24L, 23L, 22L, 21L, 20L); + checkOrder(lsr, 24L, 23L, 22L, 21L, 20L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); // Test tie breaks on fields with identical values (asc) - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("name", "asc"); - gen.writeEnd(); - } - sort = baos.toString(); + sort = sortBuilder.add("name", "asc").build().toString(); lsr = searchApi.getResults(query, null, 5, sort, fields); - checkLsrOrder(lsr, 0L, 26L, 1L, 27L, 2L); + checkOrder(lsr, 0L, 26L, 1L, 27L, 2L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("name", "asc"); - gen.write("endDate", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); + sort = sortBuilder.add("name", "asc").add("endDate", "desc").build().toString(); lsr = searchApi.getResults(query, 5, sort); - checkLsrOrder(lsr, 26L, 0L, 27L, 1L, 28L); + checkOrder(lsr, 26L, 0L, 27L, 1L, 28L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "e4", null, null, null, null, null, null), 100, + lsr = searchApi.getResults(buildQuery("Dataset", "e4", null, null, null, null, null, null), 100, null); - checkLsr(lsr, 1L, 6L, 11L, 16L, 21L, 26L); + checkResults(lsr, 1L, 6L, 11L, 16L, 21L, 26L); - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100, + lsr = searchApi.getResults(buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100, null); - checkLsr(lsr, 1L); + checkResults(lsr, 1L); - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100, + lsr = searchApi.getResults(buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100, null); - checkLsr(lsr, 1L, 27L); + checkResults(lsr, 1L, 27L); - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), + lsr = searchApi.getResults(buildQuery("Dataset", null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, null, null), 100, null); - checkLsr(lsr, 3L, 4L, 5L); + checkResults(lsr, 3L, 4L, 5L); - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), + lsr = searchApi.getResults(buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, null, null), 100, null); - checkLsr(lsr, 3L); + checkResults(lsr, 3L); List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, null, null, pojos, null, null), 100, + lsr = searchApi.getResults(buildQuery("Dataset", null, null, null, null, pojos, null, null), 100, null); - checkLsr(lsr, 4L); + checkResults(lsr, 4L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", null, null, new Date(now + 60000 * 3), + lsr = searchApi.getResults(buildQuery("Dataset", null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), pojos, null, null), 100, null); - checkLsr(lsr, 4L); + checkResults(lsr, 4L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = searchApi.getResults(SearchApi.buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), + lsr = searchApi.getResults(buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), new Date(now + 60000 * 6), pojos, null, null), 100, null); - checkLsr(lsr); - } - - private void fillParms(Queue queue, int i, String rel) throws IcatException { - int j = i % 26; - int k = (i + 5) % 26; - String name = "nm " + letters.substring(j, j + 1) + letters.substring(j, j + 1) + letters.substring(j, j + 1); - String units = "u " + letters.substring(k, k + 1) + letters.substring(k, k + 1) + letters.substring(k, k + 1); - - ParameterType dateParameterType = new ParameterType(); - dateParameterType.setId(0L); - dateParameterType.setName("D" + name); - dateParameterType.setUnits(units); - ParameterType numericParameterType = new ParameterType(); - numericParameterType.setId(0L); - numericParameterType.setName("N" + name); - numericParameterType.setUnits(units); - ParameterType stringParameterType = new ParameterType(); - stringParameterType.setId(0L); - stringParameterType.setName("S" + name); - stringParameterType.setUnits(units); - - Parameter parameter; - if (rel.equals("datafile")) { - parameter = new DatafileParameter(); - Datafile datafile = new Datafile(); - datafile.setId(new Long(i)); - ((DatafileParameter) parameter).setDatafile(datafile); - } else if (rel.equals("dataset")) { - parameter = new DatasetParameter(); - Dataset dataset = new Dataset(); - dataset.setId(new Long(i)); - ((DatasetParameter) parameter).setDataset(dataset); - } else if (rel.equals("investigation")) { - parameter = new InvestigationParameter(); - Investigation investigation = new Investigation(); - investigation.setId(new Long(i)); - ((InvestigationParameter) parameter).setInvestigation(investigation); - } else { - fail(rel + " is not valid"); - return; - } - - parameter.setId(new Long(3 * i)); - parameter.setType(dateParameterType); - parameter.setDateTimeValue(new Date(now + 60000 * k * k)); - - queue.add(SearchApi.encodeOperation("create", parameter)); - System.out.println( - rel + " " + i + " '" + "D" + name + "' '" + units + "' '" + new Date(now + 60000 * k * k) + "'"); - - parameter.setId(new Long(3 * i + 1)); - parameter.setType(numericParameterType); - parameter.setNumericValue(new Double(j * j)); - - queue.add(SearchApi.encodeOperation("create", parameter)); - System.out.println(rel + " " + i + " '" + "N" + name + "' '" + units + "' " + new Double(j * j)); - - parameter.setId(new Long(3 * i + 2)); - parameter.setType(stringParameterType); - parameter.setStringValue("v" + i * i); - - queue.add(SearchApi.encodeOperation("create", parameter)); - System.out.println(rel + " " + i + " '" + "S" + name + "' '" + units + "' 'v" + i * i + "'"); + checkResults(lsr); } @Test public void investigations() throws Exception { populate(); + JsonObjectBuilder sortBuilder = Json.createObjectBuilder(); + String sort; /* Blocked results */ - JsonObject query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, null); + JsonObject query = buildQuery("Investigation", null, null, null, null, null, null, null); List fields = Arrays.asList("startDate", "endDate", "name", "id"); SearchResult lsr = searchApi.getResults(query, null, 5, null, fields); - checkLsr(lsr, 0L, 1L, 2L, 3L, 4L); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L); checkInvestigation(lsr.getResults().get(0)); JsonValue searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); lsr = searchApi.getResults(query, searchAfter, 6, null, fields); - checkLsr(lsr, 5L, 6L, 7L, 8L, 9L); + checkResults(lsr, 5L, 6L, 7L, 8L, 9L); searchAfter = lsr.getSearchAfter(); assertNull(searchAfter); // Test searchAfter preserves the sorting of original search (asc) - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("startDate", "asc"); - gen.writeEnd(); - } - String sort = baos.toString(); + sort = sortBuilder.add("startDate", "asc").build().toString(); lsr = searchApi.getResults(query, 5, sort); - checkLsrOrder(lsr, 0L, 1L, 2L, 3L, 4L); + checkOrder(lsr, 0L, 1L, 2L, 3L, 4L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); - checkLsrOrder(lsr, 5L, 6L, 7L, 8L, 9L); + checkOrder(lsr, 5L, 6L, 7L, 8L, 9L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); // Test searchAfter preserves the sorting of original search (desc) - baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("endDate", "desc"); - gen.writeEnd(); - } - sort = baos.toString(); + sort = sortBuilder.add("endDate", "desc").build().toString(); lsr = searchApi.getResults(query, 5, sort); - checkLsrOrder(lsr, 9L, 8L, 7L, 6L, 5L); + checkOrder(lsr, 9L, 8L, 7L, 6L, 5L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); - checkLsrOrder(lsr, 4L, 3L, 2L, 1L, 0L); + checkOrder(lsr, 4L, 3L, 2L, 1L, 0L); searchAfter = lsr.getSearchAfter(); - assertNotNull("Expected searchAfter to be set, but it was null", searchAfter); + assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "b"); + query = buildQuery("Investigation", null, null, null, null, null, null, "b"); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); + checkResults(lsr, 1L, 3L, 5L, 7L, 9L); - query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN"); + query = buildQuery("Investigation", null, null, null, null, null, null, "FN"); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); + checkResults(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); - query = SearchApi.buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""); + query = buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); + checkResults(lsr, 1L, 3L, 5L, 7L, 9L); - query = SearchApi.buildQuery("Investigation", "b1", null, null, null, null, null, "b"); + query = buildQuery("Investigation", "b1", null, null, null, null, null, "b"); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 1L, 3L, 5L, 7L, 9L); + checkResults(lsr, 1L, 3L, 5L, 7L, 9L); - query = SearchApi.buildQuery("Investigation", "c1", null, null, null, null, null, "b"); + query = buildQuery("Investigation", "c1", null, null, null, null, null, "b"); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr); + checkResults(lsr); - query = SearchApi.buildQuery("Investigation", null, "l v", null, null, null, null, null); + query = buildQuery("Investigation", null, "l v", null, null, null, null, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 4L); + checkResults(lsr, 4L); - query = SearchApi.buildQuery("Investigation", "b1", "d", null, null, null, null, "b"); + query = buildQuery("Investigation", "b1", "d", null, null, null, null, "b"); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 3L); + checkResults(lsr, 3L); - query = SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), + query = buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, null, "b"); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 3L); + checkResults(lsr, 3L); - query = SearchApi.buildQuery("Investigation", null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), + query = buildQuery("Investigation", null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), null, null, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 3L, 4L, 5L); + checkResults(lsr, 3L, 4L, 5L); List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); - query = SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null); + query = buildQuery("Investigation", null, null, null, null, pojos, null, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 3L); + checkResults(lsr, 3L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); pojos.add(new ParameterPOJO(null, null, 7, 10)); pojos.add(new ParameterPOJO(null, null, new Date(now + 60000 * 63), new Date(now + 60000 * 65))); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 3L); + checkResults(lsr, 3L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); - query = SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), + query = buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), pojos, null, "b"); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 3L); + checkResults(lsr, 3L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); pojos.add(new ParameterPOJO(null, null, "v81")); - query = SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null); + query = buildQuery("Investigation", null, null, null, null, pojos, null, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr); + checkResults(lsr); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); - query = SearchApi.buildQuery("Investigation", null, null, null, null, pojos, null, null); + query = buildQuery("Investigation", null, null, null, null, pojos, null, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 3L); + checkResults(lsr, 3L); List samples = Arrays.asList("ddd", "nnn"); - query = SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null); + query = buildQuery("Investigation", null, null, null, null, null, samples, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 3L); + checkResults(lsr, 3L); samples = Arrays.asList("ddd", "mmm"); - query = SearchApi.buildQuery("Investigation", null, null, null, null, null, samples, null); + query = buildQuery("Investigation", null, null, null, null, null, samples, null); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr); + checkResults(lsr); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); samples = Arrays.asList("ddd", "nnn"); - query = SearchApi.buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), + query = buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), pojos, samples, "b"); lsr = searchApi.getResults(query, 100, null); - checkLsr(lsr, 3L); + checkResults(lsr, 3L); } - /** - * Populate UserGroup, Investigation, InvestigationParameter, - * InvestigationUser, Dataset,DatasetParameter,Datafile, DatafileParameter - * and Sample - */ - private void populate() throws IcatException { - Queue queue = new ConcurrentLinkedQueue<>(); - Long investigationUserId = 0L; - for (int i = 0; i < NUMINV; i++) { - for (int j = 0; j < NUMUSERS; j++) { - if (i % (j + 1) == 1) { - String fn = "FN " + letters.substring(j, j + 1) + " " + letters.substring(j, j + 1); - String name = letters.substring(j, j + 1) + j; - User user = new User(); - user.setId(new Long(j)); - user.setName(name); - user.setFullName(fn); - Investigation investigation = new Investigation(); - investigation.setId(new Long(i)); - InvestigationUser investigationUser = new InvestigationUser(); - investigationUser.setId(investigationUserId); - investigationUser.setUser(user); - investigationUser.setInvestigation(investigation); - - queue.add(SearchApi.encodeOperation("create", investigationUser)); - investigationUserId++; - System.out.println("'" + fn + "' " + name + " " + i); - } + @Test + public void locking() throws IcatException { + // Only LuceneApi needs manually locking + if (searchApi instanceof LuceneApi) { + logger.info("Performing locking tests for {}", searchApi.getClass().getSimpleName()); + try { + searchApi.unlock("Dataset"); + fail(); + } catch (IcatException e) { + assertEquals("Lucene is not currently locked for Dataset", e.getMessage()); } - } - - for (int i = 0; i < NUMINV; i++) { - int j = i % 26; - int k = (i + 7) % 26; - int l = (i + 17) % 26; - String word = letters.substring(j, j + 1) + " " + letters.substring(k, k + 1) + " " - + letters.substring(l, l + 1); - Investigation investigation = new Investigation(); - Facility facility = new Facility(); - facility.setName(""); - facility.setId(0L); - investigation.setFacility(facility); - InvestigationType type = new InvestigationType(); - type.setName("test"); - type.setId(0L); - investigation.setType(type); - investigation.setName(word); - investigation.setTitle(""); - investigation.setVisitId(""); - investigation.setStartDate(new Date(now + i * 60000)); - investigation.setEndDate(new Date(now + (i + 1) * 60000)); - investigation.setId(new Long(i)); - - queue.add(SearchApi.encodeOperation("create", investigation)); - System.out.println("INVESTIGATION '" + word + "' " + new Date(now + i * 60000) + " " + i); - } - - for (int i = 0; i < NUMINV; i++) { - if (i % 2 == 1) { - fillParms(queue, i, "investigation"); + searchApi.lock("Dataset"); + try { + searchApi.lock("Dataset"); + fail(); + } catch (IcatException e) { + assertEquals("Lucene already locked for Dataset", e.getMessage()); + } + searchApi.unlock("Dataset"); + try { + searchApi.unlock("Dataset"); + fail(); + } catch (IcatException e) { + assertEquals("Lucene is not currently locked for Dataset", e.getMessage()); } + } else { + logger.info("Locking tests not relevant for {}", searchApi.getClass().getSimpleName()); } + } - for (int i = 0; i < NUMDS; i++) { - int j = i % 26; - String word = "DS" + letters.substring(j, j + 1) + letters.substring(j, j + 1) - + letters.substring(j, j + 1); - - Investigation investigation = new Investigation(); - investigation.setId(new Long(i % NUMINV)); - Dataset dataset = new Dataset(); - DatasetType type = new DatasetType(); - type.setName("test"); - type.setId(0L); - dataset.setType(type); - dataset.setName(word); - dataset.setStartDate(new Date(now + i * 60000)); - dataset.setEndDate(new Date(now + (i + 1) * 60000)); - dataset.setId(new Long(i)); - dataset.setInvestigation(investigation); - - queue.add(SearchApi.encodeOperation("create", dataset)); - System.out.println("DATASET '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMINV); - } + @Test + public void modifyDatafile() throws IcatException { + // Build entities + DatafileFormat pdfFormat = datafileFormat(0, "pdf"); + DatafileFormat pngFormat = datafileFormat(0, "png"); + Investigation investigation = investigation(0, "name", date, date); + Dataset dataset = dataset(0, "name", date, date, investigation); + Datafile elephantDatafile = datafile(42, "Elephants and Aardvarks", new Date(0), dataset); + Datafile rhinoDatafile = datafile(42, "Rhinos and Aardvarks", new Date(3), dataset); + rhinoDatafile.setDatafileFormat(pdfFormat); - for (int i = 0; i < NUMDS; i++) { - if (i % 3 == 1) { - fillParms(queue, i, "dataset"); - } - } + // Build queries + JsonObject elephantQuery = buildQuery("Datafile", null, "elephant", null, null, null, null, null); + JsonObject rhinoQuery = buildQuery("Datafile", null, "rhino", null, null, null, null, null); + JsonObject pdfQuery = buildQuery("Datafile", null, "datafileFormat.name:pdf", null, null, null, null, null); + JsonObject pngQuery = buildQuery("Datafile", null, "datafileFormat.name:png", null, null, null, null, null); + JsonObject lowRange = buildFacetRangeObject("low", 0L, 2L); + JsonObject highRange = buildFacetRangeObject("high", 2L, 4L); + JsonObject rangeFacetRequest = buildFacetRangeRequest(buildFacetIdQuery("42"), "date", lowRange, highRange); + JsonObject stringFacetRequest = buildFacetStringRequest("42", "datafileFormat.name"); + FacetDimension lowFacet = new FacetDimension("", "date", new FacetLabel("low", 1L), new FacetLabel("high", 0L)); + FacetDimension highFacet = new FacetDimension("", "date", new FacetLabel("low", 0L), + new FacetLabel("high", 1L)); + FacetDimension pdfFacet = new FacetDimension("", "datafileFormat.name", new FacetLabel("pdf", 1L)); + FacetDimension pngFacet = new FacetDimension("", "datafileFormat.name", new FacetLabel("png", 1L)); - for (int i = 0; i < NUMDF; i++) { - int j = i % 26; - String word = "DF" + letters.substring(j, j + 1) + letters.substring(j, j + 1) - + letters.substring(j, j + 1); - - Investigation investigation = new Investigation(); - investigation.setId(new Long((i % NUMDS) % NUMINV)); - Dataset dataset = new Dataset(); - dataset.setId(new Long(i % NUMDS)); - dataset.setInvestigation(investigation); - Datafile datafile = new Datafile(); - datafile.setName(word); - datafile.setDatafileModTime(new Date(now + i * 60000)); - datafile.setId(new Long(i)); - datafile.setDataset(dataset); - - queue.add(SearchApi.encodeOperation("create", datafile)); - System.out.println("DATAFILE '" + word + "' " + new Date(now + i * 60000) + " " + i + " " + i % NUMDS); + // Original + modify(SearchApi.encodeOperation("create", elephantDatafile)); + checkResults(searchApi.getResults(elephantQuery, 5), 42L); + checkResults(searchApi.getResults(rhinoQuery, 5)); + checkResults(searchApi.getResults(pdfQuery, 5)); + checkResults(searchApi.getResults(pngQuery, 5)); + checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5)); + checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), lowFacet); - } + // Change name and add a format + modify(SearchApi.encodeOperation("update", rhinoDatafile)); + checkResults(searchApi.getResults(elephantQuery, 5)); + checkResults(searchApi.getResults(rhinoQuery, 5), 42L); + checkResults(searchApi.getResults(pdfQuery, 5), 42L); + checkResults(searchApi.getResults(pngQuery, 5)); + checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5), pdfFacet); + checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), highFacet); - for (int i = 0; i < NUMDF; i++) { - if (i % 4 == 1) { - fillParms(queue, i, "datafile"); - } - } + // Change just the format + modify(SearchApi.encodeOperation("update", pngFormat)); + checkResults(searchApi.getResults(elephantQuery, 5)); + checkResults(searchApi.getResults(rhinoQuery, 5), 42L); + checkResults(searchApi.getResults(pdfQuery, 5)); + checkResults(searchApi.getResults(pngQuery, 5), 42L); + checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5), pngFacet); + checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), highFacet); - for (int i = 0; i < NUMSAMP; i++) { - int j = i % 26; - String word = "SType " + letters.substring(j, j + 1) + letters.substring(j, j + 1) - + letters.substring(j, j + 1); - - Investigation investigation = new Investigation(); - investigation.setId(new Long(i % NUMINV)); - SampleType sampleType = new SampleType(); - sampleType.setId(0L); - sampleType.setName("test"); - Sample sample = new Sample(); - sample.setId(new Long(i)); - sample.setInvestigation(investigation); - sample.setType(sampleType); - sample.setName(word); - - queue.add(SearchApi.encodeOperation("create", sample)); - System.out.println("SAMPLE '" + word + "' " + i % NUMINV); - } + // Remove the format + modify(SearchApi.encodeOperation("delete", pngFormat)); + checkResults(searchApi.getResults(elephantQuery, 5)); + checkResults(searchApi.getResults(rhinoQuery, 5), 42L); + checkResults(searchApi.getResults(pdfQuery, 5)); + checkResults(searchApi.getResults(pngQuery, 5)); + checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5)); + checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), highFacet); - modifyQueue(queue); + // Remove the file + modify(SearchApi.encodeDeletion(elephantDatafile), SearchApi.encodeDeletion(rhinoDatafile)); + checkResults(searchApi.getResults(elephantQuery, 5)); + checkResults(searchApi.getResults(rhinoQuery, 5)); + checkResults(searchApi.getResults(pdfQuery, 5)); + checkResults(searchApi.getResults(pngQuery, 5)); + // Multiple commands at once + modify(SearchApi.encodeOperation("create", elephantDatafile), + SearchApi.encodeOperation("update", rhinoDatafile), + SearchApi.encodeDeletion(elephantDatafile), + SearchApi.encodeDeletion(rhinoDatafile)); + checkResults(searchApi.getResults(elephantQuery, 5)); + checkResults(searchApi.getResults(rhinoQuery, 5)); + checkResults(searchApi.getResults(pdfQuery, 5)); + checkResults(searchApi.getResults(pngQuery, 5)); } @Test public void unitConversion() throws IcatException { // Build queries for raw and SI values - JsonObject mKQuery = Json.createObjectBuilder().add("type.units", "mK").build(); - JsonObject celsiusQuery = Json.createObjectBuilder().add("type.units", "celsius").build(); - JsonObject wrongQuery = Json.createObjectBuilder().add("type.units", "wrong").build(); - JsonObject kelvinQuery = Json.createObjectBuilder().add("type.unitsSI", "Kelvin").build(); - JsonObjectBuilder lowRangeBuilder = Json.createObjectBuilder().add("from", 272.5).add("to", 273.5).add("key", "272.5_273.5"); - JsonObjectBuilder midRangeBuilder = Json.createObjectBuilder().add("from", 272999.5).add("to", 273000.5).add("key", "272999.5_273000.5"); - JsonObjectBuilder highRangeBuilder = Json.createObjectBuilder().add("from", 273272.5).add("to", 273273.5).add("key", "273272.5_273273.5"); - JsonArray ranges = Json.createArrayBuilder().add(lowRangeBuilder).add(midRangeBuilder).add(highRangeBuilder) - .build(); - JsonObject rawObject = Json.createObjectBuilder().add("dimension", "numericValue").add("ranges", ranges) - .build(); - JsonObject rawFacetQuery = Json.createObjectBuilder().add("query", mKQuery) - .add("dimensions", Json.createArrayBuilder().add(rawObject)).build(); - JsonObject systemObject = Json.createObjectBuilder().add("dimension", "numericValueSI").add("ranges", ranges) - .build(); - JsonObject systemFacetQuery = Json.createObjectBuilder().add("query", kelvinQuery) - .add("dimensions", Json.createArrayBuilder().add(systemObject)).build(); + JsonObjectBuilder objectBuilder = Json.createObjectBuilder(); + String lowKey = "272.5_273.5"; + String midKey = "272999.5_273000.5"; + String highKey = "273272.5_273273.5"; + JsonObject lowRange = buildFacetRangeObject(lowKey, 272.5, 273.5); + JsonObject midRange = buildFacetRangeObject(midKey, 272999.5, 273000.5); + JsonObject highRange = buildFacetRangeObject(highKey, 273272.5, 273273.5); + JsonObject mKQuery = objectBuilder.add("type.units", "mK").build(); + JsonObject celsiusQuery = objectBuilder.add("type.units", "celsius").build(); + JsonObject wrongQuery = objectBuilder.add("type.units", "wrong").build(); + JsonObject kelvinQuery = objectBuilder.add("type.unitsSI", "Kelvin").build(); + JsonObject mKFacetQuery = buildFacetRangeRequest(mKQuery, "numericValue", lowRange, midRange, highRange); + JsonObject celsiusFacetQuery = buildFacetRangeRequest(celsiusQuery, "numericValue", lowRange, midRange, + highRange); + JsonObject wrongFacetQuery = buildFacetRangeRequest(wrongQuery, "numericValue", lowRange, midRange, highRange); + JsonObject systemFacetQuery = buildFacetRangeRequest(kelvinQuery, "numericValueSI", lowRange, midRange, + highRange); + + // Build expected values + FacetDimension rawExpectedFacet = new FacetDimension("", "numericValue", + new FacetLabel(lowKey, 0L), new FacetLabel(midKey, 1L), new FacetLabel(highKey, 0L)); + FacetDimension lowExpectedFacet = new FacetDimension("", "numericValueSI", + new FacetLabel(lowKey, 1L), new FacetLabel(midKey, 0L), new FacetLabel(highKey, 0L)); + FacetDimension highExpectedFacet = new FacetDimension("", "numericValueSI", + new FacetLabel(lowKey, 0L), new FacetLabel(midKey, 0L), new FacetLabel(highKey, 1L)); + FacetDimension noneExpectedFacet = new FacetDimension("", "numericValueSI", + new FacetLabel(lowKey, 0L), new FacetLabel(midKey, 0L), new FacetLabel(highKey, 0L)); // Build entities - Investigation investigation = new Investigation(); - investigation.setId(0L); - investigation.setName("name"); - investigation.setVisitId("visitId"); - investigation.setTitle("title"); - investigation.setCreateTime(new Date()); - investigation.setModTime(new Date()); - Facility facility = new Facility(); - facility.setName("facility"); - facility.setId(0L); - investigation.setFacility(facility); - InvestigationType type = new InvestigationType(); - type.setName("type"); - type.setId(0L); - investigation.setType(type); - - ParameterType numericParameterType = new ParameterType(); - numericParameterType.setId(0L); - numericParameterType.setName("parameter"); - numericParameterType.setUnits("mK"); - InvestigationParameter parameter = new InvestigationParameter(); - parameter.setInvestigation(investigation); - parameter.setType(numericParameterType); - parameter.setNumericValue(273000.); - parameter.setId(0L); - - Queue queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("create", investigation)); - queue.add(SearchApi.encodeOperation("create", parameter)); - modifyQueue(queue); + Investigation investigation = investigation(0L, "name", date, date); + ParameterType parameterType = parameterType(0, "parameter", "mK"); + Parameter parameter = parameter(0, 273000, parameterType, investigation); + // Create with units of mK + modify(SearchApi.encodeOperation("create", investigation), SearchApi.encodeOperation("create", parameter)); // Assert the raw value is still 273000 (mK) - List facetDimensions = searchApi.facetSearch("InvestigationParameter", rawFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - FacetDimension facetDimension = facetDimensions.get(0); - assertEquals("numericValue", facetDimension.getDimension()); - assertEquals(3, facetDimension.getFacets().size()); - FacetLabel facetLabel = facetDimension.getFacets().get(0); - assertEquals("272.5_273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("272999.5_273000.5", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(2); - assertEquals("273272.5_273273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - + checkFacets(searchApi.facetSearch("InvestigationParameter", mKFacetQuery, 5, 5), rawExpectedFacet); // Assert the SI value is 273 (K) - facetDimensions = searchApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("numericValueSI", facetDimension.getDimension()); - assertEquals(3, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("272.5_273.5", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("272999.5_273000.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(2); - assertEquals("273272.5_273273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); + checkFacets(searchApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5), lowExpectedFacet); // Change units only to "celsius" - numericParameterType.setUnits("celsius"); - queue = new ConcurrentLinkedQueue<>(); - queue.add(SearchApi.encodeOperation("update", parameter)); - modifyQueue(queue); - rawFacetQuery = Json.createObjectBuilder().add("query", celsiusQuery) - .add("dimensions", Json.createArrayBuilder().add(rawObject)).build(); - + parameterType.setUnits("celsius"); + modify(SearchApi.encodeOperation("update", parameter)); // Assert the raw value is still 273000 (deg C) - facetDimensions = searchApi.facetSearch("InvestigationParameter", rawFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("numericValue", facetDimension.getDimension()); - assertEquals(3, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("272.5_273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("272999.5_273000.5", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(2); - assertEquals("273272.5_273273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - + checkFacets(searchApi.facetSearch("InvestigationParameter", celsiusFacetQuery, 5, 5), rawExpectedFacet); // Assert the SI value is 273273.15 (K) - facetDimensions = searchApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("numericValueSI", facetDimension.getDimension()); - assertEquals(3, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("272.5_273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("272999.5_273000.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(2); - assertEquals("273272.5_273273.5", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); + checkFacets(searchApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5), highExpectedFacet); // Change units to something wrong - numericParameterType.setUnits("wrong"); - queue = new ConcurrentLinkedQueue<>(); - // queue.add(SearchApi.encodeOperation("update", parameter)); - queue.add(SearchApi.encodeOperation("update", numericParameterType)); - modifyQueue(queue); - rawFacetQuery = Json.createObjectBuilder().add("query", wrongQuery) - .add("dimensions", Json.createArrayBuilder().add(rawObject)).build(); - + parameterType.setUnits("wrong"); + modify(SearchApi.encodeOperation("update", parameterType)); // Assert the raw value is still 273000 (wrong) - facetDimensions = searchApi.facetSearch("InvestigationParameter", rawFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("numericValue", facetDimension.getDimension()); - assertEquals(3, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("272.5_273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("272999.5_273000.5", facetLabel.getLabel()); - assertEquals(new Long(1), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(2); - assertEquals("273272.5_273273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - + checkFacets(searchApi.facetSearch("InvestigationParameter", wrongFacetQuery, 5, 5), rawExpectedFacet); // Assert that the SI value has not been set due to conversion failing - facetDimensions = searchApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5); - assertEquals(1, facetDimensions.size()); - facetDimension = facetDimensions.get(0); - assertEquals("numericValueSI", facetDimension.getDimension()); - assertEquals(3, facetDimension.getFacets().size()); - facetLabel = facetDimension.getFacets().get(0); - assertEquals("272.5_273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(1); - assertEquals("272999.5_273000.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); - facetLabel = facetDimension.getFacets().get(2); - assertEquals("273272.5_273273.5", facetLabel.getLabel()); - assertEquals(new Long(0), facetLabel.getValue()); + checkFacets(searchApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5), noneExpectedFacet); } } diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index d2ddaf60..9a203014 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -20,10 +20,10 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStream; +import java.io.StringReader; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -46,15 +46,16 @@ import javax.json.Json; import javax.json.JsonArray; +import javax.json.JsonArrayBuilder; import javax.json.JsonNumber; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; import javax.json.JsonValue; import javax.json.stream.JsonGenerator; -import org.icatproject.core.manager.ElasticsearchApi; -import org.icatproject.core.manager.LuceneApi; -import org.icatproject.core.manager.SearchApi; +import org.icatproject.core.manager.search.LuceneApi; +import org.icatproject.core.manager.search.OpensearchApi; +import org.icatproject.core.manager.search.SearchApi; import org.icatproject.icat.client.ICAT; import org.icatproject.icat.client.IcatException; import org.icatproject.icat.client.IcatException.IcatExceptionType; @@ -72,6 +73,8 @@ */ public class TestRS { + private static final String NO_DIMENSIONS = "Did not expect responseObject to contain 'dimensions', but it did"; + private static final DateFormat dft = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); private static WSession wSession; private static long end; private static long start; @@ -93,13 +96,10 @@ private static void clearSearch() String urlString = System.getProperty("luceneUrl"); URI uribase = new URI(urlString); searchApi = new LuceneApi(uribase); - } else if (searchEngine.equals("OPENSEARCH")) { + } else if (searchEngine.equals("OPENSEARCH") || searchEngine.equals("ELASTICSEARCH")) { String urlString = System.getProperty("opensearchUrl"); URI uribase = new URI(urlString); - searchApi = new SearchApi(uribase); - } else if (searchEngine.equals("ELASTICSEARCH")) { - String urlString = System.getProperty("elasticsearchUrl"); - searchApi = new ElasticsearchApi(Arrays.asList(new URL(urlString))); + searchApi = new OpensearchApi(uribase); } else { throw new RuntimeException( "searchEngine must be one of LUCENE, OPENSEARCH, ELASTICSEARCH, but it was " + searchEngine); @@ -496,6 +496,9 @@ public void testClone() throws Exception { } + /** + * Tests the old lucene/data endpoint + */ @Test public void testLuceneDatafiles() throws Exception { Session session = setupLuceneTest(); @@ -519,30 +522,23 @@ public void testLuceneDatafiles() throws Exception { checkResultFromLuceneSearch(session, "df2", array, "Datafile", "name"); } + /** + * Tests the old lucene/data endpoint + */ @Test public void testLuceneDatasets() throws Exception { - Session session = setupLuceneTest(); - DateFormat dft = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); - // All datasets searchDatasets(session, null, null, null, null, null, 20, 5); // Use the user - Set names = new HashSet<>(); JsonArray array = searchDatasets(session, "db/tr", null, null, null, null, 20, 3); - for (int i = 0; i < 3; i++) { - long n = array.getJsonObject(i).getJsonNumber("id").longValueExact(); - JsonObject result = Json.createReader(new ByteArrayInputStream(session.get("Dataset", n).getBytes())) - .readObject(); - names.add(result.getJsonObject("Dataset").getString("name")); + JsonObject result = Json.createReader(new StringReader(session.get("Dataset", n))).readObject(); + assertEquals("ds" + (i + 1), result.getJsonObject("Dataset").getString("name")); } - assertTrue(names.contains("ds1")); - assertTrue(names.contains("ds2")); - assertTrue(names.contains("ds3")); // Try a bad user searchDatasets(session, "db/fred", null, null, null, null, 20, 0); @@ -553,8 +549,8 @@ public void testLuceneDatasets() throws Exception { // Try parameters List parameters = new ArrayList<>(); ParameterForLucene stringParameter = new ParameterForLucene("colour", "name", "green"); - ParameterForLucene dateParameter = new ParameterForLucene("birthday", "date", - dft.parse("2014-05-16T16:58:26+0000"), dft.parse("2014-05-16T16:58:26+0000")); + Date birthday = dft.parse("2014-05-16T16:58:26+0000"); + ParameterForLucene dateParameter = new ParameterForLucene("birthday", "date", birthday, birthday); ParameterForLucene numericParameter = new ParameterForLucene("current", "amps", 140, 165); array = searchDatasets(session, null, null, null, null, Arrays.asList(stringParameter), 20, 1); array = searchDatasets(session, null, null, null, null, Arrays.asList(dateParameter), 20, 1); @@ -564,64 +560,72 @@ public void testLuceneDatasets() throws Exception { parameters.add(numericParameter); array = searchDatasets(session, null, null, null, null, parameters, 20, 1); - array = searchDatasets(session, null, "gamma AND ds3", dft.parse("2014-05-16T05:09:03+0000"), - dft.parse("2014-05-16T05:15:26+0000"), parameters, 20, 1); + Date lower = dft.parse("2014-05-16T05:09:03+0000"); + Date upper = dft.parse("2014-05-16T05:15:26+0000"); + array = searchDatasets(session, null, "gamma AND ds3", lower, upper, parameters, 20, 1); checkResultFromLuceneSearch(session, "gamma", array, "Dataset", "description"); } + /** + * Tests the old lucene/data endpoint + */ @Test public void testLuceneInvestigations() throws Exception { Session session = setupLuceneTest(); - DateFormat dft = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + Date lowerOrigin = dft.parse("2011-01-01T00:00:00+0000"); + Date lowerSecond = dft.parse("2011-01-01T00:00:01+0000"); + Date lowerMinute = dft.parse("2011-01-01T00:01:00+0000"); + Date upperOrigin = dft.parse("2011-12-31T23:59:59+0000"); + Date upperSecond = dft.parse("2011-12-31T23:59:58+0000"); + Date upperMinute = dft.parse("2011-12-31T23:58:00+0000"); + List samplesAnd = Arrays.asList("ford AND rust", "koh* AND diamond"); + List samplesPlus = Arrays.asList("ford + rust", "koh + diamond"); + List samplesBad = Arrays.asList("ford AND rust", "kog* AND diamond"); + String textAnd = "title AND one"; + String textTwo = "title AND two"; + String textPlus = "title + one"; searchInvestigations(session, null, null, null, null, null, null, null, 20, 3); List parameters = new ArrayList<>(); parameters.add(new ParameterForLucene("colour", "name", "green")); - JsonArray array = searchInvestigations(session, "db/tr", "title AND one", dft.parse("2011-01-01T00:00:00+0000"), - dft.parse("2011-12-31T23:59:59+0000"), parameters, Arrays.asList("ford AND rust", "koh* AND diamond"), - "Professor", 20, 1); + JsonArray array = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, + samplesAnd, "Professor", 20, 1); checkResultFromLuceneSearch(session, "one", array, "Investigation", "visitId"); // change user - searchInvestigations(session, "db/fred", "title AND one", null, null, parameters, null, null, 20, 0); + searchInvestigations(session, "db/fred", textAnd, null, null, parameters, null, null, 20, 0); // change text - searchInvestigations(session, "db/tr", "title AND two", null, null, parameters, null, null, 20, 0); + searchInvestigations(session, "db/tr", textTwo, null, null, parameters, null, null, 20, 0); // Only working to a minute - array = searchInvestigations(session, "db/tr", "title AND one", dft.parse("2011-01-01T00:00:01+0000"), - dft.parse("2011-12-31T23:59:59+0000"), parameters, null, null, 20, 1); + array = searchInvestigations(session, "db/tr", textAnd, lowerSecond, upperOrigin, parameters, null, null, 20, + 1); checkResultFromLuceneSearch(session, "one", array, "Investigation", "visitId"); - array = searchInvestigations(session, "db/tr", "title AND one", dft.parse("2011-01-01T00:00:00+0000"), - dft.parse("2011-12-31T23:59:58+0000"), parameters, null, null, 20, 1); + array = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperSecond, parameters, null, null, 20, + 1); checkResultFromLuceneSearch(session, "one", array, "Investigation", "visitId"); - searchInvestigations(session, "db/tr", "title AND one", dft.parse("2011-01-01T00:01:00+0000"), - dft.parse("2011-12-31T23:59:59+0000"), parameters, null, null, 20, 0); + searchInvestigations(session, "db/tr", textAnd, lowerMinute, upperOrigin, parameters, null, null, 20, 0); - searchInvestigations(session, "db/tr", "title AND one", dft.parse("2011-01-01T00:00:00+0000"), - dft.parse("2011-12-31T23:58:00+0000"), parameters, null, null, 20, 0); + searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperMinute, parameters, null, null, 20, 0); // Change parameters List badParameters = new ArrayList<>(); badParameters.add(new ParameterForLucene("color", "name", "green")); - searchInvestigations(session, "db/tr", "title AND one", dft.parse("2011-01-01T00:00:00+0000"), - dft.parse("2011-12-31T23:59:59+0000"), badParameters, Arrays.asList("ford + rust", "koh + diamond"), - null, 20, 0); + searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, badParameters, samplesPlus, null, 20, + 0); // Change samples - searchInvestigations(session, "db/tr", "title AND one", dft.parse("2011-01-01T00:00:00+0000"), - dft.parse("2011-12-31T23:59:59+0000"), parameters, Arrays.asList("ford AND rust", "kog* AND diamond"), - null, 20, 0); + searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, samplesBad, null, 20, 0); // Change userFullName - searchInvestigations(session, "db/tr", "title + one", dft.parse("2011-01-01T00:00:00+0000"), - dft.parse("2011-12-31T23:59:59+0000"), parameters, Arrays.asList("ford AND rust", "koh* AND diamond"), - "Doctor", 20, 0); + searchInvestigations(session, "db/tr", textPlus, lowerOrigin, upperOrigin, parameters, samplesAnd, "Doctor", 20, + 0); // Try provoking an error badParameters = new ArrayList<>(); @@ -634,6 +638,9 @@ public void testLuceneInvestigations() throws Exception { } } + /** + * Tests the new search/documents endpoint + */ @Test public void testSearchDatafiles() throws Exception { Session session = setupLuceneTest(); @@ -669,7 +676,8 @@ public void testSearchDatafiles() throws Exception { expectation.put("name", "df3"); checkResultsSource(responseObject, Arrays.asList(expectation), false); - responseObject = searchDatafiles(session, null, null, null, null, null, searchAfter.toString(), 1, sort, null, 1); + responseObject = searchDatafiles(session, null, null, null, null, null, searchAfter.toString(), 1, sort, null, + 1); searchAfter = responseObject.get("search_after"); assertNotNull(searchAfter); expectation.put("name", "df2"); @@ -705,29 +713,19 @@ public void testSearchDatafiles() throws Exception { checkResultsSource(responseObject, Arrays.asList(expectation), true); // Test searching with someone without authz for the Datafile(s) - ICAT icat = new ICAT(System.getProperty("serverUrl")); - Map credentials = new HashMap<>(); - credentials.put("username", "piOne"); - credentials.put("password", "piOne"); - Session piSession = icat.login("db", credentials); - searchDatafiles(piSession, null, null, null, null, null, null, 10, null, null, 0); + searchDatafiles(piSession(), null, null, null, null, null, null, 10, null, null, 0); // Test no facets match on Datafiles - JsonObjectBuilder target = Json.createObjectBuilder().add("target", "Datafile"); - String facets = Json.createArrayBuilder().add(target).build().toString(); + String facets = buildFacetRequest("Datafile"); responseObject = searchDatafiles(session, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); - assertFalse("Did not expect responseObject to contain 'dimensions', but it did", - responseObject.containsKey("dimensions")); + assertFalse(NO_DIMENSIONS, responseObject.containsKey("dimensions")); // Test no facets match on DatafileParameters due to lack of READ access - target = Json.createObjectBuilder() - .add("target", "DatafileParameter").add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name"))); - facets = Json.createArrayBuilder().add(target).build().toString(); + facets = buildFacetRequest("DatafileParameter"); responseObject = searchDatafiles(session, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); - assertFalse("Did not expect responseObject to contain 'dimensions', but it did", - responseObject.containsKey("dimensions")); + assertFalse(NO_DIMENSIONS, responseObject.containsKey("dimensions")); // Test facets match on DatafileParameters wSession.addRule(null, "DatafileParameter", "R"); @@ -736,10 +734,12 @@ public void testSearchDatafiles() throws Exception { checkFacets(responseObject, "DatafileParameter.type.name", Arrays.asList("colour"), Arrays.asList(1L)); } + /** + * Tests the new search/documents endpoint + */ @Test public void testSearchDatasets() throws Exception { Session session = setupLuceneTest(); - DateFormat dft = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); JsonObject responseObject; JsonValue searchAfter; Map expectation = new HashMap<>(); @@ -804,7 +804,8 @@ public void testSearchDatasets() throws Exception { assertNotNull(searchAfter); expectation.put("name", "ds4"); checkResultsSource(responseObject, Arrays.asList(expectation), false); - responseObject = searchDatasets(session, null, null, null, null, null, searchAfter.toString(), 1, sort, null, 1); + responseObject = searchDatasets(session, null, null, null, null, null, searchAfter.toString(), 1, sort, null, + 1); searchAfter = responseObject.get("search_after"); assertNotNull(searchAfter); expectation.put("name", "ds3"); @@ -841,28 +842,19 @@ public void testSearchDatasets() throws Exception { checkResultsSource(responseObject, Arrays.asList(expectation), true); // Test searching with someone without authz for the Dataset(s) - ICAT icat = new ICAT(System.getProperty("serverUrl")); - Map credentials = new HashMap<>(); - credentials.put("username", "piOne"); - credentials.put("password", "piOne"); - Session piSession = icat.login("db", credentials); - searchDatasets(piSession, null, null, null, null, null, null, 10, null, null, 0); + searchDatasets(piSession(), null, null, null, null, null, null, 10, null, null, 0); // Test facets match on Datasets - JsonObjectBuilder target = Json.createObjectBuilder() - .add("target", "Dataset").add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name"))); - String facets = Json.createArrayBuilder().add(target).build().toString(); + String facets = buildFacetRequest("Dataset"); responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5); assertFalse(responseObject.containsKey("search_after")); checkFacets(responseObject, "Dataset.type.name", Arrays.asList("calibration"), Arrays.asList(5L)); // Test no facets match on DatasetParameters due to lack of READ access - target = Json.createObjectBuilder() - .add("target", "DatasetParameter").add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name"))); - facets = Json.createArrayBuilder().add(target).build().toString(); + facets = buildFacetRequest("DatasetParameter"); responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5); assertFalse(responseObject.containsKey("search_after")); - assertFalse("Did not expect responseObject to contain 'dimensions', but it did", + assertFalse(NO_DIMENSIONS, responseObject.containsKey("dimensions")); // Test facets match on DatasetParameters @@ -873,10 +865,12 @@ public void testSearchDatasets() throws Exception { Arrays.asList(1L, 1L, 1L)); } + /** + * Tests the new search/documents endpoint + */ @Test public void testSearchInvestigations() throws Exception { Session session = setupLuceneTest(); - DateFormat dft = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); JsonObject responseObject; Map expectation = new HashMap<>(); expectation.put("name", "expt1"); @@ -952,10 +946,6 @@ public void testSearchInvestigations() throws Exception { searchInvestigations(session, "db/tr", textPlus, lowerOrigin, upperOrigin, parameters, samplesAnd, "Doctor", null, 10, null, null, 0); - // TODO currently the only field we can access is name due to how - // Investigation.getDoc() encodes, but this is the same for all the - // investigations so testing sorting is not feasible - // Test that changes to the public steps/tables are reflected in returned fields PublicStep ps = new PublicStep(); ps.setOrigin("Investigation"); @@ -990,31 +980,21 @@ public void testSearchInvestigations() throws Exception { checkResultsSource(responseObject, Arrays.asList(expectation), true); // Test searching with someone without authz for the Investigation(s) - ICAT icat = new ICAT(System.getProperty("serverUrl")); - Map credentials = new HashMap<>(); - credentials.put("username", "piOne"); - credentials.put("password", "piOne"); - Session piSession = icat.login("db", credentials); - searchInvestigations(piSession, null, null, null, null, null, null, null, null, 10, null, null, 0); + searchInvestigations(piSession(), null, null, null, null, null, null, null, null, 10, null, null, 0); // Test facets match on Investigations - JsonObjectBuilder target = Json.createObjectBuilder() - .add("target", "Investigation").add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name"))); - String facets = Json.createArrayBuilder().add(target).build().toString(); + String facets = buildFacetRequest("Investigation"); responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); checkFacets(responseObject, "Investigation.type.name", Arrays.asList("atype"), Arrays.asList(3L)); // Test no facets match on InvestigationParameters due to lack of READ access - target = Json.createObjectBuilder().add("target", "InvestigationParameter") - .add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name"))); - facets = Json.createArrayBuilder().add(target).build().toString(); + facets = buildFacetRequest("InvestigationParameter"); responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); - assertFalse("Did not expect responseObject to contain 'dimensions', but it did", - responseObject.containsKey("dimensions")); + assertFalse(NO_DIMENSIONS, responseObject.containsKey("dimensions")); // Test facets match on InvestigationParameters wSession.addRule(null, "InvestigationParameter", "R"); @@ -1057,13 +1037,22 @@ public void testSearchParameterValidation() throws Exception { } } + private String buildFacetRequest(String target) { + JsonObjectBuilder builder = Json.createObjectBuilder(); + JsonObjectBuilder dimension = Json.createObjectBuilder().add("dimension", "type.name"); + JsonArrayBuilder dimensions = Json.createArrayBuilder().add(dimension); + builder.add("target", target).add("dimensions", dimensions); + return Json.createArrayBuilder().add(builder).build().toString(); + } + private void checkFacets(JsonObject responseObject, String dimension, List expectedLabels, List expectedCounts) { - assertTrue("Expected responseObject to contain 'dimensions', but it did not", - responseObject.containsKey("dimensions")); + String dimensionsMessage = "Expected responseObject to contain 'dimensions', but it did not"; + assertTrue(dimensionsMessage, responseObject.containsKey("dimensions")); JsonObject dimensions = responseObject.getJsonObject("dimensions"); - assertTrue("Expected 'dimensions' to contain " + dimension + " but keys were " + dimensions.keySet(), - dimensions.containsKey(dimension)); + String dimensionMessage = "Expected 'dimensions' to contain " + dimension + " but keys were " + + dimensions.keySet(); + assertTrue(dimensionMessage, dimensions.containsKey(dimension)); JsonObject labelsObject = dimensions.getJsonObject(dimension); assertEquals(expectedLabels.size(), labelsObject.size()); for (int i = 0; i < expectedLabels.size(); i++) { @@ -1080,13 +1069,27 @@ private void checkResultFromLuceneSearch(Session session, String val, JsonArray assertEquals(val, result.getJsonObject(ename).getString(field)); } + private JsonArray checkResultsSize(int n, String responseString) { + JsonArray result = Json.createReader(new ByteArrayInputStream(responseString.getBytes())).readArray(); + assertEquals(n, result.size()); + return result; + } + + private JsonObject checkResultsArraySize(int n, String responseString) { + JsonObject responseObject = Json.createReader(new ByteArrayInputStream(responseString.getBytes())).readObject(); + JsonArray results = responseObject.getJsonArray("results"); + assertEquals(n, results.size()); + return responseObject; + } + private void checkResultsSource(JsonObject responseObject, List> expectations, Boolean scored) { JsonArray results = responseObject.getJsonArray("results"); assertEquals(expectations.size(), results.size()); for (int i = 0; i < expectations.size(); i++) { JsonObject result = results.getJsonObject(i); - assertTrue(result.containsKey("id")); - assertEquals(scored, result.containsKey("score")); + assertTrue("id not present in " + result.toString(), result.containsKey("id")); + String message = "score " + (scored ? "not " : "") + "present in " + result.toString(); + assertEquals(message, scored, result.containsKey("score")); assertTrue(result.containsKey("source")); JsonObject source = result.getJsonObject("source"); @@ -1108,6 +1111,15 @@ private void checkResultsSource(JsonObject responseObject, List credentials = new HashMap<>(); + credentials.put("username", "piOne"); + credentials.put("password", "piOne"); + Session piSession = icat.login("db", credentials); + return piSession; + } + private Session setupLuceneTest() throws Exception { ICAT icat = new ICAT(System.getProperty("serverUrl")); Map credentials = new HashMap<>(); @@ -1133,24 +1145,66 @@ private Session setupLuceneTest() throws Exception { return session; } - private JsonArray searchDatasets(Session session, String user, String text, Date lower, Date upper, + /** + * For use with the old lucene/data endpoint + */ + private JsonArray searchDatafiles(Session session, String user, String text, Date lower, Date upper, List parameters, int maxResults, int n) throws IcatException { - JsonArray result = Json - .createReader(new ByteArrayInputStream( - session.searchDatasets(user, text, lower, upper, parameters, maxResults).getBytes())) - .readArray(); - assertEquals(n, result.size()); - return result; + String responseString = session.searchDatafiles(user, text, lower, upper, parameters, maxResults); + return checkResultsSize(n, responseString); } - private JsonArray searchDatafiles(Session session, String user, String text, Date lower, Date upper, + /** + * For use with the old lucene/data endpoint + */ + private JsonArray searchDatasets(Session session, String user, String text, Date lower, Date upper, List parameters, int maxResults, int n) throws IcatException { - JsonArray result = Json - .createReader(new ByteArrayInputStream( - session.searchDatafiles(user, text, lower, upper, parameters, maxResults).getBytes())) - .readArray(); - assertEquals(n, result.size()); - return result; + String responseString = session.searchDatasets(user, text, lower, upper, parameters, maxResults); + return checkResultsSize(n, responseString); + } + + /** + * For use with the old lucene/data endpoint + */ + private JsonArray searchInvestigations(Session session, String user, String text, Date lower, Date upper, + List parameters, List samples, String userFullName, int maxResults, int n) + throws IcatException { + String responseString = session.searchInvestigations(user, text, lower, upper, parameters, samples, + userFullName, maxResults); + return checkResultsSize(n, responseString); + } + + /** + * For use with the new search/documents endpoint + */ + private JsonObject searchDatafiles(Session session, String user, String text, Date lower, Date upper, + List parameters, String searchAfter, int limit, String sort, String facets, int n) + throws IcatException { + String responseString = session.searchDatafiles(user, text, lower, upper, parameters, searchAfter, limit, sort, + facets); + return checkResultsArraySize(n, responseString); + } + + /** + * For use with the new search/documents endpoint + */ + private JsonObject searchDatasets(Session session, String user, String text, Date lower, Date upper, + List parameters, String searchAfter, int limit, String sort, String facets, int n) + throws IcatException { + String responseString = session.searchDatasets(user, text, lower, upper, parameters, searchAfter, limit, sort, + facets); + return checkResultsArraySize(n, responseString); + } + + /** + * For use with the new search/documents endpoint + */ + private JsonObject searchInvestigations(Session session, String user, String text, Date lower, Date upper, + List parameters, List samples, String userFullName, String searchAfter, + int limit, String sort, String facets, int n) throws IcatException { + String responseString = session.searchInvestigations(user, text, lower, upper, parameters, samples, + userFullName, searchAfter, limit, sort, facets); + return checkResultsArraySize(n, responseString); } @Test @@ -1691,50 +1745,6 @@ private JsonArray search(Session session, String query, int n) throws IcatExcept return result; } - private JsonArray searchInvestigations(Session session, String user, String text, Date lower, Date upper, - List parameters, List samples, String userFullName, int maxResults, int n) - throws IcatException { - JsonArray result = Json.createReader(new ByteArrayInputStream( - session.searchInvestigations(user, text, lower, upper, parameters, samples, userFullName, maxResults) - .getBytes())) - .readArray(); - assertEquals(n, result.size()); - return result; - } - - private JsonObject searchDatafiles(Session session, String user, String text, Date lower, Date upper, - List parameters, String searchAfter, int limit, String sort, String facets, int n) - throws IcatException { - String response = session.searchDatafiles(user, text, lower, upper, parameters, searchAfter, limit, sort, - facets); - JsonObject responseObject = Json.createReader(new ByteArrayInputStream(response.getBytes())).readObject(); - JsonArray results = responseObject.getJsonArray("results"); - assertEquals(n, results.size()); - return responseObject; - } - - private JsonObject searchDatasets(Session session, String user, String text, Date lower, Date upper, - List parameters, String searchAfter, int limit, String sort, String facets, int n) - throws IcatException { - String response = session.searchDatasets(user, text, lower, upper, parameters, searchAfter, limit, sort, - facets); - JsonObject responseObject = Json.createReader(new ByteArrayInputStream(response.getBytes())).readObject(); - JsonArray results = responseObject.getJsonArray("results"); - assertEquals(n, results.size()); - return responseObject; - } - - private JsonObject searchInvestigations(Session session, String user, String text, Date lower, Date upper, - List parameters, List samples, String userFullName, String searchAfter, - int limit, String sort, String facets, int n) throws IcatException { - String response = session.searchInvestigations(user, text, lower, upper, parameters, samples, userFullName, - searchAfter, limit, sort, facets); - JsonObject responseObject = Json.createReader(new ByteArrayInputStream(response.getBytes())).readObject(); - JsonArray results = responseObject.getJsonArray("results"); - assertEquals(n, results.size()); - return responseObject; - } - @Test public void testWriteGood() throws Exception { diff --git a/src/test/scripts/prepare_test.py b/src/test/scripts/prepare_test.py index c3fc0043..463f71cb 100644 --- a/src/test/scripts/prepare_test.py +++ b/src/test/scripts/prepare_test.py @@ -8,21 +8,18 @@ from zipfile import ZipFile import subprocess -if len(sys.argv) != 7: +if len(sys.argv) != 6: raise RuntimeError("Wrong number of arguments") containerHome = sys.argv[1] icat_url = sys.argv[2] search_engine = sys.argv[3] lucene_url = sys.argv[4] -elasticsearch_url = sys.argv[5] -opensearch_url = sys.argv[6] +opensearch_url = sys.argv[5] if search_engine == "LUCENE": search_urls = lucene_url -elif search_engine == "ELASTICSEARCH": - search_urls = elasticsearch_url -elif search_engine == "OPENSEARCH": +elif search_engine == "OPENSEARCH" or search_engine == "ELASTICSEARCH": search_urls = opensearch_url else: raise RuntimeError("Search engine %s unrecognised, " % search_engine From eb26a872f7d764b136159196a2acd9faa4bb0d78 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Wed, 8 Jun 2022 16:01:53 +0000 Subject: [PATCH 20/51] Add fields needed for DGS component #267 --- .../org/icatproject/core/entity/Datafile.java | 28 +++++++++++++++++-- .../org/icatproject/core/entity/Dataset.java | 16 +++++++++++ .../icatproject/core/entity/Instrument.java | 12 ++++++++ .../core/entity/InstrumentScientist.java | 10 +++++++ .../core/entity/Investigation.java | 6 ++++ .../core/entity/InvestigationInstrument.java | 10 +++++++ .../core/manager/PropertyHandler.java | 12 ++++---- .../core/manager/search/LuceneApi.java | 4 +-- .../core/manager/search/SearchApi.java | 2 +- .../org/icatproject/exposed/ICATRest.java | 19 +++++++++++-- 10 files changed, 104 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index 3905dc87..b58d158a 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -206,6 +206,9 @@ public void getDoc(JsonGenerator gen) { if (description != null) { SearchApi.encodeString(gen, "description", description); } + if (location != null) { + SearchApi.encodeString(gen, "location", location); + } if (doi != null) { SearchApi.encodeString(gen, "doi", doi); } @@ -220,7 +223,15 @@ public void getDoc(JsonGenerator gen) { SearchApi.encodeLong(gen, "date", modTime); } SearchApi.encodeString(gen, "id", id); - SearchApi.encodeString(gen, "investigation.id", dataset.getInvestigation().id); + if (dataset != null) { + SearchApi.encodeString(gen, "dataset.id", dataset.id); + SearchApi.encodeString(gen, "dataset.name", dataset.getName()); + Investigation investigation = dataset.getInvestigation(); + if (investigation != null) { + SearchApi.encodeString(gen, "investigation.id", investigation.id); + SearchApi.encodeString(gen, "investigation.name", investigation.getName()); + } + } } /** @@ -239,16 +250,27 @@ public static Map getDocumentFields() throws IcatExcepti EntityInfoHandler eiHandler = EntityInfoHandler.getInstance(); Relationship[] datafileFormatRelationships = { eiHandler.getRelationshipsByName(Datafile.class).get("datafileFormat") }; - Relationship[] investigationRelationships = { + Relationship[] datasetRelationships = { eiHandler.getRelationshipsByName(Datafile.class).get("dataset") }; + Relationship[] investigationRelationships = { + eiHandler.getRelationshipsByName(Datafile.class).get("dataset"), + eiHandler.getRelationshipsByName(Dataset.class).get("investigation") }; + Relationship[] instrumentRelationships = { + eiHandler.getRelationshipsByName(Investigation.class).get("investigationInstruments"), + eiHandler.getRelationshipsByName(InvestigationInstrument.class).get("instrument") }; documentFields.put("name", null); documentFields.put("description", null); + documentFields.put("location", null); documentFields.put("doi", null); documentFields.put("date", null); documentFields.put("id", null); - documentFields.put("investigation.id", investigationRelationships); + documentFields.put("dataset.id", null); + documentFields.put("dataset.name", datasetRelationships); + documentFields.put("investigation.id", datasetRelationships); + documentFields.put("investigation.name", investigationRelationships); documentFields.put("datafileFormat.id", null); documentFields.put("datafileFormat.name", datafileFormatRelationships); + documentFields.put("InvestigationInstrument instrument.id", instrumentRelationships); } return documentFields; } diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index 7ebb8980..970c0946 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -209,6 +209,13 @@ public void getDoc(JsonGenerator gen) { } SearchApi.encodeString(gen, "id", id); SearchApi.encodeString(gen, "investigation.id", investigation.id); + SearchApi.encodeString(gen, "investigation.name", investigation.getName()); + if (investigation.getStartDate() != null) { + SearchApi.encodeLong(gen, "investigation.startDate", investigation.getStartDate()); + } else { + SearchApi.encodeLong(gen, "investigation.startDate", investigation.getCreateTime()); + } + SearchApi.encodeString(gen, "investigation.title", investigation.getTitle()); if (sample != null) { sample.getDoc(gen, "sample."); @@ -234,6 +241,11 @@ public static Map getDocumentFields() throws IcatExcepti Relationship[] sampleTypeRelationships = { eiHandler.getRelationshipsByName(Dataset.class).get("sample"), eiHandler.getRelationshipsByName(Sample.class).get("type") }; Relationship[] typeRelationships = { eiHandler.getRelationshipsByName(Dataset.class).get("type") }; + Relationship[] investigationRelationships = { + eiHandler.getRelationshipsByName(Dataset.class).get("investigation") }; + Relationship[] instrumentRelationships = { + eiHandler.getRelationshipsByName(Investigation.class).get("investigationInstruments"), + eiHandler.getRelationshipsByName(InvestigationInstrument.class).get("instrument") }; documentFields.put("name", null); documentFields.put("description", null); documentFields.put("doi", null); @@ -241,6 +253,9 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("endDate", null); documentFields.put("id", null); documentFields.put("investigation.id", null); + documentFields.put("investigation.title", investigationRelationships); + documentFields.put("investigation.name", investigationRelationships); + documentFields.put("investigation.startDate", investigationRelationships); documentFields.put("sample.id", null); documentFields.put("sample.name", sampleRelationships); documentFields.put("sample.investigation.id", sampleRelationships); @@ -248,6 +263,7 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("sample.type.name", sampleTypeRelationships); documentFields.put("type.id", null); documentFields.put("type.name", typeRelationships); + documentFields.put("InvestigationInstrument instrument.id", instrumentRelationships); } return documentFields; } diff --git a/src/main/java/org/icatproject/core/entity/Instrument.java b/src/main/java/org/icatproject/core/entity/Instrument.java index 485550f4..97dc2bb4 100644 --- a/src/main/java/org/icatproject/core/entity/Instrument.java +++ b/src/main/java/org/icatproject/core/entity/Instrument.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; +import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; @@ -14,6 +15,8 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.icatproject.core.manager.search.SearchApi; + @Comment("Used by a user within an investigation") @SuppressWarnings("serial") @Entity @@ -138,4 +141,13 @@ public void setShifts(List shifts) { this.shifts = shifts; } + @Override + public void getDoc(JsonGenerator gen) { + if (fullName != null) { + SearchApi.encodeText(gen, "instrument.fullName", fullName); + } + SearchApi.encodeString(gen, "instrument.name", name); + SearchApi.encodeString(gen, "instrument.id", id); + } + } diff --git a/src/main/java/org/icatproject/core/entity/InstrumentScientist.java b/src/main/java/org/icatproject/core/entity/InstrumentScientist.java index 0cc5c794..795ada82 100644 --- a/src/main/java/org/icatproject/core/entity/InstrumentScientist.java +++ b/src/main/java/org/icatproject/core/entity/InstrumentScientist.java @@ -2,6 +2,7 @@ import java.io.Serializable; +import javax.json.stream.JsonGenerator; import javax.persistence.Entity; import javax.persistence.FetchType; import javax.persistence.JoinColumn; @@ -9,6 +10,8 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.icatproject.core.manager.search.SearchApi; + @Comment("Relationship between an ICAT user as an instrument scientist and the instrument") @SuppressWarnings("serial") @Entity @@ -43,4 +46,11 @@ public void setInstrument(Instrument instrument) { public InstrumentScientist() { } + @Override + public void getDoc(JsonGenerator gen) { + user.getDoc(gen); + SearchApi.encodeString(gen, "instrument.id", instrument.id); + SearchApi.encodeString(gen, "id", id); + } + } diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index 50ccd474..e3fc6d73 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -310,6 +310,8 @@ public static Map getDocumentFields() throws IcatExcepti Relationship[] typeRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("type") }; Relationship[] facilityRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("facility") }; + Relationship[] sampleRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("samples") }; + Relationship[] instrumentRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("investigationInstruments"), eiHandler.getRelationshipsByName(InvestigationInstrument.class).get("instrument") }; documentFields.put("name", null); documentFields.put("visitId", null); documentFields.put("title", null); @@ -322,6 +324,10 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("facility.id", null); documentFields.put("type.name", typeRelationships); documentFields.put("type.id", null); + documentFields.put("Sample name", sampleRelationships); + documentFields.put("InvestigationInstrument instrument.fullName", instrumentRelationships); + documentFields.put("InvestigationInstrument instrument.id", instrumentRelationships); + documentFields.put("InvestigationInstrument instrument.name", instrumentRelationships); } return documentFields; } diff --git a/src/main/java/org/icatproject/core/entity/InvestigationInstrument.java b/src/main/java/org/icatproject/core/entity/InvestigationInstrument.java index b8f751f4..7a07efff 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationInstrument.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationInstrument.java @@ -2,6 +2,7 @@ import java.io.Serializable; +import javax.json.stream.JsonGenerator; import javax.persistence.Entity; import javax.persistence.FetchType; import javax.persistence.JoinColumn; @@ -9,6 +10,8 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.icatproject.core.manager.search.SearchApi; + @Comment("Represents a many-to-many relationship between an investigation and the instruments assigned") @SuppressWarnings("serial") @Entity @@ -39,4 +42,11 @@ public void setInvestigation(Investigation investigation) { this.investigation = investigation; } + @Override + public void getDoc(JsonGenerator gen) { + instrument.getDoc(gen); + SearchApi.encodeString(gen, "investigation.id", investigation.id); + SearchApi.encodeString(gen, "id", id); + } + } diff --git a/src/main/java/org/icatproject/core/manager/PropertyHandler.java b/src/main/java/org/icatproject/core/manager/PropertyHandler.java index 276a28d6..f5374ba7 100644 --- a/src/main/java/org/icatproject/core/manager/PropertyHandler.java +++ b/src/main/java/org/icatproject/core/manager/PropertyHandler.java @@ -397,8 +397,9 @@ private void init() { * currently override the EntityBaseBean.getDoc() method. This should * result in no change to behaviour if the property is not specified. */ - entitiesToIndex.addAll(Arrays.asList("Datafile", "Dataset", "Investigation", "InvestigationUser", - "DatafileParameter", "DatasetParameter", "InvestigationParameter", "Sample")); + entitiesToIndex.addAll(Arrays.asList("Datafile", "Dataset", "Investigation", "InvestigationInstrument", + "InstrumentScientist", "InvestigationUser", "DatafileParameter", "DatasetParameter", + "InvestigationParameter", "Sample", "Parameter", "User")); logger.info("search.entitiesToIndex not set. Defaulting to: {}", entitiesToIndex.toString()); } formattedProps.add("search.entitiesToIndex " + entitiesToIndex.toString()); @@ -485,9 +486,9 @@ private void init() { if (searchUrls.size() != 1) { String msg = "Exactly one value for search.urls must be provided when using " + searchEngine; throw new IllegalStateException(msg); - // } else if (searchUrls.size() == 0) { - // String msg = "At least one value for search.urls must be provided"; - // throw new IllegalStateException(msg); + // } else if (searchUrls.size() == 0) { + // String msg = "At least one value for search.urls must be provided"; + // throw new IllegalStateException(msg); } formattedProps.add("search.urls" + " " + searchUrls.toString()); logger.info("Using {} as search engine with url(s) {}", searchEngine, searchUrls); @@ -514,7 +515,6 @@ private void init() { logger.info("'search.engine' entry not present so no free text search available"); } - unitAliasOptions = props.getString("units", ""); /* diff --git a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java index a422abad..93701e7c 100644 --- a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java @@ -62,7 +62,7 @@ public JsonObject buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) t builder.add("score", score); } JsonArrayBuilder arrayBuilder; - if (sort == null || sort.equals("")) { + if (sort == null || sort.equals("") || sort.equals("{}")) { arrayBuilder = Json.createArrayBuilder().add(score); } else { arrayBuilder = searchAfterArrayBuilder(lastBean, sort); @@ -99,7 +99,7 @@ public void addNow(String entityName, List ids, EntityManager manager, gen.writeEnd(); return null; } catch (Exception e) { - logger.error("About to throw internal exception because of", e); + logger.error("About to throw internal exception for ids {} because of", ids, e); throw new IcatException(IcatExceptionType.INTERNAL, e.getMessage()); } finally { manager.close(); diff --git a/src/main/java/org/icatproject/core/manager/search/SearchApi.java b/src/main/java/org/icatproject/core/manager/search/SearchApi.java index d418e64f..a7c56a29 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchApi.java @@ -172,7 +172,7 @@ public static void encodeText(JsonGenerator gen, String name, String value) { */ public JsonValue buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) throws IcatException { JsonArrayBuilder arrayBuilder; - if (sort != null && !sort.equals("")) { + if (sort != null && !sort.equals("") || sort.equals("{}")) { arrayBuilder = searchAfterArrayBuilder(lastBean, sort); } else { arrayBuilder = Json.createArrayBuilder(); diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index 4603712e..d486cc2c 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -1256,6 +1256,11 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId * "range" key should denote an array of objects with "lower" * and "upper" values. * + * @param restrict + * Whether to perform a quicker search which restricts the + * results based on an InvestigationUser or + * InstrumentScientist being able to read their "own" data. + * * @return Set of entity ids, relevance scores and Document source encoded as * json. * @@ -1267,14 +1272,15 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId @Produces(MediaType.APPLICATION_JSON) public String search(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, @QueryParam("query") String query, @QueryParam("search_after") String searchAfter, - @QueryParam("limit") int limit, @QueryParam("sort") String sort, @QueryParam("facets") String facets) + @QueryParam("limit") int limit, @QueryParam("sort") String sort, @QueryParam("facets") String facets, + @QueryParam("restrict") boolean restrict) throws IcatException { if (query == null) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "query is not set"); } String userName = beanManager.getUserName(sessionId, manager); JsonValue searchAfterValue = null; - if (searchAfter.length() > 0) { + if (searchAfter != null && searchAfter.length() > 0) { try (JsonReader jr = Json.createReader(new StringReader(searchAfter))) { searchAfterValue = jr.read(); } @@ -1282,6 +1288,13 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonReader jr = Json.createReader(new StringReader(query))) { JsonObject jo = jr.readObject(); + if (restrict && !jo.containsKey("user")) { + JsonObjectBuilder builder = Json.createObjectBuilder(); + for (Entry entry : jo.entrySet()) { + builder.add(entry.getKey(), entry.getValue()); + } + jo = builder.add("user", userName).build(); + } String target = jo.getString("target", null); if (jo.containsKey("parameters")) { for (JsonValue val : jo.getJsonArray("parameters")) { @@ -1318,7 +1331,7 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId } else { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "target:" + target + " is not expected"); } - logger.debug("Free text search with query: {}", jo.toString()); + logger.debug("Free text search with query: {}, facets: {}", jo.toString(), facets); result = beanManager.freeTextSearchDocs(userName, jo, searchAfterValue, limit, sort, facets, manager, request.getRemoteAddr(), klass); From b228aa3cf5e56efe1bfde7b7809f5631709ec622 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 10 Jun 2022 20:16:50 +0100 Subject: [PATCH 21/51] Update Search tests with Instruments #267 --- .../org/icatproject/core/entity/Dataset.java | 12 +- .../core/entity/Investigation.java | 6 +- .../core/manager/PropertyHandler.java | 11 +- .../core/manager/search/LuceneApi.java | 3 +- .../core/manager/search/OpensearchApi.java | 413 ++++++++++++------ .../search/OpensearchScriptBuilder.java | 136 ++++++ .../core/manager/search/QueryBuilder.java | 2 +- .../manager/search/ScoredEntityBaseBean.java | 11 +- .../core/manager/search/SearchManager.java | 3 +- .../core/manager/TestEntityInfo.java | 12 +- .../core/manager/TestSearchApi.java | 165 +++++-- .../org/icatproject/integration/TestRS.java | 17 +- 12 files changed, 595 insertions(+), 196 deletions(-) create mode 100644 src/main/java/org/icatproject/core/manager/search/OpensearchScriptBuilder.java diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index 970c0946..43c12057 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -210,12 +210,14 @@ public void getDoc(JsonGenerator gen) { SearchApi.encodeString(gen, "id", id); SearchApi.encodeString(gen, "investigation.id", investigation.id); SearchApi.encodeString(gen, "investigation.name", investigation.getName()); - if (investigation.getStartDate() != null) { - SearchApi.encodeLong(gen, "investigation.startDate", investigation.getStartDate()); - } else { - SearchApi.encodeLong(gen, "investigation.startDate", investigation.getCreateTime()); + if (investigation != null) { + if (investigation.getStartDate() != null) { + SearchApi.encodeLong(gen, "investigation.startDate", investigation.getStartDate()); + } else if (investigation.getCreateTime() != null) { + SearchApi.encodeLong(gen, "investigation.startDate", investigation.getCreateTime()); + } + SearchApi.encodeString(gen, "investigation.title", investigation.getTitle()); } - SearchApi.encodeString(gen, "investigation.title", investigation.getTitle()); if (sample != null) { sample.getDoc(gen, "sample."); diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index e3fc6d73..df83be9c 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -310,8 +310,9 @@ public static Map getDocumentFields() throws IcatExcepti Relationship[] typeRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("type") }; Relationship[] facilityRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("facility") }; - Relationship[] sampleRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("samples") }; - Relationship[] instrumentRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("investigationInstruments"), eiHandler.getRelationshipsByName(InvestigationInstrument.class).get("instrument") }; + Relationship[] instrumentRelationships = { + eiHandler.getRelationshipsByName(Investigation.class).get("investigationInstruments"), + eiHandler.getRelationshipsByName(InvestigationInstrument.class).get("instrument") }; documentFields.put("name", null); documentFields.put("visitId", null); documentFields.put("title", null); @@ -324,7 +325,6 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("facility.id", null); documentFields.put("type.name", typeRelationships); documentFields.put("type.id", null); - documentFields.put("Sample name", sampleRelationships); documentFields.put("InvestigationInstrument instrument.fullName", instrumentRelationships); documentFields.put("InvestigationInstrument instrument.id", instrumentRelationships); documentFields.put("InvestigationInstrument instrument.name", instrumentRelationships); diff --git a/src/main/java/org/icatproject/core/manager/PropertyHandler.java b/src/main/java/org/icatproject/core/manager/PropertyHandler.java index f5374ba7..3bd858ae 100644 --- a/src/main/java/org/icatproject/core/manager/PropertyHandler.java +++ b/src/main/java/org/icatproject/core/manager/PropertyHandler.java @@ -397,9 +397,10 @@ private void init() { * currently override the EntityBaseBean.getDoc() method. This should * result in no change to behaviour if the property is not specified. */ - entitiesToIndex.addAll(Arrays.asList("Datafile", "Dataset", "Investigation", "InvestigationInstrument", - "InstrumentScientist", "InvestigationUser", "DatafileParameter", "DatasetParameter", - "InvestigationParameter", "Sample", "Parameter", "User")); + entitiesToIndex.addAll(Arrays.asList("Datafile", "DatafileFormat", "DatafileParameter", + "Dataset", "DatasetParameter", "DatasetType", "Facility", "Instrument", "InstrumentScientist", + "Investigation", "InvestigationInstrument", "InvestigationParameter", "InvestigationType", + "InvestigationUser", "ParameterType", "Sample", "SampleType", "User")); logger.info("search.entitiesToIndex not set. Defaulting to: {}", entitiesToIndex.toString()); } formattedProps.add("search.entitiesToIndex " + entitiesToIndex.toString()); @@ -665,4 +666,8 @@ public Path getSearchDirectory() { return searchDirectory; } + public String getUnitAliasOptions() { + return unitAliasOptions; + } + } diff --git a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java index 93701e7c..8d1c4b59 100644 --- a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java @@ -170,12 +170,13 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer List resultsArray = postResponse.getJsonArray("results").getValuesAs(JsonObject.class); for (JsonObject resultObject : resultsArray) { int luceneDocId = resultObject.getInt("_id"); + int shardIndex = resultObject.getInt("_shardIndex"); Float score = Float.NaN; if (resultObject.keySet().contains("_score")) { score = resultObject.getJsonNumber("_score").bigDecimalValue().floatValue(); } JsonObject source = resultObject.getJsonObject("_source"); - ScoredEntityBaseBean result = new ScoredEntityBaseBean(luceneDocId, score, source); + ScoredEntityBaseBean result = new ScoredEntityBaseBean(luceneDocId, shardIndex, score, source); results.add(result); logger.trace("Result id {} with score {}", result.getEntityBaseBeanId(), score); } diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index e1dd83ee..284dbaae 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -95,6 +95,9 @@ public ParentRelation(RelationType relationType, String parentName, String joinF .add("type", "stemmer").add("langauge", "possessive_english")))) .build(); private static Map> relations = new HashMap<>(); + private static Map> defaultFieldsMap = new HashMap<>(); + protected static final Set indices = new HashSet<>( + Arrays.asList("datafile", "dataset", "investigation", "instrumentscientist")); static { // Non-nested children have a one to one relationship with an indexed entity and @@ -107,6 +110,17 @@ public ParentRelation(RelationType relationType, String parentName, String joinF new ParentRelation(RelationType.CHILD, "investigation", "type", InvestigationType.docFields))); relations.put("facility", Arrays.asList( new ParentRelation(RelationType.CHILD, "investigation", "facility", Facility.docFields))); + relations.put("investigation", Arrays.asList( + new ParentRelation(RelationType.CHILD, "dataset", "investigation", + new HashSet<>(Arrays.asList("investigation.name", "investigation.id", "investigation.startDate", + "investigation.title"))), + new ParentRelation(RelationType.CHILD, "datafile", "investigation", + new HashSet<>(Arrays.asList("investigation.name", "investigation.id"))))); + relations.put("dataset", Arrays.asList( + new ParentRelation(RelationType.CHILD, "datafile", "dataset", + new HashSet<>(Arrays.asList("dataset.name", "dataset.id"))))); + relations.put("user", Arrays.asList( + new ParentRelation(RelationType.CHILD, "instrumentscientist", "user", User.docFields))); // Nested children are indexed as an array of objects on their parent entity, // and know their parent's id (N.B. InvestigationUsers are also mapped to @@ -121,10 +135,14 @@ public ParentRelation(RelationType relationType, String parentName, String joinF new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null), new ParentRelation(RelationType.NESTED_CHILD, "dataset", "investigation", null), new ParentRelation(RelationType.NESTED_CHILD, "datafile", "investigation", null))); + relations.put("investigationinstrument", Arrays.asList( + new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null), + new ParentRelation(RelationType.NESTED_CHILD, "dataset", "investigation", null), + new ParentRelation(RelationType.NESTED_CHILD, "datafile", "investigation", null))); relations.put("sample", Arrays.asList( new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null))); - // Nested grandchildren are entities that are related to one of the nested + // Grandchildren are entities that are related to one of the nested // children, but do not have a direct reference to one of the indexed entities, // and so must be updated by query - they also only affect a subset of the // nested fields, rather than an entire nested object @@ -140,96 +158,70 @@ public ParentRelation(RelationType relationType, String parentName, String joinF User.docFields), new ParentRelation(RelationType.NESTED_GRANDCHILD, "dataset", "investigationuser", User.docFields), new ParentRelation(RelationType.NESTED_GRANDCHILD, "datafile", "investigationuser", User.docFields))); + relations.put("instrument", Arrays.asList( + new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "investigationinstrument", + User.docFields), + new ParentRelation(RelationType.NESTED_GRANDCHILD, "dataset", "investigationinstrument", + User.docFields), + new ParentRelation(RelationType.NESTED_GRANDCHILD, "datafile", "investigationinstrument", + User.docFields))); relations.put("sampleType", Arrays.asList( new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "sample", SampleType.docFields))); + + defaultFieldsMap.put("_all", new ArrayList<>()); + defaultFieldsMap.put("datafile", + Arrays.asList("name", "description", "doi", "location", "datafileFormat.name")); + defaultFieldsMap.put("dataset", + Arrays.asList("name", "description", "doi", "sample.name", "sample.type.name", "type.name")); + defaultFieldsMap.put("investigation", + Arrays.asList("name", "visitId", "title", "summary", "doi", "facility.name")); } - public OpensearchApi(URI server) { + public OpensearchApi(URI server) throws IcatException { super(server); icatUnits = new IcatUnits(); + initMappings(); + initScripts(); } - public OpensearchApi(URI server, String unitAliasOptions) { + public OpensearchApi(URI server, String unitAliasOptions) throws IcatException { super(server); icatUnits = new IcatUnits(unitAliasOptions); - } - - private static String buildCreateScript(String target) { - String source = "ctx._source." + target + " = params.doc"; - JsonObjectBuilder builder = Json.createObjectBuilder().add("lang", "painless").add("source", source); - return Json.createObjectBuilder().add("script", builder).build().toString(); - } - - private static String buildChildScript(Set docFields, boolean update) { - String source = ""; - for (String field : docFields) { - if (update) { - source += "ctx._source['" + field + "'] = params['" + field + "']; "; - } else { - source += "ctx._source.remove('" + field + "'); "; - } - } - JsonObjectBuilder builder = Json.createObjectBuilder().add("lang", "painless").add("source", source); - return Json.createObjectBuilder().add("script", builder).build().toString(); - } - - private static String buildNestedChildScript(String target, boolean update) { - String source = "if (ctx._source." + target + " != null) {List ids = new ArrayList(); ctx._source." + target - + ".forEach(t -> ids.add(t.id)); if (ids.contains(params.id)) {ctx._source." + target - + ".remove(ids.indexOf(params.id))}}"; - if (update) { - source += "if (ctx._source." + target + " != null) {ctx._source." + target - + ".addAll(params.doc);} else {ctx._source." + target + " = params.doc;}"; - } - JsonObjectBuilder builder = Json.createObjectBuilder().add("lang", "painless").add("source", source); - return Json.createObjectBuilder().add("script", builder).build().toString(); - } - - private static String buildNestedGrandchildScript(String target, Set docFields, boolean update) { - String source = "int listIndex; if (ctx._source." + target - + " != null) {List ids = new ArrayList(); ctx._source." + target - + ".forEach(t -> ids.add(t.id)); if (ids.contains(params.id)) {listIndex = ids.indexOf(params.id)}}"; - String childSource = "ctx._source." + target + ".get(listIndex)"; - for (String field : docFields) { - if (update) { - if (field.equals("numericValueSI")) { - source += "if (" + childSource - + ".numericValue != null && params.containsKey('conversionFactor')) {" + childSource - + ".numericValueSI = params.conversionFactor * " + childSource + ".numericValue;} else {" - + childSource + ".remove('numericValueSI');}"; - } else { - source += childSource + "['" + field + "']" + " = params['" + field + "']; "; - } - } else { - source += childSource + ".remove('" + field + "'); "; - } - } - JsonObjectBuilder builder = Json.createObjectBuilder().add("lang", "painless").add("source", source); - return Json.createObjectBuilder().add("script", builder).build().toString(); + initMappings(); + initScripts(); } private static JsonObject buildMappings(String index) { JsonObject typeLong = Json.createObjectBuilder().add("type", "long").build(); JsonObjectBuilder propertiesBuilder = Json.createObjectBuilder() - .add("id", typeLong) - .add("investigationuser", buildNestedMapping("investigation.id", "user.id")); + .add("id", typeLong); if (index.equals("investigation")) { propertiesBuilder .add("type.id", typeLong) .add("facility.id", typeLong) .add("sample", buildNestedMapping("investigation.id", "type.id")) - .add("investigationparameter", buildNestedMapping("investigation.id", "type.id")); + .add("investigationparameter", buildNestedMapping("investigation.id", "type.id")) + .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) + .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")); } else if (index.equals("dataset")) { propertiesBuilder .add("investigation.id", typeLong) .add("type.id", typeLong) .add("sample.id", typeLong) - .add("datasetparameter", buildNestedMapping("dataset.id", "type.id")); + .add("datasetparameter", buildNestedMapping("dataset.id", "type.id")) + .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) + .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")); } else if (index.equals("datafile")) { propertiesBuilder .add("investigation.id", typeLong) .add("datafileFormat.id", typeLong) - .add("datafileparameter", buildNestedMapping("datafile.id", "type.id")); + .add("datafileparameter", buildNestedMapping("datafile.id", "type.id")) + .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) + .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")); + } else if (index.equals("instrumentscientist")) { + propertiesBuilder + .add("instrument.id", typeLong) + .add("user.id", typeLong); } return Json.createObjectBuilder().add("properties", propertiesBuilder).build(); } @@ -299,7 +291,7 @@ public void clear() throws IcatException { @Override public void commit() throws IcatException { - post("/_refresh"); + post("/_refresh"); } @Override @@ -312,7 +304,7 @@ public List facetSearch(String target, JsonObject facetQuery, In } String dimensionPrefix = null; String index = target.toLowerCase(); - if (relations.containsKey(index)) { + if (!indices.contains(index) && relations.containsKey(index)) { // If we're attempting to facet a nested entity, use the parent index dimensionPrefix = index; index = relations.get(index).get(0).parentName; @@ -320,7 +312,8 @@ public List facetSearch(String target, JsonObject facetQuery, In JsonObject queryObject = facetQuery.getJsonObject("query"); JsonArray dimensions = facetQuery.getJsonArray("dimensions"); - JsonObjectBuilder bodyBuilder = parseQuery(Json.createObjectBuilder(), queryObject, index); + List defaultFields = defaultFieldsMap.get(index); + JsonObjectBuilder bodyBuilder = parseQuery(Json.createObjectBuilder(), queryObject, index, defaultFields); bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix); String body = bodyBuilder.build().toString(); @@ -328,7 +321,7 @@ public List facetSearch(String target, JsonObject facetQuery, In parameterMap.put("size", maxResults.toString()); JsonObject postResponse = postResponse("/" + index + "/_search", body, parameterMap); - + JsonObject aggregations = postResponse.getJsonObject("aggregations"); if (dimensionPrefix != null) { aggregations = aggregations.getJsonObject(dimensionPrefix); @@ -367,18 +360,17 @@ private JsonObjectBuilder parseFacets(JsonObjectBuilder bodyBuilder, JsonArray d public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer blockSize, String sort, List requestedFields) throws IcatException { String index = query.containsKey("target") ? query.getString("target").toLowerCase() : "_all"; + List defaultFields = defaultFieldsMap.get(index); JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); bodyBuilder = parseSort(bodyBuilder, sort); bodyBuilder = parseSearchAfter(bodyBuilder, searchAfter); - bodyBuilder = parseQuery(bodyBuilder, query, index); + bodyBuilder = parseQuery(bodyBuilder, query, index, defaultFields); String body = bodyBuilder.build().toString(); Map parameterMap = new HashMap<>(); - StringBuilder sb = new StringBuilder(); - requestedFields.forEach(f -> sb.append(f).append(",")); - parameterMap.put("_source", sb.toString()); - parameterMap.put("size", blockSize.toString()); + Map> joinedFields = new HashMap<>(); + buildParameterMap(blockSize, requestedFields, parameterMap, joinedFields); JsonObject postResponse = postResponse("/" + index + "/_search", body, parameterMap); @@ -391,7 +383,45 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer score = hit.getJsonNumber("_score").bigDecimalValue().floatValue(); } Integer id = new Integer(hit.getString("_id")); - entities.add(new ScoredEntityBaseBean(id, score, hit.getJsonObject("_source"))); + JsonObject source = hit.getJsonObject("_source"); + // If there are fields requested from another index, join them to the source + for (String joinedEntityName : joinedFields.keySet()) { + String joinedIndex = joinedEntityName.toLowerCase(); + Set requestedJoinedFields = joinedFields.get(joinedEntityName); + Map joinedParameterMap = new HashMap<>(); + String fld; + String parentId; + if (joinedIndex.contains("investigation")) { + // Special case to allow datafiles and datasets join via their investigation.id + // field + fld = "investigation.id"; + if (index.equals("investigation")) { + parentId = source.getString("id"); + } else { + parentId = source.getString("investigation.id"); + } + } else { + fld = joinedIndex + ".id"; + parentId = source.getString("id"); + } + // Search for joined entities matching the id + JsonObject termQuery = QueryBuilder.buildTermQuery(fld, parentId); + String joinedBody = Json.createObjectBuilder().add("query", termQuery).build().toString(); + buildParameterMap(blockSize, requestedJoinedFields, joinedParameterMap, null); + JsonObject joinedResponse = postResponse("/" + joinedIndex + "/_search", joinedBody, + joinedParameterMap); + // Parse the joined source and integrate it into the main source Json + JsonArray joinedHits = joinedResponse.getJsonObject("hits").getJsonArray("hits"); + JsonObjectBuilder sourceBuilder = Json.createObjectBuilder(); + source.entrySet().forEach(entry -> sourceBuilder.add(entry.getKey(), entry.getValue())); + JsonArrayBuilder joinedSourceBuilder = Json.createArrayBuilder(); + for (JsonValue joinedHit : joinedHits) { + JsonObject joinedHitObject = (JsonObject) joinedHit; + joinedSourceBuilder.add(joinedHitObject.getJsonObject("_source")); + } + source = sourceBuilder.add(joinedIndex, joinedSourceBuilder).build(); + } + entities.add(new ScoredEntityBaseBean(id, -1, score, source)); } // If we're returning as many results as were asked for, setSearchAfter so @@ -407,10 +437,49 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer result.setSearchAfter(Json.createArrayBuilder().add(score).add(id).build()); } } - + return result; } + /** + * Parses fields from requestedFields and set them in Map for the url + * parameters. + * + * @param blockSize The maximum number of results to return from a single + * search. + * @param requestedFields Fields that should be returned as part of the source + * @param parameterMap Map of key value pairs to be included in the url. + * @param joinedFields Map of indices to fields which should be returned that + * are NOT part of the main index/entity being searched. + * @throws IcatException if the field cannot be parsed. + */ + private void buildParameterMap(Integer blockSize, Iterable requestedFields, + Map parameterMap, Map> joinedFields) throws IcatException { + StringBuilder sb = new StringBuilder(); + for (String field : requestedFields) { + String[] splitString = field.split(" "); + if (splitString.length == 1) { + sb.append(splitString[0] + ","); + } else if (splitString.length == 2) { + if (joinedFields != null && indices.contains(splitString[0].toLowerCase())) { + if (joinedFields.containsKey(splitString[0])) { + joinedFields.get(splitString[0]).add(splitString[1]); + } else { + joinedFields.putIfAbsent(splitString[0], + new HashSet(Arrays.asList(splitString[1]))); + } + } else { + sb.append(splitString[0].toLowerCase() + ","); + } + } else { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Could not parse field: " + field); + } + } + parameterMap.put("_source", sb.toString()); + parameterMap.put("size", blockSize.toString()); + } + private JsonObjectBuilder parseSort(JsonObjectBuilder builder, String sort) { if (sort == null || sort.equals("")) { return builder.add("sort", Json.createArrayBuilder() @@ -439,24 +508,39 @@ private JsonObjectBuilder parseSearchAfter(JsonObjectBuilder builder, JsonValue } } - private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query, String index) - throws IcatException { + /** + * Parses the search query from the incoming queryRequest into Json that the + * search cluster can understand. + * + * @param builder The JsonObjectBuilder being used to create the body for + * the POST request to the cluster. + * @param queryRequest The Json object containing the information on the + * requested query, NOT formatted for the search cluster. + * @param index The index to search. + * @param defaultFields Default fields to apply parsed string queries to. + * @return The JsonObjectBuilder initially passed with the "query" added to it. + * @throws IcatException If the query cannot be parsed. + */ + private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject queryRequest, String index, + List defaultFields) throws IcatException { // In general, we use a boolean query to compound queries on individual fields JsonObjectBuilder queryBuilder = Json.createObjectBuilder(); JsonObjectBuilder boolBuilder = Json.createObjectBuilder(); - if (query.containsKey("text")) { + if (queryRequest.containsKey("text")) { // The free text is the only element we perform scoring on, so "must" occur JsonArrayBuilder mustBuilder = Json.createArrayBuilder(); - mustBuilder.add(QueryBuilder.buildStringQuery(query.getString("text"))); + mustBuilder + .add(QueryBuilder.buildStringQuery(queryRequest.getString("text"), + defaultFields.toArray(new String[0]))); boolBuilder.add("must", mustBuilder); } // Non-scored elements are added to the "filter" JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); - Long lowerTime = parseDate(query, "lower", 0, Long.MIN_VALUE); - Long upperTime = parseDate(query, "upper", 59999, Long.MAX_VALUE); + Long lowerTime = parseDate(queryRequest, "lower", 0, Long.MIN_VALUE); + Long upperTime = parseDate(queryRequest, "upper", 59999, Long.MAX_VALUE); if (lowerTime != Long.MIN_VALUE || upperTime != Long.MAX_VALUE) { if (index.equals("datafile")) { // datafile has only one date field @@ -467,24 +551,41 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query } } - List investigationUserQueries = new ArrayList<>(); - if (query.containsKey("user")) { - String name = query.getString("user"); - JsonObject nameQuery = QueryBuilder.buildMatchQuery("investigationuser.user.name", name); - investigationUserQueries.add(nameQuery); - } - if (query.containsKey("userFullName")) { - String fullName = query.getString("userFullName"); + if (queryRequest.containsKey("user")) { + String name = queryRequest.getString("user"); + // Because InstrumentScientist is on a separate index, we need to explicitly + // perform a search here + JsonObject termQuery = QueryBuilder.buildTermQuery("user.name.keyword", name); + String body = Json.createObjectBuilder().add("query", termQuery).build().toString(); + Map parameterMap = new HashMap<>(); + parameterMap.put("_source", "instrument.id"); + JsonObject postResponse = postResponse("/instrumentscientist/_search", body, parameterMap); + JsonArray hits = postResponse.getJsonObject("hits").getJsonArray("hits"); + JsonArrayBuilder instrumentIdsBuilder = Json.createArrayBuilder(); + for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { + String instrumentId = hit.getJsonObject("_source").getString("instrument.id"); + instrumentIdsBuilder.add(instrumentId); + } + JsonObject instrumentQuery = QueryBuilder.buildTermsQuery("investigationinstrument.instrument.id", + instrumentIdsBuilder.build()); + JsonObject nestedInstrumentQuery = QueryBuilder.buildNestedQuery("investigationinstrument", + instrumentQuery); + // InvestigationUser should be a nested field on the main Document + JsonObject investigationUserQuery = QueryBuilder.buildMatchQuery("investigationuser.user.name", name); + JsonObject nestedUserQuery = QueryBuilder.buildNestedQuery("investigationuser", investigationUserQuery); + // At least one of being an InstrumentScientist or an InvestigationUser is + // necessary + JsonArrayBuilder array = Json.createArrayBuilder().add(nestedInstrumentQuery).add(nestedUserQuery); + filterBuilder.add(Json.createObjectBuilder().add("bool", Json.createObjectBuilder().add("should", array))); + } + if (queryRequest.containsKey("userFullName")) { + String fullName = queryRequest.getString("userFullName"); JsonObject fullNameQuery = QueryBuilder.buildStringQuery(fullName, "investigationuser.user.fullName"); - investigationUserQueries.add(fullNameQuery); - } - if (investigationUserQueries.size() > 0) { - JsonObject[] array = investigationUserQueries.toArray(new JsonObject[0]); - filterBuilder.add(QueryBuilder.buildNestedQuery("investigationuser", array)); + filterBuilder.add(QueryBuilder.buildNestedQuery("investigationuser", fullNameQuery)); } - if (query.containsKey("samples")) { - JsonArray samples = query.getJsonArray("samples"); + if (queryRequest.containsKey("samples")) { + JsonArray samples = queryRequest.getJsonArray("samples"); for (int i = 0; i < samples.size(); i++) { String sample = samples.getString(i); JsonObject stringQuery = QueryBuilder.buildStringQuery(sample, "sample.name", "sample.type.name"); @@ -492,8 +593,8 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query } } - if (query.containsKey("parameters")) { - for (JsonObject parameterObject : query.getJsonArray("parameters").getValuesAs(JsonObject.class)) { + if (queryRequest.containsKey("parameters")) { + for (JsonObject parameterObject : queryRequest.getJsonArray("parameters").getValuesAs(JsonObject.class)) { String path = index + "parameter"; List parameterQueries = new ArrayList<>(); if (parameterObject.containsKey("name")) { @@ -522,8 +623,8 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query } } - if (query.containsKey("id")) { - filterBuilder.add(QueryBuilder.buildTermsQuery("id", query.getJsonArray("id"))); + if (queryRequest.containsKey("id")) { + filterBuilder.add(QueryBuilder.buildTermsQuery("id", queryRequest.getJsonArray("id"))); } JsonArray filterArray = filterBuilder.build(); @@ -537,14 +638,14 @@ public void initMappings() throws IcatException { for (String index : indices) { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URI uri = new URIBuilder(server).setPath("/" + index).build(); - logger.trace("Making call {}", uri); + logger.debug("Making call {}", uri); HttpHead httpHead = new HttpHead(uri); try (CloseableHttpResponse response = httpclient.execute(httpHead)) { int statusCode = response.getStatusLine().getStatusCode(); // If the index isn't present, we should get 404 and create the index if (statusCode == 200) { // If the index already exists (200), do not attempt to create it - logger.trace("{} index already exists, continue", index); + logger.debug("{} index already exists, continue", index); continue; } else if (statusCode != 404) { // If the code isn't 200 or 404, something has gone wrong @@ -561,7 +662,7 @@ public void initMappings() throws IcatException { JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); bodyBuilder.add("settings", indexSettings).add("mappings", buildMappings(index)); String body = bodyBuilder.build().toString(); - logger.trace("Making call {} with body {}", uri, body); + logger.debug("Making call {} with body {}", uri, body); httpPut.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); try (CloseableHttpResponse response = httpclient.execute(httpPut)) { Rest.checkStatus(response, IcatExceptionType.INTERNAL); @@ -574,37 +675,43 @@ public void initMappings() throws IcatException { public void initScripts() throws IcatException { for (Entry> entry : relations.entrySet()) { - String childName = entry.getKey(); + String key = entry.getKey(); ParentRelation relation = entry.getValue().get(0); + String updateScript = ""; + String deleteScript = ""; + // Each type of relation needs a different script to update switch (relation.relationType) { case CHILD: - post("/_scripts/update_" + childName, buildChildScript(relation.fields, true)); - post("/_scripts/delete_" + childName, buildChildScript(relation.fields, false)); + updateScript = OpensearchScriptBuilder.buildChildScript(relation.fields, true); + deleteScript = OpensearchScriptBuilder.buildChildScript(relation.fields, false); break; case NESTED_CHILD: - post("/_scripts/create_" + childName, buildCreateScript(childName)); - post("/_scripts/update_" + childName, buildNestedChildScript(childName, true)); - post("/_scripts/delete" + childName, buildNestedChildScript(childName, false)); + updateScript = OpensearchScriptBuilder.buildNestedChildScript(key, true); + deleteScript = OpensearchScriptBuilder.buildNestedChildScript(key, false); + String createScript = OpensearchScriptBuilder.buildCreateNestedChildScript(key); + post("/_scripts/create_" + key, createScript); break; case NESTED_GRANDCHILD: - if (childName.equals("parametertype")) { + if (key.equals("parametertype")) { // Special case, as parametertype applies to investigationparameter, // datasetparameter, datafileparameter for (String index : indices) { String target = index + "parameter"; - String updateScript = buildNestedGrandchildScript(target, relation.fields, true); - String deleteScript = buildNestedGrandchildScript(target, relation.fields, false); - post("/_scripts/update_" + index + childName, updateScript); - post("/_scripts/delete_" + index + childName, deleteScript); + updateScript = OpensearchScriptBuilder.buildGrandchildScript(target, relation.fields, + true); + deleteScript = OpensearchScriptBuilder.buildGrandchildScript(target, relation.fields, + false); } } else { - String updateScript = buildNestedGrandchildScript(relation.joinField, relation.fields, true); - String deleteScript = buildNestedGrandchildScript(relation.joinField, relation.fields, false); - post("/_scripts/update_" + childName, updateScript); - post("/_scripts/delete_" + childName, deleteScript); + updateScript = OpensearchScriptBuilder.buildGrandchildScript(relation.joinField, + relation.fields, true); + deleteScript = OpensearchScriptBuilder.buildGrandchildScript(relation.joinField, + relation.fields, false); } break; } + post("/_scripts/update_" + key, updateScript); + post("/_scripts/delete_" + key, deleteScript); } } @@ -627,15 +734,18 @@ public void modify(String json) throws IcatException { String index = innerOperation.getString("_index").toLowerCase(); String id = innerOperation.getString("_id"); JsonObject document = innerOperation.containsKey("doc") ? innerOperation.getJsonObject("doc") : null; + logger.trace("{} {} with id {}", operationKey, index, id); if (relations.containsKey(index)) { - // Entities without an index will have one or more parent indices that need to + // Related entities (with or without an index) will have one or more other + // indices that need to // be updated with their information for (ParentRelation relation : relations.get(index)) { modifyNestedEntity(sb, updatesByQuery, id, index, document, modificationType, relation); } - } else { - // Otherwise we are dealing with an indexed entity + } + if (indices.contains(index)) { + // Also modify any main, indexable entities modifyEntity(sb, investigationIds, id, index, document, modificationType); } } @@ -645,7 +755,7 @@ public void modify(String json) throws IcatException { URI uri = new URIBuilder(server).setPath("/_bulk").build(); HttpPost httpPost = new HttpPost(uri); httpPost.setEntity(new StringEntity(sb.toString(), ContentType.APPLICATION_JSON)); - // logger.trace("Making call {} with body {}", uri, sb.toString()); + logger.trace("Making call {} with body {}", uri, sb.toString()); try (CloseableHttpResponse response = httpclient.execute(httpPost)) { Rest.checkStatus(response, IcatExceptionType.INTERNAL); } @@ -680,6 +790,25 @@ public void modify(String json) throws IcatException { } } + /** + * For cases when Datasets and Datafiles are created after an Investigation, + * some nested fields such as InvestigationUser and InvestigationInstrument may + * have already been indexed on the Investigation but not the Dataset/file as + * the latter did not yet exist. This method retrieves these arrays from the + * Investigation index ensuring that all information is available on all indices + * at the time of creation. + * + * @param httpclient The client being used to send HTTP + * @param investigationId Id of an investigation which may contain relevant + * information. + * @param responseGet The response from a GET request using the + * investigationId, which may or may not contain relevant + * information in the returned _source Json. + * @throws IOException + * @throws URISyntaxException + * @throws IcatException + * @throws ClientProtocolException + */ private void extractFromInvestigation(CloseableHttpClient httpclient, String investigationId, CloseableHttpResponse responseGet) throws IOException, URISyntaxException, IcatException, ClientProtocolException { @@ -696,9 +825,29 @@ private void extractFromInvestigation(CloseableHttpClient httpclient, String inv JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); String body = bodyBuilder.add("query", queryObject).add("script", scriptBuilder).build().toString(); httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); - // logger.trace("Making call {} with body {}", uri, body); + logger.trace("Making call {} with body {}", uri, body); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + commit(); + } + } + } + if (responseObject.containsKey("investigationinstrument")) { + JsonArray jsonArray = responseObject.getJsonArray("investigationinstrument"); + for (String index : new String[] { "datafile", "dataset" }) { + URI uri = new URIBuilder(server).setPath("/" + index + "/_update_by_query").build(); + HttpPost httpPost = new HttpPost(uri); + JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("doc", jsonArray); + JsonObjectBuilder scriptBuilder = Json.createObjectBuilder(); + scriptBuilder.add("id", "create_investigationinstrument").add("params", paramsBuilder); + JsonObject queryObject = QueryBuilder.buildTermQuery("investigation.id", investigationId); + JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); + String body = bodyBuilder.add("query", queryObject).add("script", scriptBuilder).build().toString(); + logger.trace("Making call {} with body {}", uri, body); + httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); try (CloseableHttpResponse response = httpclient.execute(httpPost)) { Rest.checkStatus(response, IcatExceptionType.INTERNAL); + commit(); } } } @@ -713,6 +862,7 @@ private void modifyNestedEntity(StringBuilder sb, List updatesByQuery, if (relation.parentName.equals(relation.joinField)) { // If the target parent is the same as the joining field, we're appending the // nested child to a list of objects which can be sent as a bulk update request + // since we have the parent id document = convertDocumentUnits(document); createNestedEntity(sb, id, index, document, relation); } else if (index.equals("sampletype")) { @@ -758,8 +908,11 @@ private void updateNestedEntityByQuery(List updatesByQuery, String id, URI uri = new URIBuilder(server).setPath(path).build(); HttpPost httpPost = new HttpPost(uri); + // Determine the Id of the painless script to use String scriptId = update ? "update_" : "delete_"; scriptId += index.equals("parametertype") ? relation.parentName + index : index; + + // All updates/deletes require the entityId JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("id", id); if (update) { if (relation.fields == null) { @@ -780,7 +933,7 @@ private void updateNestedEntityByQuery(List updatesByQuery, String id, queryObject = QueryBuilder.buildTermQuery(idField, id); } JsonObject bodyJson = Json.createObjectBuilder().add("query", queryObject).add("script", scriptBuilder).build(); - // logger.trace("updateByQuery script: {}", bodyJson.toString()); + logger.trace("Making call {} with body {}", path, bodyJson.toString()); httpPost.setEntity(new StringEntity(bodyJson.toString(), ContentType.APPLICATION_JSON)); updatesByQuery.add(httpPost); } @@ -817,12 +970,14 @@ private JsonObject convertDocumentUnits(JsonObject document) { private JsonObjectBuilder convertScriptUnits(JsonObjectBuilder paramsBuilder, JsonObject document, Set fields) { for (String field : fields) { - if (field.equals("type.unitsSI")) { - convertUnits(document, paramsBuilder, "conversionFactor", 1.); - } else if (field.equals("numericValueSI")) { - continue; - } else { - paramsBuilder.add(field, document.get(field)); + if (document.containsKey(field)) { + if (field.equals("type.unitsSI")) { + convertUnits(document, paramsBuilder, "conversionFactor", 1.); + } else if (field.equals("numericValueSI")) { + continue; + } else { + paramsBuilder.add(field, document.get(field)); + } } } return paramsBuilder; @@ -838,7 +993,7 @@ private static void modifyEntity(StringBuilder sb, Set investigationIds, case CREATE: docAsUpsert = Json.createObjectBuilder().add("doc", document).add("doc_as_upsert", true).build(); sb.append(update.toString()).append("\n").append(docAsUpsert.toString()).append("\n"); - if (!index.equals("investigation")) { + if (document.containsKey("investigation.id")) { // In principle a Dataset/Datafile could be created after InvestigationUser // entities are attached to an Investigation, so need to check for those investigationIds.add(document.getString("investigation.id")); diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchScriptBuilder.java b/src/main/java/org/icatproject/core/manager/search/OpensearchScriptBuilder.java new file mode 100644 index 00000000..d0580d53 --- /dev/null +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchScriptBuilder.java @@ -0,0 +1,136 @@ +package org.icatproject.core.manager.search; + +import java.util.Set; + +import javax.json.Json; +import javax.json.JsonObjectBuilder; + +public class OpensearchScriptBuilder { + + /** + * Builds Json for creating a new script with the provided painless source code. + * + * @param source Painless source code as a String. + * @return Json for creating a new script. + */ + private static String buildScript(String source) { + JsonObjectBuilder builder = Json.createObjectBuilder().add("lang", "painless").add("source", source); + return Json.createObjectBuilder().add("script", builder).build().toString(); + } + + /** + * In order to access a specific nested child entity, access `childIndex` in + * later parts of the painless script. + * + * @param childName The name of the nested child entity. + * @return Painless code for determining the id of a given child within a nested + * array. + */ + private static String findNestedChild(String childName) { + return "int childIndex = -1; int i = 0; if (ctx._source." + childName + " != null) " + + "{while (childIndex == -1 && i < ctx._source." + childName + ".size()) " + + "{if (ctx._source." + childName + ".get(i).id == params.id) {childIndex = i;} i++;}}"; + } + + /** + * @param childName The name of the nested child entity. + * @return Painless code for removing a given child within a nested array based + * on its id. + */ + private static String removeNestedChild(String childName) { + return findNestedChild(childName) + " if (childIndex != -1) {ctx._source." + childName + + ".remove(childIndex);}"; + } + + /** + * @param field The field belonging to the child entity to be modified. + * @param ctxSource The context source where the field can be found. + * @param update If true the script will replace the field, else the + * value will be deleted. + * @return Painless code for updating one field within ctxSource. + */ + private static String updateField(String field, String ctxSource, boolean update) { + if (update) { + if (field.equals("numericValueSI")) { + return "if (" + ctxSource + ".numericValue != null && params.containsKey('conversionFactor')) {" + + ctxSource + ".numericValueSI = params.conversionFactor * " + ctxSource + + ".numericValue;} else {" + ctxSource + ".remove('numericValueSI');}"; + } else { + return ctxSource + "['" + field + "']" + " = params['" + field + "']; "; + } + } else { + return ctxSource + ".remove('" + field + "'); "; + } + } + + /** + * Builds a script which updates specific fields on a parent entity that are set + * by (at most) a single non-nested child. + * + * @param docFields The fields belonging to the child entity to be modified. + * @param update If true the script will replace the docFields, else the + * value will be deleted. + * @return The painless script as a String. + */ + public static String buildChildScript(Set docFields, boolean update) { + String source = ""; + for (String field : docFields) { + source += updateField(field, "ctx._source", update); + } + return buildScript(source); + } + + /** + * Builds a script which sets the array of nested child entities to a new array. + * Note that this will overwrite any existing nested Objects. It should not be + * used to add a new entry to an existing array, but is more efficient in cases + * where we know the array will not yet be set. + * + * @param childName The name of the nested child entity. + * @return The painless script as a String. + */ + public static String buildCreateNestedChildScript(String childName) { + String source = "ctx._source." + childName + " = params.doc"; + return buildScript(source); + } + + /** + * Builds a script which updates or removes a single specific nested entity + * based on ICAT entity Id. + * + * @param childName The name of the nested child entity. + * @param update If true the script will replace a nested entity, else the + * nested entity will be removed from the array. + * @return The painless script as a String. + */ + public static String buildNestedChildScript(String childName, boolean update) { + String source = removeNestedChild(childName); + if (update) { + source += " if (ctx._source." + childName + " != null) {ctx._source." + childName + + ".addAll(params.doc);} else {ctx._source." + childName + " = params.doc;}"; + } + return buildScript(source); + } + + /** + * Builds a script which updates specific fields on a nested child entity that + * are set + * by a single grandchild. + * + * @param childName The name of the nested child entity. + * @param docFields The fields belonging to the grandchild entity to be + * modified. + * @param update If true the script will replace a nested entity, else the + * nested entity will be removed from the array. + * @return The painless script as a String. + */ + public static String buildGrandchildScript(String childName, Set docFields, boolean update) { + String source = findNestedChild(childName); + String ctxSource = "ctx._source." + childName + ".get(childIndex)"; + for (String field : docFields) { + source += updateField(field, ctxSource, update); + } + return buildScript(source); + } + +} diff --git a/src/main/java/org/icatproject/core/manager/search/QueryBuilder.java b/src/main/java/org/icatproject/core/manager/search/QueryBuilder.java index 56460ef7..49957b2d 100644 --- a/src/main/java/org/icatproject/core/manager/search/QueryBuilder.java +++ b/src/main/java/org/icatproject/core/manager/search/QueryBuilder.java @@ -84,7 +84,7 @@ public static JsonObject buildRangeFacet(String field, JsonArray ranges) { public static JsonObject buildStringFacet(String field, int maxLabels) { JsonObjectBuilder termsBuilder = Json.createObjectBuilder(); - termsBuilder.add("field", field + ".keyword").add("size", maxLabels); + termsBuilder.add("field", field).add("size", maxLabels); return Json.createObjectBuilder().add("terms", termsBuilder).build(); } diff --git a/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java b/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java index 1fec7421..6485e97b 100644 --- a/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java +++ b/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java @@ -8,6 +8,7 @@ public class ScoredEntityBaseBean { private long entityBaseBeanId; + private int shardIndex; private int engineDocId; private float score; private JsonObject source; @@ -21,6 +22,9 @@ public class ScoredEntityBaseBean { * entityBaseBeanId. This is needed in order to enable * subsequent searches to "search after" Documents which have * already been returned once. + * @param shardIndex The index of the shard that the entity was found on. This + * is only relevant when merging results with the icat.lucene + * component. * @param score A float generated by the engine to indicate the relevance * of the returned Document to the search term(s). Higher * scores are more relevant. May be null if the results were @@ -31,13 +35,14 @@ public class ScoredEntityBaseBean { * @throws IcatException If "id" and the corresponding entityBaseBeanId are not * a key-value pair in the source JsonObject. */ - public ScoredEntityBaseBean(int engineDocId, float score, JsonObject source) throws IcatException { + public ScoredEntityBaseBean(int engineDocId, int shardIndex, float score, JsonObject source) throws IcatException { if (!source.keySet().contains("id")) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "Document source must have 'id' and the entityBaseBeanId as a key-value pair, but it was " + source.toString()); } this.engineDocId = engineDocId; + this.shardIndex = shardIndex; this.score = score; this.source = source; this.entityBaseBeanId = new Long(source.getString("id")); @@ -51,6 +56,10 @@ public int getEngineDocId() { return engineDocId; } + public int getShardIndex() { + return shardIndex; + } + public float getScore() { return score; } diff --git a/src/main/java/org/icatproject/core/manager/search/SearchManager.java b/src/main/java/org/icatproject/core/manager/search/SearchManager.java index 9ec0a658..28be1993 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchManager.java @@ -479,7 +479,8 @@ private void init() { if (searchEngine == SearchEngine.LUCENE) { searchApi = new LuceneApi(propertyHandler.getSearchUrls().get(0).toURI()); } else if (searchEngine == SearchEngine.ELASTICSEARCH || searchEngine == SearchEngine.OPENSEARCH) { - searchApi = new OpensearchApi(propertyHandler.getSearchUrls().get(0).toURI()); + String unitAliasOptions = propertyHandler.getUnitAliasOptions(); + searchApi = new OpensearchApi(propertyHandler.getSearchUrls().get(0).toURI(), unitAliasOptions); } else { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "Search engine {} not supported, must be one of " + SearchEngine.values()); diff --git a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java index c263a1eb..7c4dd84d 100644 --- a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java +++ b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java @@ -46,18 +46,18 @@ public void testBadname() throws Exception { @Test public void testHasSearchDoc() throws Exception { - Set docdbeans = new HashSet<>(Arrays.asList("Investigation", "Dataset", "Datafile", - "InvestigationParameter", "DatasetParameter", "DatafileParameter", "ParameterType", - "InvestigationUser", "User", "Sample", "SampleType", "Facility", "InvestigationType", "DatasetType", - "DatafileFormat")); + Set docdbeans = new HashSet<>(Arrays.asList("Datafile", "DatafileFormat", "DatafileParameter", + "Dataset", "DatasetParameter", "DatasetType", "Facility", "Instrument", "InstrumentScientist", + "Investigation", "InvestigationInstrument", "InvestigationParameter", "InvestigationType", + "InvestigationUser", "ParameterType", "Sample", "SampleType", "User")); for (String beanName : EntityInfoHandler.getEntityNamesList()) { @SuppressWarnings("unchecked") Class bean = (Class) Class .forName(Constants.ENTITY_PREFIX + beanName); if (docdbeans.contains(beanName)) { - assertTrue(eiHandler.hasSearchDoc(bean)); + assertTrue(beanName + " doesn't hasSearchDoc, but it should", eiHandler.hasSearchDoc(bean)); } else { - assertFalse(eiHandler.hasSearchDoc(bean)); + assertFalse(beanName + " hasSearchDoc, but is not one of " + docdbeans, eiHandler.hasSearchDoc(bean)); } } } diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 88f2a5e4..cf8df3a3 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -16,6 +16,7 @@ import java.util.Set; import javax.json.Json; +import javax.json.JsonArray; import javax.json.JsonArrayBuilder; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; @@ -30,7 +31,10 @@ import org.icatproject.core.entity.DatasetType; import org.icatproject.core.entity.EntityBaseBean; import org.icatproject.core.entity.Facility; +import org.icatproject.core.entity.Instrument; +import org.icatproject.core.entity.InstrumentScientist; import org.icatproject.core.entity.Investigation; +import org.icatproject.core.entity.InvestigationInstrument; import org.icatproject.core.entity.InvestigationParameter; import org.icatproject.core.entity.InvestigationType; import org.icatproject.core.entity.InvestigationUser; @@ -58,11 +62,19 @@ public class TestSearchApi { private static final String SEARCH_AFTER_NOT_NULL = "Expected searchAfter to be set, but it was null"; + private static final List datafileFields = Arrays.asList("id", "name", "location", "date", "dataset.id", + "dataset.name", "investigation.id", "investigation.name", "InvestigationInstrument instrument.id"); + private static final List datasetFields = Arrays.asList("id", "name", "startDate", "endDate", + "investigation.id", "investigation.name", "investigation.title", "investigation.startDate", + "InvestigationInstrument instrument.id"); + private static final List investigationFields = Arrays.asList("id", "name", "title", "startDate", "endDate", + "InvestigationInstrument instrument.id", "InvestigationInstrument instrument.name", + "InvestigationInstrument instrument.fullName"); final static Logger logger = LoggerFactory.getLogger(TestSearchApi.class); @Parameterized.Parameters - public static Iterable data() throws URISyntaxException { + public static Iterable data() throws URISyntaxException, IcatException { String luceneUrl = System.getProperty("luceneUrl"); logger.info("Using Lucene service at {}", luceneUrl); URI luceneUri = new URI(luceneUrl); @@ -183,26 +195,39 @@ private static JsonObject buildFacetStringRequest(String id, String dimension) { private void checkDatafile(ScoredEntityBaseBean datafile) { JsonObject source = datafile.getSource(); assertNotNull(source); - Set expectedKeys = new HashSet<>( - Arrays.asList("id", "investigation.id", "name", "date")); + Set expectedKeys = new HashSet<>(Arrays.asList("id", "name", "location", "date", "dataset.id", + "dataset.name", "investigation.id", "investigation.name", "investigationinstrument")); assertEquals(expectedKeys, source.keySet()); assertEquals("0", source.getString("id")); - assertEquals("0", source.getString("investigation.id")); assertEquals("DFaaa", source.getString("name")); + assertEquals("/dir/DFaaa", source.getString("location")); assertNotNull(source.getJsonNumber("date")); + assertEquals("0", source.getString("dataset.id")); + assertEquals("DSaaa", source.getString("dataset.name")); + assertEquals("0", source.getString("investigation.id")); + assertEquals("a h r", source.getString("investigation.name")); + JsonArray instruments = source.getJsonArray("investigationinstrument"); + assertEquals(1, instruments.size()); + assertEquals("0", instruments.getJsonObject(0).getString("instrument.id")); } private void checkDataset(ScoredEntityBaseBean dataset) { JsonObject source = dataset.getSource(); assertNotNull(source); - Set expectedKeys = new HashSet<>( - Arrays.asList("id", "investigation.id", "name", "startDate", "endDate")); + Set expectedKeys = new HashSet<>(Arrays.asList("id", "name", "startDate", "endDate", "investigation.id", + "investigation.name", "investigation.title", "investigation.startDate", "investigationinstrument")); assertEquals(expectedKeys, source.keySet()); assertEquals("0", source.getString("id")); - assertEquals("0", source.getString("investigation.id")); assertEquals("DSaaa", source.getString("name")); assertNotNull(source.getJsonNumber("startDate")); assertNotNull(source.getJsonNumber("endDate")); + assertEquals("0", source.getString("investigation.id")); + assertEquals("a h r", source.getString("investigation.name")); + assertEquals("title", source.getString("investigation.title")); + assertNotNull(source.getJsonNumber("investigation.startDate")); + JsonArray instruments = source.getJsonArray("investigationinstrument"); + assertEquals(1, instruments.size()); + assertEquals("0", instruments.getJsonObject(0).getString("instrument.id")); } private void checkFacets(List facetDimensions, FacetDimension... dimensions) { @@ -217,8 +242,12 @@ private void checkFacets(List facetDimensions, FacetDimension... for (int j = 0; j < expectedLabels.size(); j++) { FacetLabel expectedLabel = expectedLabels.get(j); FacetLabel actualLabel = actualLabels.get(j); - assertEquals(expectedLabel.getLabel(), actualLabel.getLabel()); - assertEquals(expectedLabel.getValue(), actualLabel.getValue()); + String label = expectedLabel.getLabel(); + Long expectedValue = expectedLabel.getValue(); + Long actualValue = actualLabel.getValue(); + assertEquals(label, actualLabel.getLabel()); + String message = "Label <" + label + ">: "; + assertEquals(message, expectedValue, actualValue); } } } @@ -226,12 +255,18 @@ private void checkFacets(List facetDimensions, FacetDimension... private void checkInvestigation(ScoredEntityBaseBean investigation) { JsonObject source = investigation.getSource(); assertNotNull(source); - Set expectedKeys = new HashSet<>(Arrays.asList("id", "name", "startDate", "endDate")); + Set expectedKeys = new HashSet<>( + Arrays.asList("id", "name", "title", "startDate", "endDate", "investigationinstrument")); assertEquals(expectedKeys, source.keySet()); assertEquals("0", source.getString("id")); assertEquals("a h r", source.getString("name")); assertNotNull(source.getJsonNumber("startDate")); assertNotNull(source.getJsonNumber("endDate")); + JsonArray instruments = source.getJsonArray("investigationinstrument"); + assertEquals(1, instruments.size()); + assertEquals("0", instruments.getJsonObject(0).getString("instrument.id")); + assertEquals("bl0", instruments.getJsonObject(0).getString("instrument.name")); + assertEquals("Beamline 0", instruments.getJsonObject(0).getString("instrument.fullName")); } private void checkResults(SearchResult lsr, Long... n) { @@ -276,10 +311,11 @@ private void checkOrder(SearchResult lsr, Long... n) { } } - private Datafile datafile(long id, String name, Date date, Dataset dataset) { + private Datafile datafile(long id, String name, String location, Date date, Dataset dataset) { Datafile datafile = new Datafile(); datafile.setId(id); datafile.setName(name); + datafile.setLocation(location); datafile.setDatafileModTime(date); datafile.setDataset(dataset); return datafile; @@ -433,6 +469,9 @@ private void populate() throws IcatException { List queue = new ArrayList<>(); Long investigationUserId = 0L; + Instrument instrumentZero = populateInstrument(queue, 0L); + Instrument instrumentOne = populateInstrument(queue, 1L); + for (int investigationId = 0; investigationId < NUMINV; investigationId++) { String word = word(investigationId % 26, (investigationId + 7) % 26, (investigationId + 17) % 26); Date startDate = new Date(now + investigationId * 60000); @@ -440,6 +479,16 @@ private void populate() throws IcatException { Investigation investigation = investigation(investigationId, word, startDate, endDate); queue.add(SearchApi.encodeOperation("create", investigation)); + InvestigationInstrument investigationInstrument = new InvestigationInstrument(); + investigationInstrument.setId(new Long(investigationId)); + if (investigationId % 2 == 0) { + investigationInstrument.setInstrument(instrumentZero); + } else { + investigationInstrument.setInstrument(instrumentOne); + } + investigationInstrument.setInvestigation(investigation); + queue.add(SearchApi.encodeOperation("create", investigationInstrument)); + for (int userId = 0; userId < NUMUSERS; userId++) { if (investigationId % (userId + 1) == 1) { String fullName = "FN " + letters.substring(userId, userId + 1) + " " @@ -487,7 +536,8 @@ private void populate() throws IcatException { break; } word = word("DF", datafileId % 26); - Datafile datafile = datafile(datafileId, word, new Date(now + datafileId * 60000), dataset); + Datafile datafile = datafile(datafileId, word, "/dir/" + word, new Date(now + datafileId * 60000), + dataset); queue.add(SearchApi.encodeOperation("create", datafile)); if (datafileId % 4 == 1) { @@ -500,6 +550,31 @@ private void populate() throws IcatException { modify(queue.toArray(new String[0])); } + /** + * Queues creation of an Instrument and a corresponding instrument scientist. + * + * @param queue Queue to add create operations to. + * @param instrumentId ICAT entity Id to use for the instrument/instrument scientist. + * @return The Instrument entity created. + * @throws IcatException + */ + private Instrument populateInstrument(List queue, long instrumentId) throws IcatException { + Instrument instrument = new Instrument(); + instrument.setId(instrumentId); + instrument.setName("bl" + instrumentId); + instrument.setFullName("Beamline " + instrumentId); + queue.add(SearchApi.encodeOperation("create", instrument)); + User user = new User(); + user.setId(new Long(NUMUSERS) + instrumentId); + user.setName("scientist_" + instrumentId); + InstrumentScientist instrumentScientist = new InstrumentScientist(); + instrumentScientist.setId(instrumentId); + instrumentScientist.setInstrument(instrument); + instrumentScientist.setUser(user); + queue.add(SearchApi.encodeOperation("create", instrumentScientist)); + return instrument; + } + private String word(int j, int k, int l) { String jString = letters.substring(j, j + 1); String kString = letters.substring(k, k + 1); @@ -525,59 +600,58 @@ public void datafiles() throws Exception { // Test size and searchAfter JsonObject query = buildQuery("Datafile", null, null, null, null, null, null, null); - List fields = Arrays.asList("date", "name", "investigation.id", "id"); - SearchResult lsr = searchApi.getResults(query, null, 5, null, fields); + SearchResult lsr = searchApi.getResults(query, null, 5, null, datafileFields); JsonValue searchAfter = lsr.getSearchAfter(); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); checkDatafile(lsr.getResults().get(0)); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - lsr = searchApi.getResults(query, searchAfter, 200, null, fields); + lsr = searchApi.getResults(query, searchAfter, 200, null, datafileFields); assertNull(lsr.getSearchAfter()); assertEquals(95, lsr.getResults().size()); // Test searchAfter preserves the sorting of original search (asc) sort = sortBuilder.add("date", "asc").build().toString(); - lsr = searchApi.getResults(query, null, 5, sort, fields); + lsr = searchApi.getResults(query, null, 5, sort, datafileFields); checkOrder(lsr, 0L, 1L, 2L, 3L, 4L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); + lsr = searchApi.getResults(query, searchAfter, 5, sort, datafileFields); checkOrder(lsr, 5L, 6L, 7L, 8L, 9L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); // Test searchAfter preserves the sorting of original search (desc) sort = sortBuilder.add("date", "desc").build().toString(); - lsr = searchApi.getResults(query, null, 5, sort, fields); + lsr = searchApi.getResults(query, null, 5, sort, datafileFields); checkOrder(lsr, 99L, 98L, 97L, 96L, 95L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); + lsr = searchApi.getResults(query, searchAfter, 5, sort, datafileFields); checkOrder(lsr, 94L, 93L, 92L, 91L, 90L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); // Test tie breaks on fields with identical values (asc) sort = sortBuilder.add("name", "asc").build().toString(); - lsr = searchApi.getResults(query, null, 5, sort, fields); + lsr = searchApi.getResults(query, null, 5, sort, datafileFields); checkOrder(lsr, 0L, 26L, 52L, 78L, 1L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); sort = sortBuilder.add("name", "asc").add("date", "desc").build().toString(); - lsr = searchApi.getResults(query, null, 5, sort, fields); + lsr = searchApi.getResults(query, null, 5, sort, datafileFields); checkOrder(lsr, 78L, 52L, 26L, 0L, 79L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); // Test tie breaks on fields with identical values (desc) sort = sortBuilder.add("name", "desc").build().toString(); - lsr = searchApi.getResults(query, null, 5, sort, fields); + lsr = searchApi.getResults(query, null, 5, sort, datafileFields); checkOrder(lsr, 25L, 51L, 77L, 24L, 50L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); sort = sortBuilder.add("name", "desc").add("date", "desc").build().toString(); - lsr = searchApi.getResults(query, null, 5, sort, fields); + lsr = searchApi.getResults(query, null, 5, sort, datafileFields); checkOrder(lsr, 77L, 51L, 25L, 76L, 50L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); @@ -587,6 +661,11 @@ public void datafiles() throws Exception { checkResults(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, 96L); + // Test instrumentScientists only see their data + query = buildQuery("Datafile", "scientist_0", null, null, null, null, null, null); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 0L, 2L, 4L, 6L, 8L); + query = buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 1L); @@ -632,13 +711,12 @@ public void datasets() throws Exception { String sort; JsonObject query = buildQuery("Dataset", null, null, null, null, null, null, null); - List fields = Arrays.asList("startDate", "endDate", "name", "investigation.id", "id"); - SearchResult lsr = searchApi.getResults(query, null, 5, null, fields); + SearchResult lsr = searchApi.getResults(query, null, 5, null, datasetFields); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); checkDataset(lsr.getResults().get(0)); JsonValue searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - lsr = searchApi.getResults(query, searchAfter, 100, null, fields); + lsr = searchApi.getResults(query, searchAfter, 100, null, datasetFields); assertNull(lsr.getSearchAfter()); checkResults(lsr, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, 25L, 26L, 27L, 28L, 29L); @@ -649,7 +727,7 @@ public void datasets() throws Exception { checkOrder(lsr, 0L, 1L, 2L, 3L, 4L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); + lsr = searchApi.getResults(query, searchAfter, 5, sort, datasetFields); checkOrder(lsr, 5L, 6L, 7L, 8L, 9L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); @@ -660,14 +738,14 @@ public void datasets() throws Exception { checkOrder(lsr, 29L, 28L, 27L, 26L, 25L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); + lsr = searchApi.getResults(query, searchAfter, 5, sort, datasetFields); checkOrder(lsr, 24L, 23L, 22L, 21L, 20L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); // Test tie breaks on fields with identical values (asc) sort = sortBuilder.add("name", "asc").build().toString(); - lsr = searchApi.getResults(query, null, 5, sort, fields); + lsr = searchApi.getResults(query, null, 5, sort, datasetFields); checkOrder(lsr, 0L, 26L, 1L, 27L, 2L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); @@ -681,6 +759,11 @@ public void datasets() throws Exception { null); checkResults(lsr, 1L, 6L, 11L, 16L, 21L, 26L); + // Test instrumentScientists only see their data + query = buildQuery("Dataset", "scientist_0", null, null, null, null, null, null); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 0L, 2L, 4L, 6L, 8L); + lsr = searchApi.getResults(buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100, null); checkResults(lsr, 1L); @@ -724,13 +807,12 @@ public void investigations() throws Exception { /* Blocked results */ JsonObject query = buildQuery("Investigation", null, null, null, null, null, null, null); - List fields = Arrays.asList("startDate", "endDate", "name", "id"); - SearchResult lsr = searchApi.getResults(query, null, 5, null, fields); + SearchResult lsr = searchApi.getResults(query, null, 5, null, investigationFields); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); checkInvestigation(lsr.getResults().get(0)); JsonValue searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - lsr = searchApi.getResults(query, searchAfter, 6, null, fields); + lsr = searchApi.getResults(query, searchAfter, 6, null, investigationFields); checkResults(lsr, 5L, 6L, 7L, 8L, 9L); searchAfter = lsr.getSearchAfter(); assertNull(searchAfter); @@ -741,7 +823,7 @@ public void investigations() throws Exception { checkOrder(lsr, 0L, 1L, 2L, 3L, 4L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); + lsr = searchApi.getResults(query, searchAfter, 5, sort, investigationFields); checkOrder(lsr, 5L, 6L, 7L, 8L, 9L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); @@ -752,11 +834,16 @@ public void investigations() throws Exception { checkOrder(lsr, 9L, 8L, 7L, 6L, 5L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - lsr = searchApi.getResults(query, searchAfter, 5, sort, fields); + lsr = searchApi.getResults(query, searchAfter, 5, sort, investigationFields); checkOrder(lsr, 4L, 3L, 2L, 1L, 0L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); + // Test instrumentScientists only see their data + query = buildQuery("Investigation", "scientist_0", null, null, null, null, null, null); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 0L, 2L, 4L, 6L, 8L); + query = buildQuery("Investigation", null, null, null, null, null, null, "b"); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 1L, 3L, 5L, 7L, 9L); @@ -884,8 +971,8 @@ public void modifyDatafile() throws IcatException { DatafileFormat pngFormat = datafileFormat(0, "png"); Investigation investigation = investigation(0, "name", date, date); Dataset dataset = dataset(0, "name", date, date, investigation); - Datafile elephantDatafile = datafile(42, "Elephants and Aardvarks", new Date(0), dataset); - Datafile rhinoDatafile = datafile(42, "Rhinos and Aardvarks", new Date(3), dataset); + Datafile elephantDatafile = datafile(42, "Elephants and Aardvarks", "/dir", new Date(0), dataset); + Datafile rhinoDatafile = datafile(42, "Rhinos and Aardvarks", "/dir", new Date(3), dataset); rhinoDatafile.setDatafileFormat(pdfFormat); // Build queries @@ -896,12 +983,12 @@ public void modifyDatafile() throws IcatException { JsonObject lowRange = buildFacetRangeObject("low", 0L, 2L); JsonObject highRange = buildFacetRangeObject("high", 2L, 4L); JsonObject rangeFacetRequest = buildFacetRangeRequest(buildFacetIdQuery("42"), "date", lowRange, highRange); - JsonObject stringFacetRequest = buildFacetStringRequest("42", "datafileFormat.name"); + JsonObject stringFacetRequest = buildFacetStringRequest("42", "datafileFormat.name.keyword"); FacetDimension lowFacet = new FacetDimension("", "date", new FacetLabel("low", 1L), new FacetLabel("high", 0L)); FacetDimension highFacet = new FacetDimension("", "date", new FacetLabel("low", 0L), new FacetLabel("high", 1L)); - FacetDimension pdfFacet = new FacetDimension("", "datafileFormat.name", new FacetLabel("pdf", 1L)); - FacetDimension pngFacet = new FacetDimension("", "datafileFormat.name", new FacetLabel("png", 1L)); + FacetDimension pdfFacet = new FacetDimension("", "datafileFormat.name.keyword", new FacetLabel("pdf", 1L)); + FacetDimension pngFacet = new FacetDimension("", "datafileFormat.name.keyword", new FacetLabel("png", 1L)); // Original modify(SearchApi.encodeOperation("create", elephantDatafile)); diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 9a203014..60323cc2 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -731,7 +731,7 @@ public void testSearchDatafiles() throws Exception { wSession.addRule(null, "DatafileParameter", "R"); responseObject = searchDatafiles(session, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); - checkFacets(responseObject, "DatafileParameter.type.name", Arrays.asList("colour"), Arrays.asList(1L)); + checkFacets(responseObject, "DatafileParameter.type.name.keyword", Arrays.asList("colour"), Arrays.asList(1L)); } /** @@ -848,7 +848,7 @@ public void testSearchDatasets() throws Exception { String facets = buildFacetRequest("Dataset"); responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5); assertFalse(responseObject.containsKey("search_after")); - checkFacets(responseObject, "Dataset.type.name", Arrays.asList("calibration"), Arrays.asList(5L)); + checkFacets(responseObject, "Dataset.type.name.keyword", Arrays.asList("calibration"), Arrays.asList(5L)); // Test no facets match on DatasetParameters due to lack of READ access facets = buildFacetRequest("DatasetParameter"); @@ -861,7 +861,8 @@ public void testSearchDatasets() throws Exception { wSession.addRule(null, "DatasetParameter", "R"); responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5); assertFalse(responseObject.containsKey("search_after")); - checkFacets(responseObject, "DatasetParameter.type.name", Arrays.asList("colour", "birthday", "current"), + checkFacets(responseObject, "DatasetParameter.type.name.keyword", + Arrays.asList("colour", "birthday", "current"), Arrays.asList(1L, 1L, 1L)); } @@ -987,7 +988,7 @@ public void testSearchInvestigations() throws Exception { responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); - checkFacets(responseObject, "Investigation.type.name", Arrays.asList("atype"), Arrays.asList(3L)); + checkFacets(responseObject, "Investigation.type.name.keyword", Arrays.asList("atype"), Arrays.asList(3L)); // Test no facets match on InvestigationParameters due to lack of READ access facets = buildFacetRequest("InvestigationParameter"); @@ -1001,7 +1002,8 @@ public void testSearchInvestigations() throws Exception { responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); - checkFacets(responseObject, "InvestigationParameter.type.name", Arrays.asList("colour"), Arrays.asList(1L)); + checkFacets(responseObject, "InvestigationParameter.type.name.keyword", Arrays.asList("colour"), + Arrays.asList(1L)); } @Test @@ -1039,7 +1041,7 @@ public void testSearchParameterValidation() throws Exception { private String buildFacetRequest(String target) { JsonObjectBuilder builder = Json.createObjectBuilder(); - JsonObjectBuilder dimension = Json.createObjectBuilder().add("dimension", "type.name"); + JsonObjectBuilder dimension = Json.createObjectBuilder().add("dimension", "type.name.keyword"); JsonArrayBuilder dimensions = Json.createArrayBuilder().add(dimension); builder.add("target", target).add("dimensions", dimensions); return Json.createArrayBuilder().add(builder).build().toString(); @@ -1047,7 +1049,8 @@ private String buildFacetRequest(String target) { private void checkFacets(JsonObject responseObject, String dimension, List expectedLabels, List expectedCounts) { - String dimensionsMessage = "Expected responseObject to contain 'dimensions', but it did not"; + String dimensionsMessage = "Expected responseObject to contain 'dimensions', but it had keys " + + responseObject.keySet(); assertTrue(dimensionsMessage, responseObject.containsKey("dimensions")); JsonObject dimensions = responseObject.getJsonObject("dimensions"); String dimensionMessage = "Expected 'dimensions' to contain " + dimension + " but keys were " From 87ee1a399983d3ed756c4af300051618d49cf3bd Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 16 Jun 2022 00:00:28 +0100 Subject: [PATCH 22/51] Sparse string faceting fix #267 --- .../core/manager/search/LuceneApi.java | 2 +- .../core/manager/search/OpensearchApi.java | 33 +++++++++++++++---- .../core/manager/TestSearchApi.java | 10 ++++-- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java index 8d1c4b59..5c897d63 100644 --- a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java @@ -56,7 +56,7 @@ public JsonObject buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) t // irrespective of the sort, override the default implementation JsonObjectBuilder builder = Json.createObjectBuilder(); builder.add("doc", lastBean.getEngineDocId()); - builder.add("shardIndex", -1); + builder.add("shardIndex", lastBean.getShardIndex()); float score = lastBean.getScore(); if (!Float.isNaN(score)) { builder.add("score", score); diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index 284dbaae..9546226d 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -96,6 +96,7 @@ public ParentRelation(RelationType relationType, String parentName, String joinF .build(); private static Map> relations = new HashMap<>(); private static Map> defaultFieldsMap = new HashMap<>(); + private static Map> defaultFacetsMap = new HashMap<>(); protected static final Set indices = new HashSet<>( Arrays.asList("datafile", "dataset", "investigation", "instrumentscientist")); @@ -175,6 +176,10 @@ public ParentRelation(RelationType relationType, String parentName, String joinF Arrays.asList("name", "description", "doi", "sample.name", "sample.type.name", "type.name")); defaultFieldsMap.put("investigation", Arrays.asList("name", "visitId", "title", "summary", "doi", "facility.name")); + + defaultFacetsMap.put("datafile", Arrays.asList("datafileFormat.name.keyword")); + defaultFacetsMap.put("dataset", Arrays.asList("type.name.keyword")); + defaultFacetsMap.put("investigation", Arrays.asList("type.name.keyword")); } public OpensearchApi(URI server) throws IcatException { @@ -298,10 +303,6 @@ public void commit() throws IcatException { public List facetSearch(String target, JsonObject facetQuery, Integer maxResults, Integer maxLabels) throws IcatException { List results = new ArrayList<>(); - if (!facetQuery.containsKey("dimensions")) { - // If no dimensions were specified, return early - return results; - } String dimensionPrefix = null; String index = target.toLowerCase(); if (!indices.contains(index) && relations.containsKey(index)) { @@ -311,10 +312,15 @@ public List facetSearch(String target, JsonObject facetQuery, In } JsonObject queryObject = facetQuery.getJsonObject("query"); - JsonArray dimensions = facetQuery.getJsonArray("dimensions"); List defaultFields = defaultFieldsMap.get(index); JsonObjectBuilder bodyBuilder = parseQuery(Json.createObjectBuilder(), queryObject, index, defaultFields); - bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix); + if (facetQuery.containsKey("dimensions")) { + JsonArray dimensions = facetQuery.getJsonArray("dimensions"); + bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix); + } else { + List dimensions = defaultFacetsMap.get(index); + bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix); + } String body = bodyBuilder.build().toString(); Map parameterMap = new HashMap<>(); @@ -345,6 +351,21 @@ private JsonObjectBuilder parseFacets(JsonObjectBuilder bodyBuilder, JsonArray d aggsBuilder.add(dimensionString, QueryBuilder.buildStringFacet(field, maxLabels)); } } + return buildFacetRequestJson(bodyBuilder, dimensionPrefix, aggsBuilder); + } + + private JsonObjectBuilder parseFacets(JsonObjectBuilder bodyBuilder, List dimensions, int maxLabels, + String dimensionPrefix) { + JsonObjectBuilder aggsBuilder = Json.createObjectBuilder(); + for (String dimensionString : dimensions) { + String field = dimensionPrefix == null ? dimensionString : dimensionPrefix + "." + dimensionString; + aggsBuilder.add(dimensionString, QueryBuilder.buildStringFacet(field, maxLabels)); + } + return buildFacetRequestJson(bodyBuilder, dimensionPrefix, aggsBuilder); + } + + private JsonObjectBuilder buildFacetRequestJson(JsonObjectBuilder bodyBuilder, String dimensionPrefix, + JsonObjectBuilder aggsBuilder) { if (dimensionPrefix == null) { bodyBuilder.add("aggs", aggsBuilder); } else { diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index cf8df3a3..13e4ca3f 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -553,8 +553,9 @@ private void populate() throws IcatException { /** * Queues creation of an Instrument and a corresponding instrument scientist. * - * @param queue Queue to add create operations to. - * @param instrumentId ICAT entity Id to use for the instrument/instrument scientist. + * @param queue Queue to add create operations to. + * @param instrumentId ICAT entity Id to use for the instrument/instrument + * scientist. * @return The Instrument entity created. * @throws IcatException */ @@ -984,6 +985,7 @@ public void modifyDatafile() throws IcatException { JsonObject highRange = buildFacetRangeObject("high", 2L, 4L); JsonObject rangeFacetRequest = buildFacetRangeRequest(buildFacetIdQuery("42"), "date", lowRange, highRange); JsonObject stringFacetRequest = buildFacetStringRequest("42", "datafileFormat.name.keyword"); + JsonObject sparseFacetRequest = Json.createObjectBuilder().add("query", buildFacetIdQuery("42")).build(); FacetDimension lowFacet = new FacetDimension("", "date", new FacetLabel("low", 1L), new FacetLabel("high", 0L)); FacetDimension highFacet = new FacetDimension("", "date", new FacetLabel("low", 0L), new FacetLabel("high", 1L)); @@ -997,6 +999,7 @@ public void modifyDatafile() throws IcatException { checkResults(searchApi.getResults(pdfQuery, 5)); checkResults(searchApi.getResults(pngQuery, 5)); checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5)); + checkFacets(searchApi.facetSearch("Datafile", sparseFacetRequest, 5, 5)); checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), lowFacet); // Change name and add a format @@ -1006,6 +1009,7 @@ public void modifyDatafile() throws IcatException { checkResults(searchApi.getResults(pdfQuery, 5), 42L); checkResults(searchApi.getResults(pngQuery, 5)); checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5), pdfFacet); + checkFacets(searchApi.facetSearch("Datafile", sparseFacetRequest, 5, 5), pdfFacet); checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), highFacet); // Change just the format @@ -1015,6 +1019,7 @@ public void modifyDatafile() throws IcatException { checkResults(searchApi.getResults(pdfQuery, 5)); checkResults(searchApi.getResults(pngQuery, 5), 42L); checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5), pngFacet); + checkFacets(searchApi.facetSearch("Datafile", sparseFacetRequest, 5, 5), pngFacet); checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), highFacet); // Remove the format @@ -1024,6 +1029,7 @@ public void modifyDatafile() throws IcatException { checkResults(searchApi.getResults(pdfQuery, 5)); checkResults(searchApi.getResults(pngQuery, 5)); checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5)); + checkFacets(searchApi.facetSearch("Datafile", sparseFacetRequest, 5, 5)); checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), highFacet); // Remove the file From 50d2dabdde79c51fc62a3b78e90dd0030838549c Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 16 Jun 2022 02:28:44 +0100 Subject: [PATCH 23/51] Filters and aborted search support #267 --- .../core/manager/EntityBeanManager.java | 6 ++++ .../core/manager/search/LuceneApi.java | 34 +++++++++++-------- .../core/manager/search/OpensearchApi.java | 13 +++++-- .../core/manager/search/SearchResult.java | 9 +++++ .../org/icatproject/exposed/ICATRest.java | 4 +++ .../core/manager/TestSearchApi.java | 34 ++++++++++++++++++- 6 files changed, 81 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index ab07ba8a..9c329f87 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -1411,6 +1411,9 @@ public List freeTextSearch(String userName, JsonObject jo, do { lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, Arrays.asList("id")); + if (lastSearchResult.isAborted()) { + break; + } allResults = lastSearchResult.getResults(); ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, limit, userName, manager, klass); if (lastBean == null) { @@ -1462,6 +1465,9 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue do { lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, fields); + if (lastSearchResult.isAborted()) { + break; + } allResults = lastSearchResult.getResults(); ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, limit, userName, manager, klass); if (lastBean == null) { diff --git a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java index 5c897d63..c720d37c 100644 --- a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java @@ -166,22 +166,26 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer JsonObject postResponse = postResponse(basePath + "/" + indexPath, queryString, parameterMap); SearchResult lsr = new SearchResult(); - List results = lsr.getResults(); - List resultsArray = postResponse.getJsonArray("results").getValuesAs(JsonObject.class); - for (JsonObject resultObject : resultsArray) { - int luceneDocId = resultObject.getInt("_id"); - int shardIndex = resultObject.getInt("_shardIndex"); - Float score = Float.NaN; - if (resultObject.keySet().contains("_score")) { - score = resultObject.getJsonNumber("_score").bigDecimalValue().floatValue(); + if (postResponse.containsKey("aborted") && postResponse.getBoolean("aborted")) { + lsr.setAborted(true); + } else { + List results = lsr.getResults(); + List resultsArray = postResponse.getJsonArray("results").getValuesAs(JsonObject.class); + for (JsonObject resultObject : resultsArray) { + int luceneDocId = resultObject.getInt("_id"); + int shardIndex = resultObject.getInt("_shardIndex"); + Float score = Float.NaN; + if (resultObject.keySet().contains("_score")) { + score = resultObject.getJsonNumber("_score").bigDecimalValue().floatValue(); + } + JsonObject source = resultObject.getJsonObject("_source"); + ScoredEntityBaseBean result = new ScoredEntityBaseBean(luceneDocId, shardIndex, score, source); + results.add(result); + logger.trace("Result id {} with score {}", result.getEntityBaseBeanId(), score); + } + if (postResponse.containsKey("search_after")) { + lsr.setSearchAfter(postResponse.getJsonObject("search_after")); } - JsonObject source = resultObject.getJsonObject("_source"); - ScoredEntityBaseBean result = new ScoredEntityBaseBean(luceneDocId, shardIndex, score, source); - results.add(result); - logger.trace("Result id {} with score {}", result.getEntityBaseBeanId(), score); - } - if (postResponse.containsKey("search_after")) { - lsr.setSearchAfter(postResponse.getJsonObject("search_after")); } return lsr; diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index 9546226d..c7aba1f5 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -548,6 +548,16 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query JsonObjectBuilder queryBuilder = Json.createObjectBuilder(); JsonObjectBuilder boolBuilder = Json.createObjectBuilder(); + // Non-scored elements are added to the "filter" + JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); + + if (queryRequest.containsKey("filter")) { + JsonObject filterObject = queryRequest.getJsonObject("filter"); + for (String fld : filterObject.keySet()) { + filterBuilder.add(QueryBuilder.buildTermQuery(fld, filterObject.getString(fld))); + } + } + if (queryRequest.containsKey("text")) { // The free text is the only element we perform scoring on, so "must" occur JsonArrayBuilder mustBuilder = Json.createArrayBuilder(); @@ -557,9 +567,6 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query boolBuilder.add("must", mustBuilder); } - // Non-scored elements are added to the "filter" - JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); - Long lowerTime = parseDate(queryRequest, "lower", 0, Long.MIN_VALUE); Long upperTime = parseDate(queryRequest, "upper", 59999, Long.MAX_VALUE); if (lowerTime != Long.MIN_VALUE || upperTime != Long.MAX_VALUE) { diff --git a/src/main/java/org/icatproject/core/manager/search/SearchResult.java b/src/main/java/org/icatproject/core/manager/search/SearchResult.java index 0bbe063c..6486e0f0 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchResult.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchResult.java @@ -10,6 +10,7 @@ public class SearchResult { private JsonValue searchAfter; private List results = new ArrayList<>(); private List dimensions; + private boolean aborted; public SearchResult() {} @@ -39,4 +40,12 @@ public void setSearchAfter(JsonValue searchAfter) { this.searchAfter = searchAfter; } + public boolean isAborted() { + return aborted; + } + + public void setAborted(boolean aborted) { + this.aborted = aborted; + } + } diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index d486cc2c..90985474 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -1337,6 +1337,10 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId JsonGenerator gen = Json.createGenerator(baos); gen.writeStartObject(); + if (result.isAborted()) { + gen.write("aborted", true).writeEnd(); + return baos.toString(); + } JsonValue newSearchAfter = result.getSearchAfter(); if (newSearchAfter != null) { diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 13e4ca3f..9b4990b3 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -61,6 +61,15 @@ @RunWith(Parameterized.class) public class TestSearchApi { + private class Filter { + private String fld; + private String value; + public Filter(String fld, String value) { + this.fld = fld; + this.value = value; + } + } + private static final String SEARCH_AFTER_NOT_NULL = "Expected searchAfter to be set, but it was null"; private static final List datafileFields = Arrays.asList("id", "name", "location", "date", "dataset.id", "dataset.name", "investigation.id", "investigation.name", "InvestigationInstrument instrument.id"); @@ -102,7 +111,7 @@ public static Iterable data() throws URISyntaxException, IcatExceptio * Utility function for building a Query from individual arguments */ public static JsonObject buildQuery(String target, String user, String text, Date lower, Date upper, - List parameters, List samples, String userFullName) { + List parameters, List samples, String userFullName, Filter... filters) { JsonObjectBuilder builder = Json.createObjectBuilder(); if (target != null) { builder.add("target", target); @@ -158,6 +167,13 @@ public static JsonObject buildQuery(String target, String user, String text, Dat if (userFullName != null) { builder.add("userFullName", userFullName); } + if (filters.length > 0 ) { + JsonObjectBuilder filterBuilder = Json.createObjectBuilder(); + for (Filter filter : filters) { + filterBuilder.add(filter.fld, filter.value); + } + builder.add("filter", filterBuilder); + } return builder.build(); } @@ -765,6 +781,14 @@ public void datasets() throws Exception { lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 2L, 4L, 6L, 8L); + // Test filter + query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("type.name.keyword", "type")); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L); + query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("type.name.keyword", "typo")); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr); + lsr = searchApi.getResults(buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100, null); checkResults(lsr, 1L); @@ -845,6 +869,14 @@ public void investigations() throws Exception { lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 2L, 4L, 6L, 8L); + // Test filter + query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("type.name.keyword", "type")); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L); + query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("type.name.keyword", "typo")); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr); + query = buildQuery("Investigation", null, null, null, null, null, null, "b"); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 1L, 3L, 5L, 7L, 9L); From cbf2cf401738019a95c67f4276213d20633c9b40 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 17 Jun 2022 12:50:56 +0000 Subject: [PATCH 24/51] Return correct Json for aborted searches #267 --- .../java/org/icatproject/core/manager/EntityBeanManager.java | 2 +- .../org/icatproject/core/manager/search/SearchManager.java | 4 +++- src/main/java/org/icatproject/exposed/ICATRest.java | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 9c329f87..435e93b1 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -1466,7 +1466,7 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue do { lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, fields); if (lastSearchResult.isAborted()) { - break; + return lastSearchResult; } allResults = lastSearchResult.getResults(); ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, limit, userName, manager, klass); diff --git a/src/main/java/org/icatproject/core/manager/search/SearchManager.java b/src/main/java/org/icatproject/core/manager/search/SearchManager.java index 28be1993..66537caf 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchManager.java @@ -332,7 +332,9 @@ public static List getPublicSearchFields(GateKeeper gateKeeper, String s buildPublicSearchFields(gateKeeper, Investigation.getDocumentFields())); gateKeeper.markPublicSearchFieldsFresh(); } - return publicSearchFields.get(simpleName); + List requestedFields = publicSearchFields.get(simpleName); + logger.trace("{} has public fields {}", simpleName, requestedFields); + return requestedFields; } public void addDocument(EntityBaseBean bean) throws IcatException { diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index 90985474..11f6f2f2 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -1338,7 +1338,7 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId JsonGenerator gen = Json.createGenerator(baos); gen.writeStartObject(); if (result.isAborted()) { - gen.write("aborted", true).writeEnd(); + gen.write("aborted", true).writeEnd().close(); return baos.toString(); } From 5af7b18e6e7dc3892c45999ed1c58bd5f419d800 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 16 Jun 2022 12:29:25 +0100 Subject: [PATCH 25/51] Enable mutlivalue string facets #267 --- .../core/manager/search/OpensearchApi.java | 21 +++++++++++++++- .../core/manager/TestSearchApi.java | 24 ++++++++++++++++--- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index c7aba1f5..2df5e1bb 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -22,6 +22,7 @@ import javax.json.JsonObject; import javax.json.JsonObjectBuilder; import javax.json.JsonReader; +import javax.json.JsonString; import javax.json.JsonValue; import javax.json.JsonValue.ValueType; import javax.json.stream.JsonGenerator; @@ -554,7 +555,25 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query if (queryRequest.containsKey("filter")) { JsonObject filterObject = queryRequest.getJsonObject("filter"); for (String fld : filterObject.keySet()) { - filterBuilder.add(QueryBuilder.buildTermQuery(fld, filterObject.getString(fld))); + ValueType valueType = filterObject.get(fld).getValueType(); + switch (valueType) { + case ARRAY: + JsonArrayBuilder shouldBuilder = Json.createArrayBuilder(); + for (JsonString value : filterObject.getJsonArray(fld).getValuesAs(JsonString.class)) { + shouldBuilder.add(QueryBuilder.buildTermQuery(fld, value.getString())); + } + filterBuilder.add(Json.createObjectBuilder().add("bool", + Json.createObjectBuilder().add("should", shouldBuilder))); + break; + + case STRING: + filterBuilder.add(QueryBuilder.buildTermQuery(fld, filterObject.getString(fld))); + break; + + default: + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "filter object values should be STRING or ARRAY, but were " + valueType); + } } } diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 9b4990b3..1b238628 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -64,9 +64,17 @@ public class TestSearchApi { private class Filter { private String fld; private String value; - public Filter(String fld, String value) { + private JsonArray array; + public Filter(String fld, String... values) { this.fld = fld; - this.value = value; + if (values.length == 1) { + this.value = values[0]; + } + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + for (String value : values) { + arrayBuilder.add(value); + } + array = arrayBuilder.build(); } } @@ -170,7 +178,11 @@ public static JsonObject buildQuery(String target, String user, String text, Dat if (filters.length > 0 ) { JsonObjectBuilder filterBuilder = Json.createObjectBuilder(); for (Filter filter : filters) { - filterBuilder.add(filter.fld, filter.value); + if (filter.value != null) { + filterBuilder.add(filter.fld, filter.value); + } else { + filterBuilder.add(filter.fld, filter.array); + } } builder.add("filter", filterBuilder); } @@ -785,6 +797,9 @@ public void datasets() throws Exception { query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("type.name.keyword", "type")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); + query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("type.name.keyword", "type", "typo")); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L); query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("type.name.keyword", "typo")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr); @@ -873,6 +888,9 @@ public void investigations() throws Exception { query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("type.name.keyword", "type")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); + query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("type.name.keyword", "type", "typo")); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L); query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("type.name.keyword", "typo")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr); From 134303c7c910763e0d01cd946738c1333d239702 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Tue, 21 Jun 2022 08:03:40 +0100 Subject: [PATCH 26/51] Refactors and Javadoc comments #267 --- .../core/entity/Investigation.java | 13 + .../core/manager/EntityBeanManager.java | 113 +++++++- .../core/manager/search/FacetLabel.java | 35 +++ .../core/manager/search/LuceneApi.java | 32 ++- .../core/manager/search/OpensearchApi.java | 264 +++++++++++++++--- ...ilder.java => OpensearchQueryBuilder.java} | 59 +++- .../core/manager/search/SearchApi.java | 224 ++++++++++++--- .../core/manager/search/SearchManager.java | 109 +++++++- .../core/manager/search/SearchResult.java | 8 +- .../core/manager/search/URIParameter.java | 0 .../org/icatproject/exposed/ICATRest.java | 117 ++++++-- .../core/manager/TestSearchApi.java | 18 +- .../org/icatproject/integration/TestRS.java | 34 +-- 13 files changed, 888 insertions(+), 138 deletions(-) rename src/main/java/org/icatproject/core/manager/search/{QueryBuilder.java => OpensearchQueryBuilder.java} (65%) delete mode 100644 src/main/java/org/icatproject/core/manager/search/URIParameter.java diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index df83be9c..34294931 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -313,6 +313,14 @@ public static Map getDocumentFields() throws IcatExcepti Relationship[] instrumentRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("investigationInstruments"), eiHandler.getRelationshipsByName(InvestigationInstrument.class).get("instrument") }; + Relationship[] parameterRelationships = { + eiHandler.getRelationshipsByName(Investigation.class).get("parameters") }; + Relationship[] parameterTypeRelationships = { + eiHandler.getRelationshipsByName(Investigation.class).get("parameters"), + eiHandler.getRelationshipsByName(InvestigationParameter.class).get("type") }; + Relationship[] sampleRelationships = { + eiHandler.getRelationshipsByName(Investigation.class).get("samples"), + eiHandler.getRelationshipsByName(Sample.class).get("type") }; documentFields.put("name", null); documentFields.put("visitId", null); documentFields.put("title", null); @@ -328,6 +336,11 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("InvestigationInstrument instrument.fullName", instrumentRelationships); documentFields.put("InvestigationInstrument instrument.id", instrumentRelationships); documentFields.put("InvestigationInstrument instrument.name", instrumentRelationships); + documentFields.put("InvestigationParameter type.name", parameterTypeRelationships); + documentFields.put("InvestigationParameter stringValue", parameterRelationships); + documentFields.put("InvestigationParameter numericValue", parameterRelationships); + documentFields.put("InvestigationParameter dateTimeValue", parameterRelationships); + documentFields.put("Sample type.name", sampleRelationships); } return documentFields; } diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 435e93b1..80d82d06 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -5,7 +5,6 @@ import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; -import java.io.StringReader; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -792,6 +791,7 @@ private ScoredEntityBaseBean filterReadAccess(List results for (ScoredEntityBaseBean sr : allResults) { long entityId = sr.getEntityBaseBeanId(); EntityBaseBean beanManaged = manager.find(klass, entityId); + logger.trace("{} {} {}", klass, entityId, beanManaged); if (beanManaged != null) { try { gateKeeper.performAuthorisation(userId, beanManaged, AccessType.READ, manager); @@ -801,6 +801,7 @@ private ScoredEntityBaseBean filterReadAccess(List results "attempt to return more than " + maxEntities + " entities"); } if (results.size() == maxCount) { + logger.debug("maxCount {} reached", maxCount); return sr; } } catch (IcatException e) { @@ -1124,6 +1125,9 @@ public String getUserName(String sessionId, EntityManager manager) throws IcatEx Session session = getSession(sessionId, manager); String userName = session.getUserName(); logger.debug("user: " + userName + " is associated with: " + sessionId); + // TODO RM + // logger.trace("userName: {}", userName); + // userName = userName.equals("db/notroot") ? "notroot" : userName; return userName; } catch (IcatException e) { logger.debug("sessionId " + sessionId + " is not associated with valid session " + e.getMessage()); @@ -1394,6 +1398,22 @@ public void searchCommit() throws IcatException { } } + /** + * Performs a search on a single entity, and authorises the results before + * returning. Does not support sorting or searchAfter. + * + * @param userName User performing the search, used for authorisation. + * @param jo JsonObject containing the details of the query to be used. + * @param limit The maximum number of results to collect before returning. If + * a batch from the search engine has more than this many + * authorised results, then the excess results will be + * discarded. + * @param manager EntityManager for finding entities from their Id. + * @param ip Used for logging only. + * @param klass Class of the entity to search. + * @return SearchResult for the query. + * @throws IcatException + */ public List freeTextSearch(String userName, JsonObject jo, int limit, String sort, EntityManager manager, String ip, Class klass) throws IcatException { long startMillis = log ? System.currentTimeMillis() : 0; @@ -1446,13 +1466,35 @@ public List freeTextSearch(String userName, JsonObject jo, return results; } - public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue searchAfter, int limit, String sort, - String facets, EntityManager manager, String ip, Class klass) - throws IcatException { + /** + * Performs a search on a single entity, and authorises the results before + * returning. + * + * @param userName User performing the search, used for authorisation. + * @param jo JsonObject containing the details of the query to be used. + * @param searchAfter JsonValue representation of the final result from a + * previous search. + * @param minCount The minimum number of results to collect before returning. + * If a batch from the search engine has at least this many + * authorised results, no further batches will be requested. + * @param maxCount The maximum number of results to collect before returning. + * If a batch from the search engine has more than this many + * authorised results, then the excess results will be + * discarded. + * @param sort String of Json representing sort criteria. + * @param manager EntityManager for finding entities from their Id. + * @param ip Used for logging only. + * @param klass Class of the entity to search. + * @return SearchResult for the query. + * @throws IcatException + */ + public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue searchAfter, int minCount, + int maxCount, String sort, EntityManager manager, String ip, + Class klass) throws IcatException { long startMillis = log ? System.currentTimeMillis() : 0; List results = new ArrayList<>(); JsonValue lastSearchAfter = null; - List dimensions = null; + List dimensions = new ArrayList<>(); if (searchActive) { SearchResult lastSearchResult = null; List allResults = Collections.emptyList(); @@ -1461,7 +1503,7 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue * As results may be rejected and maxCount may be 1 ensure that we * don't make a huge number of calls to search engine */ - int blockSize = Math.max(1000, limit); + int blockSize = Math.max(1000, maxCount); do { lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, fields); @@ -1469,7 +1511,8 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue return lastSearchResult; } allResults = lastSearchResult.getResults(); - ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, limit, userName, manager, klass); + ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, maxCount, userName, manager, + klass); if (lastBean == null) { // Haven't stopped early, so use the Lucene provided searchAfter document lastSearchAfter = lastSearchResult.getSearchAfter(); @@ -1482,11 +1525,10 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue lastSearchAfter = searchManager.buildSearchAfter(lastBean, sort); break; } - } while (results.size() < limit); + } while (results.size() < minCount); - if (facets != null && !facets.equals("") && results.size() > 0) { - JsonReader reader = Json.createReader(new StringReader(facets)); - List jsonFacets = reader.readArray().getValuesAs(JsonObject.class); + if (jo.containsKey("facets")) { + List jsonFacets = jo.getJsonArray("facets").getValuesAs(JsonObject.class); for (JsonObject jsonFacet : jsonFacets) { JsonObject facetQuery; @@ -1498,9 +1540,7 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue if (target.contains("Parameter")) { relationship = eiHandler.getRelationshipsByName(klass).get("parameters"); } else { - // TODO allow more types here, as we decide what to support - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Facet target should be a Parameter, but it was " + target); + relationship = eiHandler.getRelationshipsByName(klass).get(target.toLowerCase() + "s"); } if (gateKeeper.allowed(relationship)) { @@ -1512,7 +1552,7 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue continue; } } - dimensions = searchManager.facetSearch(target, facetQuery, results.size(), 10); + dimensions.addAll(searchManager.facetSearch(target, facetQuery, results.size(), 10)); } } } @@ -1531,6 +1571,49 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue return new SearchResult(lastSearchAfter, results, dimensions); } + /** + * Perform faceting on entities of klass using the criteria contained in jo. + * + * @param jo JsonObject containing "facets" key with a value of a JsonArray + * of JsonObjects. + * @param klass Class of the entity to facet. + * @return SearchResult with only the dimensions set. + * @throws IcatException + */ + public SearchResult facetDocs(JsonObject jo, Class klass) throws IcatException { + JsonValue lastSearchAfter = null; + List dimensions = new ArrayList<>(); + if (searchActive && jo.containsKey("facets")) { + List jsonFacets = jo.getJsonArray("facets").getValuesAs(JsonObject.class); + for (JsonObject jsonFacet : jsonFacets) { + JsonObject facetQuery; + String target = jsonFacet.getString("target"); + JsonObject filterObject = jo.getJsonObject("filter"); + if (target.equals(klass.getSimpleName())) { + facetQuery = SearchManager.buildFacetQuery(filterObject, "id", jsonFacet); + } else { + Relationship relationship; + if (target.contains("Parameter")) { + relationship = eiHandler.getRelationshipsByName(klass).get("parameters"); + } else { + relationship = eiHandler.getRelationshipsByName(klass).get(target.toLowerCase() + "s"); + } + + if (gateKeeper.allowed(relationship)) { + facetQuery = SearchManager.buildFacetQuery(filterObject, + klass.getSimpleName().toLowerCase() + ".id", jsonFacet); + } else { + logger.debug("Cannot collect facets for {} as Relationship with parent {} is not allowed", + target, klass.getSimpleName()); + continue; + } + } + dimensions.addAll(searchManager.facetSearch(target, facetQuery, 1000, 10)); + } + } + return new SearchResult(lastSearchAfter, new ArrayList<>(), dimensions); + } + public List searchGetPopulating() { if (searchActive) { return searchManager.getPopulating(); diff --git a/src/main/java/org/icatproject/core/manager/search/FacetLabel.java b/src/main/java/org/icatproject/core/manager/search/FacetLabel.java index 4a85a5d9..1e7ed288 100644 --- a/src/main/java/org/icatproject/core/manager/search/FacetLabel.java +++ b/src/main/java/org/icatproject/core/manager/search/FacetLabel.java @@ -1,5 +1,8 @@ package org.icatproject.core.manager.search; +import javax.json.JsonNumber; +import javax.json.JsonObject; + /** * Holds information for a single label value pair. * The value is the number of times the label is present in a particular facet @@ -9,12 +12,36 @@ public class FacetLabel { private String label; private Long value; + private JsonNumber from; + private JsonNumber to; public FacetLabel(String label, Long value) { this.label = label; this.value = value; } + public FacetLabel(JsonObject jsonObject) { + label = jsonObject.getString("key"); + value = jsonObject.getJsonNumber("doc_count").longValueExact(); + if (jsonObject.containsKey("from")) { + from = jsonObject.getJsonNumber("from"); + } + if (jsonObject.containsKey("to")) { + to = jsonObject.getJsonNumber("to"); + } + } + + public FacetLabel(String label, JsonObject jsonObject) { + this.label = label; + value = jsonObject.getJsonNumber("doc_count").longValueExact(); + if (jsonObject.containsKey("from")) { + from = jsonObject.getJsonNumber("from"); + } + if (jsonObject.containsKey("to")) { + to = jsonObject.getJsonNumber("to"); + } + } + public String getLabel() { return label; } @@ -23,4 +50,12 @@ public Long getValue() { return value; } + public JsonNumber getFrom() { + return from; + } + + public JsonNumber getTo() { + return to; + } + } diff --git a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java index c720d37c..6468dd0b 100644 --- a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java @@ -37,6 +37,14 @@ public class LuceneApi extends SearchApi { public String basePath = "/icat.lucene"; private static final Logger logger = LoggerFactory.getLogger(LuceneApi.class); + /** + * Gets the target index from query and checks its validity. + * + * @param query JsonObject containing the criteria to search on. + * @return The lowercase target index. + * @throws IcatException If "target" was not a key in query, or if the value was + * not a supported index. + */ private static String getTargetPath(JsonObject query) throws IcatException { if (!query.containsKey("target")) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, @@ -114,12 +122,12 @@ public void addNow(String entityName, List ids, EntityManager manager, @Override public void clear() throws IcatException { - post(basePath + "/clear"); + post(basePath + "/clear"); } @Override public void commit() throws IcatException { - post(basePath + "/commit"); + post(basePath + "/commit"); } @Override @@ -154,7 +162,7 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer if (sort != null) { parameterMap.put("sort", sort); } - + JsonObjectBuilder objectBuilder = Json.createObjectBuilder(); objectBuilder.add("query", query); if (fields != null && fields.size() > 0) { @@ -182,7 +190,7 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer ScoredEntityBaseBean result = new ScoredEntityBaseBean(luceneDocId, shardIndex, score, source); results.add(result); logger.trace("Result id {} with score {}", result.getEntityBaseBeanId(), score); - } + } if (postResponse.containsKey("search_after")) { lsr.setSearchAfter(postResponse.getJsonObject("search_after")); } @@ -191,11 +199,27 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer return lsr; } + /** + * Locks the index for entityName, removing all existing documents. While + * locked, document modifications will fail (excluding addNow as a result of a + * populate thread). + * + * @param entityName Index to lock. + * @throws IcatException + */ @Override public void lock(String entityName) throws IcatException { post(basePath + "/lock/" + entityName); } + /** + * Unlocks the index for entityName, committing all pending documents. While + * locked, document modifications will fail (excluding addNow as a result of a + * populate thread). + * + * @param entityName Index to lock. + * @throws IcatException + */ @Override public void unlock(String entityName) throws IcatException { post(basePath + "/unlock/" + entityName); diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index 2df5e1bb..b439e5ef 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -178,9 +178,9 @@ public ParentRelation(RelationType relationType, String parentName, String joinF defaultFieldsMap.put("investigation", Arrays.asList("name", "visitId", "title", "summary", "doi", "facility.name")); - defaultFacetsMap.put("datafile", Arrays.asList("datafileFormat.name.keyword")); - defaultFacetsMap.put("dataset", Arrays.asList("type.name.keyword")); - defaultFacetsMap.put("investigation", Arrays.asList("type.name.keyword")); + defaultFacetsMap.put("datafile", Arrays.asList("datafileFormat.name")); + defaultFacetsMap.put("dataset", Arrays.asList("type.name")); + defaultFacetsMap.put("investigation", Arrays.asList("type.name")); } public OpensearchApi(URI server) throws IcatException { @@ -197,6 +197,15 @@ public OpensearchApi(URI server, String unitAliasOptions) throws IcatException { initScripts(); } + /** + * Builds a JsonObject representation of the mapping of fields to their type. + * The default behaviour is for a field to be treated as text with a string + * field automatically generated with the suffix ".keyword". Therefore only + * nested and long fields need to be explicitly accounted for. + * + * @param index Index to build the mapping for. + * @return JsonObject of the document mapping. + */ private static JsonObject buildMappings(String index) { JsonObject typeLong = Json.createObjectBuilder().add("type", "long").build(); JsonObjectBuilder propertiesBuilder = Json.createObjectBuilder() @@ -232,6 +241,13 @@ private static JsonObject buildMappings(String index) { return Json.createObjectBuilder().add("properties", propertiesBuilder).build(); } + /** + * Builds a JsonObject representation of the fields on a nested object. + * + * @param idFields Id fields on the nested object which require the long type + * mapping. + * @return JsonObject for the nested object. + */ private static JsonObject buildNestedMapping(String... idFields) { JsonObject typeLong = Json.createObjectBuilder().add("type", "long").build(); JsonObjectBuilder propertiesBuilder = Json.createObjectBuilder().add("id", typeLong); @@ -241,6 +257,21 @@ private static JsonObject buildNestedMapping(String... idFields) { return Json.createObjectBuilder().add("type", "nested").add("properties", propertiesBuilder).build(); } + /** + * Extracts and parses a date value from jsonObject. If the value is a NUMBER + * (ms since epoch), then it is taken as is. If it is a STRING, then it is + * expected in the yyyyMMddHHmm format. + * + * @param jsonObject JsonObject to extract the date from. + * @param key Key of the date field to extract. + * @param offset In the event of the date being a string, we do not have + * second or ms precision. To ensure ranges are successful, + * it may be necessary to add 59999 ms to the parsed value + * as an offset. + * @param defaultValue The value to return if key is not present in jsonObject. + * @return Time since epoch in ms. + * @throws IcatException + */ private static Long parseDate(JsonObject jsonObject, String key, int offset, Long defaultValue) throws IcatException { if (jsonObject.containsKey(key)) { @@ -291,7 +322,7 @@ public void addNow(String entityName, List ids, EntityManager manager, @Override public void clear() throws IcatException { commit(); - String body = QueryBuilder.addQuery(QueryBuilder.buildMatchAllQuery()).build().toString(); + String body = OpensearchQueryBuilder.addQuery(OpensearchQueryBuilder.buildMatchAllQuery()).build().toString(); post("/_all/_delete_by_query", body); } @@ -339,6 +370,22 @@ public List facetSearch(String target, JsonObject facetQuery, In return results; } + /** + * Parses incoming Json encoding the requested facets and uses bodyBuilder to + * construct Json that can be understood by Opensearch. + * + * @param bodyBuilder JsonObjectBuilder being used to build the body of the + * request. + * @param dimensions JsonArray of JsonObjects representing dimensions to be + * faceted. + * @param maxLabels The maximum number of labels to collect for each + * dimension. + * @param dimensionPrefix Optional prefix to apply to the dimension names. This + * is needed to distinguish between potentially ambiguous + * dimensions, such as "(investigation.)type.name" and + * "(investigationparameter.)type.name". + * @return The bodyBuilder originally passed with facet information added to it. + */ private JsonObjectBuilder parseFacets(JsonObjectBuilder bodyBuilder, JsonArray dimensions, int maxLabels, String dimensionPrefix) { JsonObjectBuilder aggsBuilder = Json.createObjectBuilder(); @@ -347,24 +394,53 @@ private JsonObjectBuilder parseFacets(JsonObjectBuilder bodyBuilder, JsonArray d String field = dimensionPrefix == null ? dimensionString : dimensionPrefix + "." + dimensionString; if (dimensionObject.containsKey("ranges")) { JsonArray ranges = dimensionObject.getJsonArray("ranges"); - aggsBuilder.add(dimensionString, QueryBuilder.buildRangeFacet(field, ranges)); + aggsBuilder.add(dimensionString, OpensearchQueryBuilder.buildRangeFacet(field, ranges)); } else { - aggsBuilder.add(dimensionString, QueryBuilder.buildStringFacet(field, maxLabels)); + aggsBuilder.add(dimensionString, + OpensearchQueryBuilder.buildStringFacet(field + ".keyword", maxLabels)); } } return buildFacetRequestJson(bodyBuilder, dimensionPrefix, aggsBuilder); } + /** + * Uses bodyBuilder to construct Json for faceting string fields. + * + * @param bodyBuilder JsonObjectBuilder being used to build the body of the + * request. + * @param dimensions List of dimensions to perform string based faceting + * on. + * @param maxLabels The maximum number of labels to collect for each + * dimension. + * @param dimensionPrefix Optional prefix to apply to the dimension names. This + * is needed to distinguish between potentially ambiguous + * dimensions, such as "(investigation.)type.name" and + * "(investigationparameter.)type.name". + * @return The bodyBuilder originally passed with facet information added to it. + */ private JsonObjectBuilder parseFacets(JsonObjectBuilder bodyBuilder, List dimensions, int maxLabels, String dimensionPrefix) { JsonObjectBuilder aggsBuilder = Json.createObjectBuilder(); for (String dimensionString : dimensions) { String field = dimensionPrefix == null ? dimensionString : dimensionPrefix + "." + dimensionString; - aggsBuilder.add(dimensionString, QueryBuilder.buildStringFacet(field, maxLabels)); + aggsBuilder.add(dimensionString, OpensearchQueryBuilder.buildStringFacet(field + ".keyword", maxLabels)); } return buildFacetRequestJson(bodyBuilder, dimensionPrefix, aggsBuilder); } + /** + * Finalises the construction of faceting Json by handling the possibility of + * faceting a nested object. + * + * @param bodyBuilder JsonObjectBuilder being used to build the body of the + * request. + * @param dimensionPrefix Optional prefix to apply to the dimension names. This + * is needed to distinguish between potentially ambiguous + * dimensions, such as "(investigation.)type.name" and + * "(investigationparameter.)type.name". + * @param aggsBuilder JsonObjectBuilder that has the faceting details. + * @return The bodyBuilder originally passed with facet information added to it. + */ private JsonObjectBuilder buildFacetRequestJson(JsonObjectBuilder bodyBuilder, String dimensionPrefix, JsonObjectBuilder aggsBuilder) { if (dimensionPrefix == null) { @@ -427,7 +503,7 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer parentId = source.getString("id"); } // Search for joined entities matching the id - JsonObject termQuery = QueryBuilder.buildTermQuery(fld, parentId); + JsonObject termQuery = OpensearchQueryBuilder.buildTermQuery(fld, parentId); String joinedBody = Json.createObjectBuilder().add("query", termQuery).build().toString(); buildParameterMap(blockSize, requestedJoinedFields, joinedParameterMap, null); JsonObject joinedResponse = postResponse("/" + joinedIndex + "/_search", joinedBody, @@ -502,6 +578,13 @@ private void buildParameterMap(Integer blockSize, Iterable requestedFiel parameterMap.put("size", blockSize.toString()); } + /** + * Parse sort criteria and add it to the request body. + * + * @param builder JsonObjectBuilder being used to build the body of the request. + * @param sort String of JsonObject containing the sort criteria. + * @return The bodyBuilder originally passed with facet criteria added to it. + */ private JsonObjectBuilder parseSort(JsonObjectBuilder builder, String sort) { if (sort == null || sort.equals("")) { return builder.add("sort", Json.createArrayBuilder() @@ -522,6 +605,15 @@ private JsonObjectBuilder parseSort(JsonObjectBuilder builder, String sort) { } } + /** + * Add searchAfter to the request body. + * + * @param builder JsonObjectBuilder being used to build the body of the + * request. + * @param searchAfter Possibly null JsonValue representing the last document of + * a previous search. + * @return The bodyBuilder originally passed with searchAfter added to it. + */ private JsonObjectBuilder parseSearchAfter(JsonObjectBuilder builder, JsonValue searchAfter) { if (searchAfter == null) { return builder; @@ -556,18 +648,21 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query JsonObject filterObject = queryRequest.getJsonObject("filter"); for (String fld : filterObject.keySet()) { ValueType valueType = filterObject.get(fld).getValueType(); + String field = fld.replace(index + ".", ""); switch (valueType) { case ARRAY: JsonArrayBuilder shouldBuilder = Json.createArrayBuilder(); for (JsonString value : filterObject.getJsonArray(fld).getValuesAs(JsonString.class)) { - shouldBuilder.add(QueryBuilder.buildTermQuery(fld, value.getString())); + shouldBuilder + .add(OpensearchQueryBuilder.buildTermQuery(field + ".keyword", value.getString())); } filterBuilder.add(Json.createObjectBuilder().add("bool", Json.createObjectBuilder().add("should", shouldBuilder))); break; case STRING: - filterBuilder.add(QueryBuilder.buildTermQuery(fld, filterObject.getString(fld))); + filterBuilder.add( + OpensearchQueryBuilder.buildTermQuery(field + ".keyword", filterObject.getString(fld))); break; default: @@ -581,7 +676,7 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query // The free text is the only element we perform scoring on, so "must" occur JsonArrayBuilder mustBuilder = Json.createArrayBuilder(); mustBuilder - .add(QueryBuilder.buildStringQuery(queryRequest.getString("text"), + .add(OpensearchQueryBuilder.buildStringQuery(queryRequest.getString("text"), defaultFields.toArray(new String[0]))); boolBuilder.add("must", mustBuilder); } @@ -591,10 +686,10 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query if (lowerTime != Long.MIN_VALUE || upperTime != Long.MAX_VALUE) { if (index.equals("datafile")) { // datafile has only one date field - filterBuilder.add(QueryBuilder.buildLongRangeQuery("date", lowerTime, upperTime)); + filterBuilder.add(OpensearchQueryBuilder.buildLongRangeQuery("date", lowerTime, upperTime)); } else { - filterBuilder.add(QueryBuilder.buildLongRangeQuery("startDate", lowerTime, upperTime)); - filterBuilder.add(QueryBuilder.buildLongRangeQuery("endDate", lowerTime, upperTime)); + filterBuilder.add(OpensearchQueryBuilder.buildLongRangeQuery("startDate", lowerTime, upperTime)); + filterBuilder.add(OpensearchQueryBuilder.buildLongRangeQuery("endDate", lowerTime, upperTime)); } } @@ -602,7 +697,7 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query String name = queryRequest.getString("user"); // Because InstrumentScientist is on a separate index, we need to explicitly // perform a search here - JsonObject termQuery = QueryBuilder.buildTermQuery("user.name.keyword", name); + JsonObject termQuery = OpensearchQueryBuilder.buildTermQuery("user.name.keyword", name); String body = Json.createObjectBuilder().add("query", termQuery).build().toString(); Map parameterMap = new HashMap<>(); parameterMap.put("_source", "instrument.id"); @@ -613,13 +708,15 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query String instrumentId = hit.getJsonObject("_source").getString("instrument.id"); instrumentIdsBuilder.add(instrumentId); } - JsonObject instrumentQuery = QueryBuilder.buildTermsQuery("investigationinstrument.instrument.id", + JsonObject instrumentQuery = OpensearchQueryBuilder.buildTermsQuery("investigationinstrument.instrument.id", instrumentIdsBuilder.build()); - JsonObject nestedInstrumentQuery = QueryBuilder.buildNestedQuery("investigationinstrument", + JsonObject nestedInstrumentQuery = OpensearchQueryBuilder.buildNestedQuery("investigationinstrument", instrumentQuery); // InvestigationUser should be a nested field on the main Document - JsonObject investigationUserQuery = QueryBuilder.buildMatchQuery("investigationuser.user.name", name); - JsonObject nestedUserQuery = QueryBuilder.buildNestedQuery("investigationuser", investigationUserQuery); + JsonObject investigationUserQuery = OpensearchQueryBuilder.buildMatchQuery("investigationuser.user.name", + name); + JsonObject nestedUserQuery = OpensearchQueryBuilder.buildNestedQuery("investigationuser", + investigationUserQuery); // At least one of being an InstrumentScientist or an InvestigationUser is // necessary JsonArrayBuilder array = Json.createArrayBuilder().add(nestedInstrumentQuery).add(nestedUserQuery); @@ -627,16 +724,18 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query } if (queryRequest.containsKey("userFullName")) { String fullName = queryRequest.getString("userFullName"); - JsonObject fullNameQuery = QueryBuilder.buildStringQuery(fullName, "investigationuser.user.fullName"); - filterBuilder.add(QueryBuilder.buildNestedQuery("investigationuser", fullNameQuery)); + JsonObject fullNameQuery = OpensearchQueryBuilder.buildStringQuery(fullName, + "investigationuser.user.fullName"); + filterBuilder.add(OpensearchQueryBuilder.buildNestedQuery("investigationuser", fullNameQuery)); } if (queryRequest.containsKey("samples")) { JsonArray samples = queryRequest.getJsonArray("samples"); for (int i = 0; i < samples.size(); i++) { String sample = samples.getString(i); - JsonObject stringQuery = QueryBuilder.buildStringQuery(sample, "sample.name", "sample.type.name"); - filterBuilder.add(QueryBuilder.buildNestedQuery("sample", stringQuery)); + JsonObject stringQuery = OpensearchQueryBuilder.buildStringQuery(sample, "sample.name", + "sample.type.name"); + filterBuilder.add(OpensearchQueryBuilder.buildNestedQuery("sample", stringQuery)); } } @@ -646,34 +745,38 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query List parameterQueries = new ArrayList<>(); if (parameterObject.containsKey("name")) { String name = parameterObject.getString("name"); - parameterQueries.add(QueryBuilder.buildMatchQuery(path + ".type.name", name)); + parameterQueries.add(OpensearchQueryBuilder.buildMatchQuery(path + ".type.name", name)); } if (parameterObject.containsKey("units")) { String units = parameterObject.getString("units"); - parameterQueries.add(QueryBuilder.buildMatchQuery(path + ".type.units", units)); + parameterQueries.add(OpensearchQueryBuilder.buildMatchQuery(path + ".type.units", units)); } if (parameterObject.containsKey("stringValue")) { String stringValue = parameterObject.getString("stringValue"); - parameterQueries.add(QueryBuilder.buildMatchQuery(path + ".stringValue", stringValue)); + parameterQueries.add(OpensearchQueryBuilder.buildMatchQuery(path + ".stringValue", stringValue)); } else if (parameterObject.containsKey("lowerDateValue") && parameterObject.containsKey("upperDateValue")) { Long lower = parseDate(parameterObject, "lowerDateValue", 0, Long.MIN_VALUE); Long upper = parseDate(parameterObject, "upperDateValue", 59999, Long.MAX_VALUE); - parameterQueries.add(QueryBuilder.buildLongRangeQuery(path + ".dateTimeValue", lower, upper)); + parameterQueries + .add(OpensearchQueryBuilder.buildLongRangeQuery(path + ".dateTimeValue", lower, upper)); } else if (parameterObject.containsKey("lowerNumericValue") && parameterObject.containsKey("upperNumericValue")) { JsonNumber lower = parameterObject.getJsonNumber("lowerNumericValue"); JsonNumber upper = parameterObject.getJsonNumber("upperNumericValue"); - parameterQueries.add(QueryBuilder.buildRangeQuery(path + ".numericValue", lower, upper)); + parameterQueries.add(OpensearchQueryBuilder.buildRangeQuery(path + ".numericValue", lower, upper)); } - filterBuilder.add(QueryBuilder.buildNestedQuery(path, parameterQueries.toArray(new JsonObject[0]))); + filterBuilder.add( + OpensearchQueryBuilder.buildNestedQuery(path, parameterQueries.toArray(new JsonObject[0]))); } } if (queryRequest.containsKey("id")) { - filterBuilder.add(QueryBuilder.buildTermsQuery("id", queryRequest.getJsonArray("id"))); + filterBuilder.add(OpensearchQueryBuilder.buildTermsQuery("id", queryRequest.getJsonArray("id"))); } + // TODO add in support for specific terms? + JsonArray filterArray = filterBuilder.build(); if (filterArray.size() > 0) { boolBuilder.add("filter", filterArray); @@ -681,6 +784,11 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query return builder.add("query", queryBuilder.add("bool", boolBuilder)); } + /** + * Create mappings for indices that do not already have them. + * + * @throws IcatException + */ public void initMappings() throws IcatException { for (String index : indices) { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { @@ -720,6 +828,11 @@ public void initMappings() throws IcatException { } } + /** + * Create scripts for indices that do not already have them. + * + * @throws IcatException + */ public void initScripts() throws IcatException { for (Entry> entry : relations.entrySet()) { String key = entry.getKey(); @@ -868,7 +981,7 @@ private void extractFromInvestigation(CloseableHttpClient httpclient, String inv JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("doc", jsonArray); JsonObjectBuilder scriptBuilder = Json.createObjectBuilder(); scriptBuilder.add("id", "create_investigationuser").add("params", paramsBuilder); - JsonObject queryObject = QueryBuilder.buildTermQuery("investigation.id", investigationId); + JsonObject queryObject = OpensearchQueryBuilder.buildTermQuery("investigation.id", investigationId); JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); String body = bodyBuilder.add("query", queryObject).add("script", scriptBuilder).build().toString(); httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); @@ -887,7 +1000,7 @@ private void extractFromInvestigation(CloseableHttpClient httpclient, String inv JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("doc", jsonArray); JsonObjectBuilder scriptBuilder = Json.createObjectBuilder(); scriptBuilder.add("id", "create_investigationinstrument").add("params", paramsBuilder); - JsonObject queryObject = QueryBuilder.buildTermQuery("investigation.id", investigationId); + JsonObject queryObject = OpensearchQueryBuilder.buildTermQuery("investigation.id", investigationId); JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); String body = bodyBuilder.add("query", queryObject).add("script", scriptBuilder).build().toString(); logger.trace("Making call {} with body {}", uri, body); @@ -900,6 +1013,22 @@ private void extractFromInvestigation(CloseableHttpClient httpclient, String inv } } + /** + * Performs more complex update of an entity nested to a parent, for example + * parameters. + * + * @param sb StringBuilder used for bulk modifications. + * @param updatesByQuery List of HttpPost that cannot be bulked, and update + * existing documents based on a query. + * @param id Id of the entity. + * @param index Index of the entity. + * @param document JsonObject containing the key value pairs of the + * document fields. + * @param modificationType The type of operation to be performed. + * @param relation The relation between the nested entity and its + * parent. + * @throws URISyntaxException + */ private void modifyNestedEntity(StringBuilder sb, List updatesByQuery, String id, String index, JsonObject document, ModificationType modificationType, ParentRelation relation) throws URISyntaxException { @@ -930,9 +1059,18 @@ private void modifyNestedEntity(StringBuilder sb, List updatesByQuery, } } + /** + * Create a new nested entity in an array on its parent. + * + * @param sb StringBuilder used for bulk modifications. + * @param id Id of the entity. + * @param index Index of the entity. + * @param document JsonObject containing the key value pairs of the document + * fields. + * @param relation The relation between the nested entity and its parent. + */ private static void createNestedEntity(StringBuilder sb, String id, String index, JsonObject document, ParentRelation relation) { - String parentId = document.getString(relation.parentName + ".id"); JsonObjectBuilder innerBuilder = Json.createObjectBuilder() .add("_id", parentId).add("_index", relation.parentName); @@ -949,6 +1087,21 @@ private static void createNestedEntity(StringBuilder sb, String id, String index sb.append(payloadBuilder.build().toString()).append("\n"); } + /** + * For existing nested objects, painless scripting must be used to update or + * delete them. + * + * @param updatesByQuery List of HttpPost that cannot be bulked, and update + * existing documents based on a query. + * @param id Id of the entity. + * @param index Index of the entity. + * @param document JsonObject containing the key value pairs of the + * document fields. + * @param relation The relation between the nested entity and its parent. + * @param update Whether to update, or if false delete nested entity + * with the specified id. + * @throws URISyntaxException + */ private void updateNestedEntityByQuery(List updatesByQuery, String id, String index, JsonObject document, ParentRelation relation, boolean update) throws URISyntaxException { String path = "/" + relation.parentName + "/_update_by_query"; @@ -975,9 +1128,10 @@ private void updateNestedEntityByQuery(List updatesByQuery, String id, JsonObject queryObject; String idField = relation.joinField.equals(relation.parentName) ? "id" : relation.joinField + ".id"; if (relation.relationType.equals(RelationType.NESTED_GRANDCHILD)) { - queryObject = QueryBuilder.buildNestedQuery(relation.joinField, QueryBuilder.buildTermQuery(idField, id)); + queryObject = OpensearchQueryBuilder.buildNestedQuery(relation.joinField, + OpensearchQueryBuilder.buildTermQuery(idField, id)); } else { - queryObject = QueryBuilder.buildTermQuery(idField, id); + queryObject = OpensearchQueryBuilder.buildTermQuery(idField, id); } JsonObject bodyJson = Json.createObjectBuilder().add("query", queryObject).add("script", scriptBuilder).build(); logger.trace("Making call {} with body {}", path, bodyJson.toString()); @@ -985,6 +1139,16 @@ private void updateNestedEntityByQuery(List updatesByQuery, String id, updatesByQuery.add(httpPost); } + /** + * Gets "type.units" from the existing document, and adds "type.unitsSI" and the + * SI numeric value to the rebuilder if possible. + * + * @param document JsonObject of the original document. + * @param rebuilder JsonObjectBuilder being used to create a new document + * with converted units. + * @param valueString Field name of the numeric value. + * @param numericValue Value to possibly be converted. + */ private void convertUnits(JsonObject document, JsonObjectBuilder rebuilder, String valueString, Double numericValue) { String unitString = document.getString("type.units"); @@ -997,6 +1161,13 @@ private void convertUnits(JsonObject document, JsonObjectBuilder rebuilder, Stri } } + /** + * If appropriate, rebuilds document with conversion into SI units. + * + * @param document JsonObject containing the document field/values. + * @return Either the original JsonDocument, or a copy with SI units and values + * set. + */ private JsonObject convertDocumentUnits(JsonObject document) { if (!document.containsKey("type.units")) { return document; @@ -1014,6 +1185,15 @@ private JsonObject convertDocumentUnits(JsonObject document) { return document; } + /** + * Builds the parameters for a painless script, converting into SI units if + * appropriate. + * + * @param paramsBuilder JsonObjectBuilder for the painless script parameters. + * @param document JsonObject containing the field/values. + * @param fields List of fields to be included in the parameters. + * @return paramsBuilder with fields added. + */ private JsonObjectBuilder convertScriptUnits(JsonObjectBuilder paramsBuilder, JsonObject document, Set fields) { for (String field : fields) { @@ -1030,6 +1210,20 @@ private JsonObjectBuilder convertScriptUnits(JsonObjectBuilder paramsBuilder, Js return paramsBuilder; } + /** + * Adds modification command to sb. If relevant, also adds to the list of + * investigationIds which may contain relevant information (e.g. nested + * InvestigationUsers). + * + * @param sb StringBuilder used for bulk modifications. + * @param investigationIds List of investigationIds to check for relevant + * fields. + * @param id Id of the entity. + * @param index Index of the entity. + * @param document JsonObject containing the key value pairs of the + * document fields. + * @param modificationType The type of operation to be performed. + */ private static void modifyEntity(StringBuilder sb, Set investigationIds, String id, String index, JsonObject document, ModificationType modificationType) { diff --git a/src/main/java/org/icatproject/core/manager/search/QueryBuilder.java b/src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java similarity index 65% rename from src/main/java/org/icatproject/core/manager/search/QueryBuilder.java rename to src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java index 49957b2d..a580107e 100644 --- a/src/main/java/org/icatproject/core/manager/search/QueryBuilder.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java @@ -7,25 +7,45 @@ import javax.json.JsonObject; import javax.json.JsonObjectBuilder; -public class QueryBuilder { +/** + * Utility for building queries in Json understood by Opensearch. + */ +public class OpensearchQueryBuilder { private static JsonObject matchAllQuery = Json.createObjectBuilder().add("match_all", Json.createObjectBuilder()) .build(); + /** + * @param query JsonObject representing an Opensearch query. + * @return JsonObjectBuilder with JsonObject {"query": {...query}} + */ public static JsonObjectBuilder addQuery(JsonObject query) { return Json.createObjectBuilder().add("query", query); } + /** + * @return {"match_all": {}} + */ public static JsonObject buildMatchAllQuery() { return matchAllQuery; } + /** + * @param field Field containing the match. + * @param value Value to match. + * @return {"match": {"`field`.keyword": {"query": `value`, "operator": "and"}}} + */ public static JsonObject buildMatchQuery(String field, String value) { JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("query", value).add("operator", "and"); JsonObjectBuilder matchBuilder = Json.createObjectBuilder().add(field + ".keyword", fieldBuilder); return Json.createObjectBuilder().add("match", matchBuilder).build(); } + /** + * @param path Path to nested Object. + * @param queryObjects Any number of pre-built queries. + * @return {"nested": {"path": `path`, "query": {"bool": {"filter": [...queryObjects]}}}} + */ public static JsonObject buildNestedQuery(String path, JsonObject... queryObjects) { JsonObject builtQueries = null; if (queryObjects.length == 0) { @@ -44,6 +64,11 @@ public static JsonObject buildNestedQuery(String path, JsonObject... queryObject return Json.createObjectBuilder().add("nested", nestedBuilder).build(); } + /** + * @param value String value to query for. + * @param fields List of fields to check for value. + * @return {"query_string": {"query": `value`, "fields": [...fields]}} + */ public static JsonObject buildStringQuery(String value, String... fields) { JsonObjectBuilder queryStringBuilder = Json.createObjectBuilder().add("query", value); if (fields.length > 0) { @@ -56,32 +81,64 @@ public static JsonObject buildStringQuery(String value, String... fields) { return Json.createObjectBuilder().add("query_string", queryStringBuilder).build(); } + /** + * @param field Field containing the term. + * @param value Term to match. + * @return {"term": {`field`: `value`}} + */ public static JsonObject buildTermQuery(String field, String value) { return Json.createObjectBuilder().add("term", Json.createObjectBuilder().add(field, value)).build(); } + /** + * @param field Field containing on of the terms. + * @param values JsonArrat of possible terms. + * @return {"terms": {`field`: `values`}} + */ public static JsonObject buildTermsQuery(String field, JsonArray values) { return Json.createObjectBuilder().add("terms", Json.createObjectBuilder().add(field, values)).build(); } + /** + * @param field Field to apply the range to. + * @param lowerValue Lowest allowed value in the range. + * @param upperValue Highest allowed value in the range. + * @return {"range": {`field`: {"gte": `upperValue`, "lte": `lowerValue`}}} + */ public static JsonObject buildLongRangeQuery(String field, Long lowerValue, Long upperValue) { JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("gte", lowerValue).add("lte", upperValue); JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); return Json.createObjectBuilder().add("range", rangeBuilder).build(); } + /** + * @param field Field to apply the range to. + * @param lowerValue Lowest allowed value in the range. + * @param upperValue Highest allowed value in the range. + * @return {"range": {`field`: {"gte": `upperValue`, "lte": `lowerValue`}}} + */ public static JsonObject buildRangeQuery(String field, JsonNumber lowerValue, JsonNumber upperValue) { JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("gte", lowerValue).add("lte", upperValue); JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); return Json.createObjectBuilder().add("range", rangeBuilder).build(); } + /** + * @param field Field to facet. + * @param ranges JsonArray of ranges to allocate documents to. + * @return {"range": {"field": `field`, "keyed": true, "ranges": `ranges`}} + */ public static JsonObject buildRangeFacet(String field, JsonArray ranges) { JsonObjectBuilder rangeBuilder = Json.createObjectBuilder(); rangeBuilder.add("field", field).add("keyed", true).add("ranges", ranges); return Json.createObjectBuilder().add("range", rangeBuilder).build(); } + /** + * @param field Field to facet. + * @param maxLabels Maximum number of labels per dimension. + * @return {"terms": {"field": `field`, "size": `maxLabels`}} + */ public static JsonObject buildStringFacet(String field, int maxLabels) { JsonObjectBuilder termsBuilder = Json.createObjectBuilder(); termsBuilder.add("field", field).add("size", maxLabels); diff --git a/src/main/java/org/icatproject/core/manager/search/SearchApi.java b/src/main/java/org/icatproject/core/manager/search/SearchApi.java index a7c56a29..56b7fbc0 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchApi.java @@ -58,23 +58,6 @@ public SearchApi(URI server) { this.server = server; } - /** - * Converts String into Date object. - * - * @param value String representing a Date in the format "yyyyMMddHHmm". - * @return Date object, or null if value was null. - * @throws java.text.ParseException - */ - protected static Date decodeDate(String value) throws java.text.ParseException { - if (value == null) { - return null; - } else { - synchronized (df) { - return df.parse(value); - } - } - } - /** * Converts String into number of ms since epoch. * @@ -93,21 +76,12 @@ protected static Long decodeTime(String value) throws java.text.ParseException { } /** - * Converts Date object into String format. + * Encodes the deletion of the provided entity as Json. * - * @param dateValue Date object to be converted. - * @return String representing a Date in the format "yyyyMMddHHmm". + * @param bean Entity to be deleted from the search engine index. + * @return String of Json in the format + * {"delete": {"_index": `entityName`, "_id": `id`}} */ - protected static String encodeDate(Date dateValue) { - if (dateValue == null) { - return null; - } else { - synchronized (df) { - return df.format(dateValue); - } - } - } - public static String encodeDeletion(EntityBaseBean bean) { String entityName = bean.getClass().getSimpleName(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -119,14 +93,38 @@ public static String encodeDeletion(EntityBaseBean bean) { return baos.toString(); } + /** + * Writes a key value pair to the JsonGenerator being used to encode an entity. + * + * @param gen JsonGenerator being used to encode. + * @param name Name of the field. + * @param value Double value to encode as a double. + */ public static void encodeDouble(JsonGenerator gen, String name, Double value) { gen.write(name, value); } + /** + * Writes a key value pair to the JsonGenerator being used to encode an entity. + * + * @param gen JsonGenerator being used to encode. + * @param name Name of the field. + * @param value Long value to encode as a long. + */ public static void encodeLong(JsonGenerator gen, String name, Date value) { gen.write(name, value.getTime()); } + /** + * Encodes the creation or updating of the provided entity as Json. + * + * @param operation The operation to encode. Should either be "create" or + * "update". + * @param bean Entity to perform the operation on. + * @return String of Json in the format + * {`operation`: {"_index": `entityName`, "_id": `id`, "doc": {...}}} + * @throws IcatException + */ public static String encodeOperation(String operation, EntityBaseBean bean) throws IcatException { Long icatId = bean.getId(); if (icatId == null) { @@ -144,14 +142,36 @@ public static String encodeOperation(String operation, EntityBaseBean bean) thro return baos.toString(); } + /** + * Writes a key value pair to the JsonGenerator being used to encode an entity. + * + * @param gen JsonGenerator being used to encode. + * @param name Name of the field. + * @param value Long value to encode as a string. + */ public static void encodeString(JsonGenerator gen, String name, Long value) { gen.write(name, Long.toString(value)); } + /** + * Writes a key value pair to the JsonGenerator being used to encode an entity. + * + * @param gen JsonGenerator being used to encode. + * @param name Name of the field. + * @param value String value to encode as a string. + */ public static void encodeString(JsonGenerator gen, String name, String value) { gen.write(name, value); } + /** + * Writes a key value pair to the JsonGenerator being used to encode an entity, + * provided that value is not null. + * + * @param gen JsonGenerator being used to encode. + * @param name Name of the field. + * @param value String value to encode as a string. + */ public static void encodeText(JsonGenerator gen, String name, String value) { if (value != null) { gen.write(name, value); @@ -186,6 +206,17 @@ public JsonValue buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) th return arrayBuilder.build(); } + /** + * Builds a Json representation of the sorted fields of the final search result. + * This allows future searches to efficiently "search after" this result. + * + * @param lastBean The last ScoredEntityBaseBean of the current search results. + * @param sort String representing a JsonObject of sort criteria. + * @return JsonArray representing the sorted fields to allow future searches to + * search after it. + * @throws IcatException If one of the sort fields is not present in the source + * of the lastBean. + */ protected static JsonArrayBuilder searchAfterArrayBuilder(ScoredEntityBaseBean lastBean, String sort) throws IcatException { try (JsonReader reader = Json.createReader(new StringReader(sort))) { @@ -203,6 +234,18 @@ protected static JsonArrayBuilder searchAfterArrayBuilder(ScoredEntityBaseBean l } } + /** + * Parses the JsonObject response from the search engine into a FacetDimension, + * and adds it to results. + * + * @param results List of FacetDimensions to add the results from this + * dimension to. + * @param target The entity being targeted. + * @param dimension The dimension (field) being faceted. + * @param aggregations JsonObject containing the response from the search + * engine. + * @throws IcatException + */ protected static void parseFacetsResponse(List results, String target, String dimension, JsonObject aggregations) throws IcatException { if (dimension.equals("doc_count")) { @@ -223,8 +266,8 @@ protected static void parseFacetsResponse(List results, String t return; } for (JsonObject bucket : buckets) { - long docCount = bucket.getJsonNumber("doc_count").longValueExact(); - facets.add(new FacetLabel(bucket.getString("key"), docCount)); + FacetLabel facetLabel = new FacetLabel(bucket); + facets.add(facetLabel); } break; case OBJECT: @@ -235,8 +278,8 @@ protected static void parseFacetsResponse(List results, String t } for (String key : keySet) { JsonObject bucket = bucketsObject.getJsonObject(key); - long docCount = bucket.getJsonNumber("doc_count").longValueExact(); - facets.add(new FacetLabel(key, docCount)); + FacetLabel facetLabel = new FacetLabel(key, bucket); + facets.add(facetLabel); } break; default: @@ -246,38 +289,135 @@ protected static void parseFacetsResponse(List results, String t results.add(facetDimension); } + /** + * Adds documents to the index identified by entityName immediately. + * Practically, this should be used for populating documents from existing + * database records as opposed to adding documents as they are created. + * + * @param entityName The entity to create documents for. + * @param ids List of ids corresponding to the documents to add. + * @param manager EntityManager for finding the beans from their id. + * @param klass Class of the entity to create documents for. + * @param getBeanDocExecutor + * @throws IcatException + * @throws IOException + * @throws URISyntaxException + */ public abstract void addNow(String entityName, List ids, EntityManager manager, Class klass, ExecutorService getBeanDocExecutor) throws IcatException, IOException, URISyntaxException; + /** + * This is only for testing purposes. Other calls to the service will not + * work properly while this operation is in progress. + * + * Deletes all documents across all indices. + * + * @throws IcatException + */ public abstract void clear() throws IcatException; + /** + * Commits any pending documents to their respective index. + * + * @throws IcatException + */ public abstract void commit() throws IcatException; + /** + * Perform faceting on an entity/index. The query associated with the request + * should determine which Documents to consider, and optionally the dimensions + * to facet. If no dimensions are provided, "sparse" faceting is performed + * across relevant string fields (but no Range faceting occurs). + * + * @param target Name of the entity/index to facet on. + * @param facetQuery JsonObject containing the criteria to facet on. + * @param maxResults The maximum number of results to include in the returned + * Json. + * @param maxLabels The maximum number of labels to return for each dimension + * of the facets. + * @return List of FacetDimensions that were collected for the query. + * @throws IcatException + */ public abstract List facetSearch(String target, JsonObject facetQuery, Integer maxResults, Integer maxLabels) throws IcatException; + /** + * Gets SearchResult for query without sort or searchAfter (pagination). + * + * @param query JsonObject containing the criteria to search on. + * @param maxResults Maximum number of results to retrieve from the engine. + * @return SearchResult for the query. + * @throws IcatException + */ public SearchResult getResults(JsonObject query, int maxResults) throws IcatException { return getResults(query, null, maxResults, null, Arrays.asList("id")); } + /** + * Gets SearchResult for query without searchAfter (pagination). + * + * @param query JsonObject containing the criteria to search on. + * @param maxResults Maximum number of results to retrieve from the engine. + * @param sort String of Json representing the sort criteria. + * @return SearchResult for the query. + * @throws IcatException + */ public SearchResult getResults(JsonObject query, int maxResults, String sort) throws IcatException { return getResults(query, null, maxResults, sort, Arrays.asList("id")); } + /** + * Gets SearchResult for query. + * + * @param query JsonObject containing the criteria to search on. + * @param searchAfter JsonValue representing the last result of a previous + * search in order to skip results that have already been + * returned. + * @param blockSize Maximum number of results to retrieve from the engine. + * @param sort String of Json representing the sort criteria. + * @param requestedFields List of fields to return in the document source. + * @return SearchResult for the query. + * @throws IcatException + */ public abstract SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer blockSize, String sort, List requestedFields) throws IcatException; + /** + * Not implemented. + * + * @param entityName + * @throws IcatException + */ public void lock(String entityName) throws IcatException { logger.info("Manually locking index not supported, no request sent"); } + /** + * Not implemented. + * + * @param entityName + * @throws IcatException + */ public void unlock(String entityName) throws IcatException { logger.info("Manually unlocking index not supported, no request sent"); } + /** + * Perform one or more document modification operations. + * + * @param json String of a JsonArray containing individual create/update/delete + * operations as JsonObjects. + * @throws IcatException + */ public abstract void modify(String json) throws IcatException; + /** + * POST to path without a body or response handling. + * + * @param path Path on the search engine to POST to. + * @throws IcatException + */ protected void post(String path) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URI uri = new URIBuilder(server).setPath(path).build(); @@ -291,6 +431,13 @@ protected void post(String path) throws IcatException { } } + /** + * POST to path with a body but without response handling. + * + * @param path Path on the search engine to POST to. + * @param body String of Json to send as the request body. + * @throws IcatException + */ protected void post(String path, String body) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URI uri = new URIBuilder(server).setPath(path).build(); @@ -305,6 +452,15 @@ protected void post(String path, String body) throws IcatException { } } + /** + * POST to path with a body and response handling. + * + * @param path Path on the search engine to POST to. + * @param body String of Json to send as the request body. + * @param parameterMap Map of parameters to encode in the URI. + * @return JsonObject returned by the search engine. + * @throws IcatException + */ protected JsonObject postResponse(String path, String body, Map parameterMap) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URIBuilder builder = new URIBuilder(server).setPath(path); diff --git a/src/main/java/org/icatproject/core/manager/search/SearchManager.java b/src/main/java/org/icatproject/core/manager/search/SearchManager.java index 66537caf..18d0c734 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchManager.java @@ -323,6 +323,16 @@ public void run() { private static final Map> publicSearchFields = new HashMap<>(); + /** + * Gets (and if necessary, builds) the fields which should be returned as part + * of the document source from a search. + * + * @param gateKeeper GateKeeper instance. + * @param simpleName Name of the entity to get public fields for. + * @return List of fields which can be shown in search results provided the main + * entity is authorised. + * @throws IcatException + */ public static List getPublicSearchFields(GateKeeper gateKeeper, String simpleName) throws IcatException { if (gateKeeper.getPublicSearchFieldsStale() || publicSearchFields.size() == 0) { logger.info("Building public search fields from public tables and steps"); @@ -383,6 +393,16 @@ public void deleteDocument(EntityBaseBean bean) throws IcatException { } } + /** + * Builds a JsonObject for performing faceting against results from a previous + * search. + * + * @param results List of results from a previous search, containing entity + * ids. + * @param idField The field to perform id querying against. + * @param facetJson JsonObject containing the dimensions to facet. + * @return {"query": {`idField`: [...]}, "dimensions": [...]} + */ public static JsonObject buildFacetQuery(List results, String idField, JsonObject facetJson) { JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); results.forEach(r -> arrayBuilder.add(Long.toString(r.getEntityBaseBeanId()))); @@ -394,6 +414,32 @@ public static JsonObject buildFacetQuery(List results, Str return objectBuilder.build(); } + /** + * Builds a JsonObject for performing faceting against results from a previous + * search. + * + * @param filterObject JsonObject to be used as a query. + * @param idField The field to perform id querying against. + * @param facetJson JsonObject containing the dimensions to facet. + * @return {"query": `filterObject`, "dimensions": [...]} + */ + public static JsonObject buildFacetQuery(JsonObject filterObject, String idField, JsonObject facetJson) { + JsonObjectBuilder objectBuilder = Json.createObjectBuilder().add("query", filterObject); + if (facetJson.containsKey("dimensions")) { + objectBuilder.add("dimensions", facetJson.getJsonArray("dimensions")); + } + return objectBuilder.build(); + } + + /** + * Checks if the underlying Relationship is allowed for a field on an entity. + * + * @param gateKeeper GateKeeper instance. + * @param map Map of fields to the Relationship that must be allowed in + * order to return the fields with search results for a + * particular entity. + * @return List of fields (keys) from map that have an allowed relationship + */ private static List buildPublicSearchFields(GateKeeper gateKeeper, Map map) { List fields = new ArrayList<>(); for (Entry entry : map.entrySet()) { @@ -402,7 +448,8 @@ private static List buildPublicSearchFields(GateKeeper gateKeeper, Map buildPublicSearchFields(GateKeeper gateKeeper, Map facetSearch(String target, JsonObject facetQuery, int maxResults, int maxLabels) throws IcatException { return searchApi.facetSearch(target, facetQuery, maxResults, maxLabels); @@ -461,13 +535,36 @@ public List getPopulating() { return result; } - public SearchResult freeTextSearch(JsonObject jo, int blockSize, String sort) throws IcatException { - return searchApi.getResults(jo, blockSize, sort); + + /** + * Gets SearchResult for query without searchAfter (pagination). + * + * @param query JsonObject containing the criteria to search on. + * @param maxResults Maximum number of results to retrieve from the engine. + * @param sort String of Json representing the sort criteria. + * @return SearchResult for the query. + * @throws IcatException + */ + public SearchResult freeTextSearch(JsonObject query, int maxResults, String sort) throws IcatException { + return searchApi.getResults(query, maxResults, sort); } - public SearchResult freeTextSearch(JsonObject jo, JsonValue searchAfter, int blockSize, String sort, - List fields) throws IcatException { - return searchApi.getResults(jo, searchAfter, blockSize, sort, fields); + /** + * Gets SearchResult for query. + * + * @param query JsonObject containing the criteria to search on. + * @param searchAfter JsonValue representing the last result of a previous + * search in order to skip results that have already been + * returned. + * @param blockSize Maximum number of results to retrieve from the engine. + * @param sort String of Json representing the sort criteria. + * @param requestedFields List of fields to return in the document source. + * @return SearchResult for the query. + * @throws IcatException + */ + public SearchResult freeTextSearch(JsonObject query, JsonValue searchAfter, int blockSize, String sort, + List requestedFields) throws IcatException { + return searchApi.getResults(query, searchAfter, blockSize, sort, requestedFields); } @PostConstruct diff --git a/src/main/java/org/icatproject/core/manager/search/SearchResult.java b/src/main/java/org/icatproject/core/manager/search/SearchResult.java index 6486e0f0..146d7949 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchResult.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchResult.java @@ -5,6 +5,11 @@ import javax.json.JsonValue; +/** + * Represents the results from a single search performed against the engine. + * Stores a list of ScoredEntityBaseBean, FacetDimension, and a JsonValue + * representing the last document returned if appropriate. + */ public class SearchResult { private JsonValue searchAfter; @@ -12,7 +17,8 @@ public class SearchResult { private List dimensions; private boolean aborted; - public SearchResult() {} + public SearchResult() { + } public SearchResult(JsonValue searchAfter, List results, List dimensions) { this.searchAfter = searchAfter; diff --git a/src/main/java/org/icatproject/core/manager/search/URIParameter.java b/src/main/java/org/icatproject/core/manager/search/URIParameter.java deleted file mode 100644 index e69de29b..00000000 diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index 11f6f2f2..57d12184 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -1244,17 +1244,11 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId * then results will be returned in order of relevance to the * search query, with their search engine id as a tiebreaker. * - * @param limit + * @param minCount + * minimum number of entities to return + * + * @param maxCount * maximum number of entities to return - * @param facets - * String representing a JsonArray of JsonObjects. Each - * should define the "target" entity name, and optionally - * another JsonArray of JsonObjects representing specific - * fields to facet. If absent, then all applicable String - * fields will be faceted. These objects must have the - * "dimension" key, and if the field is numeric than the - * "range" key should denote an array of objects with "lower" - * and "upper" values. * * @param restrict * Whether to perform a quicker search which restricts the @@ -1272,12 +1266,17 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId @Produces(MediaType.APPLICATION_JSON) public String search(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, @QueryParam("query") String query, @QueryParam("search_after") String searchAfter, - @QueryParam("limit") int limit, @QueryParam("sort") String sort, @QueryParam("facets") String facets, - @QueryParam("restrict") boolean restrict) - throws IcatException { + @QueryParam("minCount") int minCount, @QueryParam("maxCount") int maxCount, @QueryParam("sort") String sort, + @QueryParam("restrict") boolean restrict) throws IcatException { if (query == null) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "query is not set"); } + if (minCount == 0) { + minCount = 10; + } + if (maxCount == 0) { + maxCount = 100; + } String userName = beanManager.getUserName(sessionId, manager); JsonValue searchAfterValue = null; if (searchAfter != null && searchAfter.length() > 0) { @@ -1331,9 +1330,9 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId } else { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "target:" + target + " is not expected"); } - logger.debug("Free text search with query: {}, facets: {}", jo.toString(), facets); - result = beanManager.freeTextSearchDocs(userName, jo, searchAfterValue, limit, sort, facets, manager, - request.getRemoteAddr(), klass); + + result = beanManager.freeTextSearchDocs(userName, jo, searchAfterValue, minCount, maxCount, sort, + manager, request.getRemoteAddr(), klass); JsonGenerator gen = Json.createGenerator(baos); gen.writeStartObject(); @@ -1377,6 +1376,92 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId } } + /** + * Performs subsequent faceting for a particular query containing a list of ids. + * + * @summary Document faceting. + * + * @param sessionId a sessionId of a user which takes the form 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 + * @param query Json of the format + * { + * "target": `target`, + * "facets": [ + * { + * "target": `facetTarget`, + * "dimensions": [ + * {"dimension": `dimension`, "ranges": [{"key": `key`, "from": `from`, "to": `to`}, ...]}, + * ... + * ] + * }, + * ... + * ], + * "filter": {`termField`: `value`, `termsField`: [...], ...} + * } + * @return Facet labels and counts for the provided query + * @throws IcatException If something goes wrong + */ + @GET + @Path("facet/documents") + @Produces(MediaType.APPLICATION_JSON) + public String facet(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, + @QueryParam("query") String query) throws IcatException { + if (query == null) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, "query is not set"); + } + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonReader jr = Json.createReader(new StringReader(query))) { + JsonObject jo = jr.readObject(); + + String target = jo.getString("target", null); + + SearchResult result; + Class klass; + + if (target.equals("Investigation")) { + klass = Investigation.class; + } else if (target.equals("Dataset")) { + klass = Dataset.class; + } else if (target.equals("Datafile")) { + klass = Datafile.class; + } else { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, "target:" + target + " is not expected"); + } + + result = beanManager.facetDocs(jo, klass); + + JsonGenerator gen = Json.createGenerator(baos); + gen.writeStartObject(); + if (result.isAborted()) { + gen.write("aborted", true).writeEnd().close(); + return baos.toString(); + } + + List dimensions = result.getDimensions(); + if (dimensions != null && dimensions.size() > 0) { + gen.writeStartObject("dimensions"); + for (FacetDimension dimension : dimensions) { + gen.writeStartObject(dimension.getTarget() + "." + dimension.getDimension()); + for (FacetLabel label : dimension.getFacets()) { + logger.debug("From and to: ", label.getFrom(), label.getTo()); + if (label.getFrom() != null && label.getTo() != null) { + gen.writeStartObject(label.getLabel()).write("from", label.getFrom()).write("to", label.getTo()).write("count", label.getValue()).writeEnd(); + } else { + gen.write(label.getLabel(), label.getValue()); + } + } + gen.writeEnd(); + } + gen.writeEnd(); + } + + gen.writeEnd().close(); + return baos.toString(); + } catch (JsonException e) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, "JsonException " + e.getMessage()); + } + } + /** * This is an internal call made by one icat instance to another in the same * cluster diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 1b238628..5fc6bea1 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -794,13 +794,13 @@ public void datasets() throws Exception { checkResults(lsr, 0L, 2L, 4L, 6L, 8L); // Test filter - query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("type.name.keyword", "type")); + query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("dataset.type.name", "type")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); - query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("type.name.keyword", "type", "typo")); + query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("dataset.type.name", "type", "typo")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); - query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("type.name.keyword", "typo")); + query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("dataset.type.name", "typo")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr); @@ -885,13 +885,13 @@ public void investigations() throws Exception { checkResults(lsr, 0L, 2L, 4L, 6L, 8L); // Test filter - query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("type.name.keyword", "type")); + query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("investigation.type.name", "type")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); - query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("type.name.keyword", "type", "typo")); + query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("investigation.type.name", "type", "typo")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); - query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("type.name.keyword", "typo")); + query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("investigation.type.name", "typo")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr); @@ -1034,13 +1034,13 @@ public void modifyDatafile() throws IcatException { JsonObject lowRange = buildFacetRangeObject("low", 0L, 2L); JsonObject highRange = buildFacetRangeObject("high", 2L, 4L); JsonObject rangeFacetRequest = buildFacetRangeRequest(buildFacetIdQuery("42"), "date", lowRange, highRange); - JsonObject stringFacetRequest = buildFacetStringRequest("42", "datafileFormat.name.keyword"); + JsonObject stringFacetRequest = buildFacetStringRequest("42", "datafileFormat.name"); JsonObject sparseFacetRequest = Json.createObjectBuilder().add("query", buildFacetIdQuery("42")).build(); FacetDimension lowFacet = new FacetDimension("", "date", new FacetLabel("low", 1L), new FacetLabel("high", 0L)); FacetDimension highFacet = new FacetDimension("", "date", new FacetLabel("low", 0L), new FacetLabel("high", 1L)); - FacetDimension pdfFacet = new FacetDimension("", "datafileFormat.name.keyword", new FacetLabel("pdf", 1L)); - FacetDimension pngFacet = new FacetDimension("", "datafileFormat.name.keyword", new FacetLabel("png", 1L)); + FacetDimension pdfFacet = new FacetDimension("", "datafileFormat.name", new FacetLabel("pdf", 1L)); + FacetDimension pngFacet = new FacetDimension("", "datafileFormat.name", new FacetLabel("png", 1L)); // Original modify(SearchApi.encodeOperation("create", elephantDatafile)); diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 60323cc2..575e6562 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -716,7 +716,7 @@ public void testSearchDatafiles() throws Exception { searchDatafiles(piSession(), null, null, null, null, null, null, 10, null, null, 0); // Test no facets match on Datafiles - String facets = buildFacetRequest("Datafile"); + JsonArray facets = buildFacetRequest("Datafile"); responseObject = searchDatafiles(session, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); assertFalse(NO_DIMENSIONS, responseObject.containsKey("dimensions")); @@ -731,7 +731,7 @@ public void testSearchDatafiles() throws Exception { wSession.addRule(null, "DatafileParameter", "R"); responseObject = searchDatafiles(session, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); - checkFacets(responseObject, "DatafileParameter.type.name.keyword", Arrays.asList("colour"), Arrays.asList(1L)); + checkFacets(responseObject, "DatafileParameter.type.name", Arrays.asList("colour"), Arrays.asList(1L)); } /** @@ -845,10 +845,10 @@ public void testSearchDatasets() throws Exception { searchDatasets(piSession(), null, null, null, null, null, null, 10, null, null, 0); // Test facets match on Datasets - String facets = buildFacetRequest("Dataset"); + JsonArray facets = buildFacetRequest("Dataset"); responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5); assertFalse(responseObject.containsKey("search_after")); - checkFacets(responseObject, "Dataset.type.name.keyword", Arrays.asList("calibration"), Arrays.asList(5L)); + checkFacets(responseObject, "Dataset.type.name", Arrays.asList("calibration"), Arrays.asList(5L)); // Test no facets match on DatasetParameters due to lack of READ access facets = buildFacetRequest("DatasetParameter"); @@ -861,7 +861,7 @@ public void testSearchDatasets() throws Exception { wSession.addRule(null, "DatasetParameter", "R"); responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5); assertFalse(responseObject.containsKey("search_after")); - checkFacets(responseObject, "DatasetParameter.type.name.keyword", + checkFacets(responseObject, "DatasetParameter.type.name", Arrays.asList("colour", "birthday", "current"), Arrays.asList(1L, 1L, 1L)); } @@ -984,11 +984,11 @@ public void testSearchInvestigations() throws Exception { searchInvestigations(piSession(), null, null, null, null, null, null, null, null, 10, null, null, 0); // Test facets match on Investigations - String facets = buildFacetRequest("Investigation"); + JsonArray facets = buildFacetRequest("Investigation"); responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); - checkFacets(responseObject, "Investigation.type.name.keyword", Arrays.asList("atype"), Arrays.asList(3L)); + checkFacets(responseObject, "Investigation.type.name", Arrays.asList("atype"), Arrays.asList(3L)); // Test no facets match on InvestigationParameters due to lack of READ access facets = buildFacetRequest("InvestigationParameter"); @@ -1002,7 +1002,7 @@ public void testSearchInvestigations() throws Exception { responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); - checkFacets(responseObject, "InvestigationParameter.type.name.keyword", Arrays.asList("colour"), + checkFacets(responseObject, "InvestigationParameter.type.name", Arrays.asList("colour"), Arrays.asList(1L)); } @@ -1039,12 +1039,12 @@ public void testSearchParameterValidation() throws Exception { } } - private String buildFacetRequest(String target) { + private JsonArray buildFacetRequest(String target) { JsonObjectBuilder builder = Json.createObjectBuilder(); - JsonObjectBuilder dimension = Json.createObjectBuilder().add("dimension", "type.name.keyword"); + JsonObjectBuilder dimension = Json.createObjectBuilder().add("dimension", "type.name"); JsonArrayBuilder dimensions = Json.createArrayBuilder().add(dimension); builder.add("target", target).add("dimensions", dimensions); - return Json.createArrayBuilder().add(builder).build().toString(); + return Json.createArrayBuilder().add(builder).build(); } private void checkFacets(JsonObject responseObject, String dimension, List expectedLabels, @@ -1181,9 +1181,9 @@ private JsonArray searchInvestigations(Session session, String user, String text * For use with the new search/documents endpoint */ private JsonObject searchDatafiles(Session session, String user, String text, Date lower, Date upper, - List parameters, String searchAfter, int limit, String sort, String facets, int n) + List parameters, String searchAfter, int maxCount, String sort, JsonArray facets, int n) throws IcatException { - String responseString = session.searchDatafiles(user, text, lower, upper, parameters, searchAfter, limit, sort, + String responseString = session.searchDatafiles(user, text, lower, upper, parameters, searchAfter, maxCount, sort, facets); return checkResultsArraySize(n, responseString); } @@ -1192,9 +1192,9 @@ private JsonObject searchDatafiles(Session session, String user, String text, Da * For use with the new search/documents endpoint */ private JsonObject searchDatasets(Session session, String user, String text, Date lower, Date upper, - List parameters, String searchAfter, int limit, String sort, String facets, int n) + List parameters, String searchAfter, int maxCount, String sort, JsonArray facets, int n) throws IcatException { - String responseString = session.searchDatasets(user, text, lower, upper, parameters, searchAfter, limit, sort, + String responseString = session.searchDatasets(user, text, lower, upper, parameters, searchAfter, maxCount, sort, facets); return checkResultsArraySize(n, responseString); } @@ -1204,9 +1204,9 @@ private JsonObject searchDatasets(Session session, String user, String text, Dat */ private JsonObject searchInvestigations(Session session, String user, String text, Date lower, Date upper, List parameters, List samples, String userFullName, String searchAfter, - int limit, String sort, String facets, int n) throws IcatException { + int maxCount, String sort, JsonArray facets, int n) throws IcatException { String responseString = session.searchInvestigations(user, text, lower, upper, parameters, samples, - userFullName, searchAfter, limit, sort, facets); + userFullName, searchAfter, maxCount, sort, facets); return checkResultsArraySize(n, responseString); } From b257f8168425106bb98eb7372ba06c25a025c975 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Wed, 13 Jul 2022 15:58:28 +0100 Subject: [PATCH 27/51] Support searches on sample name #267 --- .../org/icatproject/core/entity/Datafile.java | 26 ++ .../org/icatproject/core/entity/Dataset.java | 15 +- .../core/entity/Investigation.java | 5 + .../org/icatproject/core/entity/Sample.java | 30 +-- .../icatproject/core/entity/SampleType.java | 21 +- .../core/manager/search/OpensearchApi.java | 88 ++++++- .../core/manager/search/SearchApi.java | 13 +- .../core/manager/TestSearchApi.java | 238 +++++++++++++----- .../org/icatproject/integration/TestRS.java | 109 ++++---- 9 files changed, 371 insertions(+), 174 deletions(-) diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index b58d158a..22a605ad 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -212,6 +212,11 @@ public void getDoc(JsonGenerator gen) { if (doi != null) { SearchApi.encodeString(gen, "doi", doi); } + if (fileSize != null) { + SearchApi.encodeLong(gen, "fileSize", fileSize); + } else { + SearchApi.encodeLong(gen, "fileSize", -1L); + } if (datafileFormat != null) { datafileFormat.getDoc(gen); } @@ -226,10 +231,15 @@ public void getDoc(JsonGenerator gen) { if (dataset != null) { SearchApi.encodeString(gen, "dataset.id", dataset.id); SearchApi.encodeString(gen, "dataset.name", dataset.getName()); + Sample sample = dataset.getSample(); + if (sample != null) { + sample.getDoc(gen); + } Investigation investigation = dataset.getInvestigation(); if (investigation != null) { SearchApi.encodeString(gen, "investigation.id", investigation.id); SearchApi.encodeString(gen, "investigation.name", investigation.getName()); + SearchApi.encodeString(gen, "visitId", investigation.getVisitId()); } } } @@ -256,18 +266,34 @@ public static Map getDocumentFields() throws IcatExcepti eiHandler.getRelationshipsByName(Datafile.class).get("dataset"), eiHandler.getRelationshipsByName(Dataset.class).get("investigation") }; Relationship[] instrumentRelationships = { + eiHandler.getRelationshipsByName(Datafile.class).get("dataset"), + eiHandler.getRelationshipsByName(Dataset.class).get("investigation"), eiHandler.getRelationshipsByName(Investigation.class).get("investigationInstruments"), eiHandler.getRelationshipsByName(InvestigationInstrument.class).get("instrument") }; + Relationship[] sampleRelationships = { + eiHandler.getRelationshipsByName(Datafile.class).get("dataset"), + eiHandler.getRelationshipsByName(Dataset.class).get("sample"), + eiHandler.getRelationshipsByName(Sample.class).get("type") }; + Relationship[] sampleTypeRelationships = { + eiHandler.getRelationshipsByName(Datafile.class).get("dataset"), + eiHandler.getRelationshipsByName(Dataset.class).get("sample") }; documentFields.put("name", null); documentFields.put("description", null); documentFields.put("location", null); documentFields.put("doi", null); documentFields.put("date", null); + documentFields.put("fileSize", null); documentFields.put("id", null); documentFields.put("dataset.id", null); documentFields.put("dataset.name", datasetRelationships); + documentFields.put("sample.id", datasetRelationships); + documentFields.put("sample.name", sampleRelationships); + documentFields.put("sample.investigation.id", sampleRelationships); + documentFields.put("sample.type.id", sampleRelationships); + documentFields.put("sample.type.name", sampleTypeRelationships); documentFields.put("investigation.id", datasetRelationships); documentFields.put("investigation.name", investigationRelationships); + documentFields.put("visitId", investigationRelationships); documentFields.put("datafileFormat.id", null); documentFields.put("datafileFormat.name", datafileFormatRelationships); documentFields.put("InvestigationInstrument instrument.id", instrumentRelationships); diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index 43c12057..b170894e 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -199,28 +199,32 @@ public void getDoc(JsonGenerator gen) { } if (startDate != null) { SearchApi.encodeLong(gen, "startDate", startDate); + SearchApi.encodeLong(gen, "date", startDate); } else { SearchApi.encodeLong(gen, "startDate", createTime); + SearchApi.encodeLong(gen, "date", createTime); } if (endDate != null) { SearchApi.encodeLong(gen, "endDate", endDate); } else { SearchApi.encodeLong(gen, "endDate", modTime); } + SearchApi.encodeLong(gen, "fileSize", -1L); // This is a placeholder to allow us to dynamically build size SearchApi.encodeString(gen, "id", id); - SearchApi.encodeString(gen, "investigation.id", investigation.id); - SearchApi.encodeString(gen, "investigation.name", investigation.getName()); if (investigation != null) { + SearchApi.encodeString(gen, "investigation.id", investigation.id); + SearchApi.encodeString(gen, "investigation.name", investigation.getName()); + SearchApi.encodeString(gen, "investigation.title", investigation.getTitle()); + SearchApi.encodeString(gen, "visitId", investigation.getVisitId()); if (investigation.getStartDate() != null) { SearchApi.encodeLong(gen, "investigation.startDate", investigation.getStartDate()); } else if (investigation.getCreateTime() != null) { SearchApi.encodeLong(gen, "investigation.startDate", investigation.getCreateTime()); } - SearchApi.encodeString(gen, "investigation.title", investigation.getTitle()); } if (sample != null) { - sample.getDoc(gen, "sample."); + sample.getDoc(gen); } type.getDoc(gen); } @@ -253,11 +257,14 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("doi", null); documentFields.put("startDate", null); documentFields.put("endDate", null); + documentFields.put("date", null); + documentFields.put("fileSize", null); documentFields.put("id", null); documentFields.put("investigation.id", null); documentFields.put("investigation.title", investigationRelationships); documentFields.put("investigation.name", investigationRelationships); documentFields.put("investigation.startDate", investigationRelationships); + documentFields.put("visitId", investigationRelationships); documentFields.put("sample.id", null); documentFields.put("sample.name", sampleRelationships); documentFields.put("sample.investigation.id", sampleRelationships); diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index 34294931..59c0111a 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -278,8 +278,10 @@ public void getDoc(JsonGenerator gen) { if (startDate != null) { SearchApi.encodeLong(gen, "startDate", startDate); + SearchApi.encodeLong(gen, "date", startDate); } else { SearchApi.encodeLong(gen, "startDate", createTime); + SearchApi.encodeLong(gen, "date", createTime); } if (endDate != null) { @@ -287,6 +289,7 @@ public void getDoc(JsonGenerator gen) { } else { SearchApi.encodeLong(gen, "endDate", modTime); } + SearchApi.encodeLong(gen, "fileSize", -1L); // This is a placeholder to allow us to dynamically build size SearchApi.encodeString(gen, "id", id); facility.getDoc(gen); @@ -328,6 +331,8 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("doi", null); documentFields.put("startDate", null); documentFields.put("endDate", null); + documentFields.put("date", null); + documentFields.put("fileSize", null); documentFields.put("id", null); documentFields.put("facility.name", facilityRelationships); documentFields.put("facility.id", null); diff --git a/src/main/java/org/icatproject/core/entity/Sample.java b/src/main/java/org/icatproject/core/entity/Sample.java index 3d6a816f..cd62781c 100644 --- a/src/main/java/org/icatproject/core/entity/Sample.java +++ b/src/main/java/org/icatproject/core/entity/Sample.java @@ -2,7 +2,10 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; @@ -43,6 +46,9 @@ public class Sample extends EntityBaseBean implements Serializable { @ManyToOne(fetch = FetchType.LAZY) private SampleType type; + public static Set docFields = new HashSet<>( + Arrays.asList("sample.name", "sample.id", "sample.investigation.id")); + /* Needed for JPA */ public Sample() { } @@ -97,30 +103,12 @@ public void setType(SampleType type) { @Override public void getDoc(JsonGenerator gen) { - SearchApi.encodeString(gen, "name", name); - SearchApi.encodeString(gen, "id", id); - SearchApi.encodeString(gen, "investigation.id", investigation.id); + SearchApi.encodeString(gen, "sample.name", name); + SearchApi.encodeString(gen, "sample.id", id); + SearchApi.encodeString(gen, "sample.investigation.id", investigation.id); if (type != null) { type.getDoc(gen); } } - /** - * Alternative method for encoding that applies a prefix to potentially - * ambiguous fields: "id" and "investigation.id". In the case of a single - * Dataset Sample, these fields will already be used by the Dataset and so - * cannot be overwritten by the Sample. - * - * @param gen JsonGenerator - * @param prefix String to precede all ambiguous field names. - */ - public void getDoc(JsonGenerator gen, String prefix) { - SearchApi.encodeString(gen, prefix + "name", name); - SearchApi.encodeString(gen, prefix + "id", id); - SearchApi.encodeString(gen, prefix + "investigation.id", investigation.id); - if (type != null) { - type.getDoc(gen, prefix); - } - } - } diff --git a/src/main/java/org/icatproject/core/entity/SampleType.java b/src/main/java/org/icatproject/core/entity/SampleType.java index a8033086..b9b2d16f 100644 --- a/src/main/java/org/icatproject/core/entity/SampleType.java +++ b/src/main/java/org/icatproject/core/entity/SampleType.java @@ -46,7 +46,7 @@ public class SampleType extends EntityBaseBean implements Serializable { @OneToMany(cascade = CascadeType.ALL, mappedBy = "type") private List samples = new ArrayList<>(); - public static Set docFields = new HashSet<>(Arrays.asList("sample.type.name", "type.id")); + public static Set docFields = new HashSet<>(Arrays.asList("sample.type.name", "sample.type.id")); /* Needed for JPA */ public SampleType() { @@ -94,23 +94,8 @@ public void setSamples(List samples) { @Override public void getDoc(JsonGenerator gen) { - SearchApi.encodeString(gen, "type.name", name); - SearchApi.encodeString(gen, "type.id", id); - } - - - /** - * Alternative method for encoding that applies a prefix to potentially - * ambiguous fields: "type.id". In the case of a single - * Dataset Sample, this fields will already be used by the Dataset and so - * cannot be overwritten by the Sample. - * - * @param gen JsonGenerator - * @param prefix String to precede all ambiguous field names. - */ - public void getDoc(JsonGenerator gen, String prefix) { - SearchApi.encodeString(gen, prefix + "type.name", name); - SearchApi.encodeString(gen, prefix + "type.id", id); + SearchApi.encodeString(gen, "sample.type.name", name); + SearchApi.encodeString(gen, "sample.type.id", id); } } diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index b439e5ef..de65fa8c 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -47,6 +47,7 @@ import org.icatproject.core.entity.Facility; import org.icatproject.core.entity.InvestigationType; import org.icatproject.core.entity.ParameterType; +import org.icatproject.core.entity.Sample; import org.icatproject.core.entity.SampleType; import org.icatproject.core.entity.User; import org.icatproject.core.manager.Rest; @@ -120,9 +121,15 @@ public ParentRelation(RelationType relationType, String parentName, String joinF new HashSet<>(Arrays.asList("investigation.name", "investigation.id"))))); relations.put("dataset", Arrays.asList( new ParentRelation(RelationType.CHILD, "datafile", "dataset", - new HashSet<>(Arrays.asList("dataset.name", "dataset.id"))))); + new HashSet<>(Arrays.asList("dataset.name", "dataset.id", "sample.id"))))); relations.put("user", Arrays.asList( new ParentRelation(RelationType.CHILD, "instrumentscientist", "user", User.docFields))); + relations.put("sample", Arrays.asList( + new ParentRelation(RelationType.CHILD, "dataset", "sample", Sample.docFields), + new ParentRelation(RelationType.CHILD, "datafile", "sample", Sample.docFields))); + relations.put("sampletype", Arrays.asList( + new ParentRelation(RelationType.CHILD, "dataset", "sample.type", SampleType.docFields), + new ParentRelation(RelationType.CHILD, "datafile", "sample.type", SampleType.docFields))); // Nested children are indexed as an array of objects on their parent entity, // and know their parent's id (N.B. InvestigationUsers are also mapped to @@ -141,6 +148,7 @@ public ParentRelation(RelationType relationType, String parentName, String joinF new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null), new ParentRelation(RelationType.NESTED_CHILD, "dataset", "investigation", null), new ParentRelation(RelationType.NESTED_CHILD, "datafile", "investigation", null))); + // TODO needs to strip the openining sample. relations.put("sample", Arrays.asList( new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null))); @@ -172,7 +180,7 @@ public ParentRelation(RelationType relationType, String parentName, String joinF defaultFieldsMap.put("_all", new ArrayList<>()); defaultFieldsMap.put("datafile", - Arrays.asList("name", "description", "doi", "location", "datafileFormat.name")); + Arrays.asList("name", "description", "doi", "location", "datafileFormat.name", "sample.name")); defaultFieldsMap.put("dataset", Arrays.asList("name", "description", "doi", "sample.name", "sample.type.name", "type.name")); defaultFieldsMap.put("investigation", @@ -214,8 +222,9 @@ private static JsonObject buildMappings(String index) { propertiesBuilder .add("type.id", typeLong) .add("facility.id", typeLong) + .add("fileSize", typeLong) .add("sample", buildNestedMapping("investigation.id", "type.id")) - .add("investigationparameter", buildNestedMapping("investigation.id", "type.id")) + .add("investigationparameter", buildNestedMapping("investigation.id", "type.ichd")) .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")); } else if (index.equals("dataset")) { @@ -223,6 +232,9 @@ private static JsonObject buildMappings(String index) { .add("investigation.id", typeLong) .add("type.id", typeLong) .add("sample.id", typeLong) + .add("sample.investigaion.id", typeLong) + .add("sample.type.id", typeLong) + .add("fileSize", typeLong) .add("datasetparameter", buildNestedMapping("dataset.id", "type.id")) .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")); @@ -230,6 +242,9 @@ private static JsonObject buildMappings(String index) { propertiesBuilder .add("investigation.id", typeLong) .add("datafileFormat.id", typeLong) + .add("sample.investigaion.id", typeLong) + .add("sample.type.id", typeLong) + .add("fileSize", typeLong) .add("datafileparameter", buildNestedMapping("datafile.id", "type.id")) .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")); @@ -674,11 +689,19 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query if (queryRequest.containsKey("text")) { // The free text is the only element we perform scoring on, so "must" occur - JsonArrayBuilder mustBuilder = Json.createArrayBuilder(); - mustBuilder - .add(OpensearchQueryBuilder.buildStringQuery(queryRequest.getString("text"), - defaultFields.toArray(new String[0]))); - boolBuilder.add("must", mustBuilder); + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + String text = queryRequest.getString("text"); + arrayBuilder.add(OpensearchQueryBuilder.buildStringQuery(text, defaultFields.toArray(new String[0]))); + if (index.equals("investigation")) { + JsonObject stringQuery = OpensearchQueryBuilder.buildStringQuery(text, "sample.name", + "sample.type.name"); + arrayBuilder.add(OpensearchQueryBuilder.buildNestedQuery("sample", stringQuery)); + JsonObjectBuilder textBoolBuilder = Json.createObjectBuilder().add("should", arrayBuilder); + JsonObjectBuilder textMustBuilder = Json.createObjectBuilder().add("bool", textBoolBuilder); + boolBuilder.add("must", Json.createArrayBuilder().add(textMustBuilder)); + } else { + boolBuilder.add("must", arrayBuilder); + } } Long lowerTime = parseDate(queryRequest, "lower", 0, Long.MIN_VALUE); @@ -1011,6 +1034,31 @@ private void extractFromInvestigation(CloseableHttpClient httpclient, String inv } } } + if (responseObject.containsKey("sample")) { + JsonArray jsonArray = responseObject.getJsonArray("sample"); + for (String index : new String[] { "datafile", "dataset" }) { + URI uri = new URIBuilder(server).setPath("/" + index + "/_update_by_query").build(); + HttpPost httpPost = new HttpPost(uri); + for (JsonObject sampleObject : jsonArray.getValuesAs(JsonObject.class)) { + JsonObjectBuilder paramsBuilder = Json.createObjectBuilder(); + JsonObjectBuilder scriptBuilder = Json.createObjectBuilder(); + String sampleId = sampleObject.getString("id"); + for (String field : sampleObject.keySet()) { + paramsBuilder.add(field, sampleObject.get(field)); + } + scriptBuilder.add("id", "update_sample").add("params", paramsBuilder); + JsonObject queryObject = OpensearchQueryBuilder.buildTermQuery("sample.id", sampleId); + JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); + String body = bodyBuilder.add("query", queryObject).add("script", scriptBuilder).build().toString(); + logger.trace("Making call {} with body {}", uri, body); + httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + commit(); + } + } + } + } } /** @@ -1028,14 +1076,25 @@ private void extractFromInvestigation(CloseableHttpClient httpclient, String inv * @param relation The relation between the nested entity and its * parent. * @throws URISyntaxException + * @throws IcatException */ private void modifyNestedEntity(StringBuilder sb, List updatesByQuery, String id, String index, JsonObject document, ModificationType modificationType, ParentRelation relation) - throws URISyntaxException { + throws URISyntaxException, IcatException { switch (modificationType) { case CREATE: - if (relation.parentName.equals(relation.joinField)) { + if (index.equals("sample")) { + // In order to make searching for sample information seamless between + // Investigations and Datasets/files, need to ensure that when nesting fields + // like "sample.name" under a "sample" object, we do not end up with + // "sample.sample.name" + JsonObjectBuilder documentBuilder = Json.createObjectBuilder(); + for (Entry entry : document.entrySet()) { + documentBuilder.add(entry.getKey().replace("sample.", ""), entry.getValue()); + } + createNestedEntity(sb, id, index, documentBuilder.build(), relation); + } else if (relation.parentName.equals(relation.joinField)) { // If the target parent is the same as the joining field, we're appending the // nested child to a list of objects which can be sent as a bulk update request // since we have the parent id @@ -1068,10 +1127,15 @@ private void modifyNestedEntity(StringBuilder sb, List updatesByQuery, * @param document JsonObject containing the key value pairs of the document * fields. * @param relation The relation between the nested entity and its parent. + * @throws IcatException If parentId is missing from document. */ private static void createNestedEntity(StringBuilder sb, String id, String index, JsonObject document, - ParentRelation relation) { - String parentId = document.getString(relation.parentName + ".id"); + ParentRelation relation) throws IcatException { + if (!document.containsKey(relation.joinField + ".id")) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + relation.joinField + ".id not found in " + document.toString()); + } + String parentId = document.getString(relation.joinField + ".id"); JsonObjectBuilder innerBuilder = Json.createObjectBuilder() .add("_id", parentId).add("_index", relation.parentName); // For nested 0:* relationships, wrap single documents in an array diff --git a/src/main/java/org/icatproject/core/manager/search/SearchApi.java b/src/main/java/org/icatproject/core/manager/search/SearchApi.java index 56b7fbc0..7be69ba1 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchApi.java @@ -109,12 +109,23 @@ public static void encodeDouble(JsonGenerator gen, String name, Double value) { * * @param gen JsonGenerator being used to encode. * @param name Name of the field. - * @param value Long value to encode as a long. + * @param value Date value to encode as a long. */ public static void encodeLong(JsonGenerator gen, String name, Date value) { gen.write(name, value.getTime()); } + /** + * Writes a key value pair to the JsonGenerator being used to encode an entity. + * + * @param gen JsonGenerator being used to encode. + * @param name Name of the field. + * @param value Long value to encode as a long. + */ + public static void encodeLong(JsonGenerator gen, String name, Long value) { + gen.write(name, value); + } + /** * Encodes the creation or updating of the provided entity as Json. * diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 5fc6bea1..8b977ec4 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -65,6 +65,7 @@ private class Filter { private String fld; private String value; private JsonArray array; + public Filter(String fld, String... values) { this.fld = fld; if (values.length == 1) { @@ -119,7 +120,7 @@ public static Iterable data() throws URISyntaxException, IcatExceptio * Utility function for building a Query from individual arguments */ public static JsonObject buildQuery(String target, String user, String text, Date lower, Date upper, - List parameters, List samples, String userFullName, Filter... filters) { + List parameters, String userFullName, Filter... filters) { JsonObjectBuilder builder = Json.createObjectBuilder(); if (target != null) { builder.add("target", target); @@ -165,17 +166,10 @@ public static JsonObject buildQuery(String target, String user, String text, Dat } builder.add("parameters", parametersBuilder); } - if (samples != null && !samples.isEmpty()) { - JsonArrayBuilder samplesBuilder = Json.createArrayBuilder(); - for (String sample : samples) { - samplesBuilder.add(sample); - } - builder.add("samples", samplesBuilder); - } if (userFullName != null) { builder.add("userFullName", userFullName); } - if (filters.length > 0 ) { + if (filters.length > 0) { JsonObjectBuilder filterBuilder = Json.createObjectBuilder(); for (Filter filter : filters) { if (filter.value != null) { @@ -538,9 +532,6 @@ private void populate() throws IcatException { if (sampleId >= NUMSAMP) { break; } - word = word("SType ", sampleId % 26); - Sample sample = sample(sampleId, word, investigation); - queue.add(SearchApi.encodeOperation("create", sample)); } for (int datasetBatch = 0; datasetBatch * NUMINV < NUMDS; datasetBatch++) { @@ -552,6 +543,14 @@ private void populate() throws IcatException { endDate = new Date(now + (datasetId + 1) * 60000); word = word("DS", datasetId % 26); Dataset dataset = dataset(datasetId, word, startDate, endDate, investigation); + + if (datasetId < NUMSAMP) { + word = word("SType ", datasetId); + Sample sample = sample(datasetId, word, investigation); + queue.add(SearchApi.encodeOperation("create", sample)); + dataset.setSample(sample); + } + queue.add(SearchApi.encodeOperation("create", dataset)); if (datasetId % 3 == 1) { @@ -628,7 +627,7 @@ public void datafiles() throws Exception { String sort; // Test size and searchAfter - JsonObject query = buildQuery("Datafile", null, null, null, null, null, null, null); + JsonObject query = buildQuery("Datafile", null, null, null, null, null, null); SearchResult lsr = searchApi.getResults(query, null, 5, null, datafileFields); JsonValue searchAfter = lsr.getSearchAfter(); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); @@ -685,50 +684,78 @@ public void datafiles() throws Exception { searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - query = buildQuery("Datafile", "e4", null, null, null, null, null, null); + query = buildQuery("Datafile", "e4", null, null, null, null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 1L, 6L, 11L, 16L, 21L, 26L, 31L, 36L, 41L, 46L, 51L, 56L, 61L, 66L, 71L, 76L, 81L, 86L, 91L, 96L); // Test instrumentScientists only see their data - query = buildQuery("Datafile", "scientist_0", null, null, null, null, null, null); + query = buildQuery("Datafile", "scientist_0", null, null, null, null, null); lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 2L, 4L, 6L, 8L); - query = buildQuery("Datafile", "e4", "dfbbb", null, null, null, null, null); + query = buildQuery("Datafile", "e4", "dfbbb", null, null, null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 1L); - query = buildQuery("Datafile", null, "dfbbb", null, null, null, null, null); + query = buildQuery("Datafile", null, "dfbbb", null, null, null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 1L, 27L, 53L, 79L); query = buildQuery("Datafile", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null); + new Date(now + 60000 * 6), null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 3L, 4L, 5L, 6L); query = buildQuery("Datafile", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null); + new Date(now + 60000 * 6), null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr); + + // Target visitId + query = buildQuery("Datafile", null, "visitId:visitId", null, null, null, null); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L); + query = buildQuery("Datafile", null, "visitId:qwerty", null, null, null, null); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr); + + // Target sample.name + query = buildQuery("Datafile", null, "sample.name:ddd", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 3L, 33L, 63L, 93L); + + // Multiple samples associated with investigation 3 + query = buildQuery("Datafile", null, "ddd nnn", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 3L, 13L, 33L, 43L, 63L, 73L, 93L); + + // By default, sample ddd OR sample mmm gives two + query = buildQuery("Datafile", null, "ddd mmm", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 3L, 12L, 33L, 42L, 63L, 72L, 93L); + + // AND logic should not return any results + query = buildQuery("Datafile", null, "+ddd +mmm", null, null, null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr); List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v25")); query = buildQuery("Datafile", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null); + new Date(now + 60000 * 6), pojos, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 5L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v25")); - query = buildQuery("Datafile", null, null, null, null, pojos, null, null); + query = buildQuery("Datafile", null, null, null, null, pojos, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 5L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, "u sss", null)); - query = buildQuery("Datafile", null, null, null, null, pojos, null, null); + query = buildQuery("Datafile", null, null, null, null, pojos, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 13L, 65L); } @@ -739,7 +766,7 @@ public void datasets() throws Exception { JsonObjectBuilder sortBuilder = Json.createObjectBuilder(); String sort; - JsonObject query = buildQuery("Dataset", null, null, null, null, null, null, null); + JsonObject query = buildQuery("Dataset", null, null, null, null, null, null); SearchResult lsr = searchApi.getResults(query, null, 5, null, datasetFields); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); checkDataset(lsr.getResults().get(0)); @@ -751,7 +778,7 @@ public void datasets() throws Exception { 25L, 26L, 27L, 28L, 29L); // Test searchAfter preserves the sorting of original search (asc) - sort = sortBuilder.add("startDate", "asc").build().toString(); + sort = sortBuilder.add("date", "asc").build().toString(); lsr = searchApi.getResults(query, 5, sort); checkOrder(lsr, 0L, 1L, 2L, 3L, 4L); searchAfter = lsr.getSearchAfter(); @@ -762,7 +789,7 @@ public void datasets() throws Exception { assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); // Test searchAfter preserves the sorting of original search (desc) - sort = sortBuilder.add("endDate", "desc").build().toString(); + sort = sortBuilder.add("date", "desc").build().toString(); lsr = searchApi.getResults(query, 5, sort); checkOrder(lsr, 29L, 28L, 27L, 26L, 25L); searchAfter = lsr.getSearchAfter(); @@ -778,64 +805,93 @@ public void datasets() throws Exception { checkOrder(lsr, 0L, 26L, 1L, 27L, 2L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - sort = sortBuilder.add("name", "asc").add("endDate", "desc").build().toString(); + sort = sortBuilder.add("name", "asc").add("date", "desc").build().toString(); lsr = searchApi.getResults(query, 5, sort); checkOrder(lsr, 26L, 0L, 27L, 1L, 28L); searchAfter = lsr.getSearchAfter(); assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); - lsr = searchApi.getResults(buildQuery("Dataset", "e4", null, null, null, null, null, null), 100, + lsr = searchApi.getResults(buildQuery("Dataset", "e4", null, null, null, null, null), 100, null); checkResults(lsr, 1L, 6L, 11L, 16L, 21L, 26L); // Test instrumentScientists only see their data - query = buildQuery("Dataset", "scientist_0", null, null, null, null, null, null); + query = buildQuery("Dataset", "scientist_0", null, null, null, null, null); lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 2L, 4L, 6L, 8L); // Test filter - query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("dataset.type.name", "type")); + query = buildQuery("Dataset", null, null, null, null, null, null, new Filter("dataset.type.name", "type")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); - query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("dataset.type.name", "type", "typo")); + query = buildQuery("Dataset", null, null, null, null, null, null, + new Filter("dataset.type.name", "type", "typo")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); - query = buildQuery("Dataset", null, null, null, null, null, null, null, new Filter("dataset.type.name", "typo")); + query = buildQuery("Dataset", null, null, null, null, null, null, new Filter("dataset.type.name", "typo")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr); - lsr = searchApi.getResults(buildQuery("Dataset", "e4", "dsbbb", null, null, null, null, null), 100, + lsr = searchApi.getResults(buildQuery("Dataset", "e4", "dsbbb", null, null, null, null), 100, null); checkResults(lsr, 1L); - lsr = searchApi.getResults(buildQuery("Dataset", null, "dsbbb", null, null, null, null, null), 100, + lsr = searchApi.getResults(buildQuery("Dataset", null, "dsbbb", null, null, null, null), 100, null); checkResults(lsr, 1L, 27L); lsr = searchApi.getResults(buildQuery("Dataset", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100, null); + new Date(now + 60000 * 6), null, null), 100, null); checkResults(lsr, 3L, 4L, 5L); lsr = searchApi.getResults(buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), null, null, null), 100, null); + new Date(now + 60000 * 6), null, null), 100, null); + checkResults(lsr, 3L); + + // Target visitId + query = buildQuery("Dataset", null, "visitId:visitId", null, null, null, null); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L); + query = buildQuery("Dataset", null, "visitId:qwerty", null, null, null, null); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr); + + // Target sample.name + query = buildQuery("Dataset", null, "sample.name:ddd", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 3L); + // Multiple samples associated with investigation 3 + query = buildQuery("Dataset", null, "ddd nnn", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 3L, 13L); + + // By default, sample ddd OR sample mmm gives two + query = buildQuery("Dataset", null, "ddd mmm", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 3L, 12L); + + // AND logic should not return any results + query = buildQuery("Dataset", null, "+ddd +mmm", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr); + List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v16")); - lsr = searchApi.getResults(buildQuery("Dataset", null, null, null, null, pojos, null, null), 100, + lsr = searchApi.getResults(buildQuery("Dataset", null, null, null, null, pojos, null), 100, null); checkResults(lsr, 4L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v16")); lsr = searchApi.getResults(buildQuery("Dataset", null, null, new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null), 100, null); + new Date(now + 60000 * 6), pojos, null), 100, null); checkResults(lsr, 4L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v16")); lsr = searchApi.getResults(buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), - new Date(now + 60000 * 6), pojos, null, null), 100, null); + new Date(now + 60000 * 6), pojos, null), 100, null); checkResults(lsr); } @@ -846,7 +902,7 @@ public void investigations() throws Exception { String sort; /* Blocked results */ - JsonObject query = buildQuery("Investigation", null, null, null, null, null, null, null); + JsonObject query = buildQuery("Investigation", null, null, null, null, null, null); SearchResult lsr = searchApi.getResults(query, null, 5, null, investigationFields); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); checkInvestigation(lsr.getResults().get(0)); @@ -858,7 +914,7 @@ public void investigations() throws Exception { assertNull(searchAfter); // Test searchAfter preserves the sorting of original search (asc) - sort = sortBuilder.add("startDate", "asc").build().toString(); + sort = sortBuilder.add("date", "asc").build().toString(); lsr = searchApi.getResults(query, 5, sort); checkOrder(lsr, 0L, 1L, 2L, 3L, 4L); searchAfter = lsr.getSearchAfter(); @@ -869,7 +925,7 @@ public void investigations() throws Exception { assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); // Test searchAfter preserves the sorting of original search (desc) - sort = sortBuilder.add("endDate", "desc").build().toString(); + sort = sortBuilder.add("date", "desc").build().toString(); lsr = searchApi.getResults(query, 5, sort); checkOrder(lsr, 9L, 8L, 7L, 6L, 5L); searchAfter = lsr.getSearchAfter(); @@ -880,62 +936,65 @@ public void investigations() throws Exception { assertNotNull(SEARCH_AFTER_NOT_NULL, searchAfter); // Test instrumentScientists only see their data - query = buildQuery("Investigation", "scientist_0", null, null, null, null, null, null); + query = buildQuery("Investigation", "scientist_0", null, null, null, null, null); lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 2L, 4L, 6L, 8L); // Test filter - query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("investigation.type.name", "type")); + query = buildQuery("Investigation", null, null, null, null, null, null, + new Filter("investigation.type.name", "type")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); - query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("investigation.type.name", "type", "typo")); + query = buildQuery("Investigation", null, null, null, null, null, null, + new Filter("investigation.type.name", "type", "typo")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr, 0L, 1L, 2L, 3L, 4L); - query = buildQuery("Investigation", null, null, null, null, null, null, null, new Filter("investigation.type.name", "typo")); + query = buildQuery("Investigation", null, null, null, null, null, null, + new Filter("investigation.type.name", "typo")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr); - query = buildQuery("Investigation", null, null, null, null, null, null, "b"); + query = buildQuery("Investigation", null, null, null, null, null, "b"); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 1L, 3L, 5L, 7L, 9L); - query = buildQuery("Investigation", null, null, null, null, null, null, "FN"); + query = buildQuery("Investigation", null, null, null, null, null, "FN"); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 1L, 3L, 4L, 5L, 6L, 7L, 9L); - query = buildQuery("Investigation", null, null, null, null, null, null, "FN AND \"b b\""); + query = buildQuery("Investigation", null, null, null, null, null, "FN AND \"b b\""); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 1L, 3L, 5L, 7L, 9L); - query = buildQuery("Investigation", "b1", null, null, null, null, null, "b"); + query = buildQuery("Investigation", "b1", null, null, null, null, "b"); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 1L, 3L, 5L, 7L, 9L); - query = buildQuery("Investigation", "c1", null, null, null, null, null, "b"); + query = buildQuery("Investigation", "c1", null, null, null, null, "b"); lsr = searchApi.getResults(query, 100, null); checkResults(lsr); - query = buildQuery("Investigation", null, "l v", null, null, null, null, null); + query = buildQuery("Investigation", null, "l v", null, null, null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 4L); - query = buildQuery("Investigation", "b1", "d", null, null, null, null, "b"); + query = buildQuery("Investigation", "b1", "d", null, null, null, "b"); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 3L); query = buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), - null, null, "b"); + null, "b"); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 3L); query = buildQuery("Investigation", null, null, new Date(now + 60000 * 3), new Date(now + 60000 * 6), - null, null, null); + null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 3L, 4L, 5L); List pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); - query = buildQuery("Investigation", null, null, null, null, pojos, null, null); + query = buildQuery("Investigation", null, null, null, null, pojos, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 3L); @@ -949,38 +1008,79 @@ public void investigations() throws Exception { pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); query = buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), - pojos, null, "b"); + pojos, "b"); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 3L); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO(null, null, "v9")); pojos.add(new ParameterPOJO(null, null, "v81")); - query = buildQuery("Investigation", null, null, null, null, pojos, null, null); + query = buildQuery("Investigation", null, null, null, null, pojos, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); - query = buildQuery("Investigation", null, null, null, null, pojos, null, null); + query = buildQuery("Investigation", null, null, null, null, pojos, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 3L); + + // Target visitId + query = buildQuery("Investigation", null, "visitId:visitId", null, null, null, null); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L); + query = buildQuery("Investigation", null, "visitId:qwerty", null, null, null, null); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr); + + // Target sample.name + query = buildQuery("Investigation", null, "sample.name:ddd", null, null, null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 3L); - List samples = Arrays.asList("ddd", "nnn"); - query = buildQuery("Investigation", null, null, null, null, null, samples, null); + // Multiple samples associated with investigation 3 + query = buildQuery("Investigation", null, "ddd nnn", null, null, null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 3L); - samples = Arrays.asList("ddd", "mmm"); - query = buildQuery("Investigation", null, null, null, null, null, samples, null); + // By default, sample ddd OR sample mmm gives two investigations + query = buildQuery("Investigation", null, "ddd mmm", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 2L, 3L); + + // AND logic should not return any results + query = buildQuery("Investigation", null, "+ddd +mmm", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr); + + // Fields on Investigation and Sample + query = buildQuery("Investigation", null, "visitId ddd", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); + // ID 3 should be most relevant since it matches both terms + lsr = searchApi.getResults(query, 1, null); + checkResults(lsr, 3L); + // Specifying fields should not alter behaviour + query = buildQuery("Investigation", null, "visitId:visitId sample.name:ddd", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); + // Individual MUST should work when applied to either an Investigation or Sample field + query = buildQuery("Investigation", null, "+visitId:visitId", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); + query = buildQuery("Investigation", null, "+sample.name:ddd", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 3L); + // This query is expected to fail, as we apply both terms to Investigation and Sample + // (since we have no fields) and neither possesses both terms. + query = buildQuery("Investigation", null, "+visitId +ddd", null, null, null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); - samples = Arrays.asList("ddd", "nnn"); - query = buildQuery("Investigation", "b1", "d", new Date(now + 60000 * 3), new Date(now + 60000 * 6), - pojos, samples, "b"); + query = buildQuery("Investigation", "b1", "d ddd nnnn", new Date(now + 60000 * 3), new Date(now + 60000 * 6), + pojos, "b"); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 3L); } @@ -1027,10 +1127,10 @@ public void modifyDatafile() throws IcatException { rhinoDatafile.setDatafileFormat(pdfFormat); // Build queries - JsonObject elephantQuery = buildQuery("Datafile", null, "elephant", null, null, null, null, null); - JsonObject rhinoQuery = buildQuery("Datafile", null, "rhino", null, null, null, null, null); - JsonObject pdfQuery = buildQuery("Datafile", null, "datafileFormat.name:pdf", null, null, null, null, null); - JsonObject pngQuery = buildQuery("Datafile", null, "datafileFormat.name:png", null, null, null, null, null); + JsonObject elephantQuery = buildQuery("Datafile", null, "elephant", null, null, null, null); + JsonObject rhinoQuery = buildQuery("Datafile", null, "rhino", null, null, null, null); + JsonObject pdfQuery = buildQuery("Datafile", null, "datafileFormat.name:pdf", null, null, null, null); + JsonObject pngQuery = buildQuery("Datafile", null, "datafileFormat.name:png", null, null, null, null); JsonObject lowRange = buildFacetRangeObject("low", 0L, 2L); JsonObject highRange = buildFacetRangeObject("high", 2L, 4L); JsonObject rangeFacetRequest = buildFacetRangeRequest(buildFacetIdQuery("42"), "date", lowRange, highRange); diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 575e6562..03aeb188 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -579,9 +579,6 @@ public void testLuceneInvestigations() throws Exception { Date upperOrigin = dft.parse("2011-12-31T23:59:59+0000"); Date upperSecond = dft.parse("2011-12-31T23:59:58+0000"); Date upperMinute = dft.parse("2011-12-31T23:58:00+0000"); - List samplesAnd = Arrays.asList("ford AND rust", "koh* AND diamond"); - List samplesPlus = Arrays.asList("ford + rust", "koh + diamond"); - List samplesBad = Arrays.asList("ford AND rust", "kog* AND diamond"); String textAnd = "title AND one"; String textTwo = "title AND two"; String textPlus = "title + one"; @@ -592,7 +589,7 @@ public void testLuceneInvestigations() throws Exception { parameters.add(new ParameterForLucene("colour", "name", "green")); JsonArray array = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, - samplesAnd, "Professor", 20, 1); + null, "Professor", 20, 1); checkResultFromLuceneSearch(session, "one", array, "Investigation", "visitId"); // change user @@ -617,15 +614,10 @@ public void testLuceneInvestigations() throws Exception { // Change parameters List badParameters = new ArrayList<>(); badParameters.add(new ParameterForLucene("color", "name", "green")); - searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, badParameters, samplesPlus, null, 20, - 0); - - // Change samples - searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, samplesBad, null, 20, 0); + searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, badParameters, null, null, 20, 0); // Change userFullName - searchInvestigations(session, "db/tr", textPlus, lowerOrigin, upperOrigin, parameters, samplesAnd, "Doctor", 20, - 0); + searchInvestigations(session, "db/tr", textPlus, lowerOrigin, upperOrigin, parameters, null, "Doctor", 20, 0); // Try provoking an error badParameters = new ArrayList<>(); @@ -669,7 +661,8 @@ public void testSearchDatafiles() throws Exception { checkResultsSource(responseObject, Arrays.asList(expectation), true); // Try sorting and searchAfter - String sort = Json.createObjectBuilder().add("name", "desc").add("date", "asc").build().toString(); + String sort = Json.createObjectBuilder().add("name", "desc").add("date", "asc").add("fileSize", "asc").build() + .toString(); responseObject = searchDatafiles(session, null, null, null, null, null, null, 1, sort, null, 1); searchAfter = responseObject.get("search_after"); assertNotNull(searchAfter); @@ -798,12 +791,14 @@ public void testSearchDatasets() throws Exception { checkResultsSource(responseObject, Arrays.asList(expectation), true); // Try sorting and searchAfter - String sort = Json.createObjectBuilder().add("name", "desc").add("startDate", "asc").build().toString(); + String sort = Json.createObjectBuilder().add("name", "desc").add("date", "asc").add("fileSize", "asc").build() + .toString(); responseObject = searchDatasets(session, null, null, null, null, null, null, 1, sort, null, 1); searchAfter = responseObject.get("search_after"); assertNotNull(searchAfter); expectation.put("name", "ds4"); checkResultsSource(responseObject, Arrays.asList(expectation), false); + responseObject = searchDatasets(session, null, null, null, null, null, searchAfter.toString(), 1, sort, null, 1); searchAfter = responseObject.get("search_after"); @@ -873,6 +868,7 @@ public void testSearchDatasets() throws Exception { public void testSearchInvestigations() throws Exception { Session session = setupLuceneTest(); JsonObject responseObject; + JsonValue searchAfter; Map expectation = new HashMap<>(); expectation.put("name", "expt1"); expectation.put("startDate", "notNull"); @@ -886,120 +882,133 @@ public void testSearchInvestigations() throws Exception { Date upperOrigin = dft.parse("2011-12-31T23:59:59+0000"); Date upperSecond = dft.parse("2011-12-31T23:59:58+0000"); Date upperMinute = dft.parse("2011-12-31T23:58:00+0000"); - List samplesAnd = Arrays.asList("ford AND rust", "koh* AND diamond"); - List samplesPlus = Arrays.asList("ford + rust", "koh + diamond"); - List samplesBad = Arrays.asList("ford AND rust", "kog* AND diamond"); + String samplesSingular = "sample.name:ford AND sample.type.name:rust"; + String samplesMultiple = "sample.name:ford sample.type.name:rust sample.name:koh sample.type.name:diamond"; + String samplesBad = "sample.name:kog* AND sample.type.name:diamond"; String textAnd = "title AND one"; String textTwo = "title AND two"; String textPlus = "title + one"; - searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, null, 3); + searchInvestigations(session, null, null, null, null, null, null, null, 10, null, null, 3); List parameters = new ArrayList<>(); parameters.add(new ParameterForLucene("colour", "name", "green")); responseObject = searchInvestigations(session, "db/tr", null, lowerOrigin, upperOrigin, null, - null, null, null, 10, null, null, 1); + null, null, 10, null, null, 1); responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, null, - null, null, null, 10, null, null, 1); - responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, - null, null, null, 10, null, null, 1); + null, null, 10, null, null, 1); responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, - null, "Professor", null, 10, null, null, 1); + null, null, 10, null, null, 1); responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, - samplesAnd, "Professor", null, 10, null, null, 1); + "Professor", null, 10, null, null, 1); assertFalse(responseObject.containsKey("search_after")); checkResultsSource(responseObject, Arrays.asList(expectation), true); // change user - searchInvestigations(session, "db/fred", textAnd, null, null, parameters, null, null, null, 10, null, null, 0); + searchInvestigations(session, "db/fred", textAnd, null, null, parameters, null, null, 10, null, null, 0); // change text - searchInvestigations(session, "db/tr", textTwo, null, null, parameters, null, null, null, 10, null, null, 0); + searchInvestigations(session, "db/tr", textTwo, null, null, parameters, null, null, 10, null, null, 0); // Only working to a minute - responseObject = searchInvestigations(session, "db/tr", textAnd, lowerSecond, upperOrigin, parameters, null, + responseObject = searchInvestigations(session, "db/tr", textAnd, lowerSecond, upperOrigin, parameters, null, null, 10, null, null, 1); assertFalse(responseObject.containsKey("search_after")); checkResultsSource(responseObject, Arrays.asList(expectation), true); - responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperSecond, parameters, null, + responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperSecond, parameters, null, null, 10, null, null, 1); assertFalse(responseObject.containsKey("search_after")); checkResultsSource(responseObject, Arrays.asList(expectation), true); - searchInvestigations(session, "db/tr", textAnd, lowerMinute, upperOrigin, parameters, null, null, null, + searchInvestigations(session, "db/tr", textAnd, lowerMinute, upperOrigin, parameters, null, null, 10, null, null, 0); - searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperMinute, parameters, null, null, null, + searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperMinute, parameters, null, null, 10, null, null, 0); // Change parameters List badParameters = new ArrayList<>(); badParameters.add(new ParameterForLucene("color", "name", "green")); - searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, badParameters, samplesPlus, null, + searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, badParameters, null, null, 10, null, null, 0); // Change samples - searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, samplesBad, null, null, + searchInvestigations(session, "db/tr", samplesSingular, lowerOrigin, upperOrigin, parameters, null, null, + 10, null, null, 1); + searchInvestigations(session, "db/tr", samplesMultiple, lowerOrigin, upperOrigin, parameters, null, null, + 10, null, null, 1); + searchInvestigations(session, "db/tr", samplesBad, lowerOrigin, upperOrigin, parameters, null, null, 10, null, null, 0); // Change userFullName - searchInvestigations(session, "db/tr", textPlus, lowerOrigin, upperOrigin, parameters, samplesAnd, "Doctor", + searchInvestigations(session, "db/tr", textPlus, lowerOrigin, upperOrigin, parameters, "Doctor", null, 10, null, null, 0); + // Try sorting and searchAfter + // Note as all the investigations have the same name/date, we cannot + // meaningfully sort them, however still check that the search succeeds in + // returning a non-null searchAfter object + String sort = Json.createObjectBuilder().add("name", "desc").add("date", "asc").add("fileSize", "asc").build() + .toString(); + responseObject = searchInvestigations(session, null, null, null, null, null, null, null, 1, sort, null, 1); + searchAfter = responseObject.get("search_after"); + assertNotNull(searchAfter); + checkResultsSource(responseObject, Arrays.asList(expectation), false); + // Test that changes to the public steps/tables are reflected in returned fields PublicStep ps = new PublicStep(); ps.setOrigin("Investigation"); ps.setField("type"); ps.setId(wSession.create(ps)); - responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, + responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, 10, null, null, 1); assertFalse(responseObject.containsKey("search_after")); expectation.put("type.name", "atype"); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.addRule(null, "Facility", "R"); - responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, + responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, 10, null, null, 1); assertFalse(responseObject.containsKey("search_after")); expectation.put("facility.name", "Test port facility"); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.delete(ps); - responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, + responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, 10, null, null, 1); assertFalse(responseObject.containsKey("search_after")); expectation.put("type.name", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); wSession.delRule(null, "Facility", "R"); - responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, null, 10, null, + responseObject = searchInvestigations(session, null, textAnd, null, null, null, null, null, 10, null, null, 1); assertFalse(responseObject.containsKey("search_after")); expectation.put("facility.name", null); checkResultsSource(responseObject, Arrays.asList(expectation), true); // Test searching with someone without authz for the Investigation(s) - searchInvestigations(piSession(), null, null, null, null, null, null, null, null, 10, null, null, 0); + searchInvestigations(piSession(), null, null, null, null, null, null, null, 10, null, null, 0); // Test facets match on Investigations JsonArray facets = buildFacetRequest("Investigation"); - responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, + responseObject = searchInvestigations(session, null, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); checkFacets(responseObject, "Investigation.type.name", Arrays.asList("atype"), Arrays.asList(3L)); // Test no facets match on InvestigationParameters due to lack of READ access facets = buildFacetRequest("InvestigationParameter"); - responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, + responseObject = searchInvestigations(session, null, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); assertFalse(NO_DIMENSIONS, responseObject.containsKey("dimensions")); // Test facets match on InvestigationParameters wSession.addRule(null, "InvestigationParameter", "R"); - responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets, + responseObject = searchInvestigations(session, null, null, null, null, null, null, null, 10, null, facets, 3); assertFalse(responseObject.containsKey("search_after")); checkFacets(responseObject, "InvestigationParameter.type.name", Arrays.asList("colour"), @@ -1013,7 +1022,7 @@ public void testSearchParameterValidation() throws Exception { badParameters = Arrays.asList(new ParameterForLucene(null, null, null)); try { - searchInvestigations(session, null, null, null, null, badParameters, null, null, null, 10, null, null, 0); + searchInvestigations(session, null, null, null, null, badParameters, null, null, 10, null, null, 0); fail("BAD_PARAMETER exception not caught"); } catch (IcatException e) { assertEquals(IcatExceptionType.BAD_PARAMETER, e.getType()); @@ -1022,7 +1031,7 @@ public void testSearchParameterValidation() throws Exception { badParameters = Arrays.asList(new ParameterForLucene("color", null, null)); try { - searchInvestigations(session, null, null, null, null, badParameters, null, null, null, 10, null, null, 0); + searchInvestigations(session, null, null, null, null, badParameters, null, null,10, null, null, 0); fail("BAD_PARAMETER exception not caught"); } catch (IcatException e) { assertEquals(IcatExceptionType.BAD_PARAMETER, e.getType()); @@ -1031,7 +1040,7 @@ public void testSearchParameterValidation() throws Exception { badParameters = Arrays.asList(new ParameterForLucene("color", "string", null)); try { - searchInvestigations(session, null, null, null, null, badParameters, null, null, null, 10, null, null, 0); + searchInvestigations(session, null, null, null, null, badParameters, null, null, 10, null, null, 0); fail("BAD_PARAMETER exception not caught"); } catch (IcatException e) { assertEquals(IcatExceptionType.BAD_PARAMETER, e.getType()); @@ -1183,7 +1192,8 @@ private JsonArray searchInvestigations(Session session, String user, String text private JsonObject searchDatafiles(Session session, String user, String text, Date lower, Date upper, List parameters, String searchAfter, int maxCount, String sort, JsonArray facets, int n) throws IcatException { - String responseString = session.searchDatafiles(user, text, lower, upper, parameters, searchAfter, maxCount, sort, + String responseString = session.searchDatafiles(user, text, lower, upper, parameters, searchAfter, maxCount, + sort, facets); return checkResultsArraySize(n, responseString); } @@ -1194,7 +1204,8 @@ private JsonObject searchDatafiles(Session session, String user, String text, Da private JsonObject searchDatasets(Session session, String user, String text, Date lower, Date upper, List parameters, String searchAfter, int maxCount, String sort, JsonArray facets, int n) throws IcatException { - String responseString = session.searchDatasets(user, text, lower, upper, parameters, searchAfter, maxCount, sort, + String responseString = session.searchDatasets(user, text, lower, upper, parameters, searchAfter, maxCount, + sort, facets); return checkResultsArraySize(n, responseString); } @@ -1203,10 +1214,10 @@ private JsonObject searchDatasets(Session session, String user, String text, Dat * For use with the new search/documents endpoint */ private JsonObject searchInvestigations(Session session, String user, String text, Date lower, Date upper, - List parameters, List samples, String userFullName, String searchAfter, - int maxCount, String sort, JsonArray facets, int n) throws IcatException { - String responseString = session.searchInvestigations(user, text, lower, upper, parameters, samples, - userFullName, searchAfter, maxCount, sort, facets); + List parameters, String userFullName, String searchAfter, int maxCount, String sort, + JsonArray facets, int n) throws IcatException { + String responseString = session.searchInvestigations(user, text, lower, upper, parameters, userFullName, + searchAfter, maxCount, sort, facets); return checkResultsArraySize(n, responseString); } From c7d2a7ae1dee7d8fafacaa0c3e19ed177a23385b Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 22 Jul 2022 13:28:35 +0100 Subject: [PATCH 28/51] SampleParameter, fileCount, value in range #267 --- .../org/icatproject/core/entity/Datafile.java | 10 +- .../org/icatproject/core/entity/Dataset.java | 4 +- .../core/entity/Investigation.java | 10 +- .../icatproject/core/entity/Parameter.java | 6 + .../core/entity/SampleParameter.java | 8 + .../core/manager/EntityBeanManager.java | 153 ++++-- .../core/manager/PropertyHandler.java | 2 +- .../core/manager/search/OpensearchApi.java | 476 +++++++++++++++--- .../search/OpensearchQueryBuilder.java | 69 ++- .../search/OpensearchScriptBuilder.java | 92 +++- .../core/manager/search/SearchManager.java | 37 +- .../core/manager/TestEntityInfo.java | 2 +- .../core/manager/TestSearchApi.java | 159 ++++++ .../org/icatproject/integration/TestRS.java | 2 + 14 files changed, 902 insertions(+), 128 deletions(-) diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index 22a605ad..e41a84d9 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -215,8 +215,9 @@ public void getDoc(JsonGenerator gen) { if (fileSize != null) { SearchApi.encodeLong(gen, "fileSize", fileSize); } else { - SearchApi.encodeLong(gen, "fileSize", -1L); + SearchApi.encodeLong(gen, "fileSize", 0L); } + SearchApi.encodeLong(gen, "fileCount", 1L); // Always 1, but makes sorting on fields consistent if (datafileFormat != null) { datafileFormat.getDoc(gen); } @@ -240,6 +241,11 @@ public void getDoc(JsonGenerator gen) { SearchApi.encodeString(gen, "investigation.id", investigation.id); SearchApi.encodeString(gen, "investigation.name", investigation.getName()); SearchApi.encodeString(gen, "visitId", investigation.getVisitId()); + if (investigation.getStartDate() != null) { + SearchApi.encodeLong(gen, "investigation.startDate", investigation.getStartDate()); + } else if (investigation.getCreateTime() != null) { + SearchApi.encodeLong(gen, "investigation.startDate", investigation.getCreateTime()); + } } } } @@ -283,6 +289,7 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("doi", null); documentFields.put("date", null); documentFields.put("fileSize", null); + documentFields.put("fileCount", null); documentFields.put("id", null); documentFields.put("dataset.id", null); documentFields.put("dataset.name", datasetRelationships); @@ -293,6 +300,7 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("sample.type.name", sampleTypeRelationships); documentFields.put("investigation.id", datasetRelationships); documentFields.put("investigation.name", investigationRelationships); + documentFields.put("investigation.startDate", investigationRelationships); documentFields.put("visitId", investigationRelationships); documentFields.put("datafileFormat.id", null); documentFields.put("datafileFormat.name", datafileFormatRelationships); diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index b170894e..eebef4de 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -209,7 +209,8 @@ public void getDoc(JsonGenerator gen) { } else { SearchApi.encodeLong(gen, "endDate", modTime); } - SearchApi.encodeLong(gen, "fileSize", -1L); // This is a placeholder to allow us to dynamically build size + SearchApi.encodeLong(gen, "fileSize", 0L); // This is a placeholder to allow us to dynamically build size + SearchApi.encodeLong(gen, "fileCount", 0L); // This is a placeholder to allow us to dynamically build count SearchApi.encodeString(gen, "id", id); if (investigation != null) { SearchApi.encodeString(gen, "investigation.id", investigation.id); @@ -259,6 +260,7 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("endDate", null); documentFields.put("date", null); documentFields.put("fileSize", null); + documentFields.put("fileCount", null); documentFields.put("id", null); documentFields.put("investigation.id", null); documentFields.put("investigation.title", investigationRelationships); diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index 59c0111a..c0cec85b 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -289,7 +289,8 @@ public void getDoc(JsonGenerator gen) { } else { SearchApi.encodeLong(gen, "endDate", modTime); } - SearchApi.encodeLong(gen, "fileSize", -1L); // This is a placeholder to allow us to dynamically build size + SearchApi.encodeLong(gen, "fileSize", 0L); // This is a placeholder to allow us to dynamically build size + SearchApi.encodeLong(gen, "fileCount", 0L); // This is a placeholder to allow us to dynamically build count SearchApi.encodeString(gen, "id", id); facility.getDoc(gen); @@ -322,6 +323,8 @@ public static Map getDocumentFields() throws IcatExcepti eiHandler.getRelationshipsByName(Investigation.class).get("parameters"), eiHandler.getRelationshipsByName(InvestigationParameter.class).get("type") }; Relationship[] sampleRelationships = { + eiHandler.getRelationshipsByName(Investigation.class).get("samples") }; + Relationship[] sampleTypeRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("samples"), eiHandler.getRelationshipsByName(Sample.class).get("type") }; documentFields.put("name", null); @@ -333,6 +336,7 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("endDate", null); documentFields.put("date", null); documentFields.put("fileSize", null); + documentFields.put("fileCount", null); documentFields.put("id", null); documentFields.put("facility.name", facilityRelationships); documentFields.put("facility.id", null); @@ -345,7 +349,9 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("InvestigationParameter stringValue", parameterRelationships); documentFields.put("InvestigationParameter numericValue", parameterRelationships); documentFields.put("InvestigationParameter dateTimeValue", parameterRelationships); - documentFields.put("Sample type.name", sampleRelationships); + documentFields.put("Sample sample.id", sampleRelationships); + documentFields.put("Sample sample.name", sampleRelationships); + documentFields.put("Sample type.name", sampleTypeRelationships); } return documentFields; } diff --git a/src/main/java/org/icatproject/core/entity/Parameter.java b/src/main/java/org/icatproject/core/entity/Parameter.java index feb877ab..6e7dd8d6 100644 --- a/src/main/java/org/icatproject/core/entity/Parameter.java +++ b/src/main/java/org/icatproject/core/entity/Parameter.java @@ -170,6 +170,12 @@ public void getDoc(JsonGenerator gen) { } else if (dateTimeValue != null) { SearchApi.encodeLong(gen, "dateTimeValue", dateTimeValue); } + if (rangeTop != null) { + SearchApi.encodeDouble(gen, "rangeTop", rangeTop); + } + if (rangeBottom != null) { + SearchApi.encodeDouble(gen, "rangeBottom", rangeBottom); + } type.getDoc(gen); SearchApi.encodeString(gen, "id", id); } diff --git a/src/main/java/org/icatproject/core/entity/SampleParameter.java b/src/main/java/org/icatproject/core/entity/SampleParameter.java index 2af4d254..8b2236ea 100644 --- a/src/main/java/org/icatproject/core/entity/SampleParameter.java +++ b/src/main/java/org/icatproject/core/entity/SampleParameter.java @@ -2,6 +2,7 @@ import java.io.Serializable; +import javax.json.stream.JsonGenerator; import javax.persistence.Entity; import javax.persistence.EntityManager; import javax.persistence.JoinColumn; @@ -12,6 +13,7 @@ import org.icatproject.core.IcatException; import org.icatproject.core.manager.EntityBeanManager.PersistMode; +import org.icatproject.core.manager.search.SearchApi; import org.icatproject.core.manager.GateKeeper; @Comment("A parameter associated with a sample") @@ -51,4 +53,10 @@ public void setSample(Sample sample) { this.sample = sample; } + @Override + public void getDoc(JsonGenerator gen) { + super.getDoc(gen); + SearchApi.encodeString(gen, "sample.id", sample.id); + } + } \ No newline at end of file diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 80d82d06..91773ff2 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -61,8 +61,10 @@ import org.icatproject.core.IcatException; import org.icatproject.core.IcatException.IcatExceptionType; import org.icatproject.core.entity.Datafile; +import org.icatproject.core.entity.Dataset; import org.icatproject.core.entity.EntityBaseBean; import org.icatproject.core.entity.ParameterValueType; +import org.icatproject.core.entity.Sample; import org.icatproject.core.entity.Session; import org.icatproject.core.manager.EntityInfoHandler.Relationship; import org.icatproject.core.manager.PropertyHandler.CallType; @@ -1529,30 +1531,12 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue if (jo.containsKey("facets")) { List jsonFacets = jo.getJsonArray("facets").getValuesAs(JsonObject.class); - for (JsonObject jsonFacet : jsonFacets) { - JsonObject facetQuery; String target = jsonFacet.getString("target"); - if (target.equals(klass.getSimpleName())) { - facetQuery = SearchManager.buildFacetQuery(results, "id", jsonFacet); - } else { - Relationship relationship; - if (target.contains("Parameter")) { - relationship = eiHandler.getRelationshipsByName(klass).get("parameters"); - } else { - relationship = eiHandler.getRelationshipsByName(klass).get(target.toLowerCase() + "s"); - } - - if (gateKeeper.allowed(relationship)) { - facetQuery = SearchManager.buildFacetQuery(results, - klass.getSimpleName().toLowerCase() + ".id", jsonFacet); - } else { - logger.debug("Cannot collect facets for {} as Relationship with parent {} is not allowed", - target, klass.getSimpleName()); - continue; - } + JsonObject facetQuery = buildFacetQuery(klass, target, results, jsonFacet); + if (facetQuery != null) { + dimensions.addAll(searchManager.facetSearch(target, facetQuery, results.size(), 10)); } - dimensions.addAll(searchManager.facetSearch(target, facetQuery, results.size(), 10)); } } } @@ -1586,32 +1570,125 @@ public SearchResult facetDocs(JsonObject jo, Class kla if (searchActive && jo.containsKey("facets")) { List jsonFacets = jo.getJsonArray("facets").getValuesAs(JsonObject.class); for (JsonObject jsonFacet : jsonFacets) { - JsonObject facetQuery; String target = jsonFacet.getString("target"); JsonObject filterObject = jo.getJsonObject("filter"); - if (target.equals(klass.getSimpleName())) { - facetQuery = SearchManager.buildFacetQuery(filterObject, "id", jsonFacet); + JsonObject facetQuery = buildFacetQuery(klass, target, filterObject, jsonFacet); + if (facetQuery != null) { + dimensions.addAll(searchManager.facetSearch(target, facetQuery, 1000, 10)); + } + } + } + return new SearchResult(lastSearchAfter, new ArrayList<>(), dimensions); + } + + /** + * Formats Json for requesting faceting. Performs the logic needed to ensure + * that we do not facet on a field that should not be visible. + * + * @param klass Class of the entity to facet. + * @param target The entity which directly posses the dimensions of + * interest. Note this may be different than the klass, for + * example if klass is Investigation then target might be + * InvestigationParameter. + * @param filterObject JsonObject to be used as the query. + * @param jsonFacet JsonObject containing the dimensions to facet. + * @return JsonObject with the format + * {"query": `filterObject`, "dimensions": [...]} + * @throws IcatException + */ + private JsonObject buildFacetQuery(Class klass, String target, JsonObject filterObject, + JsonObject jsonFacet) throws IcatException { + if (target.equals(klass.getSimpleName())) { + return SearchManager.buildFacetQuery(filterObject, jsonFacet); + } else { + Relationship relationship; + if (target.equals("SampleParameter")) { + Relationship sampleRelationship; + if (klass.getSimpleName().equals("Investigation")) { + sampleRelationship = eiHandler.getRelationshipsByName(klass).get("samples"); } else { - Relationship relationship; - if (target.contains("Parameter")) { - relationship = eiHandler.getRelationshipsByName(klass).get("parameters"); - } else { - relationship = eiHandler.getRelationshipsByName(klass).get(target.toLowerCase() + "s"); + if (klass.getSimpleName().equals("Datafile")) { + Relationship datasetRelationship = eiHandler.getRelationshipsByName(klass).get("dataset"); + if (!gateKeeper.allowed(datasetRelationship)) { + return null; + } } + sampleRelationship = eiHandler.getRelationshipsByName(Dataset.class).get("sample"); + } + Relationship parameterRelationship = eiHandler.getRelationshipsByName(Sample.class).get("parameters"); + if (!gateKeeper.allowed(sampleRelationship) || !gateKeeper.allowed(parameterRelationship)) { + return null; + } + return SearchManager.buildFacetQuery(filterObject, jsonFacet); + } else if (target.contains("Parameter")) { + relationship = eiHandler.getRelationshipsByName(klass).get("parameters"); + } else { + relationship = eiHandler.getRelationshipsByName(klass).get(target.toLowerCase() + "s"); + } - if (gateKeeper.allowed(relationship)) { - facetQuery = SearchManager.buildFacetQuery(filterObject, - klass.getSimpleName().toLowerCase() + ".id", jsonFacet); - } else { - logger.debug("Cannot collect facets for {} as Relationship with parent {} is not allowed", - target, klass.getSimpleName()); - continue; + if (gateKeeper.allowed(relationship)) { + return SearchManager.buildFacetQuery(filterObject, jsonFacet); + } else { + logger.debug("Cannot collect facets for {} as Relationship with parent {} is not allowed", + target, klass.getSimpleName()); + return null; + } + } + } + + /** + * Formats Json for requesting faceting. Performs the logic needed to ensure + * that we do not facet on a field that should not be visible. + * + * @param klass Class of the entity to facet. + * @param target The entity which directly posses the dimensions of interest. + * Note this may be different than the klass, for example if + * klass is Investigation then target might be + * InvestigationParameter. + * @param results List of results from a previous search, containing entity + * ids. + * @param jsonFacet JsonObject containing the dimensions to facet. + * @return {"query": {`idField`: [...]}, "dimensions": [...]} + * @throws IcatException + */ + private JsonObject buildFacetQuery(Class klass, String target, + List results, JsonObject jsonFacet) throws IcatException { + if (target.equals(klass.getSimpleName())) { + return SearchManager.buildFacetQuery(results, "id", jsonFacet); + } else { + Relationship relationship; + if (target.equals("SampleParameter")) { + Relationship sampleRelationship; + if (klass.getSimpleName().equals("Investigation")) { + sampleRelationship = eiHandler.getRelationshipsByName(klass).get("samples"); + } else { + if (klass.getSimpleName().equals("Datafile")) { + Relationship datasetRelationship = eiHandler.getRelationshipsByName(klass).get("dataset"); + if (!gateKeeper.allowed(datasetRelationship)) { + return null; + } } + sampleRelationship = eiHandler.getRelationshipsByName(Dataset.class).get("sample"); + } + Relationship parameterRelationship = eiHandler.getRelationshipsByName(Sample.class).get("parameters"); + if (!gateKeeper.allowed(sampleRelationship) || !gateKeeper.allowed(parameterRelationship)) { + return null; } - dimensions.addAll(searchManager.facetSearch(target, facetQuery, 1000, 10)); + return SearchManager.buildSampleFacetQuery(results, jsonFacet); + } else if (target.contains("Parameter")) { + relationship = eiHandler.getRelationshipsByName(klass).get("parameters"); + } else { + relationship = eiHandler.getRelationshipsByName(klass).get(target.toLowerCase() + "s"); + } + + if (gateKeeper.allowed(relationship)) { + return SearchManager.buildFacetQuery(results, klass.getSimpleName().toLowerCase() + ".id", jsonFacet); + } else { + logger.debug("Cannot collect facets for {} as Relationship with parent {} is not allowed", + target, klass.getSimpleName()); + return null; } } - return new SearchResult(lastSearchAfter, new ArrayList<>(), dimensions); } public List searchGetPopulating() { diff --git a/src/main/java/org/icatproject/core/manager/PropertyHandler.java b/src/main/java/org/icatproject/core/manager/PropertyHandler.java index 3bd858ae..5935ac1f 100644 --- a/src/main/java/org/icatproject/core/manager/PropertyHandler.java +++ b/src/main/java/org/icatproject/core/manager/PropertyHandler.java @@ -400,7 +400,7 @@ private void init() { entitiesToIndex.addAll(Arrays.asList("Datafile", "DatafileFormat", "DatafileParameter", "Dataset", "DatasetParameter", "DatasetType", "Facility", "Instrument", "InstrumentScientist", "Investigation", "InvestigationInstrument", "InvestigationParameter", "InvestigationType", - "InvestigationUser", "ParameterType", "Sample", "SampleType", "User")); + "InvestigationUser", "ParameterType", "Sample", "SampleType", "SampleParameter", "User")); logger.info("search.entitiesToIndex not set. Defaulting to: {}", entitiesToIndex.toString()); } formattedProps.add("search.entitiesToIndex " + entitiesToIndex.toString()); diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index de65fa8c..615fd3a5 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -126,10 +126,13 @@ public ParentRelation(RelationType relationType, String parentName, String joinF new ParentRelation(RelationType.CHILD, "instrumentscientist", "user", User.docFields))); relations.put("sample", Arrays.asList( new ParentRelation(RelationType.CHILD, "dataset", "sample", Sample.docFields), - new ParentRelation(RelationType.CHILD, "datafile", "sample", Sample.docFields))); + new ParentRelation(RelationType.CHILD, "datafile", "sample", Sample.docFields), + new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null))); relations.put("sampletype", Arrays.asList( new ParentRelation(RelationType.CHILD, "dataset", "sample.type", SampleType.docFields), - new ParentRelation(RelationType.CHILD, "datafile", "sample.type", SampleType.docFields))); + new ParentRelation(RelationType.CHILD, "datafile", "sample.type", SampleType.docFields), + new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "investigation", + SampleType.docFields))); // Nested children are indexed as an array of objects on their parent entity, // and know their parent's id (N.B. InvestigationUsers are also mapped to @@ -140,6 +143,10 @@ public ParentRelation(RelationType relationType, String parentName, String joinF new ParentRelation(RelationType.NESTED_CHILD, "dataset", "dataset", null))); relations.put("investigationparameter", Arrays.asList( new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null))); + relations.put("sampleparameter", Arrays.asList( + new ParentRelation(RelationType.NESTED_CHILD, "investigation", "sample", null), // Must be first + new ParentRelation(RelationType.NESTED_CHILD, "dataset", "sample", null), + new ParentRelation(RelationType.NESTED_CHILD, "datafile", "sample", null))); relations.put("investigationuser", Arrays.asList( new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null), new ParentRelation(RelationType.NESTED_CHILD, "dataset", "investigation", null), @@ -148,9 +155,6 @@ public ParentRelation(RelationType relationType, String parentName, String joinF new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null), new ParentRelation(RelationType.NESTED_CHILD, "dataset", "investigation", null), new ParentRelation(RelationType.NESTED_CHILD, "datafile", "investigation", null))); - // TODO needs to strip the openining sample. - relations.put("sample", Arrays.asList( - new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null))); // Grandchildren are entities that are related to one of the nested // children, but do not have a direct reference to one of the indexed entities, @@ -159,9 +163,15 @@ public ParentRelation(RelationType relationType, String parentName, String joinF relations.put("parametertype", Arrays.asList( new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "investigationparameter", ParameterType.docFields), + new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "sampleparameter", + ParameterType.docFields), new ParentRelation(RelationType.NESTED_GRANDCHILD, "dataset", "datasetparameter", ParameterType.docFields), + new ParentRelation(RelationType.NESTED_GRANDCHILD, "dataset", "sampleparameter", + ParameterType.docFields), new ParentRelation(RelationType.NESTED_GRANDCHILD, "datafile", "datafileparameter", + ParameterType.docFields), + new ParentRelation(RelationType.NESTED_GRANDCHILD, "datafile", "sampleparameter", ParameterType.docFields))); relations.put("user", Arrays.asList( new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "investigationuser", @@ -175,8 +185,6 @@ public ParentRelation(RelationType relationType, String parentName, String joinF User.docFields), new ParentRelation(RelationType.NESTED_GRANDCHILD, "datafile", "investigationinstrument", User.docFields))); - relations.put("sampleType", Arrays.asList( - new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "sample", SampleType.docFields))); defaultFieldsMap.put("_all", new ArrayList<>()); defaultFieldsMap.put("datafile", @@ -223,8 +231,10 @@ private static JsonObject buildMappings(String index) { .add("type.id", typeLong) .add("facility.id", typeLong) .add("fileSize", typeLong) + .add("fileCount", typeLong) .add("sample", buildNestedMapping("investigation.id", "type.id")) - .add("investigationparameter", buildNestedMapping("investigation.id", "type.ichd")) + .add("sampleparameter", buildNestedMapping("sample.id", "type.id")) + .add("investigationparameter", buildNestedMapping("investigation.id", "type.id")) .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")); } else if (index.equals("dataset")) { @@ -235,9 +245,11 @@ private static JsonObject buildMappings(String index) { .add("sample.investigaion.id", typeLong) .add("sample.type.id", typeLong) .add("fileSize", typeLong) + .add("fileCount", typeLong) .add("datasetparameter", buildNestedMapping("dataset.id", "type.id")) .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) - .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")); + .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")) + .add("sampleparameter", buildNestedMapping("sample.id", "type.id")); } else if (index.equals("datafile")) { propertiesBuilder .add("investigation.id", typeLong) @@ -245,9 +257,11 @@ private static JsonObject buildMappings(String index) { .add("sample.investigaion.id", typeLong) .add("sample.type.id", typeLong) .add("fileSize", typeLong) + .add("fileCount", typeLong) .add("datafileparameter", buildNestedMapping("datafile.id", "type.id")) .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) - .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")); + .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")) + .add("sampleparameter", buildNestedMapping("sample.id", "type.id")); } else if (index.equals("instrumentscientist")) { propertiesBuilder .add("instrument.id", typeLong) @@ -261,15 +275,24 @@ private static JsonObject buildMappings(String index) { * * @param idFields Id fields on the nested object which require the long type * mapping. - * @return JsonObject for the nested object. + * @return JsonObjectBuilder for the nested object. */ - private static JsonObject buildNestedMapping(String... idFields) { + private static JsonObjectBuilder buildNestedMapping(String... idFields) { + JsonObjectBuilder propertiesBuilder = propertiesBuilder(idFields); + return buildNestedMapping(propertiesBuilder); + } + + private static JsonObjectBuilder buildNestedMapping(JsonObjectBuilder propertiesBuilder) { + return Json.createObjectBuilder().add("type", "nested").add("properties", propertiesBuilder); + } + + private static JsonObjectBuilder propertiesBuilder(String... idFields) { JsonObject typeLong = Json.createObjectBuilder().add("type", "long").build(); JsonObjectBuilder propertiesBuilder = Json.createObjectBuilder().add("id", typeLong); for (String idField : idFields) { propertiesBuilder.add(idField, typeLong); } - return Json.createObjectBuilder().add("type", "nested").add("properties", propertiesBuilder).build(); + return propertiesBuilder; } /** @@ -662,27 +685,23 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query if (queryRequest.containsKey("filter")) { JsonObject filterObject = queryRequest.getJsonObject("filter"); for (String fld : filterObject.keySet()) { - ValueType valueType = filterObject.get(fld).getValueType(); + JsonValue value = filterObject.get(fld); String field = fld.replace(index + ".", ""); - switch (valueType) { + switch (value.getValueType()) { case ARRAY: - JsonArrayBuilder shouldBuilder = Json.createArrayBuilder(); - for (JsonString value : filterObject.getJsonArray(fld).getValuesAs(JsonString.class)) { - shouldBuilder - .add(OpensearchQueryBuilder.buildTermQuery(field + ".keyword", value.getString())); + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + for (JsonValue arrayValue : ((JsonArray) value).getValuesAs(JsonString.class)) { + parseFilter(arrayBuilder, field, arrayValue); } + // If the key was just a nested entity (no ".") then we should FILTER all of our + // queries on that entity. + String occur = fld.contains(".") ? "should" : "filter"; filterBuilder.add(Json.createObjectBuilder().add("bool", - Json.createObjectBuilder().add("should", shouldBuilder))); - break; - - case STRING: - filterBuilder.add( - OpensearchQueryBuilder.buildTermQuery(field + ".keyword", filterObject.getString(fld))); + Json.createObjectBuilder().add(occur, arrayBuilder))); break; default: - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "filter object values should be STRING or ARRAY, but were " + valueType); + parseFilter(filterBuilder, field, value); } } } @@ -807,6 +826,130 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query return builder.add("query", queryBuilder.add("bool", boolBuilder)); } + /** + * Parses a filter object applied to a single field. Note that in the case that + * this field is actually a nested object, more complex logic will be applied to + * ensure that only object matching all nested filters are returned. + * + * @param filterBuilder Builder for the array of queries to filter by. + * @param field Field to apply the filter to. In the case of nested + * queries, this should only be the name of the top level + * field. For example "investigationparameter". + * @param value JsonValue representing the filter query. This can be a + * STRING for simple terms, or an OBJECT containing nested + * "value", "exact" or "range" filters. + * @throws IcatException + */ + private void parseFilter(JsonArrayBuilder filterBuilder, String field, JsonValue value) throws IcatException { + ValueType valueType = value.getValueType(); + switch (valueType) { + case STRING: + filterBuilder.add( + OpensearchQueryBuilder.buildTermQuery(field + ".keyword", ((JsonString) value).getString())); + return; + case OBJECT: + JsonObject valueObject = (JsonObject) value; + if (valueObject.containsKey("filter")) { + List queryObjectsList = new ArrayList<>(); + for (JsonObject nestedFilter : valueObject.getJsonArray("filter").getValuesAs(JsonObject.class)) { + String nestedField = nestedFilter.getString("field"); + if (nestedFilter.containsKey("value")) { + // String based term query + String stringValue = nestedFilter.getString("value"); + queryObjectsList.add( + OpensearchQueryBuilder.buildTermQuery(field + "." + nestedField + ".keyword", + stringValue)); + } else if (nestedFilter.containsKey("exact")) { + JsonNumber exact = nestedFilter.getJsonNumber("exact"); + String units = nestedFilter.getString("units", null); + if (units != null) { + SystemValue exactValue = icatUnits.new SystemValue(exact.doubleValue(), units); + if (exactValue.value != null) { + // If we were able to parse the units, apply query to the SI value + JsonObject bottomQuery = OpensearchQueryBuilder + .buildDoubleRangeQuery(field + ".rangeBottomSI", null, exactValue.value); + JsonObject topQuery = OpensearchQueryBuilder + .buildDoubleRangeQuery(field + ".rangeTopSI", exactValue.value, null); + JsonObject inRangeQuery = OpensearchQueryBuilder + .buildBoolQuery(Arrays.asList(bottomQuery, topQuery), null); + JsonObject exactQuery = OpensearchQueryBuilder + .buildTermQuery(field + "." + nestedField + "SI", exactValue.value); + queryObjectsList.add( + OpensearchQueryBuilder.buildBoolQuery(null, + Arrays.asList(inRangeQuery, exactQuery))); + } else { + // If units could not be parsed, make them part of the query on the raw data + JsonObject bottomQuery = OpensearchQueryBuilder + .buildRangeQuery(field + ".rangeBottom", null, exact); + JsonObject topQuery = OpensearchQueryBuilder.buildRangeQuery(field + ".rangeTop", + exact, null); + JsonObject inRangeQuery = OpensearchQueryBuilder + .buildBoolQuery(Arrays.asList(bottomQuery, topQuery), null); + JsonObject exactQuery = OpensearchQueryBuilder + .buildTermQuery(field + "." + nestedField, exact); + queryObjectsList.add( + OpensearchQueryBuilder.buildBoolQuery(null, + Arrays.asList(inRangeQuery, exactQuery))); + queryObjectsList.add( + OpensearchQueryBuilder.buildTermQuery(field + ".type.units.keyword", + units)); + } + } else { + // If units were not provided, just apply to the raw data + JsonObject bottomQuery = OpensearchQueryBuilder.buildRangeQuery(field + ".rangeBottom", + null, exact); + JsonObject topQuery = OpensearchQueryBuilder.buildRangeQuery(field + ".rangeTop", exact, + null); + JsonObject inRangeQuery = OpensearchQueryBuilder + .buildBoolQuery(Arrays.asList(bottomQuery, topQuery), null); + JsonObject exactQuery = OpensearchQueryBuilder.buildTermQuery(field + "." + nestedField, + exact); + queryObjectsList.add( + OpensearchQueryBuilder.buildBoolQuery(null, + Arrays.asList(inRangeQuery, exactQuery))); + } + } else { + JsonNumber from = nestedFilter.getJsonNumber("from"); + JsonNumber to = nestedFilter.getJsonNumber("to"); + String units = nestedFilter.getString("units", null); + if (units != null) { + SystemValue fromValue = icatUnits.new SystemValue(from.doubleValue(), units); + SystemValue toValue = icatUnits.new SystemValue(to.doubleValue(), units); + if (fromValue.value != null && toValue.value != null) { + // If we were able to parse the units, apply query to the SI value + queryObjectsList.add(OpensearchQueryBuilder.buildDoubleRangeQuery( + field + "." + nestedField + "SI", fromValue.value, toValue.value)); + } else { + // If units could not be parsed, make them part of the query on the raw data + queryObjectsList.add( + OpensearchQueryBuilder.buildRangeQuery(field + "." + nestedField, from, + to)); + queryObjectsList.add( + OpensearchQueryBuilder.buildTermQuery(field + ".type.units.keyword", + units)); + } + } else { + // If units were not provided, just apply to the raw data + queryObjectsList.add( + OpensearchQueryBuilder.buildRangeQuery(field + "." + nestedField, from, to)); + } + } + } + JsonObject[] queryObjects = queryObjectsList.toArray(new JsonObject[0]); + filterBuilder.add(OpensearchQueryBuilder.buildNestedQuery(field, queryObjects)); + } else { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "expected an ARRAY with the key 'filter', but received " + valueObject.toString()); + } + return; + + default: + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "filter values should be STRING, OBJECT or and ARRAY of the former, but were " + valueType); + } + + } + /** * Create mappings for indices that do not already have them. * @@ -860,6 +1003,37 @@ public void initScripts() throws IcatException { for (Entry> entry : relations.entrySet()) { String key = entry.getKey(); ParentRelation relation = entry.getValue().get(0); + // Special cases + if (key.equals("parametertype")) { + // ParameterType can apply to 4 different nested objects + post("/_scripts/update_parametertype", + OpensearchScriptBuilder.buildParameterTypeScript(ParameterType.docFields, true)); + post("/_scripts/delete_parametertype", + OpensearchScriptBuilder.buildParameterTypeScript(ParameterType.docFields, false)); + continue; + } else if (key.equals("sample")) { + // Sample is a child of Datafile and Dataset... + post("/_scripts/update_sample", OpensearchScriptBuilder.buildChildScript(Sample.docFields, true)); + post("/_scripts/delete_sample", OpensearchScriptBuilder.buildChildScript(Sample.docFields, false)); + // ...but a nested child of Investigations + post("/_scripts/update_nestedsample", OpensearchScriptBuilder.buildNestedChildScript(key, true)); + post("/_scripts/delete_nestedsample", OpensearchScriptBuilder.buildNestedChildScript(key, false)); + String createScript = OpensearchScriptBuilder.buildCreateNestedChildScript(key); + post("/_scripts/create_" + key, createScript); + continue; + } else if (key.equals("sampletype")) { + // SampleType is a child of Datafile and Dataset... + post("/_scripts/update_sampletype", + OpensearchScriptBuilder.buildChildScript(SampleType.docFields, true)); + post("/_scripts/delete_sampletype", + OpensearchScriptBuilder.buildChildScript(SampleType.docFields, false)); + // ...but a nested grandchild of Investigations + post("/_scripts/update_nestedsampletype", + OpensearchScriptBuilder.buildGrandchildScript("sample", SampleType.docFields, true)); + post("/_scripts/delete_nestedsampletype", + OpensearchScriptBuilder.buildGrandchildScript("sample", SampleType.docFields, false)); + continue; + } String updateScript = ""; String deleteScript = ""; // Each type of relation needs a different script to update @@ -875,33 +1049,24 @@ public void initScripts() throws IcatException { post("/_scripts/create_" + key, createScript); break; case NESTED_GRANDCHILD: - if (key.equals("parametertype")) { - // Special case, as parametertype applies to investigationparameter, - // datasetparameter, datafileparameter - for (String index : indices) { - String target = index + "parameter"; - updateScript = OpensearchScriptBuilder.buildGrandchildScript(target, relation.fields, - true); - deleteScript = OpensearchScriptBuilder.buildGrandchildScript(target, relation.fields, - false); - } - } else { - updateScript = OpensearchScriptBuilder.buildGrandchildScript(relation.joinField, - relation.fields, true); - deleteScript = OpensearchScriptBuilder.buildGrandchildScript(relation.joinField, - relation.fields, false); - } + updateScript = OpensearchScriptBuilder.buildGrandchildScript(relation.joinField, + relation.fields, true); + deleteScript = OpensearchScriptBuilder.buildGrandchildScript(relation.joinField, + relation.fields, false); break; } post("/_scripts/update_" + key, updateScript); post("/_scripts/delete_" + key, deleteScript); } + post("/_scripts/fileSize", OpensearchScriptBuilder.buildFileSizeScript()); } public void modify(String json) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { List updatesByQuery = new ArrayList<>(); Set investigationIds = new HashSet<>(); + Map investigationAggregations = new HashMap<>(); + Map datasetAggregations = new HashMap<>(); StringBuilder sb = new StringBuilder(); JsonReader jsonReader = Json.createReader(new StringReader(json)); JsonArray outerArray = jsonReader.readArray(); @@ -929,7 +1094,8 @@ public void modify(String json) throws IcatException { } if (indices.contains(index)) { // Also modify any main, indexable entities - modifyEntity(sb, investigationIds, id, index, document, modificationType); + modifyEntity(httpclient, sb, investigationIds, investigationAggregations, datasetAggregations, id, + index, document, modificationType); } } @@ -968,11 +1134,77 @@ public void modify(String json) throws IcatException { } } } + + StringBuilder fileSizeStringBuilder = new StringBuilder(); + buildFileSizeUpdates("investigation", investigationAggregations, fileSizeStringBuilder); + buildFileSizeUpdates("dataset", datasetAggregations, fileSizeStringBuilder); + if (fileSizeStringBuilder.toString().length() > 0) { + // Perform simple bulk modifications + URI uri = new URIBuilder(server).setPath("/_bulk").build(); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(fileSizeStringBuilder.toString(), ContentType.APPLICATION_JSON)); + logger.trace("Making call {} with body {}", uri, fileSizeStringBuilder.toString()); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + } + } } catch (IOException | URISyntaxException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } } + /** + * Builds commands for updating the fileSizes of the entities keyed in + * aggregations. + * + * @param entity Name of the entity/index to be updated. + * @param aggregations Map of aggregated fileSize changes with the + * entity ids as keys. + * @param fileSizeStringBuilder StringBuilder for constructing the bulk updates. + */ + private void buildFileSizeUpdates(String entity, Map aggregations, + StringBuilder fileSizeStringBuilder) { + if (aggregations.size() > 0) { + for (String id : aggregations.keySet()) { + JsonObject targetObject = Json.createObjectBuilder().add("_id", new Long(id)).add("_index", entity) + .build(); + JsonObject update = Json.createObjectBuilder().add("update", targetObject).build(); + Long deltaFileSize = aggregations.get(id)[0]; + Long deltaFileCount = aggregations.get(id)[1]; + JsonObjectBuilder paramsBuilder = Json.createObjectBuilder(); + JsonObjectBuilder scriptBuilder = Json.createObjectBuilder(); + paramsBuilder.add("deltaFileSize", deltaFileSize).add("deltaFileCount", deltaFileCount); + scriptBuilder.add("id", "fileSize").add("params", paramsBuilder); + JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); + String body = bodyBuilder.add("script", scriptBuilder).build().toString(); + fileSizeStringBuilder.append(update.toString()).append("\n").append(body).append("\n"); + } + } + } + + /** + * Gets the source of a Datafile and returns it. + * + * @param httpclient The client being used to send HTTP + * @param id ICAT entity id of the Datafile. + * @return The Datafile source. + * @throws IOException + * @throws URISyntaxException + * @throws ClientProtocolException + */ + private JsonObject extractSource(CloseableHttpClient httpclient, String id) + throws IOException, URISyntaxException, ClientProtocolException { + URI uriGet = new URIBuilder(server).setPath("/datafile/_source/" + id) + .build(); + HttpGet httpGet = new HttpGet(uriGet); + try (CloseableHttpResponse responseGet = httpclient.execute(httpGet)) { + if (responseGet.getStatusLine().getStatusCode() == 200) { + return Json.createReader(responseGet.getEntity().getContent()).readObject(); + } + } + return null; + } + /** * For cases when Datasets and Datafiles are created after an Investigation, * some nested fields such as InvestigationUser and InvestigationInstrument may @@ -1044,7 +1276,7 @@ private void extractFromInvestigation(CloseableHttpClient httpclient, String inv JsonObjectBuilder scriptBuilder = Json.createObjectBuilder(); String sampleId = sampleObject.getString("id"); for (String field : sampleObject.keySet()) { - paramsBuilder.add(field, sampleObject.get(field)); + paramsBuilder.add("sample." + field, sampleObject.get(field)); } scriptBuilder.add("id", "update_sample").add("params", paramsBuilder); JsonObject queryObject = OpensearchQueryBuilder.buildTermQuery("sample.id", sampleId); @@ -1084,22 +1316,24 @@ private void modifyNestedEntity(StringBuilder sb, List updatesByQuery, switch (modificationType) { case CREATE: - if (index.equals("sample")) { - // In order to make searching for sample information seamless between - // Investigations and Datasets/files, need to ensure that when nesting fields - // like "sample.name" under a "sample" object, we do not end up with - // "sample.sample.name" - JsonObjectBuilder documentBuilder = Json.createObjectBuilder(); - for (Entry entry : document.entrySet()) { - documentBuilder.add(entry.getKey().replace("sample.", ""), entry.getValue()); - } - createNestedEntity(sb, id, index, documentBuilder.build(), relation); - } else if (relation.parentName.equals(relation.joinField)) { + if (relation.parentName.equals(relation.joinField)) { // If the target parent is the same as the joining field, we're appending the // nested child to a list of objects which can be sent as a bulk update request // since we have the parent id document = convertDocumentUnits(document); - createNestedEntity(sb, id, index, document, relation); + if (index.equals("sample")) { + // In order to make searching for sample information seamless between + // Investigations and Datasets/files, need to ensure that when nesting fields + // like "sample.name" under a "sample" object, we do not end up with + // "sample.sample.name" + JsonObjectBuilder documentBuilder = Json.createObjectBuilder(); + for (Entry entry : document.entrySet()) { + documentBuilder.add(entry.getKey().replace("sample.", ""), entry.getValue()); + } + createNestedEntity(sb, id, index, documentBuilder.build(), relation); + } else { + createNestedEntity(sb, id, index, document, relation); + } } else if (index.equals("sampletype")) { // Otherwise, in most cases we don't need to update, as User and ParameterType // cannot be null on their parent InvestigationUser or InvestigationParameter @@ -1107,6 +1341,15 @@ private void modifyNestedEntity(StringBuilder sb, List updatesByQuery, // SampleType can be null upon creation of a Sample, need to account for the // creation of a SampleType at a later date. updateNestedEntityByQuery(updatesByQuery, id, index, document, relation, true); + } else if (index.equals("sampleparameter")) { + // SampleParameter requires specific logic, as the join is performed using the + // Sample id rather than the SampleParameter id or the parent id. + logger.debug("index: {}, parent: {}, joinField: {}, doc: {}", index, relation.parentName, + relation.joinField, document.toString()); + if (document.containsKey("sample.id")) { + String sampleId = document.getString("sample.id"); + updateNestedEntityByQuery(updatesByQuery, sampleId, index, document, relation, true); + } } break; case UPDATE: @@ -1141,8 +1384,12 @@ private static void createNestedEntity(StringBuilder sb, String id, String index // For nested 0:* relationships, wrap single documents in an array JsonArray docArray = Json.createArrayBuilder().add(document).build(); JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("id", id).add("doc", docArray); - // ParameterType is a special case where script needs to include the parentName - String scriptId = index.equals("parametertype") ? "update_" + relation.parentName + index : "update_" + index; + String scriptId; + if (index.equals("sample") || index.equals("sampletype") && relation.parentName.equals("investigation")) { + scriptId = "update_nested" + index; + } else { + scriptId = "update_" + index; + } JsonObjectBuilder scriptBuilder = Json.createObjectBuilder().add("id", scriptId).add("params", paramsBuilder); JsonObjectBuilder upsertBuilder = Json.createObjectBuilder().add(index, docArray); JsonObjectBuilder payloadBuilder = Json.createObjectBuilder() @@ -1174,7 +1421,11 @@ private void updateNestedEntityByQuery(List updatesByQuery, String id, // Determine the Id of the painless script to use String scriptId = update ? "update_" : "delete_"; - scriptId += index.equals("parametertype") ? relation.parentName + index : index; + if (index.equals("sample") || index.equals("sampletype") && relation.parentName.equals("investigation")) { + scriptId += "nested" + index; + } else { + scriptId += index; + } // All updates/deletes require the entityId JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("id", id); @@ -1191,7 +1442,10 @@ private void updateNestedEntityByQuery(List updatesByQuery, String id, JsonObjectBuilder scriptBuilder = Json.createObjectBuilder().add("id", scriptId).add("params", paramsBuilder); JsonObject queryObject; String idField = relation.joinField.equals(relation.parentName) ? "id" : relation.joinField + ".id"; - if (relation.relationType.equals(RelationType.NESTED_GRANDCHILD)) { + // sample.id is a nested field on investigations, so need a nested query to + // successfully add sampleparameter + if (relation.relationType.equals(RelationType.NESTED_GRANDCHILD) + || index.equals("sampleparameter") && relation.parentName.equals("investigation")) { queryObject = OpensearchQueryBuilder.buildNestedQuery(relation.joinField, OpensearchQueryBuilder.buildTermQuery(idField, id)); } else { @@ -1244,7 +1498,15 @@ private JsonObject convertDocumentUnits(JsonObject document) { Double numericValue = document.containsKey("numericValue") ? document.getJsonNumber("numericValue").doubleValue() : null; + Double rangeBottom = document.containsKey("rangeBottom") + ? document.getJsonNumber("rangeBottom").doubleValue() + : null; + Double rangeTop = document.containsKey("rangeTop") + ? document.getJsonNumber("rangeTop").doubleValue() + : null; convertUnits(document, rebuilder, "numericValueSI", numericValue); + convertUnits(document, rebuilder, "rangeBottomSI", rangeBottom); + convertUnits(document, rebuilder, "rangeTopSI", rangeTop); document = rebuilder.build(); return document; } @@ -1279,17 +1541,29 @@ private JsonObjectBuilder convertScriptUnits(JsonObjectBuilder paramsBuilder, Js * investigationIds which may contain relevant information (e.g. nested * InvestigationUsers). * - * @param sb StringBuilder used for bulk modifications. - * @param investigationIds List of investigationIds to check for relevant - * fields. - * @param id Id of the entity. - * @param index Index of the entity. - * @param document JsonObject containing the key value pairs of the - * document fields. - * @param modificationType The type of operation to be performed. + * @param httpclient The client being used to send HTTP + * @param sb StringBuilder used for bulk modifications. + * @param investigationIds List of investigationIds to check for + * relevant + * fields. + * @param investigationAggregations Map of aggregated fileSize changes with the + * Investigation ids as keys. + * @param datasetAggregations Map of aggregated fileSize changes with the + * Dataset ids as keys. + * @param id Id of the entity. + * @param index Index of the entity. + * @param document JsonObject containing the key value pairs of + * the + * document fields. + * @param modificationType The type of operation to be performed. + * @throws URISyntaxException + * @throws IOException + * @throws ClientProtocolException */ - private static void modifyEntity(StringBuilder sb, Set investigationIds, String id, String index, - JsonObject document, ModificationType modificationType) { + private void modifyEntity(CloseableHttpClient httpclient, StringBuilder sb, Set investigationIds, + Map investigationAggregations, Map datasetAggregations, String id, + String index, JsonObject document, ModificationType modificationType) + throws ClientProtocolException, IOException, URISyntaxException { JsonObject targetObject = Json.createObjectBuilder().add("_id", new Long(id)).add("_index", index).build(); JsonObject update = Json.createObjectBuilder().add("update", targetObject).build(); @@ -1303,13 +1577,73 @@ private static void modifyEntity(StringBuilder sb, Set investigationIds, // entities are attached to an Investigation, so need to check for those investigationIds.add(document.getString("investigation.id")); } + if (index.equals("datafile") && document.containsKey("fileSize")) { + long newFileSize = document.getJsonNumber("fileSize").longValueExact(); + if (document.containsKey("investigation.id")) { + String investigationId = document.getString("investigation.id"); + Long[] runningFileSize = investigationAggregations.getOrDefault(investigationId, + new Long[] { 0L, 0L }); + Long[] newValue = new Long[] { runningFileSize[0] + newFileSize, runningFileSize[1] + 1L }; + investigationAggregations.put(investigationId, newValue); + } + if (document.containsKey("dataset.id")) { + String datasetId = document.getString("dataset.id"); + Long[] runningFileSize = datasetAggregations.getOrDefault(datasetId, new Long[] { 0L, 0L }); + Long[] newValue = new Long[] { runningFileSize[0] + newFileSize, runningFileSize[1] + 1L }; + datasetAggregations.put(datasetId, newValue); + } + } break; case UPDATE: docAsUpsert = Json.createObjectBuilder().add("doc", document).add("doc_as_upsert", true).build(); sb.append(update.toString()).append("\n").append(docAsUpsert.toString()).append("\n"); + if (index.equals("datafile") && document.containsKey("fileSize")) { + long newFileSize = document.getJsonNumber("fileSize").longValueExact(); + long oldFileSize; + JsonObject source = extractSource(httpclient, id); + if (source != null && source.containsKey("fileSize")) { + oldFileSize = source.getJsonNumber("fileSize").longValueExact(); + } else { + oldFileSize = 0; + } + if (newFileSize != oldFileSize) { + if (document.containsKey("investigation.id")) { + String investigationId = document.getString("investigation.id"); + Long[] runningFileSize = investigationAggregations.getOrDefault(investigationId, + new Long[] { 0L, 0L }); + Long[] newValue = new Long[] { runningFileSize[0] + newFileSize - oldFileSize, runningFileSize[1] }; + investigationAggregations.put(investigationId, newValue); + } + if (document.containsKey("dataset.id")) { + String datasetId = document.getString("dataset.id"); + Long[] runningFileSize = datasetAggregations.getOrDefault(datasetId, new Long[] { 0L, 0L }); + Long[] newValue = new Long[] { runningFileSize[0] + newFileSize - oldFileSize, runningFileSize[1] }; + datasetAggregations.put(datasetId, newValue); + } + } + } break; case DELETE: sb.append(Json.createObjectBuilder().add("delete", targetObject).build().toString()).append("\n"); + if (index.equals("datafile")) { + JsonObject source = extractSource(httpclient, id); + if (source != null && source.containsKey("fileSize")) { + long oldFileSize = source.getJsonNumber("fileSize").longValueExact(); + if (source.containsKey("investigation.id")) { + String investigationId = source.getString("investigation.id"); + Long[] runningFileSize = investigationAggregations.getOrDefault(investigationId, + new Long[] { 0L, 0L }); + Long[] newValue = new Long[] { runningFileSize[0] - oldFileSize, runningFileSize[1] - 1 }; + investigationAggregations.put(investigationId, newValue); + } + if (source.containsKey("dataset.id")) { + String datasetId = source.getString("dataset.id"); + Long[] runningFileSize = datasetAggregations.getOrDefault(datasetId, new Long[] { 0L, 0L }); + Long[] newValue = new Long[] { runningFileSize[0] - oldFileSize, runningFileSize[1] - 1 }; + datasetAggregations.put(datasetId, newValue); + } + } + } break; } } diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java b/src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java index a580107e..dfb2d32b 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java @@ -1,5 +1,7 @@ package org.icatproject.core.manager.search; +import java.util.List; + import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonArrayBuilder; @@ -23,6 +25,33 @@ public static JsonObjectBuilder addQuery(JsonObject query) { return Json.createObjectBuilder().add("query", query); } + /** + * @param filter Path to nested Object. + * @param should Any number of pre-built queries. + * @return {"bool": {"filter": [...filter], "should": [...should]}} + */ + public static JsonObject buildBoolQuery(List filter, List should) { + JsonObjectBuilder boolBuilder = Json.createObjectBuilder(); + buildBoolArray("should", should, boolBuilder); + buildBoolArray("filter", filter, boolBuilder); + return Json.createObjectBuilder().add("bool", boolBuilder).build(); + } + + /** + * @param occur String of an occurance keyword ("filter", "should", "must" etc.) + * @param queries List of JsonObjects representing the queries to occur. + * @param boolBuilder Builder of the main boolean query. + */ + private static void buildBoolArray(String occur, List queries, JsonObjectBuilder boolBuilder) { + if (queries != null && queries.size() > 0) { + JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); + for (JsonObject queryObject : queries) { + filterBuilder.add(queryObject); + } + boolBuilder.add(occur, filterBuilder); + } + } + /** * @return {"match_all": {}} */ @@ -90,6 +119,24 @@ public static JsonObject buildTermQuery(String field, String value) { return Json.createObjectBuilder().add("term", Json.createObjectBuilder().add(field, value)).build(); } + /** + * @param field Field containing the number. + * @param value Number to match. + * @return {"term": {`field`: `value`}} + */ + public static JsonObject buildTermQuery(String field, JsonNumber value) { + return Json.createObjectBuilder().add("term", Json.createObjectBuilder().add(field, value)).build(); + } + + /** + * @param field Field containing the double value. + * @param value Double to match. + * @return {"term": {`field`: `value`}} + */ + public static JsonObject buildTermQuery(String field, double value) { + return Json.createObjectBuilder().add("term", Json.createObjectBuilder().add(field, value)).build(); + } + /** * @param field Field containing on of the terms. * @param values JsonArrat of possible terms. @@ -99,6 +146,20 @@ public static JsonObject buildTermsQuery(String field, JsonArray values) { return Json.createObjectBuilder().add("terms", Json.createObjectBuilder().add(field, values)).build(); } + /** + * @param field Field to apply the range to. + * @param lowerValue Lowest allowed value in the range. + * @param upperValue Highest allowed value in the range. + * @return {"range": {`field`: {"gte": `upperValue`, "lte": `lowerValue`}}} + */ + public static JsonObject buildDoubleRangeQuery(String field, Double lowerValue, Double upperValue) { + JsonObjectBuilder fieldBuilder = Json.createObjectBuilder(); + if (lowerValue != null) fieldBuilder.add("gte", lowerValue); + if (upperValue != null) fieldBuilder.add("lte", upperValue); + JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); + return Json.createObjectBuilder().add("range", rangeBuilder).build(); + } + /** * @param field Field to apply the range to. * @param lowerValue Lowest allowed value in the range. @@ -106,7 +167,9 @@ public static JsonObject buildTermsQuery(String field, JsonArray values) { * @return {"range": {`field`: {"gte": `upperValue`, "lte": `lowerValue`}}} */ public static JsonObject buildLongRangeQuery(String field, Long lowerValue, Long upperValue) { - JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("gte", lowerValue).add("lte", upperValue); + JsonObjectBuilder fieldBuilder = Json.createObjectBuilder(); + if (lowerValue != null) fieldBuilder.add("gte", lowerValue); + if (upperValue != null) fieldBuilder.add("lte", upperValue); JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); return Json.createObjectBuilder().add("range", rangeBuilder).build(); } @@ -118,7 +181,9 @@ public static JsonObject buildLongRangeQuery(String field, Long lowerValue, Long * @return {"range": {`field`: {"gte": `upperValue`, "lte": `lowerValue`}}} */ public static JsonObject buildRangeQuery(String field, JsonNumber lowerValue, JsonNumber upperValue) { - JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("gte", lowerValue).add("lte", upperValue); + JsonObjectBuilder fieldBuilder = Json.createObjectBuilder(); + if (lowerValue != null) fieldBuilder.add("gte", lowerValue); + if (upperValue != null) fieldBuilder.add("lte", upperValue); JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); return Json.createObjectBuilder().add("range", rangeBuilder).build(); } diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchScriptBuilder.java b/src/main/java/org/icatproject/core/manager/search/OpensearchScriptBuilder.java index d0580d53..2e8468fa 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchScriptBuilder.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchScriptBuilder.java @@ -22,12 +22,20 @@ private static String buildScript(String source) { * In order to access a specific nested child entity, access `childIndex` in * later parts of the painless script. * - * @param childName The name of the nested child entity. + * @param childName The name of the nested child entity. + * @param declareChildId Should be true for only the first time a child is found + * during a script so that the variable can be reused. * @return Painless code for determining the id of a given child within a nested * array. */ - private static String findNestedChild(String childName) { - return "int childIndex = -1; int i = 0; if (ctx._source." + childName + " != null) " + private static String findNestedChild(String childName, boolean declareChildId) { + String source; + if (declareChildId) { + source = "int childIndex = -1; int i = 0;"; + } else { + source = "childIndex = -1; i = 0;"; + } + return source + " if (ctx._source." + childName + " != null) " + "{while (childIndex == -1 && i < ctx._source." + childName + ".size()) " + "{if (ctx._source." + childName + ".get(i).id == params.id) {childIndex = i;} i++;}}"; } @@ -38,7 +46,7 @@ private static String findNestedChild(String childName) { * on its id. */ private static String removeNestedChild(String childName) { - return findNestedChild(childName) + " if (childIndex != -1) {ctx._source." + childName + return findNestedChild(childName, true) + " if (childIndex != -1) {ctx._source." + childName + ".remove(childIndex);}"; } @@ -114,8 +122,7 @@ public static String buildNestedChildScript(String childName, boolean update) { /** * Builds a script which updates specific fields on a nested child entity that - * are set - * by a single grandchild. + * are set by a single grandchild. * * @param childName The name of the nested child entity. * @param docFields The fields belonging to the grandchild entity to be @@ -125,12 +132,79 @@ public static String buildNestedChildScript(String childName, boolean update) { * @return The painless script as a String. */ public static String buildGrandchildScript(String childName, Set docFields, boolean update) { - String source = findNestedChild(childName); + String source = findNestedChild(childName, true); String ctxSource = "ctx._source." + childName + ".get(childIndex)"; - for (String field : docFields) { - source += updateField(field, ctxSource, update); + if (docFields != null) { + source += "if (childIndex != -1) { "; + for (String field : docFields) { + source += updateField(field, ctxSource, update); + } + source += " } "; } return buildScript(source); } + /** + * Builds a script which increments fileSize by deltaFileSize. If + * fileSize is null then deltaFileSize is taken as its new value. + * + * @return The painless script as a String. + */ + public static String buildFileSizeScript() { + String source = "if (ctx._source.fileSize != null) "; + source += "{ctx._source.fileSize += params.deltaFileSize;} else {ctx._source.fileSize = params.deltaFileSize;}"; + source += "if (ctx._source.fileCount != null) "; + source += "{ctx._source.fileCount += params.deltaFileCount;} else {ctx._source.fileCount = params.deltaFileCount;}"; + return buildScript(source); + } + + /** + * Modifies ParameterTypes with logic to ensure the update is applied to all + * possible Parameters (Investigation, Dataset, Datafile, Sample). + * + * @param fields The fields belonging to the ParameterType to be + * modified. + * @param update If true the script will replace a nested entity, else the + * nested entity will be removed from the array. + * @return + */ + public static String buildParameterTypeScript(Set docFields, boolean update) { + String source = findNestedChild("investigationparameter", true); + String ctxSource = "ctx._source.investigationparameter.get(childIndex)"; + if (docFields != null) { + source += "if (childIndex != -1) { "; + for (String field : docFields) { + source += updateField(field, ctxSource, update); + } + source += " } "; + } + source += findNestedChild("datasetparameter", false); + ctxSource = "ctx._source.datasetparameter.get(childIndex)"; + if (docFields != null) { + source += "if (childIndex != -1) { "; + for (String field : docFields) { + source += updateField(field, ctxSource, update); + } + source += " } "; + } + source += findNestedChild("datafileparameter", false); + ctxSource = "ctx._source.datafileparameter.get(childIndex)"; + if (docFields != null) { + source += "if (childIndex != -1) { "; + for (String field : docFields) { + source += updateField(field, ctxSource, update); + } + source += " } "; + } + source += findNestedChild("sampleparameter", false); + ctxSource = "ctx._source.sampleparameter.get(childIndex)"; + if (docFields != null) { + source += "if (childIndex != -1) { "; + for (String field : docFields) { + source += updateField(field, ctxSource, update); + } + source += " } "; + } + return buildScript(source); + } } diff --git a/src/main/java/org/icatproject/core/manager/search/SearchManager.java b/src/main/java/org/icatproject/core/manager/search/SearchManager.java index 18d0c734..99ce1d97 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchManager.java @@ -34,7 +34,9 @@ import javax.json.JsonArrayBuilder; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; +import javax.json.JsonString; import javax.json.JsonValue; +import javax.json.JsonValue.ValueType; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; import javax.persistence.PersistenceUnit; @@ -414,16 +416,47 @@ public static JsonObject buildFacetQuery(List results, Str return objectBuilder.build(); } + /** + * Builds a JsonObject for performing faceting against results from a previous + * search. Has specific logic for handling the nesting of Samples. + * + * @param results List of results from a previous search, containing sample + * ids. + * @param facetJson JsonObject containing the dimensions to facet. + * @return + */ + public static JsonObject buildSampleFacetQuery(List results, JsonObject facetJson) { + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + results.forEach(r -> { + JsonObject source = r.getSource(); + if (source.containsKey("sample.id")) { + ValueType valueType = source.get("sample.id").getValueType(); + if (valueType.equals(ValueType.STRING)) { + arrayBuilder.add(source.getString("sample.id")); + } else if (valueType.equals(ValueType.ARRAY)) { + source.getJsonArray("sample.id").getValuesAs(JsonString.class).forEach(sampleId -> { + arrayBuilder.add(sampleId); + }); + } + } + }); + JsonObject terms = Json.createObjectBuilder().add("sample.id", arrayBuilder.build()).build(); + JsonObjectBuilder objectBuilder = Json.createObjectBuilder().add("query", terms); + if (facetJson.containsKey("dimensions")) { + objectBuilder.add("dimensions", facetJson.getJsonArray("dimensions")); + } + return objectBuilder.build(); + } + /** * Builds a JsonObject for performing faceting against results from a previous * search. * * @param filterObject JsonObject to be used as a query. - * @param idField The field to perform id querying against. * @param facetJson JsonObject containing the dimensions to facet. * @return {"query": `filterObject`, "dimensions": [...]} */ - public static JsonObject buildFacetQuery(JsonObject filterObject, String idField, JsonObject facetJson) { + public static JsonObject buildFacetQuery(JsonObject filterObject, JsonObject facetJson) { JsonObjectBuilder objectBuilder = Json.createObjectBuilder().add("query", filterObject); if (facetJson.containsKey("dimensions")) { objectBuilder.add("dimensions", facetJson.getJsonArray("dimensions")); diff --git a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java index 7c4dd84d..37748cc0 100644 --- a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java +++ b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java @@ -49,7 +49,7 @@ public void testHasSearchDoc() throws Exception { Set docdbeans = new HashSet<>(Arrays.asList("Datafile", "DatafileFormat", "DatafileParameter", "Dataset", "DatasetParameter", "DatasetType", "Facility", "Instrument", "InstrumentScientist", "Investigation", "InvestigationInstrument", "InvestigationParameter", "InvestigationType", - "InvestigationUser", "ParameterType", "Sample", "SampleType", "User")); + "InvestigationUser", "ParameterType", "Sample", "SampleType", "SampleParameter", "User")); for (String beanName : EntityInfoHandler.getEntityNamesList()) { @SuppressWarnings("unchecked") Class bean = (Class) Class diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 8b977ec4..5eef20b1 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -41,6 +41,7 @@ import org.icatproject.core.entity.Parameter; import org.icatproject.core.entity.ParameterType; import org.icatproject.core.entity.Sample; +import org.icatproject.core.entity.SampleParameter; import org.icatproject.core.entity.SampleType; import org.icatproject.core.entity.User; import org.icatproject.core.manager.search.FacetDimension; @@ -77,6 +78,15 @@ public Filter(String fld, String... values) { } array = arrayBuilder.build(); } + + public Filter(String fld, JsonObject... values) { + this.fld = fld; + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + for (JsonObject value : values) { + arrayBuilder.add(value); + } + array = arrayBuilder.build(); + } } private static final String SEARCH_AFTER_NOT_NULL = "Expected searchAfter to be set, but it was null"; @@ -414,6 +424,14 @@ private Parameter parameter(long id, double value, ParameterType parameterType, return parameter; } + private Parameter parameter(long id, String value, double rangeBottom, double rangeTop, ParameterType parameterType, EntityBaseBean parent) { + Parameter parameter = parameter(id, parameterType, parent); + parameter.setStringValue(value); + parameter.setRangeBottom(rangeBottom); + parameter.setRangeTop(rangeTop); + return parameter; + } + private Parameter parameter(long id, ParameterType parameterType, EntityBaseBean parent) { Parameter parameter; if (parent instanceof Datafile) { @@ -425,6 +443,9 @@ private Parameter parameter(long id, ParameterType parameterType, EntityBaseBean } else if (parent instanceof Investigation) { parameter = new InvestigationParameter(); ((InvestigationParameter) parameter).setInvestigation((Investigation) parent); + } else if (parent instanceof Sample) { + parameter = new SampleParameter(); + ((SampleParameter) parameter).setSample((Sample) parent); } else { fail(parent.getClass().getSimpleName() + " is not valid"); return null; @@ -1115,6 +1136,50 @@ public void locking() throws IcatException { } } + @Test + public void fileSizeAggregation() throws IcatException { + // Build entities + Investigation investigation = investigation(0, "name", date, date); + Dataset dataset = dataset(0, "name", date, date, investigation); + Datafile datafile = datafile(0, "name", "/dir", new Date(0), dataset); + datafile.setFileSize(123L); + + // Build queries + JsonObject datafileQuery = buildQuery("Datafile", null, "*", null, null, null, null); + JsonObject datasetQuery = buildQuery("Dataset", null, "*", null, null, null, null); + JsonObject investigationQuery = buildQuery("Investigation", null, "*", null, null, null, null); + List fields = Arrays.asList("id", "fileSize", "fileCount"); + + // Create + modify(SearchApi.encodeOperation("create", investigation), SearchApi.encodeOperation("create", dataset), SearchApi.encodeOperation("create", datafile)); + checkFileSize(datafileQuery, fields, 123, 1); + checkFileSize(datasetQuery, fields, 123, 1); + checkFileSize(investigationQuery, fields, 123, 1); + + // Update + datafile.setFileSize(456L); + modify(SearchApi.encodeOperation("update", datafile)); + checkFileSize(datafileQuery, fields, 456, 1); + checkFileSize(datasetQuery, fields, 456, 1); + checkFileSize(investigationQuery, fields, 456, 1); + + // Delete + modify(SearchApi.encodeOperation("delete", datafile)); + checkFileSize(datasetQuery, fields, 0, 0); + checkFileSize(investigationQuery, fields, 0, 0); + } + + private void checkFileSize(JsonObject query, List fields, long expectedFileSize, long expectedFileCount) throws IcatException { + SearchResult results = searchApi.getResults(query, null, 5, null, fields); + checkResults(results, 0L); + JsonObject source = results.getResults().get(0).getSource(); + long fileSize = source.getJsonNumber("fileSize").longValueExact(); + long fileCount = source.getJsonNumber("fileCount").longValueExact(); + assertEquals(expectedFileSize,fileSize); + assertEquals(expectedFileCount,fileCount); + } + + @Test public void modifyDatafile() throws IcatException { // Build entities @@ -1260,4 +1325,98 @@ public void unitConversion() throws IcatException { checkFacets(searchApi.facetSearch("InvestigationParameter", systemFacetQuery, 5, 5), noneExpectedFacet); } + @Test + public void exactFilter() throws IcatException { + // Build entities + Investigation numericInvestigation = investigation(0, "numeric", date, date); + Investigation rangeInvestigation = investigation(1, "range", date, date); + ParameterType numericParameterType = parameterType(0, "numericParameter", "K"); + ParameterType rangeParameterType = parameterType(1, "rangeParameter", "K"); + Parameter numericParameter = parameter(0, 273, numericParameterType, numericInvestigation); + Parameter rangeParameter = parameter(1, "270 - 275", 270, 275, rangeParameterType, rangeInvestigation); + + JsonObjectBuilder filterBuilder = Json.createObjectBuilder(); + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + JsonObject value = Json.createObjectBuilder().add("field", "numericValue").add("exact", 273).build(); + JsonObject numericName = Json.createObjectBuilder().add("field", "type.name").add("value", "numericParameter").build(); + arrayBuilder.add(numericName).add(value); + filterBuilder.add("key", "key").add("label", "label").add("filter", arrayBuilder); + JsonObject numericFilter = filterBuilder.build(); + + filterBuilder = Json.createObjectBuilder(); + arrayBuilder = Json.createArrayBuilder(); + JsonObject rangeName = Json.createObjectBuilder().add("field", "type.name").add("value", "rangeParameter").build(); + arrayBuilder.add(rangeName).add(value); + filterBuilder.add("key", "key").add("label", "label").add("filter", arrayBuilder); + JsonObject rangeFilter = filterBuilder.build(); + + // Create + modify(SearchApi.encodeOperation("create", numericInvestigation), + SearchApi.encodeOperation("create", rangeInvestigation), + SearchApi.encodeOperation("create", numericParameter), + SearchApi.encodeOperation("create", rangeParameter)); + + JsonObject query = buildQuery("Investigation", null, null, null, null, null, null, + new Filter("investigationparameter", numericFilter)); + SearchResult lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 0L); + + query = buildQuery("Investigation", null, null, null, null, null, null, + new Filter("investigationparameter", rangeFilter)); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 1L); + } + + @Test + public void sampleParameters() throws IcatException { + // Build entities + Investigation investigation = investigation(0, "investigation", date, date); + Dataset dataset = dataset(1, "dataset", date, date, investigation); + Datafile datafile = datafile(2, "datafile", "datafile.txt", date, dataset); + Sample sample = sample(3, "sample", investigation); + ParameterType parameterType = parameterType(4, "parameter", "K"); + SampleParameter parameter = (SampleParameter) parameter(5, "stringValue", parameterType, sample); + dataset.setSample(sample); + + // Queries and expected responses + JsonObjectBuilder query = Json.createObjectBuilder().add("sample.id", Json.createArrayBuilder().add("3")); + JsonObjectBuilder dimension = Json.createObjectBuilder().add("dimension", "type.name"); + JsonArrayBuilder dimensions = Json.createArrayBuilder().add(dimension); + JsonObject facet = Json.createObjectBuilder().add("query", query).add("dimensions", dimensions).build(); + + JsonObjectBuilder filterBuilder = Json.createObjectBuilder(); + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + JsonObject value = Json.createObjectBuilder().add("field", "stringValue").add("value", "stringValue").build(); + JsonObject numericName = Json.createObjectBuilder().add("field", "type.name").add("value", "parameter").build(); + arrayBuilder.add(numericName).add(value); + filterBuilder.add("key", "key").add("label", "label").add("filter", arrayBuilder); + JsonObject filter = filterBuilder.build(); + + FacetDimension expectedFacet = new FacetDimension("", "type.name", new FacetLabel("parameter", 1L)); + JsonObject investigationQuery = buildQuery("Investigation", null, null, null, null, null, null, + new Filter("sampleparameter", filter)); + JsonObject datasetQuery = buildQuery("Dataset", null, null, null, null, null, null, + new Filter("sampleparameter", filter)); + JsonObject datafileQuery = buildQuery("Datafile", null, null, null, null, null, null, + new Filter("sampleparameter", filter)); + + // Create + modify(SearchApi.encodeOperation("create", investigation), + SearchApi.encodeOperation("create", dataset), + SearchApi.encodeOperation("create", datafile), + SearchApi.encodeOperation("create", sample), + SearchApi.encodeOperation("create", parameterType), + SearchApi.encodeOperation("create", parameter)); + + // Test + checkFacets(searchApi.facetSearch("SampleParameter", facet, 5, 5), expectedFacet); + + SearchResult lsr = searchApi.getResults(investigationQuery, null, 5, null, investigationFields); + checkResults(lsr, 0L); + lsr = searchApi.getResults(datasetQuery, null, 5, null, datasetFields); + checkResults(lsr, 1L); + lsr = searchApi.getResults(datafileQuery, null, 5, null, datafileFields); + checkResults(lsr, 2L); + } + } diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 03aeb188..862ce0fb 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -893,6 +893,8 @@ public void testSearchInvestigations() throws Exception { List parameters = new ArrayList<>(); parameters.add(new ParameterForLucene("colour", "name", "green")); + responseObject = searchInvestigations(session, "db/tr", null, null, null, null, + null, null, 10, null, null, 2); responseObject = searchInvestigations(session, "db/tr", null, lowerOrigin, upperOrigin, null, null, null, 10, null, null, 1); responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, null, From c141e426f85a05dab749dbe08d7748ebc3b0e6db Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Tue, 2 Aug 2022 15:05:31 +0000 Subject: [PATCH 29/51] Add utility to population and timed aggregation #267 --- .../core/manager/EntityBeanManager.java | 5 +- .../core/manager/PropertyHandler.java | 8 + .../core/manager/search/LuceneApi.java | 10 +- .../core/manager/search/SearchApi.java | 30 ++- .../core/manager/search/SearchManager.java | 187 +++++++++++++++--- .../org/icatproject/exposed/ICATRest.java | 49 +++-- src/main/resources/run.properties | 1 + 7 files changed, 235 insertions(+), 55 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 91773ff2..edb1dec2 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -1699,14 +1699,15 @@ public List searchGetPopulating() { } } - public void searchPopulate(String entityName, long minid, EntityManager manager) throws IcatException { + public void searchPopulate(String entityName, Long minId, Long maxId, boolean delete, EntityManager manager) + throws IcatException { if (searchActive) { try { Class.forName(Constants.ENTITY_PREFIX + entityName); } catch (ClassNotFoundException e) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, e.getMessage()); } - searchManager.populate(entityName, minid); + searchManager.populate(entityName, minId, maxId, delete); } } diff --git a/src/main/java/org/icatproject/core/manager/PropertyHandler.java b/src/main/java/org/icatproject/core/manager/PropertyHandler.java index 5935ac1f..80f2c673 100644 --- a/src/main/java/org/icatproject/core/manager/PropertyHandler.java +++ b/src/main/java/org/icatproject/core/manager/PropertyHandler.java @@ -308,6 +308,7 @@ public int getLifetimeMinutes() { private int searchPopulateBlockSize; private Path searchDirectory; private long searchBacklogHandlerIntervalMillis; + private long searchAggregateFilesIntervalMillis; private String unitAliasOptions; private Map cluster = new HashMap<>(); private long searchEnqueuedRequestIntervalMillis; @@ -512,6 +513,9 @@ private void init() { searchEnqueuedRequestIntervalMillis = props.getPositiveLong("search.enqueuedRequestIntervalSeconds"); formattedProps.add("search.enqueuedRequestIntervalSeconds" + " " + searchEnqueuedRequestIntervalMillis); searchEnqueuedRequestIntervalMillis *= 1000; + + searchAggregateFilesIntervalMillis = props.getNonNegativeLong("search.searchAggregateFilesIntervalSeconds"); + searchAggregateFilesIntervalMillis *= 1000; } else { logger.info("'search.engine' entry not present so no free text search available"); } @@ -662,6 +666,10 @@ public long getSearchEnqueuedRequestIntervalMillis() { return searchEnqueuedRequestIntervalMillis; } + public long getSearchAggregateFilesIntervalMillis() { + return searchAggregateFilesIntervalMillis; + } + public Path getSearchDirectory() { return searchDirectory; } diff --git a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java index 6468dd0b..72b77efa 100644 --- a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java @@ -200,16 +200,20 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer } /** - * Locks the index for entityName, removing all existing documents. While + * Locks the index for entityName, optionally removing all existing documents. While * locked, document modifications will fail (excluding addNow as a result of a * populate thread). * * @param entityName Index to lock. + * @param delete If true, all existing documents of entityName are deleted. + * @return The largest ICAT id currently stored in the index. * @throws IcatException */ @Override - public void lock(String entityName) throws IcatException { - post(basePath + "/lock/" + entityName); + public long lock(String entityName, boolean delete) throws IcatException { + String json = Json.createObjectBuilder().add("delete", delete).build().toString(); + JsonObject postResponse = postResponse(basePath + "/lock/" + entityName, json); + return postResponse.getJsonNumber("currentId").longValueExact(); } /** diff --git a/src/main/java/org/icatproject/core/manager/search/SearchApi.java b/src/main/java/org/icatproject/core/manager/search/SearchApi.java index 7be69ba1..240222c9 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchApi.java @@ -398,10 +398,13 @@ public abstract SearchResult getResults(JsonObject query, JsonValue searchAfter, * Not implemented. * * @param entityName + * @param delete + * @return long * @throws IcatException */ - public void lock(String entityName) throws IcatException { + public long lock(String entityName, boolean delete) throws IcatException { logger.info("Manually locking index not supported, no request sent"); + return 0; } /** @@ -463,6 +466,31 @@ protected void post(String path, String body) throws IcatException { } } + /** + * POST to path with a body and response handling. + * + * @param path Path on the search engine to POST to. + * @param body String of Json to send as the request body. + * @return JsonObject returned by the search engine. + * @throws IcatException + */ + protected JsonObject postResponse(String path, String body) throws IcatException { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URI uri = new URIBuilder(server).setPath(path).build(); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + logger.trace("Making call {} with body {}", uri, body); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + JsonReader jsonReader = Json.createReader(response.getEntity().getContent()); + return jsonReader.readObject(); + } + } catch (URISyntaxException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + + /** * POST to path with a body and response handling. * diff --git a/src/main/java/org/icatproject/core/manager/search/SearchManager.java b/src/main/java/org/icatproject/core/manager/search/SearchManager.java index 99ce1d97..77779a89 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchManager.java @@ -9,6 +9,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Set; import java.util.List; import java.util.Map; @@ -92,14 +93,10 @@ public void run() { // Record failures in a flat file to be examined periodically logger.error("Search engine failed to modify documents with error {} : {}", e.getClass(), e.getMessage()); - synchronized (backlogHandlerFileLock) { - try { - FileWriter output = new FileWriter(backlogHandlerFile, true); - output.write(sb.toString() + "\n"); - output.close(); - } catch (IOException e2) { - logger.error("Problems writing to {} : {}", backlogHandlerFile, e2.getMessage()); - } + try { + synchronizedWrite(sb.toString(), backlogHandlerFileLock, backlogHandlerFile); + } catch (IcatException e2) { + // Already logged the error } } finally { queueFile.delete(); @@ -121,7 +118,7 @@ public class IndexSome implements Callable { public IndexSome(String entityName, List ids, EntityManagerFactory entityManagerFactory, long start) throws IcatException { try { - logger.debug("About to index {} {} records", ids.size(), entityName); + logger.debug("About to index {} {} records after id {}", ids.size(), entityName, start); this.entityName = entityName; klass = (Class) Class.forName(Constants.ENTITY_PREFIX + entityName); this.ids = ids; @@ -169,10 +166,78 @@ public void run() { } } + /** + * Handles the the aggregation of the fileSize and fileCount fields for Dataset + * and Investigation entities. + */ + private class AggregateFilesHandler extends TimerTask { + + @Override + public void run() { + EntityManager entityManager = entityManagerFactory.createEntityManager(); + aggregate(entityManager, datasetAggregationFileLock, datasetAggregationFile, Dataset.class); + aggregate(entityManager, investigationAggregationFileLock, investigationAggregationFile, + Investigation.class); + } + + /** + * Performs aggregation by reading the unique id values from file and querying + * the DB for the full entity (including fileSize and fileCount fields). This is + * then submitted as an update to the search engine. + * + * @param entityManager JPQL EntityManager for querying + * @param fileLock Lock for the file + * @param file File to read the ids of entities from + * @param klass Class of the entity to be aggregated + */ + private void aggregate(EntityManager entityManager, Long fileLock, File file, + Class klass) { + String entityName = klass.getSimpleName(); + synchronized (fileLock) { + if (file.length() != 0) { + logger.debug("Will attempt to process {}", file); + try (BufferedReader reader = new BufferedReader(new FileReader(file))) { + String line; + Set datasetIds = new HashSet<>(); + while ((line = reader.readLine()) != null) { + if (datasetIds.add(line)) { + String query = "SELECT e FROM " + entityName + " e WHERE e.id = " + line; + EntityBaseBean dataset = entityManager.createQuery(query, klass).getSingleResult(); + updateDocument(dataset); + } + } + file.delete(); + logger.info(entityName + " aggregations performed"); + } catch (IOException e) { + logger.error("Problems reading from {} : {}", file, e.getMessage()); + } catch (Throwable e) { + logger.error("Something unexpected happened " + e.getClass() + " " + e.getMessage()); + } + logger.debug("finish processing"); + } + } + } + } + private enum PopState { STOPPING, STOPPED } + /** + * Holds relevant values for a Populate thread. + */ + private class PopulateBucket { + private Long minId; + private Long maxId; + private boolean delete; + + public PopulateBucket(Long minId, Long maxId, boolean delete) { + this.minId = minId; + this.maxId = maxId; + this.delete = delete; + } + } + public class PopulateThread extends Thread { private EntityManager manager; @@ -193,9 +258,15 @@ public void run() { populatingClassEntry = populateMap.firstEntry(); if (populatingClassEntry != null) { - searchApi.lock(populatingClassEntry.getKey()); - - Long start = populatingClassEntry.getValue(); + PopulateBucket bucket = populatingClassEntry.getValue(); + Long start = bucket.minId != null ? bucket.minId : 0; + long currentId = searchApi.lock(populatingClassEntry.getKey(), bucket.delete); + if (currentId > start) { + searchApi.unlock(populatingClassEntry.getKey()); + populateMap.remove(populatingClassEntry.getKey()); + throw new IcatException(IcatExceptionType.BAD_PARAMETER, "Populate called starting from id " + + start + " but current id is greater: " + currentId); + } logger.info("Search engine populating " + populatingClassEntry); @@ -209,9 +280,16 @@ public void run() { break; } /* Get next block of ids */ + String query = "SELECT e.id from " + populatingClassEntry.getKey() + " e"; + if (bucket.maxId != null) { + // Add 1 from lower limit to get a half interval + query += " WHERE e.id BETWEEN " + (start + 1) + " AND " + (bucket.maxId); + } else { + query += " WHERE e.id > " + start; + } + query += " ORDER BY e.id"; List ids = manager - .createQuery("SELECT e.id from " + populatingClassEntry.getKey() - + " e WHERE e.id > " + start + " ORDER BY e.id", Long.class) + .createQuery(query, Long.class) .setMaxResults(populateBlockSize).getResultList(); if (ids.size() == 0) { break; @@ -222,7 +300,8 @@ public void run() { while ((fut = threads.poll()) != null) { Long s = fut.get(); if (s.equals(tasks.first())) { - populateMap.put(populatingClassEntry.getKey(), s); + PopulateBucket populateBucket = new PopulateBucket(s, bucket.maxId, bucket.delete); + populateMap.put(populatingClassEntry.getKey(), populateBucket); } tasks.remove(s); } @@ -232,12 +311,14 @@ public void run() { fut = threads.take(); Long s = fut.get(); if (s.equals(tasks.first())) { - populateMap.put(populatingClassEntry.getKey(), s); + PopulateBucket populateBucket = new PopulateBucket(s, bucket.maxId, bucket.delete); + populateMap.put(populatingClassEntry.getKey(), populateBucket); } tasks.remove(s); } - logger.debug("About to submit " + ids.size() + " " + populatingClassEntry + " documents"); + logger.debug("About to submit {} {} documents from id {} onwards", ids.size(), + populatingClassEntry, start); threads.submit( new IndexSome(populatingClassEntry.getKey(), ids, entityManagerFactory, start)); tasks.add(start); @@ -252,7 +333,8 @@ public void run() { fut = threads.take(); Long s = fut.get(); if (s.equals(tasks.first())) { - populateMap.put(populatingClassEntry.getKey(), s); + PopulateBucket populateBucket = new PopulateBucket(s, bucket.maxId, bucket.delete); + populateMap.put(populatingClassEntry.getKey(), populateBucket); } tasks.remove(s); } @@ -282,11 +364,11 @@ public void run() { /** * The Set of classes for which population is requested */ - private ConcurrentSkipListMap populateMap = new ConcurrentSkipListMap<>(); + private ConcurrentSkipListMap populateMap = new ConcurrentSkipListMap<>(); /** The thread which does the population */ private PopulateThread populateThread; - private Entry populatingClassEntry; + private Entry populatingClassEntry; @PersistenceUnit(unitName = "icat") private EntityManagerFactory entityManagerFactory; @@ -307,10 +389,16 @@ public void run() { private boolean active; + private long aggregateFilesIntervalMillis; + private Long backlogHandlerFileLock = 0L; private Long queueFileLock = 0L; + private Long datasetAggregationFileLock = 0L; + + private Long investigationAggregationFileLock = 0L; + private Timer timer; private Set entitiesToIndex; @@ -319,6 +407,10 @@ public void run() { private File queueFile; + private File datasetAggregationFile; + + private File investigationAggregationFile; + private SearchEngine searchEngine; private List urls; @@ -353,14 +445,25 @@ public void addDocument(EntityBaseBean bean) throws IcatException { Class klass = bean.getClass(); if (eiHandler.hasSearchDoc(klass) && entitiesToIndex.contains(klass.getSimpleName())) { enqueue(SearchApi.encodeOperation("create", bean)); + enqueueAggregation(bean); } } - public void enqueue(String json) throws IcatException { - synchronized (queueFileLock) { + private void enqueue(String json) throws IcatException { + synchronizedWrite(json, queueFileLock, queueFile); + } + + /** + * @param line String to write to file, followed by \n. + * @param fileLock Lock for the file + * @param file File to write to + * @throws IcatException + */ + private void synchronizedWrite(String line, Long fileLock, File file) throws IcatException { + synchronized (fileLock) { try { - FileWriter output = new FileWriter(queueFile, true); - output.write(json + "\n"); + FileWriter output = new FileWriter(file, true); + output.write(line + "\n"); output.close(); } catch (IOException e) { String msg = "Problems writing to " + queueFile + " " + e.getMessage(); @@ -368,7 +471,28 @@ public void enqueue(String json) throws IcatException { throw new IcatException(IcatExceptionType.INTERNAL, msg); } } + } + /** + * If bean is a Datafile and an aggregation interval is set, then the Datafile's + * Dataset and Investigation ids are written to file to be aggregated at a later + * date. + * + * @param bean Entity to consider for aggregation. + * @throws IcatException + */ + private void enqueueAggregation(EntityBaseBean bean) throws IcatException { + if (bean.getClass().getSimpleName().equals("Datafile") && aggregateFilesIntervalMillis > 0) { + Dataset dataset = ((Datafile) bean).getDataset(); + if (dataset != null) { + synchronizedWrite(dataset.getId().toString(), datasetAggregationFileLock, datasetAggregationFile); + Investigation investigation = dataset.getInvestigation(); + if (investigation != null) { + synchronizedWrite(investigation.getId().toString(), investigationAggregationFileLock, + investigationAggregationFile); + } + } + } } public void clear() throws IcatException { @@ -392,6 +516,7 @@ public void commit() throws IcatException { public void deleteDocument(EntityBaseBean bean) throws IcatException { if (eiHandler.hasSearchDoc(bean.getClass())) { enqueue(SearchApi.encodeDeletion(bean)); + enqueueAggregation(bean); } } @@ -562,13 +687,12 @@ public List facetSearch(String target, JsonObject facetQuery, in public List getPopulating() { List result = new ArrayList<>(); - for (Entry e : populateMap.entrySet()) { + for (Entry e : populateMap.entrySet()) { result.add(e.getKey() + " " + e.getValue()); } return result; } - /** * Gets SearchResult for query without searchAfter (pagination). * @@ -622,6 +746,8 @@ private void init() { Path searchDirectory = propertyHandler.getSearchDirectory(); backlogHandlerFile = searchDirectory.resolve("backLog").toFile(); queueFile = searchDirectory.resolve("queue").toFile(); + datasetAggregationFile = searchDirectory.resolve("datasetAggregation").toFile(); + investigationAggregationFile = searchDirectory.resolve("investigationAggregation").toFile(); maxThreads = Runtime.getRuntime().availableProcessors(); populateExecutor = Executors.newWorkStealingPool(maxThreads); getBeanDocExecutor = Executors.newCachedThreadPool(); @@ -630,6 +756,10 @@ private void init() { propertyHandler.getSearchBacklogHandlerIntervalMillis()); timer.schedule(new EnqueuedSearchRequestHandler(), 0L, propertyHandler.getSearchEnqueuedRequestIntervalMillis()); + aggregateFilesIntervalMillis = propertyHandler.getSearchAggregateFilesIntervalMillis(); + if (aggregateFilesIntervalMillis > 0) { + timer.schedule(new AggregateFilesHandler(), 0L, aggregateFilesIntervalMillis); + } entitiesToIndex = propertyHandler.getEntitiesToIndex(); logger.info("Initialised SearchManager at {}", urls); } catch (Exception e) { @@ -645,7 +775,7 @@ public boolean isActive() { return active; } - public void populate(String entityName, long minid) throws IcatException { + public void populate(String entityName, Long minId, Long maxId, boolean delete) throws IcatException { if (popState == PopState.STOPPING) { while (populateThread != null && populateThread.getState() != Thread.State.TERMINATED) { try { @@ -655,7 +785,7 @@ public void populate(String entityName, long minid) throws IcatException { } } } - if (populateMap.put(entityName, minid) == null) { + if (populateMap.put(entityName, new PopulateBucket(minId, maxId, delete)) == null) { logger.debug("Search engine population of {} requested", entityName); } else { throw new IcatException(IcatExceptionType.OBJECT_ALREADY_EXISTS, @@ -671,6 +801,7 @@ public void updateDocument(EntityBaseBean bean) throws IcatException { Class klass = bean.getClass(); if (eiHandler.hasSearchDoc(klass) && entitiesToIndex.contains(klass.getSimpleName())) { enqueue(SearchApi.encodeOperation("update", bean)); + enqueueAggregation(bean); } } diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index 57d12184..d23ed616 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -1381,9 +1381,10 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId * * @summary Document faceting. * - * @param sessionId a sessionId of a user which takes the form 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 - * @param query Json of the format - * { + * @param sessionId a sessionId of a user which takes the form + * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 + * @param query Json of the format + * { * "target": `target`, * "facets": [ * { @@ -1445,7 +1446,11 @@ public String facet(@Context HttpServletRequest request, @QueryParam("sessionId" for (FacetLabel label : dimension.getFacets()) { logger.debug("From and to: ", label.getFrom(), label.getTo()); if (label.getFrom() != null && label.getTo() != null) { - gen.writeStartObject(label.getLabel()).write("from", label.getFrom()).write("to", label.getTo()).write("count", label.getValue()).writeEnd(); + gen.writeStartObject(label.getLabel()); + gen.write("from", label.getFrom()); + gen.write("to", label.getTo()); + gen.write("count", label.getValue()); + gen.writeEnd(); } else { gen.write(label.getLabel(), label.getValue()); } @@ -1584,27 +1589,29 @@ public void waitMillis(@FormParam("sessionId") String sessionId, @FormParam("ms" } /** - * Clear and repopulate search engine documents for the specified entityName - * - * @summary Search engine populate - * - * @param sessionId - * a sessionId of a user listed in rootUserNames - * @param entityName - * the name of the entity - * @param minid - * only process entities with id values greater than this - * value - * - * @throws IcatException - * when something is wrong + * Populates search engine documents for the specified entityName. + * + * Optionally, this will also delete all existing documents of entityName. This + * should only be used when repopulating from scratch is needed. + * + * @param sessionId a sessionId of a user listed in rootUserNames + * @param entityName the name of the entity + * @param minId Process entities with id values greater than (NOT equal to) + * this value + * @param maxId Process entities up to and including with id up to and + * including this value + * @param delete If true, then all existing documents of this type will be + * deleted before adding new ones. + * @throws IcatException when something is wrong */ @POST - @Path("lucene/db/{entityName}/{minid}") + @Consumes(MediaType.APPLICATION_FORM_URLENCODED) + @Path("lucene/db/{entityName}") public void searchPopulate(@FormParam("sessionId") String sessionId, @PathParam("entityName") String entityName, - @PathParam("minid") long minid) throws IcatException { + @FormParam("minId") Long minId, @FormParam("maxId") Long maxId, @FormParam("delete") boolean delete) + throws IcatException { checkRoot(sessionId); - beanManager.searchPopulate(entityName, minid, manager); + beanManager.searchPopulate(entityName, minId, maxId, delete, manager); } /** diff --git a/src/main/resources/run.properties b/src/main/resources/run.properties index 4cf67f9c..a6186eae 100644 --- a/src/main/resources/run.properties +++ b/src/main/resources/run.properties @@ -22,6 +22,7 @@ search.populateBlockSize = 10000 search.directory = ${HOME}/data/search search.backlogHandlerIntervalSeconds = 60 search.enqueuedRequestIntervalSeconds = 3 +search.aggregateFilesIntervalSeconds = 3600 # Configure this option to prevent certain entities being indexed # For example, remove Datafile and DatafileParameter !search.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample From 42bcef0cd28b38355d66b8b443017d7b7f70dae8 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Sun, 24 Jul 2022 08:00:58 +0100 Subject: [PATCH 30/51] Improve timed file aggregation #267 --- .../core/manager/PropertyHandler.java | 2 +- .../core/manager/search/OpensearchApi.java | 10 ++-- .../core/manager/search/SearchManager.java | 50 ++++++++++++------- .../core/manager/TestSearchApi.java | 8 +-- .../org/icatproject/integration/TestRS.java | 6 +-- src/test/scripts/prepare_test.py | 1 + 6 files changed, 48 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/PropertyHandler.java b/src/main/java/org/icatproject/core/manager/PropertyHandler.java index 80f2c673..e48c4e43 100644 --- a/src/main/java/org/icatproject/core/manager/PropertyHandler.java +++ b/src/main/java/org/icatproject/core/manager/PropertyHandler.java @@ -514,7 +514,7 @@ private void init() { formattedProps.add("search.enqueuedRequestIntervalSeconds" + " " + searchEnqueuedRequestIntervalMillis); searchEnqueuedRequestIntervalMillis *= 1000; - searchAggregateFilesIntervalMillis = props.getNonNegativeLong("search.searchAggregateFilesIntervalSeconds"); + searchAggregateFilesIntervalMillis = props.getNonNegativeLong("search.aggregateFilesIntervalSeconds"); searchAggregateFilesIntervalMillis *= 1000; } else { logger.info("'search.engine' entry not present so no free text search available"); diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index 615fd3a5..df6a4674 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -80,6 +80,7 @@ public ParentRelation(RelationType relationType, String parentName, String joinF } } + private boolean aggregateFiles = false; private IcatUnits icatUnits; protected static final Logger logger = LoggerFactory.getLogger(OpensearchApi.class); private static JsonObject indexSettings = Json.createObjectBuilder().add("analysis", Json.createObjectBuilder() @@ -206,9 +207,10 @@ public OpensearchApi(URI server) throws IcatException { initScripts(); } - public OpensearchApi(URI server, String unitAliasOptions) throws IcatException { + public OpensearchApi(URI server, String unitAliasOptions, boolean aggregateFiles) throws IcatException { super(server); icatUnits = new IcatUnits(unitAliasOptions); + this.aggregateFiles = aggregateFiles; initMappings(); initScripts(); } @@ -1577,7 +1579,7 @@ private void modifyEntity(CloseableHttpClient httpclient, StringBuilder sb, Set< // entities are attached to an Investigation, so need to check for those investigationIds.add(document.getString("investigation.id")); } - if (index.equals("datafile") && document.containsKey("fileSize")) { + if (aggregateFiles && index.equals("datafile") && document.containsKey("fileSize")) { long newFileSize = document.getJsonNumber("fileSize").longValueExact(); if (document.containsKey("investigation.id")) { String investigationId = document.getString("investigation.id"); @@ -1597,7 +1599,7 @@ private void modifyEntity(CloseableHttpClient httpclient, StringBuilder sb, Set< case UPDATE: docAsUpsert = Json.createObjectBuilder().add("doc", document).add("doc_as_upsert", true).build(); sb.append(update.toString()).append("\n").append(docAsUpsert.toString()).append("\n"); - if (index.equals("datafile") && document.containsKey("fileSize")) { + if (aggregateFiles && index.equals("datafile") && document.containsKey("fileSize")) { long newFileSize = document.getJsonNumber("fileSize").longValueExact(); long oldFileSize; JsonObject source = extractSource(httpclient, id); @@ -1625,7 +1627,7 @@ private void modifyEntity(CloseableHttpClient httpclient, StringBuilder sb, Set< break; case DELETE: sb.append(Json.createObjectBuilder().add("delete", targetObject).build().toString()).append("\n"); - if (index.equals("datafile")) { + if (aggregateFiles && index.equals("datafile")) { JsonObject source = extractSource(httpclient, id); if (source != null && source.containsKey("fileSize")) { long oldFileSize = source.getJsonNumber("fileSize").longValueExact(); diff --git a/src/main/java/org/icatproject/core/manager/search/SearchManager.java b/src/main/java/org/icatproject/core/manager/search/SearchManager.java index 77779a89..84a4bce3 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchManager.java @@ -5,6 +5,7 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.net.URI; import java.net.URL; import java.nio.file.Path; import java.util.ArrayList; @@ -88,6 +89,7 @@ public void run() { try { searchApi.modify(sb.toString()); + logger.info("Enqueued search documents now all indexed"); } catch (Exception e) { // Catch all exceptions so the Timer doesn't end unexpectedly // Record failures in a flat file to be examined periodically @@ -100,6 +102,7 @@ public void run() { } } finally { queueFile.delete(); + logger.debug("finish processing, queue File removed"); } } } @@ -172,12 +175,16 @@ public void run() { */ private class AggregateFilesHandler extends TimerTask { + private EntityManager entityManager; + + public AggregateFilesHandler(EntityManager entityManager) { + this.entityManager = entityManager; + } + @Override public void run() { - EntityManager entityManager = entityManagerFactory.createEntityManager(); - aggregate(entityManager, datasetAggregationFileLock, datasetAggregationFile, Dataset.class); - aggregate(entityManager, investigationAggregationFileLock, investigationAggregationFile, - Investigation.class); + aggregate(datasetAggregationFileLock, datasetAggregationFile, Dataset.class); + aggregate(investigationAggregationFileLock, investigationAggregationFile, Investigation.class); } /** @@ -185,25 +192,27 @@ public void run() { * the DB for the full entity (including fileSize and fileCount fields). This is * then submitted as an update to the search engine. * - * @param entityManager JPQL EntityManager for querying - * @param fileLock Lock for the file - * @param file File to read the ids of entities from - * @param klass Class of the entity to be aggregated + * @param fileLock Lock for the file + * @param file File to read the ids of entities from + * @param klass Class of the entity to be aggregated */ - private void aggregate(EntityManager entityManager, Long fileLock, File file, - Class klass) { + private void aggregate(Long fileLock, File file, Class klass) { String entityName = klass.getSimpleName(); synchronized (fileLock) { if (file.length() != 0) { logger.debug("Will attempt to process {}", file); try (BufferedReader reader = new BufferedReader(new FileReader(file))) { String line; - Set datasetIds = new HashSet<>(); + Set ids = new HashSet<>(); while ((line = reader.readLine()) != null) { - if (datasetIds.add(line)) { + if (ids.add(line)) { // True if id not yet encountered String query = "SELECT e FROM " + entityName + " e WHERE e.id = " + line; - EntityBaseBean dataset = entityManager.createQuery(query, klass).getSingleResult(); - updateDocument(dataset); + try { + EntityBaseBean entity = entityManager.createQuery(query, klass).getSingleResult(); + updateDocument(entity); + } catch (Exception e) { + logger.error("{} with id {} not found, continue", entityName, line); + } } } file.delete(); @@ -259,7 +268,7 @@ public void run() { if (populatingClassEntry != null) { PopulateBucket bucket = populatingClassEntry.getValue(); - Long start = bucket.minId != null ? bucket.minId : 0; + Long start = bucket.minId != null && bucket.minId > 0 ? bucket.minId : 0; long currentId = searchApi.lock(populatingClassEntry.getKey(), bucket.delete); if (currentId > start) { searchApi.unlock(populatingClassEntry.getKey()); @@ -732,11 +741,15 @@ private void init() { active = urls != null && urls.size() > 0; if (active) { try { + URI uri = propertyHandler.getSearchUrls().get(0).toURI(); if (searchEngine == SearchEngine.LUCENE) { - searchApi = new LuceneApi(propertyHandler.getSearchUrls().get(0).toURI()); + searchApi = new LuceneApi(uri); } else if (searchEngine == SearchEngine.ELASTICSEARCH || searchEngine == SearchEngine.OPENSEARCH) { String unitAliasOptions = propertyHandler.getUnitAliasOptions(); - searchApi = new OpensearchApi(propertyHandler.getSearchUrls().get(0).toURI(), unitAliasOptions); + // If interval is not set then aggregate in real time + long aggregateFilesInterval = propertyHandler.getSearchAggregateFilesIntervalMillis(); + boolean aggregateFiles = aggregateFilesInterval == 0; + searchApi = new OpensearchApi(uri, unitAliasOptions, aggregateFiles); } else { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "Search engine {} not supported, must be one of " + SearchEngine.values()); @@ -758,7 +771,8 @@ private void init() { propertyHandler.getSearchEnqueuedRequestIntervalMillis()); aggregateFilesIntervalMillis = propertyHandler.getSearchAggregateFilesIntervalMillis(); if (aggregateFilesIntervalMillis > 0) { - timer.schedule(new AggregateFilesHandler(), 0L, aggregateFilesIntervalMillis); + EntityManager entityManager = entityManagerFactory.createEntityManager(); + timer.schedule(new AggregateFilesHandler(entityManager), 0L, aggregateFilesIntervalMillis); } entitiesToIndex = propertyHandler.getEntitiesToIndex(); logger.info("Initialised SearchManager at {}", urls); diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 5eef20b1..5810dd35 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -53,6 +53,7 @@ import org.icatproject.core.manager.search.SearchApi; import org.icatproject.core.manager.search.SearchResult; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -111,7 +112,7 @@ public static Iterable data() throws URISyntaxException, IcatExceptio logger.info("Using Opensearch/Elasticsearch service at {}", opensearchUrl); URI opensearchUri = new URI(opensearchUrl); - return Arrays.asList(new LuceneApi(luceneUri), new OpensearchApi(opensearchUri, "\u2103: celsius")); + return Arrays.asList(new LuceneApi(luceneUri), new OpensearchApi(opensearchUri, "\u2103: celsius", false)); } @Parameterized.Parameter @@ -1117,9 +1118,9 @@ public void locking() throws IcatException { } catch (IcatException e) { assertEquals("Lucene is not currently locked for Dataset", e.getMessage()); } - searchApi.lock("Dataset"); + searchApi.lock("Dataset", true); try { - searchApi.lock("Dataset"); + searchApi.lock("Dataset", true); fail(); } catch (IcatException e) { assertEquals("Lucene already locked for Dataset", e.getMessage()); @@ -1136,6 +1137,7 @@ public void locking() throws IcatException { } } + @Ignore // Aggregating in real time is really slow, so don't test @Test public void fileSizeAggregation() throws IcatException { // Build entities diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 862ce0fb..a17d0bbd 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -2417,9 +2417,9 @@ public void testLucenePopulate() throws Exception { assertTrue(session.luceneGetPopulating().isEmpty()); - session.lucenePopulate("Dataset", -1); - session.lucenePopulate("Datafile", -1); - session.lucenePopulate("Investigation", -1); + session.lucenePopulate("Dataset", 0); + session.lucenePopulate("Datafile", 0); + session.lucenePopulate("Investigation", 0); do { Thread.sleep(1000); diff --git a/src/test/scripts/prepare_test.py b/src/test/scripts/prepare_test.py index 463f71cb..89333a03 100644 --- a/src/test/scripts/prepare_test.py +++ b/src/test/scripts/prepare_test.py @@ -54,6 +54,7 @@ "search.directory = %s/data/search" % subst["HOME"], "search.backlogHandlerIntervalSeconds = 60", "search.enqueuedRequestIntervalSeconds = 3", + "search.aggregateFilesIntervalSeconds = 3600", "key = wombat" ] f.write("\n".join(contents)) From e7a322cb8e0609116e12976192125af773818ce0 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 5 Aug 2022 08:50:15 +0000 Subject: [PATCH 31/51] Improved timeout and search syntax errors #267 --- .../core/manager/EntityBeanManager.java | 20 ++++++----- .../core/manager/PropertyHandler.java | 9 +++++ .../core/manager/search/LuceneApi.java | 34 ++++++++----------- .../core/manager/search/SearchApi.java | 3 +- .../core/manager/search/SearchResult.java | 9 ----- .../org/icatproject/exposed/ICATRest.java | 10 ------ 6 files changed, 38 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index edb1dec2..f42bdf53 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -150,6 +150,7 @@ public enum PersistMode { private Map notificationRequests; private boolean searchActive; + private long searchMaxSearchTimeMillis; private int maxEntities; @@ -1163,6 +1164,7 @@ void init() { log = !logRequests.isEmpty(); notificationRequests = propertyHandler.getNotificationRequests(); searchActive = searchManager.isActive(); + searchMaxSearchTimeMillis = propertyHandler.getSearchMaxSearchTimeMillis(); maxEntities = propertyHandler.getMaxEntities(); exportCacheSize = propertyHandler.getImportCacheSize(); rootUserNames = propertyHandler.getRootUserNames(); @@ -1418,7 +1420,7 @@ public void searchCommit() throws IcatException { */ public List freeTextSearch(String userName, JsonObject jo, int limit, String sort, EntityManager manager, String ip, Class klass) throws IcatException { - long startMillis = log ? System.currentTimeMillis() : 0; + long startMillis = System.currentTimeMillis(); List results = new ArrayList<>(); JsonValue searchAfter = null; JsonValue lastSearchAfter = null; @@ -1433,9 +1435,6 @@ public List freeTextSearch(String userName, JsonObject jo, do { lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, Arrays.asList("id")); - if (lastSearchResult.isAborted()) { - break; - } allResults = lastSearchResult.getResults(); ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, limit, userName, manager, klass); if (lastBean == null) { @@ -1450,6 +1449,10 @@ public List freeTextSearch(String userName, JsonObject jo, lastSearchAfter = searchManager.buildSearchAfter(lastBean, sort); break; } + if (System.currentTimeMillis() - startMillis > searchMaxSearchTimeMillis) { + String msg = "Search cancelled for exceeding " + searchMaxSearchTimeMillis / 1000 + " seconds"; + throw new IcatException(IcatExceptionType.INTERNAL, msg); + } } while (results.size() < limit); } @@ -1493,7 +1496,7 @@ public List freeTextSearch(String userName, JsonObject jo, public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue searchAfter, int minCount, int maxCount, String sort, EntityManager manager, String ip, Class klass) throws IcatException { - long startMillis = log ? System.currentTimeMillis() : 0; + long startMillis = System.currentTimeMillis(); List results = new ArrayList<>(); JsonValue lastSearchAfter = null; List dimensions = new ArrayList<>(); @@ -1509,9 +1512,6 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue do { lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, fields); - if (lastSearchResult.isAborted()) { - return lastSearchResult; - } allResults = lastSearchResult.getResults(); ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, maxCount, userName, manager, klass); @@ -1527,6 +1527,10 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue lastSearchAfter = searchManager.buildSearchAfter(lastBean, sort); break; } + if (System.currentTimeMillis() - startMillis > searchMaxSearchTimeMillis) { + String msg = "Search cancelled for exceeding " + searchMaxSearchTimeMillis / 1000 + " seconds"; + throw new IcatException(IcatExceptionType.INTERNAL, msg); + } } while (results.size() < minCount); if (jo.containsKey("facets")) { diff --git a/src/main/java/org/icatproject/core/manager/PropertyHandler.java b/src/main/java/org/icatproject/core/manager/PropertyHandler.java index e48c4e43..70174cf2 100644 --- a/src/main/java/org/icatproject/core/manager/PropertyHandler.java +++ b/src/main/java/org/icatproject/core/manager/PropertyHandler.java @@ -309,6 +309,7 @@ public int getLifetimeMinutes() { private Path searchDirectory; private long searchBacklogHandlerIntervalMillis; private long searchAggregateFilesIntervalMillis; + private long searchMaxSearchTimeMillis; private String unitAliasOptions; private Map cluster = new HashMap<>(); private long searchEnqueuedRequestIntervalMillis; @@ -516,6 +517,10 @@ private void init() { searchAggregateFilesIntervalMillis = props.getNonNegativeLong("search.aggregateFilesIntervalSeconds"); searchAggregateFilesIntervalMillis *= 1000; + + searchMaxSearchTimeMillis = props.getPositiveLong("search.maxSearchTimeSeconds"); + formattedProps.add("search.maxSearchTimeSeconds" + " " + searchMaxSearchTimeMillis); + searchMaxSearchTimeMillis *= 1000; } else { logger.info("'search.engine' entry not present so no free text search available"); } @@ -670,6 +675,10 @@ public long getSearchAggregateFilesIntervalMillis() { return searchAggregateFilesIntervalMillis; } + public long getSearchMaxSearchTimeMillis() { + return searchMaxSearchTimeMillis; + } + public Path getSearchDirectory() { return searchDirectory; } diff --git a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java index 72b77efa..cdd4e811 100644 --- a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java @@ -174,26 +174,22 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer JsonObject postResponse = postResponse(basePath + "/" + indexPath, queryString, parameterMap); SearchResult lsr = new SearchResult(); - if (postResponse.containsKey("aborted") && postResponse.getBoolean("aborted")) { - lsr.setAborted(true); - } else { - List results = lsr.getResults(); - List resultsArray = postResponse.getJsonArray("results").getValuesAs(JsonObject.class); - for (JsonObject resultObject : resultsArray) { - int luceneDocId = resultObject.getInt("_id"); - int shardIndex = resultObject.getInt("_shardIndex"); - Float score = Float.NaN; - if (resultObject.keySet().contains("_score")) { - score = resultObject.getJsonNumber("_score").bigDecimalValue().floatValue(); - } - JsonObject source = resultObject.getJsonObject("_source"); - ScoredEntityBaseBean result = new ScoredEntityBaseBean(luceneDocId, shardIndex, score, source); - results.add(result); - logger.trace("Result id {} with score {}", result.getEntityBaseBeanId(), score); - } - if (postResponse.containsKey("search_after")) { - lsr.setSearchAfter(postResponse.getJsonObject("search_after")); + List results = lsr.getResults(); + List resultsArray = postResponse.getJsonArray("results").getValuesAs(JsonObject.class); + for (JsonObject resultObject : resultsArray) { + int luceneDocId = resultObject.getInt("_id"); + int shardIndex = resultObject.getInt("_shardIndex"); + Float score = Float.NaN; + if (resultObject.keySet().contains("_score")) { + score = resultObject.getJsonNumber("_score").bigDecimalValue().floatValue(); } + JsonObject source = resultObject.getJsonObject("_source"); + ScoredEntityBaseBean result = new ScoredEntityBaseBean(luceneDocId, shardIndex, score, source); + results.add(result); + logger.trace("Result id {} with score {}", result.getEntityBaseBeanId(), score); + } + if (postResponse.containsKey("search_after")) { + lsr.setSearchAfter(postResponse.getJsonObject("search_after")); } return lsr; diff --git a/src/main/java/org/icatproject/core/manager/search/SearchApi.java b/src/main/java/org/icatproject/core/manager/search/SearchApi.java index 240222c9..dcab42e7 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchApi.java @@ -511,7 +511,8 @@ protected JsonObject postResponse(String path, String body, Map httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); logger.trace("Making call {} with body {}", uri, body); try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); + int code = response.getStatusLine().getStatusCode(); + Rest.checkStatus(response, code == 400 ? IcatExceptionType.BAD_PARAMETER : IcatExceptionType.INTERNAL); JsonReader jsonReader = Json.createReader(response.getEntity().getContent()); return jsonReader.readObject(); } diff --git a/src/main/java/org/icatproject/core/manager/search/SearchResult.java b/src/main/java/org/icatproject/core/manager/search/SearchResult.java index 146d7949..db8ad31d 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchResult.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchResult.java @@ -15,7 +15,6 @@ public class SearchResult { private JsonValue searchAfter; private List results = new ArrayList<>(); private List dimensions; - private boolean aborted; public SearchResult() { } @@ -46,12 +45,4 @@ public void setSearchAfter(JsonValue searchAfter) { this.searchAfter = searchAfter; } - public boolean isAborted() { - return aborted; - } - - public void setAborted(boolean aborted) { - this.aborted = aborted; - } - } diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index d23ed616..7d0d085b 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -1336,11 +1336,6 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId JsonGenerator gen = Json.createGenerator(baos); gen.writeStartObject(); - if (result.isAborted()) { - gen.write("aborted", true).writeEnd().close(); - return baos.toString(); - } - JsonValue newSearchAfter = result.getSearchAfter(); if (newSearchAfter != null) { gen.write("search_after", newSearchAfter); @@ -1433,11 +1428,6 @@ public String facet(@Context HttpServletRequest request, @QueryParam("sessionId" JsonGenerator gen = Json.createGenerator(baos); gen.writeStartObject(); - if (result.isAborted()) { - gen.write("aborted", true).writeEnd().close(); - return baos.toString(); - } - List dimensions = result.getDimensions(); if (dimensions != null && dimensions.size() > 0) { gen.writeStartObject("dimensions"); From 1f59002aec1cca0479c85fb2b58e8c598479d42f Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 5 Aug 2022 08:52:29 +0000 Subject: [PATCH 32/51] search.maxSearchTimeSeconds added to run.properties #267 --- src/main/resources/run.properties | 1 + src/test/scripts/prepare_test.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/resources/run.properties b/src/main/resources/run.properties index a6186eae..4e9f2c3a 100644 --- a/src/main/resources/run.properties +++ b/src/main/resources/run.properties @@ -23,6 +23,7 @@ search.directory = ${HOME}/data/search search.backlogHandlerIntervalSeconds = 60 search.enqueuedRequestIntervalSeconds = 3 search.aggregateFilesIntervalSeconds = 3600 +search.maxSearchTimeSeconds = 5 # Configure this option to prevent certain entities being indexed # For example, remove Datafile and DatafileParameter !search.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample diff --git a/src/test/scripts/prepare_test.py b/src/test/scripts/prepare_test.py index 89333a03..8627dbc3 100644 --- a/src/test/scripts/prepare_test.py +++ b/src/test/scripts/prepare_test.py @@ -55,6 +55,7 @@ "search.backlogHandlerIntervalSeconds = 60", "search.enqueuedRequestIntervalSeconds = 3", "search.aggregateFilesIntervalSeconds = 3600", + "search.maxSearchTimeSeconds = 5", "key = wombat" ] f.write("\n".join(contents)) From fbc9b2aecfdbc5d0c8096d3f76d9eaccf0d12b14 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Tue, 9 Aug 2022 14:45:25 +0000 Subject: [PATCH 33/51] Range check for lock #267 --- .../core/manager/search/LuceneApi.java | 39 ++++++++++++++++--- .../core/manager/search/SearchApi.java | 6 +-- .../core/manager/search/SearchManager.java | 11 ++---- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java index cdd4e811..f8e025d0 100644 --- a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java @@ -200,16 +200,45 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer * locked, document modifications will fail (excluding addNow as a result of a * populate thread). * + * A check is also performed against the minId and maxId used for population. + * This ensures that no data is duplicated in the index. + * * @param entityName Index to lock. + * @param minId The exclusive minimum ICAT id being populated for. If + * Documents already exist with an id greater than this, the + * lock will fail. If null, treated as if it were + * Long.MIN_VALUE + * @param maxId The inclusive maximum ICAT id being populated for. If + * Documents already exist with an id less than or equal to + * this, the lock will fail. If null, treated as if it were + * Long.MAX_VALUE * @param delete If true, all existing documents of entityName are deleted. - * @return The largest ICAT id currently stored in the index. * @throws IcatException */ @Override - public long lock(String entityName, boolean delete) throws IcatException { - String json = Json.createObjectBuilder().add("delete", delete).build().toString(); - JsonObject postResponse = postResponse(basePath + "/lock/" + entityName, json); - return postResponse.getJsonNumber("currentId").longValueExact(); + public void lock(String entityName, Long minId, Long maxId, Boolean delete) throws IcatException { + String path = basePath + "/lock/" + entityName; + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URIBuilder builder = new URIBuilder(server).setPath(path); + if (minId != null) { + builder.addParameter("minId", minId.toString()); + } + if (maxId != null) { + builder.addParameter("maxId", maxId.toString()); + } + if (delete != null) { + builder.addParameter("delete", delete.toString()); + } + URI uri = builder.build(); + logger.debug("Making call {}", uri); + HttpPost httpPost = new HttpPost(uri); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + int code = response.getStatusLine().getStatusCode(); + Rest.checkStatus(response, code == 400 ? IcatExceptionType.BAD_PARAMETER : IcatExceptionType.INTERNAL); + } + } catch (URISyntaxException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } } /** diff --git a/src/main/java/org/icatproject/core/manager/search/SearchApi.java b/src/main/java/org/icatproject/core/manager/search/SearchApi.java index dcab42e7..467ee0e8 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchApi.java @@ -398,13 +398,13 @@ public abstract SearchResult getResults(JsonObject query, JsonValue searchAfter, * Not implemented. * * @param entityName + * @param minId + * @param maxId * @param delete - * @return long * @throws IcatException */ - public long lock(String entityName, boolean delete) throws IcatException { + public void lock(String entityName, Long minId, Long maxId, Boolean delete) throws IcatException { logger.info("Manually locking index not supported, no request sent"); - return 0; } /** diff --git a/src/main/java/org/icatproject/core/manager/search/SearchManager.java b/src/main/java/org/icatproject/core/manager/search/SearchManager.java index 84a4bce3..93ff9541 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchManager.java @@ -269,13 +269,7 @@ public void run() { if (populatingClassEntry != null) { PopulateBucket bucket = populatingClassEntry.getValue(); Long start = bucket.minId != null && bucket.minId > 0 ? bucket.minId : 0; - long currentId = searchApi.lock(populatingClassEntry.getKey(), bucket.delete); - if (currentId > start) { - searchApi.unlock(populatingClassEntry.getKey()); - populateMap.remove(populatingClassEntry.getKey()); - throw new IcatException(IcatExceptionType.BAD_PARAMETER, "Populate called starting from id " - + start + " but current id is greater: " + currentId); - } + searchApi.lock(populatingClassEntry.getKey(), bucket.minId, bucket.maxId, bucket.delete); logger.info("Search engine populating " + populatingClassEntry); @@ -327,7 +321,7 @@ public void run() { } logger.debug("About to submit {} {} documents from id {} onwards", ids.size(), - populatingClassEntry, start); + populatingClassEntry.getKey(), start); threads.submit( new IndexSome(populatingClassEntry.getKey(), ids, entityManagerFactory, start)); tasks.add(start); @@ -357,6 +351,7 @@ public void run() { } } catch (Throwable t) { logger.error("Problem encountered in", t); + populateMap.remove(populatingClassEntry.getKey()); } finally { manager.close(); popState = PopState.STOPPED; From 37668c52f3da2ae1fef040502e5c206222b5d6f5 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Wed, 7 Sep 2022 22:55:19 +0100 Subject: [PATCH 34/51] Add support for faceting DatasetTechnique #267 --- .../core/entity/DatasetTechnique.java | 10 + .../org/icatproject/core/entity/Facility.java | 6 + .../icatproject/core/entity/Technique.java | 17 + .../core/manager/EntityBeanManager.java | 2 + .../core/manager/search/FacetLabel.java | 4 + .../core/manager/search/OpensearchApi.java | 299 ++++++++++-------- .../search/OpensearchQueryBuilder.java | 2 +- .../core/manager/TestEntityInfo.java | 13 +- .../core/manager/TestSearchApi.java | 105 ++++-- 9 files changed, 304 insertions(+), 154 deletions(-) diff --git a/src/main/java/org/icatproject/core/entity/DatasetTechnique.java b/src/main/java/org/icatproject/core/entity/DatasetTechnique.java index 1b371414..a9d51dc1 100644 --- a/src/main/java/org/icatproject/core/entity/DatasetTechnique.java +++ b/src/main/java/org/icatproject/core/entity/DatasetTechnique.java @@ -2,6 +2,7 @@ import java.io.Serializable; +import javax.json.stream.JsonGenerator; import javax.persistence.Entity; import javax.persistence.FetchType; import javax.persistence.JoinColumn; @@ -9,6 +10,8 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.icatproject.core.manager.search.SearchApi; + @Comment("Represents a many-to-many relationship between a dataset and the experimental technique being used to create that Dataset") @SuppressWarnings("serial") @Entity @@ -38,4 +41,11 @@ public void setDataset(Dataset dataset) { public void setTechnique(Technique technique) { this.technique = technique; } + + @Override + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "id", id); + SearchApi.encodeString(gen, "dataset.id", dataset.id); + technique.getDoc(gen); + } } diff --git a/src/main/java/org/icatproject/core/entity/Facility.java b/src/main/java/org/icatproject/core/entity/Facility.java index e0e6f4e6..5b9b1362 100644 --- a/src/main/java/org/icatproject/core/entity/Facility.java +++ b/src/main/java/org/icatproject/core/entity/Facility.java @@ -199,4 +199,10 @@ public void setUrl(String url) { this.url = url; } + @Override + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "facility.name", name); + SearchApi.encodeString(gen, "facility.id", id); + } + } diff --git a/src/main/java/org/icatproject/core/entity/Technique.java b/src/main/java/org/icatproject/core/entity/Technique.java index 98a8a11c..6c8088cf 100644 --- a/src/main/java/org/icatproject/core/entity/Technique.java +++ b/src/main/java/org/icatproject/core/entity/Technique.java @@ -2,8 +2,12 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import javax.json.stream.JsonGenerator; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; @@ -11,6 +15,8 @@ import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.icatproject.core.manager.search.SearchApi; + @Comment("Represents an experimental technique") @SuppressWarnings("serial") @Entity @@ -30,6 +36,9 @@ public class Technique extends EntityBaseBean implements Serializable { @OneToMany(cascade = CascadeType.ALL, mappedBy = "technique") private List datasetTechniques = new ArrayList(); + public static Set docFields = new HashSet<>( + Arrays.asList("technique.id", "technique.name", "technique.description", "technique.pid")); + public String getName() { return name; } @@ -61,4 +70,12 @@ public void setDescription(String description) { public void setDatasetTechniques(List datasetTechniques) { this.datasetTechniques = datasetTechniques; } + + @Override + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "technique.id", id); + SearchApi.encodeString(gen, "technique.name", name); + SearchApi.encodeString(gen, "technique.description", description); + SearchApi.encodeString(gen, "technique.pid", pid); + } } diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index f42bdf53..5a00dc61 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -1626,6 +1626,8 @@ private JsonObject buildFacetQuery(Class klass, String return SearchManager.buildFacetQuery(filterObject, jsonFacet); } else if (target.contains("Parameter")) { relationship = eiHandler.getRelationshipsByName(klass).get("parameters"); + } else if (target.contains("DatasetTechnique")) { + relationship = eiHandler.getRelationshipsByName(klass).get("datasetTechniques"); } else { relationship = eiHandler.getRelationshipsByName(klass).get(target.toLowerCase() + "s"); } diff --git a/src/main/java/org/icatproject/core/manager/search/FacetLabel.java b/src/main/java/org/icatproject/core/manager/search/FacetLabel.java index 1e7ed288..2e4743ca 100644 --- a/src/main/java/org/icatproject/core/manager/search/FacetLabel.java +++ b/src/main/java/org/icatproject/core/manager/search/FacetLabel.java @@ -58,4 +58,8 @@ public JsonNumber getTo() { return to; } + public String toString() { + return label + ": " + value; + } + } diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index df6a4674..646d7c3a 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -49,6 +49,7 @@ import org.icatproject.core.entity.ParameterType; import org.icatproject.core.entity.Sample; import org.icatproject.core.entity.SampleType; +import org.icatproject.core.entity.Technique; import org.icatproject.core.entity.User; import org.icatproject.core.manager.Rest; import org.icatproject.utils.IcatUnits; @@ -142,6 +143,8 @@ public ParentRelation(RelationType relationType, String parentName, String joinF new ParentRelation(RelationType.NESTED_CHILD, "datafile", "datafile", null))); relations.put("datasetparameter", Arrays.asList( new ParentRelation(RelationType.NESTED_CHILD, "dataset", "dataset", null))); + relations.put("datasettechnique", Arrays.asList( + new ParentRelation(RelationType.NESTED_CHILD, "dataset", "dataset", null))); relations.put("investigationparameter", Arrays.asList( new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null))); relations.put("sampleparameter", Arrays.asList( @@ -174,6 +177,9 @@ public ParentRelation(RelationType relationType, String parentName, String joinF ParameterType.docFields), new ParentRelation(RelationType.NESTED_GRANDCHILD, "datafile", "sampleparameter", ParameterType.docFields))); + relations.put("technique", Arrays.asList( + new ParentRelation(RelationType.NESTED_GRANDCHILD, "dataset", "datasettechnique", + Technique.docFields))); relations.put("user", Arrays.asList( new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "investigationuser", User.docFields), @@ -249,6 +255,7 @@ private static JsonObject buildMappings(String index) { .add("fileSize", typeLong) .add("fileCount", typeLong) .add("datasetparameter", buildNestedMapping("dataset.id", "type.id")) + .add("datasettechnique", buildNestedMapping("dataset.id", "technique.id")) .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")) .add("sampleparameter", buildNestedMapping("sample.id", "type.id")); @@ -385,7 +392,7 @@ public List facetSearch(String target, JsonObject facetQuery, In JsonObject queryObject = facetQuery.getJsonObject("query"); List defaultFields = defaultFieldsMap.get(index); - JsonObjectBuilder bodyBuilder = parseQuery(Json.createObjectBuilder(), queryObject, index, defaultFields); + JsonObjectBuilder bodyBuilder = parseQuery(Json.createObjectBuilder(), queryObject, index, dimensionPrefix, defaultFields); if (facetQuery.containsKey("dimensions")) { JsonArray dimensions = facetQuery.getJsonArray("dimensions"); bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix); @@ -503,7 +510,7 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); bodyBuilder = parseSort(bodyBuilder, sort); bodyBuilder = parseSearchAfter(bodyBuilder, searchAfter); - bodyBuilder = parseQuery(bodyBuilder, query, index, defaultFields); + bodyBuilder = parseQuery(bodyBuilder, query, index, null, defaultFields); String body = bodyBuilder.build().toString(); Map parameterMap = new HashMap<>(); @@ -671,12 +678,13 @@ private JsonObjectBuilder parseSearchAfter(JsonObjectBuilder builder, JsonValue * @param queryRequest The Json object containing the information on the * requested query, NOT formatted for the search cluster. * @param index The index to search. + * @param dimensionPrefix Used to build nested queries for arbitrary fields. * @param defaultFields Default fields to apply parsed string queries to. * @return The JsonObjectBuilder initially passed with the "query" added to it. * @throws IcatException If the query cannot be parsed. */ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject queryRequest, String index, - List defaultFields) throws IcatException { + String dimensionPrefix, List defaultFields) throws IcatException { // In general, we use a boolean query to compound queries on individual fields JsonObjectBuilder queryBuilder = Json.createObjectBuilder(); JsonObjectBuilder boolBuilder = Json.createObjectBuilder(); @@ -684,49 +692,172 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query // Non-scored elements are added to the "filter" JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); - if (queryRequest.containsKey("filter")) { - JsonObject filterObject = queryRequest.getJsonObject("filter"); - for (String fld : filterObject.keySet()) { - JsonValue value = filterObject.get(fld); - String field = fld.replace(index + ".", ""); - switch (value.getValueType()) { - case ARRAY: - JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); - for (JsonValue arrayValue : ((JsonArray) value).getValuesAs(JsonString.class)) { - parseFilter(arrayBuilder, field, arrayValue); + Long lowerTime = Long.MIN_VALUE; + Long upperTime = Long.MAX_VALUE; + for (String queryKey: queryRequest.keySet()) { + switch (queryKey) { + case "target": + break; // Avoid using the target index as a term in the search + case "lower": + lowerTime = parseDate(queryRequest, "lower", 0, Long.MIN_VALUE); + break; + case "upper": + upperTime = parseDate(queryRequest, "upper", 59999, Long.MAX_VALUE); + break; + case "filter": + JsonObject filterObject = queryRequest.getJsonObject("filter"); + for (String fld : filterObject.keySet()) { + JsonValue value = filterObject.get(fld); + String field = fld.replace(index + ".", ""); + switch (value.getValueType()) { + case ARRAY: + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + for (JsonValue arrayValue : ((JsonArray) value).getValuesAs(JsonString.class)) { + parseFilter(arrayBuilder, field, arrayValue); + } + // If the key was just a nested entity (no ".") then we should FILTER all of our + // queries on that entity. + String occur = fld.contains(".") ? "should" : "filter"; + filterBuilder.add(Json.createObjectBuilder().add("bool", + Json.createObjectBuilder().add(occur, arrayBuilder))); + break; + + default: + parseFilter(filterBuilder, field, value); } - // If the key was just a nested entity (no ".") then we should FILTER all of our - // queries on that entity. - String occur = fld.contains(".") ? "should" : "filter"; - filterBuilder.add(Json.createObjectBuilder().add("bool", - Json.createObjectBuilder().add(occur, arrayBuilder))); - break; - - default: - parseFilter(filterBuilder, field, value); - } - } - } - - if (queryRequest.containsKey("text")) { - // The free text is the only element we perform scoring on, so "must" occur - JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); - String text = queryRequest.getString("text"); - arrayBuilder.add(OpensearchQueryBuilder.buildStringQuery(text, defaultFields.toArray(new String[0]))); - if (index.equals("investigation")) { - JsonObject stringQuery = OpensearchQueryBuilder.buildStringQuery(text, "sample.name", - "sample.type.name"); - arrayBuilder.add(OpensearchQueryBuilder.buildNestedQuery("sample", stringQuery)); - JsonObjectBuilder textBoolBuilder = Json.createObjectBuilder().add("should", arrayBuilder); - JsonObjectBuilder textMustBuilder = Json.createObjectBuilder().add("bool", textBoolBuilder); - boolBuilder.add("must", Json.createArrayBuilder().add(textMustBuilder)); - } else { - boolBuilder.add("must", arrayBuilder); + } + break; + case "text": + // The free text is the only element we perform scoring on, so "must" occur + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + String text = queryRequest.getString("text"); + arrayBuilder.add(OpensearchQueryBuilder.buildStringQuery(text, defaultFields.toArray(new String[0]))); + if (index.equals("investigation")) { + JsonObject stringQuery = OpensearchQueryBuilder.buildStringQuery(text, "sample.name", + "sample.type.name"); + arrayBuilder.add(OpensearchQueryBuilder.buildNestedQuery("sample", stringQuery)); + JsonObjectBuilder textBoolBuilder = Json.createObjectBuilder().add("should", arrayBuilder); + JsonObjectBuilder textMustBuilder = Json.createObjectBuilder().add("bool", textBoolBuilder); + boolBuilder.add("must", Json.createArrayBuilder().add(textMustBuilder)); + } else { + boolBuilder.add("must", arrayBuilder); + } + break; + case "user": + String user = queryRequest.getString("user"); + // Because InstrumentScientist is on a separate index, we need to explicitly + // perform a search here + JsonObject termQuery = OpensearchQueryBuilder.buildTermQuery("user.name.keyword", user); + String body = Json.createObjectBuilder().add("query", termQuery).build().toString(); + Map parameterMap = new HashMap<>(); + parameterMap.put("_source", "instrument.id"); + JsonObject postResponse = postResponse("/instrumentscientist/_search", body, parameterMap); + JsonArray hits = postResponse.getJsonObject("hits").getJsonArray("hits"); + JsonArrayBuilder instrumentIdsBuilder = Json.createArrayBuilder(); + for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { + String instrumentId = hit.getJsonObject("_source").getString("instrument.id"); + instrumentIdsBuilder.add(instrumentId); + } + JsonObject instrumentQuery = OpensearchQueryBuilder.buildTermsQuery("investigationinstrument.instrument.id", + instrumentIdsBuilder.build()); + JsonObject nestedInstrumentQuery = OpensearchQueryBuilder.buildNestedQuery("investigationinstrument", + instrumentQuery); + // InvestigationUser should be a nested field on the main Document + JsonObject investigationUserQuery = OpensearchQueryBuilder.buildMatchQuery("investigationuser.user.name", + user); + JsonObject nestedUserQuery = OpensearchQueryBuilder.buildNestedQuery("investigationuser", + investigationUserQuery); + // At least one of being an InstrumentScientist or an InvestigationUser is + // necessary + JsonArrayBuilder array = Json.createArrayBuilder().add(nestedInstrumentQuery).add(nestedUserQuery); + filterBuilder.add(Json.createObjectBuilder().add("bool", Json.createObjectBuilder().add("should", array))); + break; + case "userFullName": + String fullName = queryRequest.getString("userFullName"); + JsonObject fullNameQuery = OpensearchQueryBuilder.buildStringQuery(fullName, + "investigationuser.user.fullName"); + filterBuilder.add(OpensearchQueryBuilder.buildNestedQuery("investigationuser", fullNameQuery)); + break; + case "samples": + JsonArray samples = queryRequest.getJsonArray("samples"); + for (int i = 0; i < samples.size(); i++) { + String sample = samples.getString(i); + JsonObject stringQuery = OpensearchQueryBuilder.buildStringQuery(sample, "sample.name", + "sample.type.name"); + filterBuilder.add(OpensearchQueryBuilder.buildNestedQuery("sample", stringQuery)); + } + break; + case "parameters": + for (JsonObject parameterObject : queryRequest.getJsonArray("parameters").getValuesAs(JsonObject.class)) { + String path = index + "parameter"; + List parameterQueries = new ArrayList<>(); + if (parameterObject.containsKey("name")) { + String name = parameterObject.getString("name"); + parameterQueries.add(OpensearchQueryBuilder.buildMatchQuery(path + ".type.name", name)); + } + if (parameterObject.containsKey("units")) { + String units = parameterObject.getString("units"); + parameterQueries.add(OpensearchQueryBuilder.buildMatchQuery(path + ".type.units", units)); + } + if (parameterObject.containsKey("stringValue")) { + String stringValue = parameterObject.getString("stringValue"); + parameterQueries.add(OpensearchQueryBuilder.buildMatchQuery(path + ".stringValue", stringValue)); + } else if (parameterObject.containsKey("lowerDateValue") + && parameterObject.containsKey("upperDateValue")) { + Long lower = parseDate(parameterObject, "lowerDateValue", 0, Long.MIN_VALUE); + Long upper = parseDate(parameterObject, "upperDateValue", 59999, Long.MAX_VALUE); + parameterQueries + .add(OpensearchQueryBuilder.buildLongRangeQuery(path + ".dateTimeValue", lower, upper)); + } else if (parameterObject.containsKey("lowerNumericValue") + && parameterObject.containsKey("upperNumericValue")) { + JsonNumber lower = parameterObject.getJsonNumber("lowerNumericValue"); + JsonNumber upper = parameterObject.getJsonNumber("upperNumericValue"); + parameterQueries.add(OpensearchQueryBuilder.buildRangeQuery(path + ".numericValue", lower, upper)); + } + filterBuilder.add( + OpensearchQueryBuilder.buildNestedQuery(path, parameterQueries.toArray(new JsonObject[0]))); + } + break; + default: + // If the term doesn't require special logic, handle according to type + JsonObject defaultTermQuery; + String field = queryKey; + if (dimensionPrefix != null) { + field = dimensionPrefix + "." + field; + } + ValueType valueType = queryRequest.get(queryKey).getValueType(); + // if (queryKey.contains(".")) { + // int pathEnd = queryKey.indexOf("."); + // path = queryKey.substring(0, pathEnd); + // if (path.equals(index)) { + // // e.g. "dataset.id" should be interpretted as "id" iff we're searching Datasets + // field = queryKey.substring(pathEnd + 1); + // } + // } + switch (valueType) { + case STRING: + defaultTermQuery = OpensearchQueryBuilder.buildTermQuery(field + ".keyword", queryRequest.getString(queryKey)); + break; + case NUMBER: + defaultTermQuery = OpensearchQueryBuilder.buildTermQuery(field, queryRequest.getJsonNumber(queryKey)); + break; + case ARRAY: + // Only support array of String as list of ICAT ids is currently only use case + defaultTermQuery = OpensearchQueryBuilder.buildTermsQuery(field, queryRequest.getJsonArray(queryKey)); + break; + default: + throw new IcatException(IcatExceptionType.BAD_PARAMETER, "Query values should be ARRAY, STRING or NUMBER, but had value of type " + valueType); + } + if (dimensionPrefix != null) { + // e.g. "sample.id" should use a nested query as sample is nested on other entities + filterBuilder.add(OpensearchQueryBuilder.buildNestedQuery(dimensionPrefix, defaultTermQuery)); + } else { + // Otherwise, we can associate the query directly with the searched entity + filterBuilder.add(defaultTermQuery); + } } } - Long lowerTime = parseDate(queryRequest, "lower", 0, Long.MIN_VALUE); - Long upperTime = parseDate(queryRequest, "upper", 59999, Long.MAX_VALUE); if (lowerTime != Long.MIN_VALUE || upperTime != Long.MAX_VALUE) { if (index.equals("datafile")) { // datafile has only one date field @@ -737,90 +868,6 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query } } - if (queryRequest.containsKey("user")) { - String name = queryRequest.getString("user"); - // Because InstrumentScientist is on a separate index, we need to explicitly - // perform a search here - JsonObject termQuery = OpensearchQueryBuilder.buildTermQuery("user.name.keyword", name); - String body = Json.createObjectBuilder().add("query", termQuery).build().toString(); - Map parameterMap = new HashMap<>(); - parameterMap.put("_source", "instrument.id"); - JsonObject postResponse = postResponse("/instrumentscientist/_search", body, parameterMap); - JsonArray hits = postResponse.getJsonObject("hits").getJsonArray("hits"); - JsonArrayBuilder instrumentIdsBuilder = Json.createArrayBuilder(); - for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { - String instrumentId = hit.getJsonObject("_source").getString("instrument.id"); - instrumentIdsBuilder.add(instrumentId); - } - JsonObject instrumentQuery = OpensearchQueryBuilder.buildTermsQuery("investigationinstrument.instrument.id", - instrumentIdsBuilder.build()); - JsonObject nestedInstrumentQuery = OpensearchQueryBuilder.buildNestedQuery("investigationinstrument", - instrumentQuery); - // InvestigationUser should be a nested field on the main Document - JsonObject investigationUserQuery = OpensearchQueryBuilder.buildMatchQuery("investigationuser.user.name", - name); - JsonObject nestedUserQuery = OpensearchQueryBuilder.buildNestedQuery("investigationuser", - investigationUserQuery); - // At least one of being an InstrumentScientist or an InvestigationUser is - // necessary - JsonArrayBuilder array = Json.createArrayBuilder().add(nestedInstrumentQuery).add(nestedUserQuery); - filterBuilder.add(Json.createObjectBuilder().add("bool", Json.createObjectBuilder().add("should", array))); - } - if (queryRequest.containsKey("userFullName")) { - String fullName = queryRequest.getString("userFullName"); - JsonObject fullNameQuery = OpensearchQueryBuilder.buildStringQuery(fullName, - "investigationuser.user.fullName"); - filterBuilder.add(OpensearchQueryBuilder.buildNestedQuery("investigationuser", fullNameQuery)); - } - - if (queryRequest.containsKey("samples")) { - JsonArray samples = queryRequest.getJsonArray("samples"); - for (int i = 0; i < samples.size(); i++) { - String sample = samples.getString(i); - JsonObject stringQuery = OpensearchQueryBuilder.buildStringQuery(sample, "sample.name", - "sample.type.name"); - filterBuilder.add(OpensearchQueryBuilder.buildNestedQuery("sample", stringQuery)); - } - } - - if (queryRequest.containsKey("parameters")) { - for (JsonObject parameterObject : queryRequest.getJsonArray("parameters").getValuesAs(JsonObject.class)) { - String path = index + "parameter"; - List parameterQueries = new ArrayList<>(); - if (parameterObject.containsKey("name")) { - String name = parameterObject.getString("name"); - parameterQueries.add(OpensearchQueryBuilder.buildMatchQuery(path + ".type.name", name)); - } - if (parameterObject.containsKey("units")) { - String units = parameterObject.getString("units"); - parameterQueries.add(OpensearchQueryBuilder.buildMatchQuery(path + ".type.units", units)); - } - if (parameterObject.containsKey("stringValue")) { - String stringValue = parameterObject.getString("stringValue"); - parameterQueries.add(OpensearchQueryBuilder.buildMatchQuery(path + ".stringValue", stringValue)); - } else if (parameterObject.containsKey("lowerDateValue") - && parameterObject.containsKey("upperDateValue")) { - Long lower = parseDate(parameterObject, "lowerDateValue", 0, Long.MIN_VALUE); - Long upper = parseDate(parameterObject, "upperDateValue", 59999, Long.MAX_VALUE); - parameterQueries - .add(OpensearchQueryBuilder.buildLongRangeQuery(path + ".dateTimeValue", lower, upper)); - } else if (parameterObject.containsKey("lowerNumericValue") - && parameterObject.containsKey("upperNumericValue")) { - JsonNumber lower = parameterObject.getJsonNumber("lowerNumericValue"); - JsonNumber upper = parameterObject.getJsonNumber("upperNumericValue"); - parameterQueries.add(OpensearchQueryBuilder.buildRangeQuery(path + ".numericValue", lower, upper)); - } - filterBuilder.add( - OpensearchQueryBuilder.buildNestedQuery(path, parameterQueries.toArray(new JsonObject[0]))); - } - } - - if (queryRequest.containsKey("id")) { - filterBuilder.add(OpensearchQueryBuilder.buildTermsQuery("id", queryRequest.getJsonArray("id"))); - } - - // TODO add in support for specific terms? - JsonArray filterArray = filterBuilder.build(); if (filterArray.size() > 0) { boolBuilder.add("filter", filterArray); diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java b/src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java index dfb2d32b..0f262f75 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java @@ -139,7 +139,7 @@ public static JsonObject buildTermQuery(String field, double value) { /** * @param field Field containing on of the terms. - * @param values JsonArrat of possible terms. + * @param values JsonArray of possible terms. * @return {"terms": {`field`: `values`}} */ public static JsonObject buildTermsQuery(String field, JsonArray values) { diff --git a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java index 805592b5..02815783 100644 --- a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java +++ b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java @@ -47,9 +47,10 @@ public void testBadname() throws Exception { @Test public void testHasSearchDoc() throws Exception { Set docdbeans = new HashSet<>(Arrays.asList("Datafile", "DatafileFormat", "DatafileParameter", - "Dataset", "DatasetParameter", "DatasetType", "Facility", "Instrument", "InstrumentScientist", - "Investigation", "InvestigationInstrument", "InvestigationParameter", "InvestigationType", - "InvestigationUser", "ParameterType", "Sample", "SampleType", "SampleParameter", "User")); + "Dataset", "DatasetParameter", "DatasetTechnique", "DatasetType", "Facility", "Instrument", + "InstrumentScientist", "Investigation", "InvestigationInstrument", "InvestigationParameter", + "InvestigationType", "InvestigationUser", "ParameterType", "Sample", "SampleType", "SampleParameter", + "Technique", "User")); for (String beanName : EntityInfoHandler.getEntityNamesList()) { @SuppressWarnings("unchecked") Class bean = (Class) Class @@ -180,8 +181,10 @@ public void testFields() throws Exception { + "startDate,studyInvestigations,summary,title,type,visitId", Investigation.class); testField("complete,dataCollectionDatasets,datafiles,datasetInstruments,datasetTechniques,description," - + "doi,endDate,fileCount,fileSize,investigation,location,name,parameters,sample,startDate,type", Dataset.class); - testField("dataCollectionDatafiles,dataCollectionDatasets,dataCollectionInvestigations,dataPublications,doi,jobsAsInput,jobsAsOutput,parameters", + + "doi,endDate,fileCount,fileSize,investigation,location,name,parameters,sample,startDate,type", + Dataset.class); + testField( + "dataCollectionDatafiles,dataCollectionDatasets,dataCollectionInvestigations,dataPublications,doi,jobsAsInput,jobsAsOutput,parameters", DataCollection.class); testField("application,arguments,inputDataCollection,outputDataCollection", Job.class); testField("description,endDate,name,pid,startDate,status,studyInvestigations,user", Study.class); diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 5810dd35..1fb2c61e 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -28,6 +28,7 @@ import org.icatproject.core.entity.DatafileParameter; import org.icatproject.core.entity.Dataset; import org.icatproject.core.entity.DatasetParameter; +import org.icatproject.core.entity.DatasetTechnique; import org.icatproject.core.entity.DatasetType; import org.icatproject.core.entity.EntityBaseBean; import org.icatproject.core.entity.Facility; @@ -43,6 +44,7 @@ import org.icatproject.core.entity.Sample; import org.icatproject.core.entity.SampleParameter; import org.icatproject.core.entity.SampleType; +import org.icatproject.core.entity.Technique; import org.icatproject.core.entity.User; import org.icatproject.core.manager.search.FacetDimension; import org.icatproject.core.manager.search.FacetLabel; @@ -194,8 +196,8 @@ public static JsonObject buildQuery(String target, String user, String text, Dat return builder.build(); } - private static JsonObject buildFacetIdQuery(String id) { - return Json.createObjectBuilder().add("id", Json.createArrayBuilder().add(id)).build(); + private static JsonObject buildFacetIdQuery(String idField, String idValue) { + return Json.createObjectBuilder().add(idField, Json.createArrayBuilder().add(idValue)).build(); } private static JsonObject buildFacetRangeObject(String key, double from, double to) { @@ -218,8 +220,8 @@ private static JsonObject buildFacetRangeRequest(JsonObject queryObject, String return Json.createObjectBuilder().add("query", queryObject).add("dimensions", rangedDimensionsBuilder).build(); } - private static JsonObject buildFacetStringRequest(String id, String dimension) { - JsonObject idQuery = buildFacetIdQuery(id); + private static JsonObject buildFacetStringRequest(String idField, String idValue, String dimension) { + JsonObject idQuery = buildFacetIdQuery(idField, idValue); JsonObjectBuilder stringDimensionBuilder = Json.createObjectBuilder().add("dimension", dimension); JsonArrayBuilder stringDimensionsBuilder = Json.createArrayBuilder().add(stringDimensionBuilder); return Json.createObjectBuilder().add("query", idQuery).add("dimensions", stringDimensionsBuilder).build(); @@ -271,7 +273,8 @@ private void checkFacets(List facetDimensions, FacetDimension... assertEquals(expectedFacet.getDimension(), actualFacet.getDimension()); List expectedLabels = expectedFacet.getFacets(); List actualLabels = actualFacet.getFacets(); - assertEquals(expectedLabels.size(), actualLabels.size()); + String message = "Expected " + expectedLabels.toString() + " but got " + actualLabels.toString(); + assertEquals(message, expectedLabels.size(), actualLabels.size()); for (int j = 0; j < expectedLabels.size(); j++) { FacetLabel expectedLabel = expectedLabels.get(j); FacetLabel actualLabel = actualLabels.get(j); @@ -279,7 +282,7 @@ private void checkFacets(List facetDimensions, FacetDimension... Long expectedValue = expectedLabel.getValue(); Long actualValue = actualLabel.getValue(); assertEquals(label, actualLabel.getLabel()); - String message = "Label <" + label + ">: "; + message = "Label <" + label + ">: "; assertEquals(message, expectedValue, actualValue); } } @@ -425,7 +428,8 @@ private Parameter parameter(long id, double value, ParameterType parameterType, return parameter; } - private Parameter parameter(long id, String value, double rangeBottom, double rangeTop, ParameterType parameterType, EntityBaseBean parent) { + private Parameter parameter(long id, String value, double rangeBottom, double rangeTop, ParameterType parameterType, + EntityBaseBean parent) { Parameter parameter = parameter(id, parameterType, parent); parameter.setStringValue(value); parameter.setRangeBottom(rangeBottom); @@ -515,6 +519,8 @@ private void populate() throws IcatException { Instrument instrumentZero = populateInstrument(queue, 0L); Instrument instrumentOne = populateInstrument(queue, 1L); + Technique techniqueZero = populateTechnique(queue, 0L); + Technique techniqueOne = populateTechnique(queue, 1L); for (int investigationId = 0; investigationId < NUMINV; investigationId++) { String word = word(investigationId % 26, (investigationId + 7) % 26, (investigationId + 17) % 26); @@ -566,6 +572,12 @@ private void populate() throws IcatException { word = word("DS", datasetId % 26); Dataset dataset = dataset(datasetId, word, startDate, endDate, investigation); + if (datasetId % 2 == 0) { + populateDatasetTechnique(queue, techniqueZero, dataset); + } else { + populateDatasetTechnique(queue, techniqueOne, dataset); + } + if (datasetId < NUMSAMP) { word = word("SType ", datasetId); Sample sample = sample(datasetId, word, investigation); @@ -625,6 +637,41 @@ private Instrument populateInstrument(List queue, long instrumentId) thr return instrument; } + /** + * Queues creation of an Technique. + * + * @param queue Queue to add create operations to. + * @param techniqueId ICAT entity Id to use for the Technique. + * @return The Technique entity created. + * @throws IcatException + */ + private Technique populateTechnique(List queue, long techniqueId) throws IcatException { + Technique technique = new Technique(); + technique.setId(techniqueId); + technique.setName("technique" + techniqueId); + technique.setDescription("Technique number " + techniqueId); + technique.setPid(Long.toString(techniqueId)); + queue.add(SearchApi.encodeOperation("create", technique)); + return technique; + } + + /** + * Queues creation of an DatasetTechnique. + * + * @param queue Queue to add create operations to. + * @return The DatasetTechnique entity created. + * @throws IcatException + */ + private DatasetTechnique populateDatasetTechnique(List queue, Technique technique, Dataset dataset) + throws IcatException { + DatasetTechnique datasetTechnique = new DatasetTechnique(); + datasetTechnique.setId(technique.getId() * 100 + dataset.getId()); + datasetTechnique.setTechnique(technique); + datasetTechnique.setDataset(dataset); + queue.add(SearchApi.encodeOperation("create", datasetTechnique)); + return datasetTechnique; + } + private String word(int j, int k, int l) { String jString = letters.substring(j, j + 1); String kString = letters.substring(k, k + 1); @@ -915,6 +962,14 @@ public void datasets() throws Exception { lsr = searchApi.getResults(buildQuery("Dataset", "b1", "dsddd", new Date(now + 60000 * 3), new Date(now + 60000 * 6), pojos, null), 100, null); checkResults(lsr); + + // Test DatasetTechnique Facets + JsonObject stringFacetRequestZero = buildFacetStringRequest("dataset.id", "0", "technique.name"); + JsonObject stringFacetRequestOne = buildFacetStringRequest("dataset.id", "1", "technique.name"); + FacetDimension facetZero = new FacetDimension("", "technique.name", new FacetLabel("technique0", 1L)); + FacetDimension facetOne = new FacetDimension("", "technique.name", new FacetLabel("technique1", 1L)); + checkFacets(searchApi.facetSearch("DatasetTechnique", stringFacetRequestZero, 5, 5), facetZero); + checkFacets(searchApi.facetSearch("DatasetTechnique", stringFacetRequestOne, 5, 5), facetOne); } @Test @@ -1086,15 +1141,15 @@ public void investigations() throws Exception { query = buildQuery("Investigation", null, "visitId:visitId sample.name:ddd", null, null, null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); - // Individual MUST should work when applied to either an Investigation or Sample field + // Individual MUST should work when applied to either an Investigation or Sample query = buildQuery("Investigation", null, "+visitId:visitId", null, null, null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); query = buildQuery("Investigation", null, "+sample.name:ddd", null, null, null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 3L); - // This query is expected to fail, as we apply both terms to Investigation and Sample - // (since we have no fields) and neither possesses both terms. + // This query is expected to fail, as we apply both terms to Investigation and + // Sample (since we have no fields) and neither possesses both terms. query = buildQuery("Investigation", null, "+visitId +ddd", null, null, null, null); lsr = searchApi.getResults(query, 100, null); checkResults(lsr); @@ -1118,9 +1173,9 @@ public void locking() throws IcatException { } catch (IcatException e) { assertEquals("Lucene is not currently locked for Dataset", e.getMessage()); } - searchApi.lock("Dataset", true); + searchApi.lock("Dataset", 0L, 1L, true); try { - searchApi.lock("Dataset", true); + searchApi.lock("Dataset", 0L, 1L, true); fail(); } catch (IcatException e) { assertEquals("Lucene already locked for Dataset", e.getMessage()); @@ -1153,7 +1208,10 @@ public void fileSizeAggregation() throws IcatException { List fields = Arrays.asList("id", "fileSize", "fileCount"); // Create - modify(SearchApi.encodeOperation("create", investigation), SearchApi.encodeOperation("create", dataset), SearchApi.encodeOperation("create", datafile)); + String createInvestigation = SearchApi.encodeOperation("create", investigation); + String createDataset = SearchApi.encodeOperation("create", dataset); + String createDatafile = SearchApi.encodeOperation("create", datafile); + modify(createInvestigation, createDataset, createDatafile); checkFileSize(datafileQuery, fields, 123, 1); checkFileSize(datasetQuery, fields, 123, 1); checkFileSize(investigationQuery, fields, 123, 1); @@ -1171,17 +1229,17 @@ public void fileSizeAggregation() throws IcatException { checkFileSize(investigationQuery, fields, 0, 0); } - private void checkFileSize(JsonObject query, List fields, long expectedFileSize, long expectedFileCount) throws IcatException { + private void checkFileSize(JsonObject query, List fields, long expectedFileSize, long expectedFileCount) + throws IcatException { SearchResult results = searchApi.getResults(query, null, 5, null, fields); checkResults(results, 0L); JsonObject source = results.getResults().get(0).getSource(); long fileSize = source.getJsonNumber("fileSize").longValueExact(); long fileCount = source.getJsonNumber("fileCount").longValueExact(); - assertEquals(expectedFileSize,fileSize); - assertEquals(expectedFileCount,fileCount); + assertEquals(expectedFileSize, fileSize); + assertEquals(expectedFileCount, fileCount); } - @Test public void modifyDatafile() throws IcatException { // Build entities @@ -1200,9 +1258,10 @@ public void modifyDatafile() throws IcatException { JsonObject pngQuery = buildQuery("Datafile", null, "datafileFormat.name:png", null, null, null, null); JsonObject lowRange = buildFacetRangeObject("low", 0L, 2L); JsonObject highRange = buildFacetRangeObject("high", 2L, 4L); - JsonObject rangeFacetRequest = buildFacetRangeRequest(buildFacetIdQuery("42"), "date", lowRange, highRange); - JsonObject stringFacetRequest = buildFacetStringRequest("42", "datafileFormat.name"); - JsonObject sparseFacetRequest = Json.createObjectBuilder().add("query", buildFacetIdQuery("42")).build(); + JsonObject facetIdQuery = buildFacetIdQuery("id", "42"); + JsonObject rangeFacetRequest = buildFacetRangeRequest(facetIdQuery, "date", lowRange,highRange); + JsonObject stringFacetRequest = buildFacetStringRequest("id", "42", "datafileFormat.name"); + JsonObject sparseFacetRequest = Json.createObjectBuilder().add("query", facetIdQuery).build(); FacetDimension lowFacet = new FacetDimension("", "date", new FacetLabel("low", 1L), new FacetLabel("high", 0L)); FacetDimension highFacet = new FacetDimension("", "date", new FacetLabel("low", 0L), new FacetLabel("high", 1L)); @@ -1340,14 +1399,16 @@ public void exactFilter() throws IcatException { JsonObjectBuilder filterBuilder = Json.createObjectBuilder(); JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); JsonObject value = Json.createObjectBuilder().add("field", "numericValue").add("exact", 273).build(); - JsonObject numericName = Json.createObjectBuilder().add("field", "type.name").add("value", "numericParameter").build(); + JsonObject numericName = Json.createObjectBuilder().add("field", "type.name").add("value", "numericParameter") + .build(); arrayBuilder.add(numericName).add(value); filterBuilder.add("key", "key").add("label", "label").add("filter", arrayBuilder); JsonObject numericFilter = filterBuilder.build(); filterBuilder = Json.createObjectBuilder(); arrayBuilder = Json.createArrayBuilder(); - JsonObject rangeName = Json.createObjectBuilder().add("field", "type.name").add("value", "rangeParameter").build(); + JsonObject rangeName = Json.createObjectBuilder().add("field", "type.name").add("value", "rangeParameter") + .build(); arrayBuilder.add(rangeName).add(value); filterBuilder.add("key", "key").add("label", "label").add("filter", arrayBuilder); JsonObject rangeFilter = filterBuilder.build(); From de8fd2d73b54de68c52c5e8479750edcb4669392 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 9 Sep 2022 05:07:35 +0100 Subject: [PATCH 35/51] Add deprecation warnings #267 --- pom.xml | 16 ++-- src/main/config/run.properties.example | 1 + .../core/manager/search/OpensearchApi.java | 94 ++++++++++++------- .../org/icatproject/exposed/ICATRest.java | 52 +++++++--- src/main/resources/run.properties | 1 + .../core/manager/TestSearchApi.java | 24 +++-- .../org/icatproject/integration/TestRS.java | 6 +- src/test/scripts/prepare_test.py | 11 +-- 8 files changed, 128 insertions(+), 77 deletions(-) diff --git a/pom.xml b/pom.xml index 63480b22..2e2cde6a 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.icatproject icat.server - 5.0.1-SNAPSHOT + 5.1.0-SNAPSHOT war ICAT Server A metadata catalogue to support Large Facility experimental data, @@ -113,13 +113,13 @@ org.icatproject icat.utils - 4.16.2-SNAPSHOT + 4.17.0-SNAPSHOT org.icatproject icat.client - 5.0.1-SNAPSHOT + 5.1.0-SNAPSHOT test @@ -226,8 +226,8 @@ ${javax.net.ssl.trustStore} - ${luceneUrl} - ${opensearchUrl} + ${searchEngine} + ${searchUrls} false @@ -246,8 +246,7 @@ ${javax.net.ssl.trustStore} ${serverUrl} ${searchEngine} - ${luceneUrl} - ${opensearchUrl} + ${searchUrls} @@ -328,8 +327,7 @@ ${containerHome} ${serverUrl} ${searchEngine} - ${luceneUrl} - ${opensearchUrl} + ${searchUrls} diff --git a/src/main/config/run.properties.example b/src/main/config/run.properties.example index 5ba348be..bd0ed307 100644 --- a/src/main/config/run.properties.example +++ b/src/main/config/run.properties.example @@ -43,6 +43,7 @@ notification.Datafile = CU log.list = SESSION WRITE READ INFO # Search Engine +# LUCENE, OPENSEARCH and ELASTICSEARCH engines are supported, however the latter two are considered experimental search.engine = LUCENE search.urls = https://localhost:8181 search.populateBlockSize = 10000 diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index 646d7c3a..8c10e232 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -57,6 +57,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * The interface to Opensearch/Elasticsearch clusters is currently considered to + * be experimental. For the more widely used and extensively tested Lucene based + * engine, see {@link LuceneApi}. + */ public class OpensearchApi extends SearchApi { private static enum ModificationType { @@ -392,7 +397,8 @@ public List facetSearch(String target, JsonObject facetQuery, In JsonObject queryObject = facetQuery.getJsonObject("query"); List defaultFields = defaultFieldsMap.get(index); - JsonObjectBuilder bodyBuilder = parseQuery(Json.createObjectBuilder(), queryObject, index, dimensionPrefix, defaultFields); + JsonObjectBuilder bodyBuilder = parseQuery(Json.createObjectBuilder(), queryObject, index, dimensionPrefix, + defaultFields); if (facetQuery.containsKey("dimensions")) { JsonArray dimensions = facetQuery.getJsonArray("dimensions"); bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix); @@ -673,13 +679,14 @@ private JsonObjectBuilder parseSearchAfter(JsonObjectBuilder builder, JsonValue * Parses the search query from the incoming queryRequest into Json that the * search cluster can understand. * - * @param builder The JsonObjectBuilder being used to create the body for - * the POST request to the cluster. - * @param queryRequest The Json object containing the information on the - * requested query, NOT formatted for the search cluster. - * @param index The index to search. + * @param builder The JsonObjectBuilder being used to create the body + * for + * the POST request to the cluster. + * @param queryRequest The Json object containing the information on the + * requested query, NOT formatted for the search cluster. + * @param index The index to search. * @param dimensionPrefix Used to build nested queries for arbitrary fields. - * @param defaultFields Default fields to apply parsed string queries to. + * @param defaultFields Default fields to apply parsed string queries to. * @return The JsonObjectBuilder initially passed with the "query" added to it. * @throws IcatException If the query cannot be parsed. */ @@ -694,8 +701,8 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query Long lowerTime = Long.MIN_VALUE; Long upperTime = Long.MAX_VALUE; - for (String queryKey: queryRequest.keySet()) { - switch (queryKey) { + for (String queryKey : queryRequest.keySet()) { + switch (queryKey) { case "target": break; // Avoid using the target index as a term in the search case "lower": @@ -704,7 +711,7 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query case "upper": upperTime = parseDate(queryRequest, "upper", 59999, Long.MAX_VALUE); break; - case "filter": + case "filter": JsonObject filterObject = queryRequest.getJsonObject("filter"); for (String fld : filterObject.keySet()) { JsonValue value = filterObject.get(fld); @@ -721,7 +728,7 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query filterBuilder.add(Json.createObjectBuilder().add("bool", Json.createObjectBuilder().add(occur, arrayBuilder))); break; - + default: parseFilter(filterBuilder, field, value); } @@ -731,7 +738,8 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query // The free text is the only element we perform scoring on, so "must" occur JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); String text = queryRequest.getString("text"); - arrayBuilder.add(OpensearchQueryBuilder.buildStringQuery(text, defaultFields.toArray(new String[0]))); + arrayBuilder + .add(OpensearchQueryBuilder.buildStringQuery(text, defaultFields.toArray(new String[0]))); if (index.equals("investigation")) { JsonObject stringQuery = OpensearchQueryBuilder.buildStringQuery(text, "sample.name", "sample.type.name"); @@ -758,19 +766,23 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query String instrumentId = hit.getJsonObject("_source").getString("instrument.id"); instrumentIdsBuilder.add(instrumentId); } - JsonObject instrumentQuery = OpensearchQueryBuilder.buildTermsQuery("investigationinstrument.instrument.id", + JsonObject instrumentQuery = OpensearchQueryBuilder.buildTermsQuery( + "investigationinstrument.instrument.id", instrumentIdsBuilder.build()); - JsonObject nestedInstrumentQuery = OpensearchQueryBuilder.buildNestedQuery("investigationinstrument", + JsonObject nestedInstrumentQuery = OpensearchQueryBuilder.buildNestedQuery( + "investigationinstrument", instrumentQuery); // InvestigationUser should be a nested field on the main Document - JsonObject investigationUserQuery = OpensearchQueryBuilder.buildMatchQuery("investigationuser.user.name", + JsonObject investigationUserQuery = OpensearchQueryBuilder.buildMatchQuery( + "investigationuser.user.name", user); JsonObject nestedUserQuery = OpensearchQueryBuilder.buildNestedQuery("investigationuser", investigationUserQuery); // At least one of being an InstrumentScientist or an InvestigationUser is // necessary JsonArrayBuilder array = Json.createArrayBuilder().add(nestedInstrumentQuery).add(nestedUserQuery); - filterBuilder.add(Json.createObjectBuilder().add("bool", Json.createObjectBuilder().add("should", array))); + filterBuilder.add( + Json.createObjectBuilder().add("bool", Json.createObjectBuilder().add("should", array))); break; case "userFullName": String fullName = queryRequest.getString("userFullName"); @@ -788,7 +800,8 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query } break; case "parameters": - for (JsonObject parameterObject : queryRequest.getJsonArray("parameters").getValuesAs(JsonObject.class)) { + for (JsonObject parameterObject : queryRequest.getJsonArray("parameters") + .getValuesAs(JsonObject.class)) { String path = index + "parameter"; List parameterQueries = new ArrayList<>(); if (parameterObject.containsKey("name")) { @@ -801,21 +814,25 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query } if (parameterObject.containsKey("stringValue")) { String stringValue = parameterObject.getString("stringValue"); - parameterQueries.add(OpensearchQueryBuilder.buildMatchQuery(path + ".stringValue", stringValue)); + parameterQueries + .add(OpensearchQueryBuilder.buildMatchQuery(path + ".stringValue", stringValue)); } else if (parameterObject.containsKey("lowerDateValue") && parameterObject.containsKey("upperDateValue")) { Long lower = parseDate(parameterObject, "lowerDateValue", 0, Long.MIN_VALUE); Long upper = parseDate(parameterObject, "upperDateValue", 59999, Long.MAX_VALUE); parameterQueries - .add(OpensearchQueryBuilder.buildLongRangeQuery(path + ".dateTimeValue", lower, upper)); + .add(OpensearchQueryBuilder.buildLongRangeQuery(path + ".dateTimeValue", lower, + upper)); } else if (parameterObject.containsKey("lowerNumericValue") && parameterObject.containsKey("upperNumericValue")) { JsonNumber lower = parameterObject.getJsonNumber("lowerNumericValue"); JsonNumber upper = parameterObject.getJsonNumber("upperNumericValue"); - parameterQueries.add(OpensearchQueryBuilder.buildRangeQuery(path + ".numericValue", lower, upper)); + parameterQueries + .add(OpensearchQueryBuilder.buildRangeQuery(path + ".numericValue", lower, upper)); } filterBuilder.add( - OpensearchQueryBuilder.buildNestedQuery(path, parameterQueries.toArray(new JsonObject[0]))); + OpensearchQueryBuilder.buildNestedQuery(path, + parameterQueries.toArray(new JsonObject[0]))); } break; default: @@ -827,29 +844,36 @@ private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject query } ValueType valueType = queryRequest.get(queryKey).getValueType(); // if (queryKey.contains(".")) { - // int pathEnd = queryKey.indexOf("."); - // path = queryKey.substring(0, pathEnd); - // if (path.equals(index)) { - // // e.g. "dataset.id" should be interpretted as "id" iff we're searching Datasets - // field = queryKey.substring(pathEnd + 1); - // } + // int pathEnd = queryKey.indexOf("."); + // path = queryKey.substring(0, pathEnd); + // if (path.equals(index)) { + // // e.g. "dataset.id" should be interpretted as "id" iff we're searching + // Datasets + // field = queryKey.substring(pathEnd + 1); + // } // } switch (valueType) { case STRING: - defaultTermQuery = OpensearchQueryBuilder.buildTermQuery(field + ".keyword", queryRequest.getString(queryKey)); + defaultTermQuery = OpensearchQueryBuilder.buildTermQuery(field + ".keyword", + queryRequest.getString(queryKey)); break; case NUMBER: - defaultTermQuery = OpensearchQueryBuilder.buildTermQuery(field, queryRequest.getJsonNumber(queryKey)); + defaultTermQuery = OpensearchQueryBuilder.buildTermQuery(field, + queryRequest.getJsonNumber(queryKey)); break; case ARRAY: // Only support array of String as list of ICAT ids is currently only use case - defaultTermQuery = OpensearchQueryBuilder.buildTermsQuery(field, queryRequest.getJsonArray(queryKey)); + defaultTermQuery = OpensearchQueryBuilder.buildTermsQuery(field, + queryRequest.getJsonArray(queryKey)); break; default: - throw new IcatException(IcatExceptionType.BAD_PARAMETER, "Query values should be ARRAY, STRING or NUMBER, but had value of type " + valueType); + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Query values should be ARRAY, STRING or NUMBER, but had value of type " + + valueType); } if (dimensionPrefix != null) { - // e.g. "sample.id" should use a nested query as sample is nested on other entities + // e.g. "sample.id" should use a nested query as sample is nested on other + // entities filterBuilder.add(OpensearchQueryBuilder.buildNestedQuery(dimensionPrefix, defaultTermQuery)); } else { // Otherwise, we can associate the query directly with the searched entity @@ -1660,13 +1684,15 @@ private void modifyEntity(CloseableHttpClient httpclient, StringBuilder sb, Set< String investigationId = document.getString("investigation.id"); Long[] runningFileSize = investigationAggregations.getOrDefault(investigationId, new Long[] { 0L, 0L }); - Long[] newValue = new Long[] { runningFileSize[0] + newFileSize - oldFileSize, runningFileSize[1] }; + Long[] newValue = new Long[] { runningFileSize[0] + newFileSize - oldFileSize, + runningFileSize[1] }; investigationAggregations.put(investigationId, newValue); } if (document.containsKey("dataset.id")) { String datasetId = document.getString("dataset.id"); Long[] runningFileSize = datasetAggregations.getOrDefault(datasetId, new Long[] { 0L, 0L }); - Long[] newValue = new Long[] { runningFileSize[0] + newFileSize - oldFileSize, runningFileSize[1] }; + Long[] newValue = new Long[] { runningFileSize[0] + newFileSize - oldFileSize, + runningFileSize[1] }; datasetAggregations.put(datasetId, newValue); } } diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index 7d0d085b..a3f23ed5 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -934,6 +934,10 @@ public void logout(@Context HttpServletRequest request, @PathParam("sessionId") * * @summary Free text id search. * + * @deprecated in favour of {@link #searchDocuments}, which offers more + * functionality and returns full documents rather than just ICAT + * ids. + * * @param sessionId * a sessionId of a user which takes the form * 0d9a3706-80d4-4d29-9ff3-4d65d4308a24 @@ -1028,14 +1032,6 @@ public void logout(@Context HttpServletRequest request, @PathParam("sessionId") * >lucene parser but avoid trying to use fields. This is * only respected in the case of an investigation search. * - * @param sort - * JSON encoded sort object. Each key should be a field on the - * targeted Document, with a value of "asc" or "desc" to - * specify the order of the results. Multiple pairs can be - * provided, in which case each subsequent sort is used as a - * tiebreaker for the previous one. If no sort is specified, - * then results will be returned in order of relevance to the - * search query, with their search engine id as a tiebreaker. * * @param maxCount * maximum number of entities to return @@ -1048,8 +1044,9 @@ public void logout(@Context HttpServletRequest request, @PathParam("sessionId") @GET @Path("lucene/data") @Produces(MediaType.APPLICATION_JSON) - public String search(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, - @QueryParam("query") String query, @QueryParam("maxCount") int maxCount, @QueryParam("sort") String sort) + @Deprecated + public String lucene(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, + @QueryParam("query") String query, @QueryParam("maxCount") int maxCount) throws IcatException { if (query == null) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "query is not set"); @@ -1094,7 +1091,7 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId throw new IcatException(IcatExceptionType.BAD_PARAMETER, "target:" + target + " is not expected"); } logger.debug("Free text search with query: {}", jo.toString()); - objects = beanManager.freeTextSearch(userName, jo, maxCount, sort, manager, request.getRemoteAddr(), klass); + objects = beanManager.freeTextSearch(userName, jo, maxCount, null, manager, request.getRemoteAddr(), klass); JsonGenerator gen = Json.createGenerator(baos); gen.writeStartArray(); for (ScoredEntityBaseBean sb : objects) { @@ -1264,7 +1261,7 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId @GET @Path("search/documents") @Produces(MediaType.APPLICATION_JSON) - public String search(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, + public String searchDocuments(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, @QueryParam("query") String query, @QueryParam("search_after") String searchAfter, @QueryParam("minCount") int minCount, @QueryParam("maxCount") int maxCount, @QueryParam("sort") String sort, @QueryParam("restrict") boolean restrict) throws IcatException { @@ -1399,7 +1396,7 @@ public String search(@Context HttpServletRequest request, @QueryParam("sessionId @GET @Path("facet/documents") @Produces(MediaType.APPLICATION_JSON) - public String facet(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, + public String facetDocuments(@Context HttpServletRequest request, @QueryParam("sessionId") String sessionId, @QueryParam("query") String query) throws IcatException { if (query == null) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "query is not set"); @@ -1578,6 +1575,35 @@ public void waitMillis(@FormParam("sessionId") String sessionId, @FormParam("ms" } } + /** + * Clear and repopulate lucene documents for the specified entityName + * + * @deprecated in favour of {@link #searchPopulate}, which allows an upper limit + * on population to be set and makes deletion of existing documents + * optional. + * + * @summary Lucene Populate + * + * @param sessionId + * a sessionId of a user listed in rootUserNames + * @param entityName + * the name of the entity + * @param minid + * only process entities with id values greater than this + * value + * + * @throws IcatException + * when something is wrong + */ + @POST + @Path("lucene/db/{entityName}/{minid}") + @Deprecated + public void lucenePopulate(@FormParam("sessionId") String sessionId, @PathParam("entityName") String entityName, + @PathParam("minid") long minid) throws IcatException { + checkRoot(sessionId); + beanManager.searchPopulate(entityName, minid, null, true, manager); + } + /** * Populates search engine documents for the specified entityName. * diff --git a/src/main/resources/run.properties b/src/main/resources/run.properties index 4e9f2c3a..e871b393 100644 --- a/src/main/resources/run.properties +++ b/src/main/resources/run.properties @@ -16,6 +16,7 @@ notification.Datafile = CU log.list = SESSION WRITE READ INFO +# LUCENE, OPENSEARCH and ELASTICSEARCH engines are supported, however the latter two are considered experimental search.engine = lucene search.urls = https://localhost.localdomain:8181 search.populateBlockSize = 10000 diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 1fb2c61e..54d06ec4 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -23,6 +23,7 @@ import javax.json.JsonValue; import org.icatproject.core.IcatException; +import org.icatproject.core.IcatException.IcatExceptionType; import org.icatproject.core.entity.Datafile; import org.icatproject.core.entity.DatafileFormat; import org.icatproject.core.entity.DatafileParameter; @@ -106,15 +107,20 @@ public Filter(String fld, JsonObject... values) { @Parameterized.Parameters public static Iterable data() throws URISyntaxException, IcatException { - String luceneUrl = System.getProperty("luceneUrl"); - logger.info("Using Lucene service at {}", luceneUrl); - URI luceneUri = new URI(luceneUrl); - - String opensearchUrl = System.getProperty("opensearchUrl"); - logger.info("Using Opensearch/Elasticsearch service at {}", opensearchUrl); - URI opensearchUri = new URI(opensearchUrl); - - return Arrays.asList(new LuceneApi(luceneUri), new OpensearchApi(opensearchUri, "\u2103: celsius", false)); + String searchEngine = System.getProperty("searchEngine"); + String searchUrls = System.getProperty("searchUrls"); + URI searchUri = new URI(searchUrls); + logger.info("Using {} service at {}", searchEngine, searchUrls); + switch (searchEngine) { + case "LUCENE": + return Arrays.asList(new LuceneApi(searchUri)); + case "OPENSEARCH": + case "ELASTICSEARCH": + return Arrays.asList(new OpensearchApi(searchUri, "\u2103: celsius", false)); + default: + String msg = "Search engine must be one of LUCENE, OPENSEARCH or ELASTICSEARCH but was " + searchEngine; + throw new IcatException(IcatExceptionType.BAD_PARAMETER, msg); + } } @Parameterized.Parameter diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 01cdfdee..ad2baaa4 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -92,13 +92,11 @@ private static void clearSearch() throws URISyntaxException, MalformedURLException, org.icatproject.core.IcatException { if (searchApi == null) { String searchEngine = System.getProperty("searchEngine"); + String urlString = System.getProperty("searchUrls"); + URI uribase = new URI(urlString); if (searchEngine.equals("LUCENE")) { - String urlString = System.getProperty("luceneUrl"); - URI uribase = new URI(urlString); searchApi = new LuceneApi(uribase); } else if (searchEngine.equals("OPENSEARCH") || searchEngine.equals("ELASTICSEARCH")) { - String urlString = System.getProperty("opensearchUrl"); - URI uribase = new URI(urlString); searchApi = new OpensearchApi(uribase); } else { throw new RuntimeException( diff --git a/src/test/scripts/prepare_test.py b/src/test/scripts/prepare_test.py index 3b8bb409..f068d7e1 100644 --- a/src/test/scripts/prepare_test.py +++ b/src/test/scripts/prepare_test.py @@ -8,20 +8,15 @@ from zipfile import ZipFile import subprocess -if len(sys.argv) != 6: +if len(sys.argv) != 5: raise RuntimeError("Wrong number of arguments") containerHome = sys.argv[1] icat_url = sys.argv[2] search_engine = sys.argv[3] -lucene_url = sys.argv[4] -opensearch_url = sys.argv[5] +search_urls = sys.argv[4] -if search_engine == "LUCENE": - search_urls = lucene_url -elif search_engine == "OPENSEARCH" or search_engine == "ELASTICSEARCH": - search_urls = opensearch_url -else: +if search_engine not in ["LUCENE", "OPENSEARCH", "ELASTICSEARCH"]: raise RuntimeError("Search engine %s unrecognised, " % search_engine + "should be one of LUCENE, ELASTICSEARCH, OPENSEARCH") From 60f30a6c684a07b1c08b19ab3afeb9c87305167a Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Mon, 3 Oct 2022 15:19:12 +0100 Subject: [PATCH 36/51] Refactors in EntityBeanManager and SearchManager #267 --- .../core/manager/EntityBeanManager.java | 173 +++++++++--------- .../core/manager/search/FacetDimension.java | 5 +- .../core/manager/search/FacetLabel.java | 9 +- .../core/manager/search/SearchManager.java | 14 +- .../core/manager/search/SearchResult.java | 4 + .../org/icatproject/exposed/ICATRest.java | 2 +- 6 files changed, 98 insertions(+), 109 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 5a00dc61..53f28755 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -1128,9 +1128,6 @@ public String getUserName(String sessionId, EntityManager manager) throws IcatEx Session session = getSession(sessionId, manager); String userName = session.getUserName(); logger.debug("user: " + userName + " is associated with: " + sessionId); - // TODO RM - // logger.trace("userName: {}", userName); - // userName = userName.equals("db/notroot") ? "notroot" : userName; return userName; } catch (IcatException e) { logger.debug("sessionId " + sessionId + " is not associated with valid session " + e.getMessage()); @@ -1408,7 +1405,7 @@ public void searchCommit() throws IcatException { * * @param userName User performing the search, used for authorisation. * @param jo JsonObject containing the details of the query to be used. - * @param limit The maximum number of results to collect before returning. If + * @param maxCount The maximum number of results to collect before returning. If * a batch from the search engine has more than this many * authorised results, then the excess results will be * discarded. @@ -1418,56 +1415,15 @@ public void searchCommit() throws IcatException { * @return SearchResult for the query. * @throws IcatException */ - public List freeTextSearch(String userName, JsonObject jo, int limit, String sort, + public List freeTextSearch(String userName, JsonObject jo, int maxCount, EntityManager manager, String ip, Class klass) throws IcatException { long startMillis = System.currentTimeMillis(); List results = new ArrayList<>(); - JsonValue searchAfter = null; - JsonValue lastSearchAfter = null; if (searchActive) { - SearchResult lastSearchResult = null; - List allResults = Collections.emptyList(); - /* - * As results may be rejected and maxCount may be 1 ensure that we - * don't make a huge number of calls to search engine - */ - int blockSize = Math.max(1000, limit); - - do { - lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, Arrays.asList("id")); - allResults = lastSearchResult.getResults(); - ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, limit, userName, manager, klass); - if (lastBean == null) { - // Haven't stopped early, so use the Lucene provided searchAfter document - lastSearchAfter = lastSearchResult.getSearchAfter(); - if (lastSearchAfter == null) { - break; // If searchAfter is null, we ran out of results so stop here - } - searchAfter = lastSearchAfter; - } else { - // Have stopped early by reaching the limit, so build a searchAfter document - lastSearchAfter = searchManager.buildSearchAfter(lastBean, sort); - break; - } - if (System.currentTimeMillis() - startMillis > searchMaxSearchTimeMillis) { - String msg = "Search cancelled for exceeding " + searchMaxSearchTimeMillis / 1000 + " seconds"; - throw new IcatException(IcatExceptionType.INTERNAL, msg); - } - } while (results.size() < limit); - } - - if (logRequests.contains("R")) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) { - gen.write("userName", userName); - if (results.size() > 0) { - gen.write("entityId", results.get(0).getEntityBaseBeanId()); - } - gen.writeEnd(); - } - transmitter.processMessage("freeTextSearch", ip, baos.toString(), startMillis); + searchDocuments(userName, jo, null, maxCount, maxCount, null, manager, klass, + startMillis, results, Arrays.asList("id")); } - logger.debug("Returning {} results", results.size()); + logSearch(userName, ip, startMillis, results, "freeTextSearch"); return results; } @@ -1494,44 +1450,16 @@ public List freeTextSearch(String userName, JsonObject jo, * @throws IcatException */ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue searchAfter, int minCount, - int maxCount, String sort, EntityManager manager, String ip, - Class klass) throws IcatException { + int maxCount, String sort, EntityManager manager, String ip, Class klass) + throws IcatException { long startMillis = System.currentTimeMillis(); - List results = new ArrayList<>(); JsonValue lastSearchAfter = null; + List results = new ArrayList<>(); List dimensions = new ArrayList<>(); if (searchActive) { - SearchResult lastSearchResult = null; - List allResults = Collections.emptyList(); List fields = SearchManager.getPublicSearchFields(gateKeeper, klass.getSimpleName()); - /* - * As results may be rejected and maxCount may be 1 ensure that we - * don't make a huge number of calls to search engine - */ - int blockSize = Math.max(1000, maxCount); - - do { - lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, fields); - allResults = lastSearchResult.getResults(); - ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, maxCount, userName, manager, - klass); - if (lastBean == null) { - // Haven't stopped early, so use the Lucene provided searchAfter document - lastSearchAfter = lastSearchResult.getSearchAfter(); - if (lastSearchAfter == null) { - break; // If searchAfter is null, we ran out of results so stop here - } - searchAfter = lastSearchAfter; - } else { - // Have stopped early by reaching the limit, so build a searchAfter document - lastSearchAfter = searchManager.buildSearchAfter(lastBean, sort); - break; - } - if (System.currentTimeMillis() - startMillis > searchMaxSearchTimeMillis) { - String msg = "Search cancelled for exceeding " + searchMaxSearchTimeMillis / 1000 + " seconds"; - throw new IcatException(IcatExceptionType.INTERNAL, msg); - } - } while (results.size() < minCount); + lastSearchAfter = searchDocuments(userName, jo, searchAfter, maxCount, minCount, sort, manager, klass, + startMillis, results, fields); if (jo.containsKey("facets")) { List jsonFacets = jo.getJsonArray("facets").getValuesAs(JsonObject.class); @@ -1544,6 +1472,21 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue } } } + logSearch(userName, ip, startMillis, results, "freeTextSearchDocs"); + return new SearchResult(lastSearchAfter, results, dimensions); + } + + /** + * Performs logging dependent on the value of logRequests. + * + * @param userName User performing the search + * @param ip Used for logging only + * @param startMillis The start time of the search in milliseconds + * @param results List of authorised search results + * @param operation Name of the calling function + */ + private void logSearch(String userName, String ip, long startMillis, List results, + String operation) { if (logRequests.contains("R")) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) { @@ -1553,10 +1496,69 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue } gen.writeEnd(); } - transmitter.processMessage("freeTextSearchDocs", ip, baos.toString(), startMillis); + transmitter.processMessage(operation, ip, baos.toString(), startMillis); } logger.debug("Returning {} results", results.size()); - return new SearchResult(lastSearchAfter, results, dimensions); + } + + /** + * Performs batches of searches, the results of which are authorised. Results + * are collected until they run out, minCount is reached, or too much time + * elapses. + * + * @param userName User performing the search, used for authorisation. + * @param jo JsonObject containing the details of the query to be used. + * @param searchAfter JsonValue representation of the final result from a + * previous search. + * @param minCount The minimum number of results to collect before returning. + * If a batch from the search engine has at least this many + * authorised results, no further batches will be requested. + * @param maxCount The maximum number of results to collect before returning. + * If a batch from the search engine has more than this many + * authorised results, then the excess results will be + * discarded. + * @param sort String of Json representing sort criteria. + * @param manager EntityManager for finding entities from their Id. + * @param klass Class of the entity to search. + * @param startMillis The start time of the search in milliseconds + * @param results List of results from the search. Authorised results will + * be appended to this List. + * @param fields Fields to include in the returned Documents. + * @return JsonValue representing the last result of the search, formatted to + * allow future searches to "search after" this result. May be null. + * @throws IcatException If the search exceeds the maximum allowed time. + */ + private JsonValue searchDocuments(String userName, JsonObject jo, JsonValue searchAfter, int maxCount, int minCount, + String sort, EntityManager manager, Class klass, long startMillis, + List results, List fields) throws IcatException { + /* + * As results may be rejected and maxCount may be 1 ensure that we + * don't make a huge number of calls to search engine + */ + int blockSize = Math.max(1000, maxCount); + JsonValue lastSearchAfter; + do { + SearchResult lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, fields); + List allResults = lastSearchResult.getResults(); + ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, maxCount, userName, manager, + klass); + if (lastBean == null) { + // Haven't stopped early, so use the Lucene provided searchAfter document + lastSearchAfter = lastSearchResult.getSearchAfter(); + if (lastSearchAfter == null) { + return null; // If searchAfter is null, we ran out of results so stop here + } + searchAfter = lastSearchAfter; + } else { + // Have stopped early by reaching the limit, so build a searchAfter document + return searchManager.buildSearchAfter(lastBean, sort); + } + if (System.currentTimeMillis() - startMillis > searchMaxSearchTimeMillis) { + String msg = "Search cancelled for exceeding " + searchMaxSearchTimeMillis / 1000 + " seconds"; + throw new IcatException(IcatExceptionType.INTERNAL, msg); + } + } while (results.size() < minCount); + return lastSearchAfter; } /** @@ -1569,7 +1571,6 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue * @throws IcatException */ public SearchResult facetDocs(JsonObject jo, Class klass) throws IcatException { - JsonValue lastSearchAfter = null; List dimensions = new ArrayList<>(); if (searchActive && jo.containsKey("facets")) { List jsonFacets = jo.getJsonArray("facets").getValuesAs(JsonObject.class); @@ -1582,7 +1583,7 @@ public SearchResult facetDocs(JsonObject jo, Class kla } } } - return new SearchResult(lastSearchAfter, new ArrayList<>(), dimensions); + return new SearchResult(dimensions); } /** diff --git a/src/main/java/org/icatproject/core/manager/search/FacetDimension.java b/src/main/java/org/icatproject/core/manager/search/FacetDimension.java index 870d89b4..d5308ba6 100644 --- a/src/main/java/org/icatproject/core/manager/search/FacetDimension.java +++ b/src/main/java/org/icatproject/core/manager/search/FacetDimension.java @@ -1,6 +1,7 @@ package org.icatproject.core.manager.search; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -23,9 +24,7 @@ public FacetDimension(String target, String dimension) { public FacetDimension(String target, String dimension, FacetLabel... labels) { this.target = target; this.dimension = dimension; - for (FacetLabel label : labels) { - facets.add(label); - } + Collections.addAll(facets, labels); } public List getFacets() { diff --git a/src/main/java/org/icatproject/core/manager/search/FacetLabel.java b/src/main/java/org/icatproject/core/manager/search/FacetLabel.java index 2e4743ca..0ee9c443 100644 --- a/src/main/java/org/icatproject/core/manager/search/FacetLabel.java +++ b/src/main/java/org/icatproject/core/manager/search/FacetLabel.java @@ -21,14 +21,7 @@ public FacetLabel(String label, Long value) { } public FacetLabel(JsonObject jsonObject) { - label = jsonObject.getString("key"); - value = jsonObject.getJsonNumber("doc_count").longValueExact(); - if (jsonObject.containsKey("from")) { - from = jsonObject.getJsonNumber("from"); - } - if (jsonObject.containsKey("to")) { - to = jsonObject.getJsonNumber("to"); - } + this(jsonObject.getString("key"), jsonObject); } public FacetLabel(String label, JsonObject jsonObject) { diff --git a/src/main/java/org/icatproject/core/manager/search/SearchManager.java b/src/main/java/org/icatproject/core/manager/search/SearchManager.java index 93ff9541..bbc60f2b 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchManager.java @@ -538,11 +538,7 @@ public static JsonObject buildFacetQuery(List results, Str JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); results.forEach(r -> arrayBuilder.add(Long.toString(r.getEntityBaseBeanId()))); JsonObject terms = Json.createObjectBuilder().add(idField, arrayBuilder.build()).build(); - JsonObjectBuilder objectBuilder = Json.createObjectBuilder().add("query", terms); - if (facetJson.containsKey("dimensions")) { - objectBuilder.add("dimensions", facetJson.getJsonArray("dimensions")); - } - return objectBuilder.build(); + return buildFacetQuery(terms, facetJson); } /** @@ -570,11 +566,7 @@ public static JsonObject buildSampleFacetQuery(List result } }); JsonObject terms = Json.createObjectBuilder().add("sample.id", arrayBuilder.build()).build(); - JsonObjectBuilder objectBuilder = Json.createObjectBuilder().add("query", terms); - if (facetJson.containsKey("dimensions")) { - objectBuilder.add("dimensions", facetJson.getJsonArray("dimensions")); - } - return objectBuilder.build(); + return buildFacetQuery(terms, facetJson); } /** @@ -605,7 +597,7 @@ public static JsonObject buildFacetQuery(JsonObject filterObject, JsonObject fac private static List buildPublicSearchFields(GateKeeper gateKeeper, Map map) { List fields = new ArrayList<>(); for (Entry entry : map.entrySet()) { - Boolean includeField = true; + boolean includeField = true; if (entry.getValue() != null) { for (Relationship relationship : entry.getValue()) { if (!gateKeeper.allowed(relationship)) { diff --git a/src/main/java/org/icatproject/core/manager/search/SearchResult.java b/src/main/java/org/icatproject/core/manager/search/SearchResult.java index db8ad31d..caa2a5f9 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchResult.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchResult.java @@ -19,6 +19,10 @@ public class SearchResult { public SearchResult() { } + public SearchResult(List dimensions) { + this.dimensions = dimensions; + } + public SearchResult(JsonValue searchAfter, List results, List dimensions) { this.searchAfter = searchAfter; this.results = results; diff --git a/src/main/java/org/icatproject/exposed/ICATRest.java b/src/main/java/org/icatproject/exposed/ICATRest.java index a3f23ed5..bd431629 100644 --- a/src/main/java/org/icatproject/exposed/ICATRest.java +++ b/src/main/java/org/icatproject/exposed/ICATRest.java @@ -1091,7 +1091,7 @@ public String lucene(@Context HttpServletRequest request, @QueryParam("sessionId throw new IcatException(IcatExceptionType.BAD_PARAMETER, "target:" + target + " is not expected"); } logger.debug("Free text search with query: {}", jo.toString()); - objects = beanManager.freeTextSearch(userName, jo, maxCount, null, manager, request.getRemoteAddr(), klass); + objects = beanManager.freeTextSearch(userName, jo, maxCount, manager, request.getRemoteAddr(), klass); JsonGenerator gen = Json.createGenerator(baos); gen.writeStartArray(); for (ScoredEntityBaseBean sb : objects) { From 6c9477a836be357467f3857c4c89dd82a11dabc6 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Wed, 5 Oct 2022 11:11:01 +0100 Subject: [PATCH 37/51] Expand TestRS to cover getReadableIds change #277 --- .../org/icatproject/integration/TestRS.java | 353 +++++++++++++++--- 1 file changed, 292 insertions(+), 61 deletions(-) diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 2e0c80f9..0350ff43 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -19,7 +19,9 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStream; +import java.io.StringReader; import java.net.URI; +import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -88,14 +90,28 @@ public void clearSession() throws Exception { wSession.clearAuthz(); } - @Ignore("Test fails because of bug in eclipselink") - @Test - public void testDistinctBehaviour() throws Exception { + private Session rootSession() throws URISyntaxException, IcatException { ICAT icat = new ICAT(System.getProperty("serverUrl")); Map credentials = new HashMap<>(); credentials.put("username", "root"); credentials.put("password", "password"); Session session = icat.login("db", credentials); + return session; + } + + private Session piOneSession() throws URISyntaxException, IcatException { + ICAT icat = new ICAT(System.getProperty("serverUrl")); + Map credentials = new HashMap<>(); + credentials.put("username", "piOne"); + credentials.put("password", "piOne"); + Session session = icat.login("db", credentials); + return session; + } + + @Ignore("Test fails because of bug in eclipselink") + @Test + public void testDistinctBehaviour() throws Exception { + Session session = rootSession(); Path path = Paths.get(ClassLoader.class.getResource("/icat.port").toURI()); session.importMetaData(path, DuplicateAction.CHECK, Attributes.USER); @@ -115,97 +131,244 @@ public void TestJsoniseBean() throws Exception { DateFormat dft = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); Session session = createAndPopulate(); - /* Expected: <[{"User":{"id":8148,"createId":"db/notroot","createTime":"2019-03-11T14:14:47.000Z","modId":"db/notroot","modTime":"2019-03-11T14:14:47.000Z","affiliation":"Unseen University","familyName":"Worblehat","fullName":"Dr. Horace Worblehat","givenName":"Horace","instrumentScientists":[],"investigationUsers":[],"name":"db/lib","studies":[],"userGroups":[]}}]> */ + /* + * Expected: <[{"User":{"id":8148,"createId":"db/notroot","createTime": + * "2019-03-11T14:14:47.000Z","modId":"db/notroot","modTime": + * "2019-03-11T14:14:47.000Z","affiliation":"Unseen University","familyName": + * "Worblehat","fullName":"Dr. Horace Worblehat","givenName":"Horace", + * "instrumentScientists":[],"investigationUsers":[],"name":"db/lib","studies":[ + * ],"userGroups":[]}}]> + */ JsonArray user_response = search(session, "SELECT u from User u WHERE u.name = 'db/lib'", 1); collector.checkThat(user_response.getJsonObject(0).containsKey("User"), is(true)); JsonObject user = user_response.getJsonObject(0).getJsonObject("User"); - collector.checkThat(user.getJsonNumber("id").isIntegral(), is(true)); // Check Integer conversion - collector.checkThat(user.getString("createId"), is("db/notroot")); // Check String conversion - - /* Expected: <[{"Facility":{"id":2852,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","applications":[],"datafileFormats":[],"datasetTypes":[],"daysUntilRelease":90,"facilityCycles":[],"instruments":[],"investigationTypes":[],"investigations":[],"name":"Test port facility","parameterTypes":[],"sampleTypes":[]}}]> */ + collector.checkThat(user.getJsonNumber("id").isIntegral(), is(true)); // Check Integer conversion + collector.checkThat(user.getString("createId"), is("db/notroot")); // Check String conversion + + /* + * Expected: <[{"Facility":{"id":2852,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","applications":[],"datafileFormats":[], + * "datasetTypes":[],"daysUntilRelease":90,"facilityCycles":[],"instruments":[], + * "investigationTypes":[],"investigations":[],"name":"Test port facility" + * ,"parameterTypes":[],"sampleTypes":[]}}]> + */ JsonArray fac_response = search(session, "SELECT f from Facility f WHERE f.name = 'Test port facility'", 1); collector.checkThat(fac_response.getJsonObject(0).containsKey("Facility"), is(true)); - /* Expected: <[{"Instrument":{"id":1449,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","fullName":"EDDI - Energy Dispersive Diffraction","instrumentScientists":[],"investigationInstruments":[],"name":"EDDI","pid":"ig:0815","shifts":[]}}]> */ + /* + * Expected: <[{"Instrument":{"id":1449,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","fullName":"EDDI - Energy Dispersive Diffraction" + * ,"instrumentScientists":[],"investigationInstruments":[],"name":"EDDI","pid": + * "ig:0815","shifts":[]}}]> + */ JsonArray inst_response = search(session, "SELECT i from Instrument i WHERE i.name = 'EDDI'", 1); collector.checkThat(inst_response.getJsonObject(0).containsKey("Instrument"), is(true)); - /* Expected: <[{"InvestigationType":{"id":3401,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","investigations":[],"name":"atype"}}]> */ + /* + * Expected: + * <[{"InvestigationType":{"id":3401,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","investigations":[],"name":"atype"}}]> + */ JsonArray it_response = search(session, "SELECT it from InvestigationType it WHERE it.name = 'atype'", 1); collector.checkThat(it_response.getJsonObject(0).containsKey("InvestigationType"), is(true)); - /* Expected: <[{"ParameterType":{"id":5373,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","applicableToDataCollection":false,"applicableToDatafile":true,"applicableToDataset":true,"applicableToInvestigation":true,"applicableToSample":false,"dataCollectionParameters":[],"datafileParameters":[],"datasetParameters":[],"enforced":false,"investigationParameters":[],"minimumNumericValue":73.4,"name":"temp","permissibleStringValues":[],"pid":"pt:25c","sampleParameters":[],"units":"degrees Kelvin","valueType":"NUMERIC","verified":false}}]> */ + /* + * Expected: <[{"ParameterType":{"id":5373,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","applicableToDataCollection":false, + * "applicableToDatafile":true,"applicableToDataset":true, + * "applicableToInvestigation":true,"applicableToSample":false, + * "dataCollectionParameters":[],"datafileParameters":[],"datasetParameters":[], + * "enforced":false,"investigationParameters":[],"minimumNumericValue":73.4, + * "name":"temp","permissibleStringValues":[],"pid":"pt:25c","sampleParameters": + * [],"units":"degrees Kelvin","valueType":"NUMERIC","verified":false}}]> + */ JsonArray pt_response = search(session, "SELECT pt from ParameterType pt WHERE pt.name = 'temp'", 1); collector.checkThat(pt_response.getJsonObject(0).containsKey("ParameterType"), is(true)); - collector.checkThat((Double) pt_response.getJsonObject(0).getJsonObject("ParameterType").getJsonNumber("minimumNumericValue").doubleValue(), is(73.4)); // Check Double conversion - collector.checkThat((Boolean) pt_response.getJsonObject(0).getJsonObject("ParameterType").getBoolean("enforced"), is(Boolean.FALSE)); // Check boolean conversion - collector.checkThat(pt_response.getJsonObject(0).getJsonObject("ParameterType").getJsonString("valueType").getString(), is("NUMERIC")); // Check ParameterValueType conversion - - /* Expected: <[{"Investigation":{"id":4814,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"endDate":"2010-12-31T23:59:59.000Z","investigationGroups":[],"investigationInstruments":[],"investigationUsers":[],"keywords":[],"name":"expt1","parameters":[],"publications":[],"samples":[],"shifts":[],"startDate":"2010-01-01T00:00:00.000Z","studyInvestigations":[],"title":"a title at the beginning","visitId":"zero"}},{"Investigation":{"id":4815,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"endDate":"2011-12-31T23:59:59.000Z","investigationGroups":[],"investigationInstruments":[],"investigationUsers":[],"keywords":[],"name":"expt1","parameters":[],"publications":[],"samples":[],"shifts":[],"startDate":"2011-01-01T00:00:00.000Z","studyInvestigations":[],"title":"a title in the middle","visitId":"one"}},{"Investigation":{"id":4816,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"endDate":"2012-12-31T23:59:59.000Z","investigationGroups":[],"investigationInstruments":[],"investigationUsers":[],"keywords":[],"name":"expt1","parameters":[],"publications":[],"samples":[],"shifts":[],"startDate":"2012-01-01T00:00:00.000Z","studyInvestigations":[],"title":"a title at the end","visitId":"two"}}]> */ + collector.checkThat((Double) pt_response.getJsonObject(0).getJsonObject("ParameterType") + .getJsonNumber("minimumNumericValue").doubleValue(), is(73.4)); // Check Double conversion + collector.checkThat( + (Boolean) pt_response.getJsonObject(0).getJsonObject("ParameterType").getBoolean("enforced"), + is(Boolean.FALSE)); // Check boolean conversion + collector.checkThat( + pt_response.getJsonObject(0).getJsonObject("ParameterType").getJsonString("valueType").getString(), + is("NUMERIC")); // Check ParameterValueType conversion + + /* + * Expected: <[{"Investigation":{"id":4814,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","datasets":[],"endDate":"2010-12-31T23:59:59.000Z" + * ,"investigationGroups":[],"investigationInstruments":[],"investigationUsers": + * [],"keywords":[],"name":"expt1","parameters":[],"publications":[],"samples":[ + * ],"shifts":[],"startDate":"2010-01-01T00:00:00.000Z","studyInvestigations":[] + * ,"title":"a title at the beginning","visitId":"zero"}},{"Investigation":{"id" + * :4815,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId" + * :"db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"endDate": + * "2011-12-31T23:59:59.000Z","investigationGroups":[], + * "investigationInstruments":[],"investigationUsers":[],"keywords":[],"name": + * "expt1","parameters":[],"publications":[],"samples":[],"shifts":[], + * "startDate":"2011-01-01T00:00:00.000Z","studyInvestigations":[], + * "title":"a title in the middle","visitId":"one"}},{"Investigation":{"id":4816 + * ,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId": + * "db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"endDate": + * "2012-12-31T23:59:59.000Z","investigationGroups":[], + * "investigationInstruments":[],"investigationUsers":[],"keywords":[],"name": + * "expt1","parameters":[],"publications":[],"samples":[],"shifts":[], + * "startDate":"2012-01-01T00:00:00.000Z","studyInvestigations":[], + * "title":"a title at the end","visitId":"two"}}]> + */ JsonArray inv_response = search(session, "SELECT inv from Investigation inv WHERE inv.name = 'expt1'", 3); collector.checkThat(inv_response.getJsonObject(0).containsKey("Investigation"), is(true)); - /* Expected: <[{"InvestigationUser":{"id":4723,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","role":"troublemaker"}}]> */ - JsonArray invu_response = search(session, "SELECT invu from InvestigationUser invu WHERE invu.role = 'troublemaker'", 1); + /* + * Expected: + * <[{"InvestigationUser":{"id":4723,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","role":"troublemaker"}}]> + */ + JsonArray invu_response = search(session, + "SELECT invu from InvestigationUser invu WHERE invu.role = 'troublemaker'", 1); collector.checkThat(invu_response.getJsonObject(0).containsKey("InvestigationUser"), is(true)); - /* Expected: <[{"Shift":{"id":2995,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","comment":"waiting","endDate":"2013-12-31T22:59:59.000Z","startDate":"2013-12-31T11:00:00.000Z"}}]> */ + /* + * Expected: <[{"Shift":{"id":2995,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","comment":"waiting","endDate": + * "2013-12-31T22:59:59.000Z","startDate":"2013-12-31T11:00:00.000Z"}}]> + */ JsonArray shift_response = search(session, "SELECT shift from Shift shift WHERE shift.comment = 'waiting'", 1); collector.checkThat(shift_response.getJsonObject(0).containsKey("Shift"), is(true)); - /* Expected: <[{"SampleType":{"id":3220,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","molecularFormula":"C","name":"diamond","safetyInformation":"fairly harmless","samples":[]}}]> */ + /* + * Expected: <[{"SampleType":{"id":3220,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","molecularFormula":"C","name":"diamond", + * "safetyInformation":"fairly harmless","samples":[]}}]> + */ JsonArray st_response = search(session, "SELECT st from SampleType st WHERE st.name = 'diamond'", 1); collector.checkThat(st_response.getJsonObject(0).containsKey("SampleType"), is(true)); - /* Expected: <[{"Sample":{"id":2181,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"name":"Koh-I-Noor","parameters":[],"pid":"sdb:374717"}}]> */ + /* + * Expected: <[{"Sample":{"id":2181,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","datasets":[],"name":"Koh-I-Noor","parameters":[], + * "pid":"sdb:374717"}}]> + */ JsonArray s_response = search(session, "SELECT s from Sample s WHERE s.name = 'Koh-I-Noor'", 1); collector.checkThat(s_response.getJsonObject(0).containsKey("Sample"), is(true)); - /* Expected: <[{"InvestigationParameter":{"id":1123,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","stringValue":"green"}}]> */ - JsonArray invp_response = search(session, "SELECT invp from InvestigationParameter invp WHERE invp.stringValue = 'green'", 1); + /* + * Expected: + * <[{"InvestigationParameter":{"id":1123,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","stringValue":"green"}}]> + */ + JsonArray invp_response = search(session, + "SELECT invp from InvestigationParameter invp WHERE invp.stringValue = 'green'", 1); collector.checkThat(invp_response.size(), equalTo(1)); collector.checkThat(invp_response.getJsonObject(0).containsKey("InvestigationParameter"), is(true)); - /* Expected: <[{"DatasetType":{"id":1754,"createId":"db/notroot","createTime":"2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime":"2019-03-11T15:58:33.000Z","datasets":[],"name":"calibration"}}]> */ + /* + * Expected: <[{"DatasetType":{"id":1754,"createId":"db/notroot","createTime": + * "2019-03-11T15:58:33.000Z","modId":"db/notroot","modTime": + * "2019-03-11T15:58:33.000Z","datasets":[],"name":"calibration"}}]> + */ JsonArray dst_response = search(session, "SELECT dst from DatasetType dst WHERE dst.name = 'calibration'", 1); collector.checkThat(dst_response.getJsonObject(0).containsKey("DatasetType"), is(true)); - /* Expected: <[{"Dataset":{"id":8128,"createId":"db/notroot","createTime":"2019-03-12T11:40:26.000Z","modId":"db/notroot","modTime":"2019-03-12T11:40:26.000Z","complete":true,"dataCollectionDatasets":[],"datafiles":[],"description":"alpha","endDate":"2014-05-16T04:28:26.000Z","name":"ds1","parameters":[],"startDate":"2014-05-16T04:28:26.000Z"}}]> */ + /* + * Expected: <[{"Dataset":{"id":8128,"createId":"db/notroot","createTime": + * "2019-03-12T11:40:26.000Z","modId":"db/notroot","modTime": + * "2019-03-12T11:40:26.000Z","complete":true,"dataCollectionDatasets":[], + * "datafiles":[],"description":"alpha","endDate":"2014-05-16T04:28:26.000Z", + * "name":"ds1","parameters":[],"startDate":"2014-05-16T04:28:26.000Z"}}]> + */ JsonArray ds_response = search(session, "SELECT ds from Dataset ds WHERE ds.name = 'ds1'", 1); collector.checkThat(ds_response.getJsonObject(0).containsKey("Dataset"), is(true)); - collector.checkThat(dft.parse(ds_response.getJsonObject(0).getJsonObject("Dataset").getString("startDate")), isA(Date.class)); //Check Date conversion - - /* Expected: <[{"DatasetParameter":{"id":4632,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","stringValue":"green"}}]> */ - JsonArray dsp_response = search(session, "SELECT dsp from DatasetParameter dsp WHERE dsp.stringValue = 'green'", 1); + collector.checkThat(dft.parse(ds_response.getJsonObject(0).getJsonObject("Dataset").getString("startDate")), + isA(Date.class)); // Check Date conversion + + /* + * Expected: + * <[{"DatasetParameter":{"id":4632,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z","stringValue":"green"}}]> + */ + JsonArray dsp_response = search(session, "SELECT dsp from DatasetParameter dsp WHERE dsp.stringValue = 'green'", + 1); collector.checkThat(dsp_response.size(), equalTo(1)); collector.checkThat(dsp_response.getJsonObject(0).containsKey("DatasetParameter"), is(true)); - /* Expected: <[{"Datafile":{"id":15643,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","dataCollectionDatafiles":[],"destDatafiles":[],"fileSize":17,"name":"df2","parameters":[],"sourceDatafiles":[]}}]> */ + /* + * Expected: <[{"Datafile":{"id":15643,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z","dataCollectionDatafiles":[],"destDatafiles":[], + * "fileSize":17,"name":"df2","parameters":[],"sourceDatafiles":[]}}]> + */ JsonArray df_response = search(session, "SELECT df from Datafile df WHERE df.name = 'df2'", 1); collector.checkThat(df_response.size(), equalTo(1)); collector.checkThat(df_response.getJsonObject(0).containsKey("Datafile"), is(true)); - /* Expected: <[{"DatafileParameter":{"id":1938,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","stringValue":"green"}}]> */ - JsonArray dfp_response = search(session, "SELECT dfp from DatafileParameter dfp WHERE dfp.stringValue = 'green'", 1); + /* + * Expected: + * <[{"DatafileParameter":{"id":1938,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z","stringValue":"green"}}]> + */ + JsonArray dfp_response = search(session, + "SELECT dfp from DatafileParameter dfp WHERE dfp.stringValue = 'green'", 1); collector.checkThat(dfp_response.size(), equalTo(1)); collector.checkThat(dfp_response.getJsonObject(0).containsKey("DatafileParameter"), is(true)); - /* Expected: <[{"Application":{"id":2972,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","jobs":[],"name":"aprog","version":"1.2.3"}},{"Application":{"id":2973,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","jobs":[],"name":"aprog","version":"1.2.6"}}]> */ + /* + * Expected: <[{"Application":{"id":2972,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z","jobs":[],"name":"aprog","version":"1.2.3"}},{ + * "Application":{"id":2973,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z","jobs":[],"name":"aprog","version":"1.2.6"}}]> + */ JsonArray a_response = search(session, "SELECT a from Application a WHERE a.name = 'aprog'", 2); collector.checkThat(a_response.size(), equalTo(2)); collector.checkThat(a_response.getJsonObject(0).containsKey("Application"), is(true)); - /* Expected: <[{DataCollection":{"id":4485,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","dataCollectionDatafiles":[],"dataCollectionDatasets":[],"jobsAsInput":[],"jobsAsOutput":[],"parameters":[]}},{"DataCollection":{"id":4486,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","dataCollectionDatafiles":[],"dataCollectionDatasets":[],"jobsAsInput":[],"jobsAsOutput":[],"parameters":[]}},{"DataCollection":{"id":4487,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z","dataCollectionDatafiles":[],"dataCollectionDatasets":[],"jobsAsInput":[],"jobsAsOutput":[],"parameters":[]}}]> */ + /* + * Expected: + * <[{DataCollection":{"id":4485,"createId":"db/notroot","createTime":"2019-03- + * 12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33. + * 000Z","dataCollectionDatafiles":[],"dataCollectionDatasets":[],"jobsAsInput":[],"jobsAsOutput":[],"parameters":[]}},{"DataCollection":{"id":4486,"createId":"db + * /notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/ + * notroot","modTime":"2019-03-12T13:30:33. + * 000Z","dataCollectionDatafiles":[],"dataCollectionDatasets":[],"jobsAsInput":[],"jobsAsOutput":[],"parameters":[]}},{"DataCollection":{"id":4487,"createId":"db + * /notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/ + * notroot","modTime":"2019-03-12T13:30:33. + * 000Z","dataCollectionDatafiles":[],"dataCollectionDatasets":[],"jobsAsInput":[],"jobsAsOutput":[],"parameters + * ":[]}}]> + */ JsonArray dc_response = search(session, "SELECT dc from DataCollection dc", 3); collector.checkThat(dc_response.size(), equalTo(3)); collector.checkThat(dc_response.getJsonObject(0).containsKey("DataCollection"), is(true)); - /* Expected: <[{"DataCollectionDatafile":{"id":4362,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z"}},{"DataCollectionDatafile":{"id":4363,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z"}}]> */ + /* + * Expected: + * <[{"DataCollectionDatafile":{"id":4362,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z"}},{"DataCollectionDatafile":{"id":4363,"createId": + * "db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot", + * "modTime":"2019-03-12T13:30:33.000Z"}}]> + */ JsonArray dcdf_response = search(session, "SELECT dcdf from DataCollectionDatafile dcdf", 2); collector.checkThat(dcdf_response.getJsonObject(0).containsKey("DataCollectionDatafile"), is(true)); - /* Expected: <[{"Job":{"id":1634,"createId":"db/notroot","createTime":"2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime":"2019-03-12T13:30:33.000Z"}}]> */ + /* + * Expected: <[{"Job":{"id":1634,"createId":"db/notroot","createTime": + * "2019-03-12T13:30:33.000Z","modId":"db/notroot","modTime": + * "2019-03-12T13:30:33.000Z"}}]> + */ JsonArray j_response = search(session, "SELECT j from Job j", 1); collector.checkThat(j_response.getJsonObject(0).containsKey("Job"), is(true)); } @@ -329,6 +492,10 @@ public void testLuceneDatafiles() throws Exception { // Set text and parameters array = searchDatafiles(session, null, "df2", null, null, parameters, 20, 1); checkResultFromLuceneSearch(session, "df2", array, "Datafile", "name"); + + // Search with a user who should not see any results + Session piOneSession = piOneSession(); + searchDatafiles(piOneSession, null, null, null, null, null, 20, 0); } @Test @@ -374,6 +541,10 @@ public void testLuceneDatasets() throws Exception { array = searchDatasets(session, null, "gamma AND ds3", dft.parse("2014-05-16T05:09:03+0000"), dft.parse("2014-05-16T05:15:26+0000"), parameters, 20, 1); checkResultFromLuceneSearch(session, "gamma", array, "Dataset", "description"); + + // Search with a user who should not see any results + Session piOneSession = piOneSession(); + searchDatasets(piOneSession, null, null, null, null, null, 20, 0); } @Test @@ -439,6 +610,10 @@ public void testLuceneInvestigations() throws Exception { } catch (IcatException e) { assertEquals(IcatExceptionType.BAD_PARAMETER, e.getType()); } + + // Search with a user who should not see any results + Session piOneSession = piOneSession(); + searchInvestigations(piOneSession, null, null, null, null, null, null, null, 20, 0); } private void checkResultFromLuceneSearch(Session session, String val, JsonArray array, String ename, String field) @@ -517,9 +692,10 @@ public void testGet() throws Exception { long fid = search(session, "Facility.id", 1).getJsonNumber(0).longValueExact(); + String query = "Facility INCLUDE InvestigationType"; JsonObject fac = Json .createReader( - new ByteArrayInputStream(session.get("Facility INCLUDE InvestigationType", fid).getBytes())) + new ByteArrayInputStream(session.get(query, fid).getBytes())) .readObject().getJsonObject("Facility"); assertEquals("Test port facility", fac.getString("name")); @@ -533,6 +709,13 @@ public void testGet() throws Exception { } Collections.sort(names); assertEquals(Arrays.asList("atype", "btype"), names); + + // Search with a user who should not see any results + Session piOneSession = piOneSession(); + wSession.addRule(null, "Facility", "R"); + fac = Json.createReader(new StringReader(piOneSession.get(query, fid))).readObject().getJsonObject("Facility"); + its = fac.getJsonArray("investigationTypes"); + assertEquals(0, its.size()); } @Test @@ -583,11 +766,7 @@ public void testSearchWithNew() throws Exception { @Test public void testWait() throws Exception { - ICAT icat = new ICAT(System.getProperty("serverUrl")); - Map credentials = new HashMap<>(); - credentials.put("username", "root"); - credentials.put("password", "password"); - Session rootSession = icat.login("db", credentials); + Session rootSession = rootSession(); long t = System.currentTimeMillis(); rootSession.waitMillis(1000L); System.out.println(System.currentTimeMillis() - t); @@ -607,14 +786,15 @@ public void testSearch() throws Exception { JsonArray array; - JsonObject user = search(session, "SELECT u FROM User u WHERE u.name = 'db/lib'", 1).getJsonObject(0).getJsonObject("User"); + JsonObject user = search(session, "SELECT u FROM User u WHERE u.name = 'db/lib'", 1).getJsonObject(0) + .getJsonObject("User"); assertEquals("Horace", user.getString("givenName")); assertEquals("Worblehat", user.getString("familyName")); assertEquals("Unseen University", user.getString("affiliation")); String query = "SELECT inv FROM Investigation inv JOIN inv.shifts AS s " - + "WHERE s.instrument.pid = 'ig:0815' AND s.comment = 'beamtime' " - + "AND s.startDate <= '2014-01-01 12:00:00' AND s.endDate >= '2014-01-01 12:00:00'"; + + "WHERE s.instrument.pid = 'ig:0815' AND s.comment = 'beamtime' " + + "AND s.startDate <= '2014-01-01 12:00:00' AND s.endDate >= '2014-01-01 12:00:00'"; JsonObject inv = search(session, query, 1).getJsonObject(0).getJsonObject("Investigation"); assertEquals("expt1", inv.getString("name")); assertEquals("zero", inv.getString("visitId")); @@ -716,6 +896,14 @@ public void testSearch() throws Exception { Collections.sort(names); assertEquals(Arrays.asList("atype", "btype"), names); } + + // Search with a user who should not see any results + Session piOneSession = piOneSession(); + wSession.addRule(null, "Facility", "R"); + JsonObject searchResult = search(piOneSession, "Facility INCLUDE InvestigationType", 1).getJsonObject(0); + JsonArray investigationTypes = searchResult.getJsonObject("Facility").getJsonArray("investigationTypes"); + System.out.println(investigationTypes); + assertEquals(0, investigationTypes.size()); } @Test @@ -1071,7 +1259,7 @@ public void testWriteGood() throws Exception { JsonArray array = search(session, "SELECT it.name, it.facility.name FROM InvestigationType it WHERE it.id = " + newInvTypeId, 1) - .getJsonArray(0); + .getJsonArray(0); assertEquals("ztype", array.getString(0)); assertEquals("Test port facility", array.getString(1)); @@ -1533,14 +1721,65 @@ private void exportMetaDataDump(Map credentials) throws Exceptio Files.delete(dump2); } + @Test + public void exportMetaDataQueryUser() throws Exception { + Session rootSession = rootSession(); + Session piOneSession = piOneSession(); + Path path = Paths.get(ClassLoader.class.getResource("/icat.port").toURI()); + + // Get known configuration + rootSession.importMetaData(path, DuplicateAction.CHECK, Attributes.ALL); + String query = "Investigation INCLUDE Facility, Dataset"; + Path dump1 = Files.createTempFile("dump1", ".tmp"); + Path dump2 = Files.createTempFile("dump2", ".tmp"); + + // piOne should only be able to dump the Investigation, but not have R access to + // Dataset, Facility + wSession.addRule(null, "Investigation", "R"); + try (InputStream stream = piOneSession.exportMetaData(query, Attributes.USER)) { + Files.copy(stream, dump1, StandardCopyOption.REPLACE_EXISTING); + } + // piOne should now be able to dump all due to rules giving R access + wSession.addRule(null, "Facility", "R"); + wSession.addRule(null, "Dataset", "R"); + try (InputStream stream = piOneSession.exportMetaData(query, Attributes.USER)) { + Files.copy(stream, dump2, StandardCopyOption.REPLACE_EXISTING); + } + List restrictedLines = Files.readAllLines(dump1); + List permissiveLines = Files.readAllLines(dump2); + String restrictiveMessage = " appeared in export, but piOne use should not have access"; + String permissiveMessage = " did not appear in export, but piOne should have access"; + + boolean containsInvestigations = false; + for (String line : restrictedLines) { + System.out.println(line); + containsInvestigations = containsInvestigations || line.startsWith("Investigation"); + assertFalse("Dataset" + restrictiveMessage, line.startsWith("Dataset")); + assertFalse("Facility" + restrictiveMessage, line.startsWith("Facility")); + } + assertTrue("Investigation" + permissiveMessage, containsInvestigations); + + containsInvestigations = false; + boolean containsDatasets = false; + boolean containsFacilities = false; + for (String line : permissiveLines) { + System.out.println(line); + containsInvestigations = containsInvestigations || line.startsWith("Investigation"); + containsDatasets = containsDatasets || line.startsWith("Dataset"); + containsFacilities = containsFacilities || line.startsWith("Facility"); + } + assertTrue("Investigation" + permissiveMessage, containsInvestigations); + assertTrue("Dataset" + permissiveMessage, containsDatasets); + assertTrue("Facility" + permissiveMessage, containsFacilities); + + Files.delete(dump1); + Files.delete(dump2); + } + @Ignore("Test fails - appears brittle to differences in timezone") @Test public void exportMetaDataQuery() throws Exception { - ICAT icat = new ICAT(System.getProperty("serverUrl")); - Map credentials = new HashMap<>(); - credentials.put("username", "root"); - credentials.put("password", "password"); - Session session = icat.login("db", credentials); + Session session = rootSession(); Path path = Paths.get(ClassLoader.class.getResource("/icat.port").toURI()); // Get known configuration @@ -1571,11 +1810,7 @@ public void exportMetaDataQuery() throws Exception { @Test public void importMetaDataAllNotRoot() throws Exception { - ICAT icat = new ICAT(System.getProperty("serverUrl")); - Map credentials = new HashMap<>(); - credentials.put("username", "piOne"); - credentials.put("password", "piOne"); - Session session = icat.login("db", credentials); + Session session = piOneSession(); Path path = Paths.get(ClassLoader.class.getResource("/icat.port").toURI()); try { session.importMetaData(path, DuplicateAction.CHECK, Attributes.ALL); @@ -1586,11 +1821,7 @@ public void importMetaDataAllNotRoot() throws Exception { } private void importMetaData(Attributes attributes, String userName) throws Exception { - ICAT icat = new ICAT(System.getProperty("serverUrl")); - Map credentials = new HashMap<>(); - credentials.put("username", "root"); - credentials.put("password", "password"); - Session session = icat.login("db", credentials); + Session session = rootSession(); Path path = Paths.get(ClassLoader.class.getResource("/icat.port").toURI()); start = System.currentTimeMillis(); From de4746775dc6badc2844c4b8829976e987a1a2f9 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Wed, 12 Oct 2022 19:21:40 +0100 Subject: [PATCH 38/51] Review changes, Opensearch refactors and fixes #267 --- src/main/config/run.properties.example | 2 + .../core/manager/search/FacetLabel.java | 6 +- .../core/manager/search/LuceneApi.java | 6 +- .../core/manager/search/OpensearchApi.java | 1326 ++++++----------- .../core/manager/search/OpensearchBulk.java | 57 + .../core/manager/search/OpensearchQuery.java | 827 ++++++++++ .../search/OpensearchQueryBuilder.java | 213 --- .../search/OpensearchScriptBuilder.java | 58 +- .../manager/search/ScoredEntityBaseBean.java | 4 +- .../core/manager/search/SearchApi.java | 58 +- .../core/manager/search/SearchManager.java | 1 + .../core/manager/TestSearchApi.java | 21 +- .../org/icatproject/integration/TestWS.java | 2 +- 13 files changed, 1384 insertions(+), 1197 deletions(-) create mode 100644 src/main/java/org/icatproject/core/manager/search/OpensearchBulk.java create mode 100644 src/main/java/org/icatproject/core/manager/search/OpensearchQuery.java delete mode 100644 src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java diff --git a/src/main/config/run.properties.example b/src/main/config/run.properties.example index bd0ed307..e5b0ee32 100644 --- a/src/main/config/run.properties.example +++ b/src/main/config/run.properties.example @@ -50,6 +50,8 @@ search.populateBlockSize = 10000 search.directory = ${HOME}/data/icat/search search.backlogHandlerIntervalSeconds = 60 search.enqueuedRequestIntervalSeconds = 5 +search.aggregateFilesIntervalSeconds = 3600 +search.maxSearchTimeSeconds = 5 # The entities to index with the search engine. For example, remove 'Datafile' and 'DatafileParameter' if the number of datafiles exceeds lucene's limit of 2^32 entries in an index !search.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample diff --git a/src/main/java/org/icatproject/core/manager/search/FacetLabel.java b/src/main/java/org/icatproject/core/manager/search/FacetLabel.java index 0ee9c443..6508b1f6 100644 --- a/src/main/java/org/icatproject/core/manager/search/FacetLabel.java +++ b/src/main/java/org/icatproject/core/manager/search/FacetLabel.java @@ -11,11 +11,11 @@ public class FacetLabel { private String label; - private Long value; + private long value; private JsonNumber from; private JsonNumber to; - public FacetLabel(String label, Long value) { + public FacetLabel(String label, long value) { this.label = label; this.value = value; } @@ -39,7 +39,7 @@ public String getLabel() { return label; } - public Long getValue() { + public long getValue() { return value; } diff --git a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java index f8e025d0..9c6a7ac5 100644 --- a/src/main/java/org/icatproject/core/manager/search/LuceneApi.java +++ b/src/main/java/org/icatproject/core/manager/search/LuceneApi.java @@ -48,7 +48,7 @@ public class LuceneApi extends SearchApi { private static String getTargetPath(JsonObject query) throws IcatException { if (!query.containsKey("target")) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "'target' must be present in query for LuceneApi, but it was " + query.toString()); + "'target' must be present in query for LuceneApi, but it was " + query); } String path = query.getString("target").toLowerCase(); if (!indices.contains(path)) { @@ -179,8 +179,8 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer for (JsonObject resultObject : resultsArray) { int luceneDocId = resultObject.getInt("_id"); int shardIndex = resultObject.getInt("_shardIndex"); - Float score = Float.NaN; - if (resultObject.keySet().contains("_score")) { + float score = Float.NaN; + if (resultObject.containsKey("_score")) { score = resultObject.getJsonNumber("_score").bigDecimalValue().floatValue(); } JsonObject source = resultObject.getJsonObject("_source"); diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index 8c10e232..757dcae0 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -18,13 +18,10 @@ import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonArrayBuilder; -import javax.json.JsonNumber; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; import javax.json.JsonReader; -import javax.json.JsonString; import javax.json.JsonValue; -import javax.json.JsonValue.ValueType; import javax.json.stream.JsonGenerator; import javax.persistence.EntityManager; @@ -87,7 +84,7 @@ public ParentRelation(RelationType relationType, String parentName, String joinF } private boolean aggregateFiles = false; - private IcatUnits icatUnits; + public IcatUnits icatUnits; protected static final Logger logger = LoggerFactory.getLogger(OpensearchApi.class); private static JsonObject indexSettings = Json.createObjectBuilder().add("analysis", Json.createObjectBuilder() .add("analyzer", Json.createObjectBuilder() @@ -130,7 +127,11 @@ public ParentRelation(RelationType relationType, String parentName, String joinF new ParentRelation(RelationType.CHILD, "datafile", "dataset", new HashSet<>(Arrays.asList("dataset.name", "dataset.id", "sample.id"))))); relations.put("user", Arrays.asList( - new ParentRelation(RelationType.CHILD, "instrumentscientist", "user", User.docFields))); + new ParentRelation(RelationType.CHILD, "instrumentscientist", "user", User.docFields), + new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "investigationuser", + User.docFields), + new ParentRelation(RelationType.NESTED_GRANDCHILD, "dataset", "investigationuser", User.docFields), + new ParentRelation(RelationType.NESTED_GRANDCHILD, "datafile", "investigationuser", User.docFields))); relations.put("sample", Arrays.asList( new ParentRelation(RelationType.CHILD, "dataset", "sample", Sample.docFields), new ParentRelation(RelationType.CHILD, "datafile", "sample", Sample.docFields), @@ -138,8 +139,7 @@ public ParentRelation(RelationType relationType, String parentName, String joinF relations.put("sampletype", Arrays.asList( new ParentRelation(RelationType.CHILD, "dataset", "sample.type", SampleType.docFields), new ParentRelation(RelationType.CHILD, "datafile", "sample.type", SampleType.docFields), - new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "investigation", - SampleType.docFields))); + new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "sample", SampleType.docFields))); // Nested children are indexed as an array of objects on their parent entity, // and know their parent's id (N.B. InvestigationUsers are also mapped to @@ -185,11 +185,6 @@ public ParentRelation(RelationType relationType, String parentName, String joinF relations.put("technique", Arrays.asList( new ParentRelation(RelationType.NESTED_GRANDCHILD, "dataset", "datasettechnique", Technique.docFields))); - relations.put("user", Arrays.asList( - new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "investigationuser", - User.docFields), - new ParentRelation(RelationType.NESTED_GRANDCHILD, "dataset", "investigationuser", User.docFields), - new ParentRelation(RelationType.NESTED_GRANDCHILD, "datafile", "investigationuser", User.docFields))); relations.put("instrument", Arrays.asList( new ParentRelation(RelationType.NESTED_GRANDCHILD, "investigation", "investigationinstrument", User.docFields), @@ -237,49 +232,57 @@ public OpensearchApi(URI server, String unitAliasOptions, boolean aggregateFiles */ private static JsonObject buildMappings(String index) { JsonObject typeLong = Json.createObjectBuilder().add("type", "long").build(); - JsonObjectBuilder propertiesBuilder = Json.createObjectBuilder() - .add("id", typeLong); - if (index.equals("investigation")) { - propertiesBuilder - .add("type.id", typeLong) - .add("facility.id", typeLong) - .add("fileSize", typeLong) - .add("fileCount", typeLong) - .add("sample", buildNestedMapping("investigation.id", "type.id")) - .add("sampleparameter", buildNestedMapping("sample.id", "type.id")) - .add("investigationparameter", buildNestedMapping("investigation.id", "type.id")) - .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) - .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")); - } else if (index.equals("dataset")) { - propertiesBuilder - .add("investigation.id", typeLong) - .add("type.id", typeLong) - .add("sample.id", typeLong) - .add("sample.investigaion.id", typeLong) - .add("sample.type.id", typeLong) - .add("fileSize", typeLong) - .add("fileCount", typeLong) - .add("datasetparameter", buildNestedMapping("dataset.id", "type.id")) - .add("datasettechnique", buildNestedMapping("dataset.id", "technique.id")) - .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) - .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")) - .add("sampleparameter", buildNestedMapping("sample.id", "type.id")); - } else if (index.equals("datafile")) { - propertiesBuilder - .add("investigation.id", typeLong) - .add("datafileFormat.id", typeLong) - .add("sample.investigaion.id", typeLong) - .add("sample.type.id", typeLong) - .add("fileSize", typeLong) - .add("fileCount", typeLong) - .add("datafileparameter", buildNestedMapping("datafile.id", "type.id")) - .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) - .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")) - .add("sampleparameter", buildNestedMapping("sample.id", "type.id")); - } else if (index.equals("instrumentscientist")) { - propertiesBuilder - .add("instrument.id", typeLong) - .add("user.id", typeLong); + JsonObjectBuilder propertiesBuilder = Json.createObjectBuilder().add("id", typeLong); + switch (index) { + case "investigation": + propertiesBuilder + .add("type.id", typeLong) + .add("facility.id", typeLong) + .add("fileSize", typeLong) + .add("fileCount", typeLong) + .add("sample", buildNestedMapping("investigation.id", "type.id")) + .add("sampleparameter", buildNestedMapping("sample.id", "type.id")) + .add("investigationparameter", buildNestedMapping("investigation.id", "type.id")) + .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) + .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")); + break; + + case "dataset": + propertiesBuilder + .add("investigation.id", typeLong) + .add("type.id", typeLong) + .add("sample.id", typeLong) + .add("sample.investigaion.id", typeLong) + .add("sample.type.id", typeLong) + .add("fileSize", typeLong) + .add("fileCount", typeLong) + .add("datasetparameter", buildNestedMapping("dataset.id", "type.id")) + .add("datasettechnique", buildNestedMapping("dataset.id", "technique.id")) + .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) + .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")) + .add("sampleparameter", buildNestedMapping("sample.id", "type.id")); + break; + + case "datafile": + propertiesBuilder + .add("investigation.id", typeLong) + .add("datafileFormat.id", typeLong) + .add("sample.investigaion.id", typeLong) + .add("sample.type.id", typeLong) + .add("fileSize", typeLong) + .add("fileCount", typeLong) + .add("datafileparameter", buildNestedMapping("datafile.id", "type.id")) + .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) + .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")) + .add("sampleparameter", buildNestedMapping("sample.id", "type.id")); + break; + + case "instrumentscientist": + propertiesBuilder + .add("instrument.id", typeLong) + .add("user.id", typeLong); + break; + } return Json.createObjectBuilder().add("properties", propertiesBuilder).build(); } @@ -309,44 +312,6 @@ private static JsonObjectBuilder propertiesBuilder(String... idFields) { return propertiesBuilder; } - /** - * Extracts and parses a date value from jsonObject. If the value is a NUMBER - * (ms since epoch), then it is taken as is. If it is a STRING, then it is - * expected in the yyyyMMddHHmm format. - * - * @param jsonObject JsonObject to extract the date from. - * @param key Key of the date field to extract. - * @param offset In the event of the date being a string, we do not have - * second or ms precision. To ensure ranges are successful, - * it may be necessary to add 59999 ms to the parsed value - * as an offset. - * @param defaultValue The value to return if key is not present in jsonObject. - * @return Time since epoch in ms. - * @throws IcatException - */ - private static Long parseDate(JsonObject jsonObject, String key, int offset, Long defaultValue) - throws IcatException { - if (jsonObject.containsKey(key)) { - ValueType valueType = jsonObject.get(key).getValueType(); - switch (valueType) { - case STRING: - String dateString = jsonObject.getString(key); - try { - return decodeTime(dateString) + offset; - } catch (Exception e) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Could not parse date " + dateString + " using expected format yyyyMMddHHmm"); - } - case NUMBER: - return jsonObject.getJsonNumber(key).longValueExact(); - default: - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Dates should be represented by a NUMBER or STRING JsonValue, but got " + valueType); - } - } - return defaultValue; - } - @Override public void addNow(String entityName, List ids, EntityManager manager, Class klass, ExecutorService getBeanDocExecutor) @@ -356,7 +321,7 @@ public void addNow(String entityName, List ids, EntityManager manager, ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartArray(); - for (Long id : ids) { + for (long id : ids) { EntityBaseBean bean = (EntityBaseBean) manager.find(klass, id); if (bean != null) { gen.writeStartObject().writeStartObject("create"); @@ -374,7 +339,7 @@ public void addNow(String entityName, List ids, EntityManager manager, @Override public void clear() throws IcatException { commit(); - String body = OpensearchQueryBuilder.addQuery(OpensearchQueryBuilder.buildMatchAllQuery()).build().toString(); + String body = OpensearchQuery.matchAllQuery.toString(); post("/_all/_delete_by_query", body); } @@ -397,16 +362,16 @@ public List facetSearch(String target, JsonObject facetQuery, In JsonObject queryObject = facetQuery.getJsonObject("query"); List defaultFields = defaultFieldsMap.get(index); - JsonObjectBuilder bodyBuilder = parseQuery(Json.createObjectBuilder(), queryObject, index, dimensionPrefix, - defaultFields); + OpensearchQuery opensearchQuery = new OpensearchQuery(this); + opensearchQuery.parseQuery(queryObject, index, dimensionPrefix, defaultFields); if (facetQuery.containsKey("dimensions")) { JsonArray dimensions = facetQuery.getJsonArray("dimensions"); - bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix); + opensearchQuery.parseFacets(dimensions, maxLabels, dimensionPrefix); } else { List dimensions = defaultFacetsMap.get(index); - bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix); + opensearchQuery.parseFacets(dimensions, maxLabels, dimensionPrefix); } - String body = bodyBuilder.build().toString(); + String body = opensearchQuery.body(); Map parameterMap = new HashMap<>(); parameterMap.put("size", maxResults.toString()); @@ -423,101 +388,17 @@ public List facetSearch(String target, JsonObject facetQuery, In return results; } - /** - * Parses incoming Json encoding the requested facets and uses bodyBuilder to - * construct Json that can be understood by Opensearch. - * - * @param bodyBuilder JsonObjectBuilder being used to build the body of the - * request. - * @param dimensions JsonArray of JsonObjects representing dimensions to be - * faceted. - * @param maxLabels The maximum number of labels to collect for each - * dimension. - * @param dimensionPrefix Optional prefix to apply to the dimension names. This - * is needed to distinguish between potentially ambiguous - * dimensions, such as "(investigation.)type.name" and - * "(investigationparameter.)type.name". - * @return The bodyBuilder originally passed with facet information added to it. - */ - private JsonObjectBuilder parseFacets(JsonObjectBuilder bodyBuilder, JsonArray dimensions, int maxLabels, - String dimensionPrefix) { - JsonObjectBuilder aggsBuilder = Json.createObjectBuilder(); - for (JsonObject dimensionObject : dimensions.getValuesAs(JsonObject.class)) { - String dimensionString = dimensionObject.getString("dimension"); - String field = dimensionPrefix == null ? dimensionString : dimensionPrefix + "." + dimensionString; - if (dimensionObject.containsKey("ranges")) { - JsonArray ranges = dimensionObject.getJsonArray("ranges"); - aggsBuilder.add(dimensionString, OpensearchQueryBuilder.buildRangeFacet(field, ranges)); - } else { - aggsBuilder.add(dimensionString, - OpensearchQueryBuilder.buildStringFacet(field + ".keyword", maxLabels)); - } - } - return buildFacetRequestJson(bodyBuilder, dimensionPrefix, aggsBuilder); - } - - /** - * Uses bodyBuilder to construct Json for faceting string fields. - * - * @param bodyBuilder JsonObjectBuilder being used to build the body of the - * request. - * @param dimensions List of dimensions to perform string based faceting - * on. - * @param maxLabels The maximum number of labels to collect for each - * dimension. - * @param dimensionPrefix Optional prefix to apply to the dimension names. This - * is needed to distinguish between potentially ambiguous - * dimensions, such as "(investigation.)type.name" and - * "(investigationparameter.)type.name". - * @return The bodyBuilder originally passed with facet information added to it. - */ - private JsonObjectBuilder parseFacets(JsonObjectBuilder bodyBuilder, List dimensions, int maxLabels, - String dimensionPrefix) { - JsonObjectBuilder aggsBuilder = Json.createObjectBuilder(); - for (String dimensionString : dimensions) { - String field = dimensionPrefix == null ? dimensionString : dimensionPrefix + "." + dimensionString; - aggsBuilder.add(dimensionString, OpensearchQueryBuilder.buildStringFacet(field + ".keyword", maxLabels)); - } - return buildFacetRequestJson(bodyBuilder, dimensionPrefix, aggsBuilder); - } - - /** - * Finalises the construction of faceting Json by handling the possibility of - * faceting a nested object. - * - * @param bodyBuilder JsonObjectBuilder being used to build the body of the - * request. - * @param dimensionPrefix Optional prefix to apply to the dimension names. This - * is needed to distinguish between potentially ambiguous - * dimensions, such as "(investigation.)type.name" and - * "(investigationparameter.)type.name". - * @param aggsBuilder JsonObjectBuilder that has the faceting details. - * @return The bodyBuilder originally passed with facet information added to it. - */ - private JsonObjectBuilder buildFacetRequestJson(JsonObjectBuilder bodyBuilder, String dimensionPrefix, - JsonObjectBuilder aggsBuilder) { - if (dimensionPrefix == null) { - bodyBuilder.add("aggs", aggsBuilder); - } else { - bodyBuilder.add("aggs", Json.createObjectBuilder() - .add(dimensionPrefix, Json.createObjectBuilder() - .add("nested", Json.createObjectBuilder().add("path", dimensionPrefix)) - .add("aggs", aggsBuilder))); - } - return bodyBuilder; - } - @Override public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer blockSize, String sort, List requestedFields) throws IcatException { String index = query.containsKey("target") ? query.getString("target").toLowerCase() : "_all"; List defaultFields = defaultFieldsMap.get(index); - JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); - bodyBuilder = parseSort(bodyBuilder, sort); - bodyBuilder = parseSearchAfter(bodyBuilder, searchAfter); - bodyBuilder = parseQuery(bodyBuilder, query, index, null, defaultFields); - String body = bodyBuilder.build().toString(); + OpensearchQuery opensearchQuery = new OpensearchQuery(this); + opensearchQuery.parseQuery(query, index, null, defaultFields); + opensearchQuery.parseSort(sort); + opensearchQuery.parseSearchAfter(searchAfter); + String body = opensearchQuery.body(); Map parameterMap = new HashMap<>(); Map> joinedFields = new HashMap<>(); @@ -529,7 +410,7 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer List entities = result.getResults(); JsonArray hits = postResponse.getJsonObject("hits").getJsonArray("hits"); for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { - Float score = Float.NaN; + float score = Float.NaN; if (!hit.isNull("_score")) { score = hit.getJsonNumber("_score").bigDecimalValue().floatValue(); } @@ -556,7 +437,7 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer parentId = source.getString("id"); } // Search for joined entities matching the id - JsonObject termQuery = OpensearchQueryBuilder.buildTermQuery(fld, parentId); + JsonObject termQuery = OpensearchQuery.buildTermQuery(fld, parentId); String joinedBody = Json.createObjectBuilder().add("query", termQuery).build().toString(); buildParameterMap(blockSize, requestedJoinedFields, joinedParameterMap, null); JsonObject joinedResponse = postResponse("/" + joinedIndex + "/_search", joinedBody, @@ -616,8 +497,7 @@ private void buildParameterMap(Integer blockSize, Iterable requestedFiel if (joinedFields.containsKey(splitString[0])) { joinedFields.get(splitString[0]).add(splitString[1]); } else { - joinedFields.putIfAbsent(splitString[0], - new HashSet(Arrays.asList(splitString[1]))); + joinedFields.putIfAbsent(splitString[0], new HashSet<>(Arrays.asList(splitString[1]))); } } else { sb.append(splitString[0].toLowerCase() + ","); @@ -631,398 +511,6 @@ private void buildParameterMap(Integer blockSize, Iterable requestedFiel parameterMap.put("size", blockSize.toString()); } - /** - * Parse sort criteria and add it to the request body. - * - * @param builder JsonObjectBuilder being used to build the body of the request. - * @param sort String of JsonObject containing the sort criteria. - * @return The bodyBuilder originally passed with facet criteria added to it. - */ - private JsonObjectBuilder parseSort(JsonObjectBuilder builder, String sort) { - if (sort == null || sort.equals("")) { - return builder.add("sort", Json.createArrayBuilder() - .add(Json.createObjectBuilder().add("_score", "desc")) - .add(Json.createObjectBuilder().add("id", "asc")).build()); - } else { - JsonObject sortObject = Json.createReader(new StringReader(sort)).readObject(); - JsonArrayBuilder sortArrayBuilder = Json.createArrayBuilder(); - for (String key : sortObject.keySet()) { - if (key.toLowerCase().contains("date")) { - sortArrayBuilder.add(Json.createObjectBuilder().add(key, sortObject.getString(key))); - } else { - sortArrayBuilder.add(Json.createObjectBuilder() - .add(key + ".keyword", sortObject.getString(key))); - } - } - return builder.add("sort", sortArrayBuilder.add(Json.createObjectBuilder().add("id", "asc")).build()); - } - } - - /** - * Add searchAfter to the request body. - * - * @param builder JsonObjectBuilder being used to build the body of the - * request. - * @param searchAfter Possibly null JsonValue representing the last document of - * a previous search. - * @return The bodyBuilder originally passed with searchAfter added to it. - */ - private JsonObjectBuilder parseSearchAfter(JsonObjectBuilder builder, JsonValue searchAfter) { - if (searchAfter == null) { - return builder; - } else { - return builder.add("search_after", searchAfter); - } - } - - /** - * Parses the search query from the incoming queryRequest into Json that the - * search cluster can understand. - * - * @param builder The JsonObjectBuilder being used to create the body - * for - * the POST request to the cluster. - * @param queryRequest The Json object containing the information on the - * requested query, NOT formatted for the search cluster. - * @param index The index to search. - * @param dimensionPrefix Used to build nested queries for arbitrary fields. - * @param defaultFields Default fields to apply parsed string queries to. - * @return The JsonObjectBuilder initially passed with the "query" added to it. - * @throws IcatException If the query cannot be parsed. - */ - private JsonObjectBuilder parseQuery(JsonObjectBuilder builder, JsonObject queryRequest, String index, - String dimensionPrefix, List defaultFields) throws IcatException { - // In general, we use a boolean query to compound queries on individual fields - JsonObjectBuilder queryBuilder = Json.createObjectBuilder(); - JsonObjectBuilder boolBuilder = Json.createObjectBuilder(); - - // Non-scored elements are added to the "filter" - JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); - - Long lowerTime = Long.MIN_VALUE; - Long upperTime = Long.MAX_VALUE; - for (String queryKey : queryRequest.keySet()) { - switch (queryKey) { - case "target": - break; // Avoid using the target index as a term in the search - case "lower": - lowerTime = parseDate(queryRequest, "lower", 0, Long.MIN_VALUE); - break; - case "upper": - upperTime = parseDate(queryRequest, "upper", 59999, Long.MAX_VALUE); - break; - case "filter": - JsonObject filterObject = queryRequest.getJsonObject("filter"); - for (String fld : filterObject.keySet()) { - JsonValue value = filterObject.get(fld); - String field = fld.replace(index + ".", ""); - switch (value.getValueType()) { - case ARRAY: - JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); - for (JsonValue arrayValue : ((JsonArray) value).getValuesAs(JsonString.class)) { - parseFilter(arrayBuilder, field, arrayValue); - } - // If the key was just a nested entity (no ".") then we should FILTER all of our - // queries on that entity. - String occur = fld.contains(".") ? "should" : "filter"; - filterBuilder.add(Json.createObjectBuilder().add("bool", - Json.createObjectBuilder().add(occur, arrayBuilder))); - break; - - default: - parseFilter(filterBuilder, field, value); - } - } - break; - case "text": - // The free text is the only element we perform scoring on, so "must" occur - JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); - String text = queryRequest.getString("text"); - arrayBuilder - .add(OpensearchQueryBuilder.buildStringQuery(text, defaultFields.toArray(new String[0]))); - if (index.equals("investigation")) { - JsonObject stringQuery = OpensearchQueryBuilder.buildStringQuery(text, "sample.name", - "sample.type.name"); - arrayBuilder.add(OpensearchQueryBuilder.buildNestedQuery("sample", stringQuery)); - JsonObjectBuilder textBoolBuilder = Json.createObjectBuilder().add("should", arrayBuilder); - JsonObjectBuilder textMustBuilder = Json.createObjectBuilder().add("bool", textBoolBuilder); - boolBuilder.add("must", Json.createArrayBuilder().add(textMustBuilder)); - } else { - boolBuilder.add("must", arrayBuilder); - } - break; - case "user": - String user = queryRequest.getString("user"); - // Because InstrumentScientist is on a separate index, we need to explicitly - // perform a search here - JsonObject termQuery = OpensearchQueryBuilder.buildTermQuery("user.name.keyword", user); - String body = Json.createObjectBuilder().add("query", termQuery).build().toString(); - Map parameterMap = new HashMap<>(); - parameterMap.put("_source", "instrument.id"); - JsonObject postResponse = postResponse("/instrumentscientist/_search", body, parameterMap); - JsonArray hits = postResponse.getJsonObject("hits").getJsonArray("hits"); - JsonArrayBuilder instrumentIdsBuilder = Json.createArrayBuilder(); - for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { - String instrumentId = hit.getJsonObject("_source").getString("instrument.id"); - instrumentIdsBuilder.add(instrumentId); - } - JsonObject instrumentQuery = OpensearchQueryBuilder.buildTermsQuery( - "investigationinstrument.instrument.id", - instrumentIdsBuilder.build()); - JsonObject nestedInstrumentQuery = OpensearchQueryBuilder.buildNestedQuery( - "investigationinstrument", - instrumentQuery); - // InvestigationUser should be a nested field on the main Document - JsonObject investigationUserQuery = OpensearchQueryBuilder.buildMatchQuery( - "investigationuser.user.name", - user); - JsonObject nestedUserQuery = OpensearchQueryBuilder.buildNestedQuery("investigationuser", - investigationUserQuery); - // At least one of being an InstrumentScientist or an InvestigationUser is - // necessary - JsonArrayBuilder array = Json.createArrayBuilder().add(nestedInstrumentQuery).add(nestedUserQuery); - filterBuilder.add( - Json.createObjectBuilder().add("bool", Json.createObjectBuilder().add("should", array))); - break; - case "userFullName": - String fullName = queryRequest.getString("userFullName"); - JsonObject fullNameQuery = OpensearchQueryBuilder.buildStringQuery(fullName, - "investigationuser.user.fullName"); - filterBuilder.add(OpensearchQueryBuilder.buildNestedQuery("investigationuser", fullNameQuery)); - break; - case "samples": - JsonArray samples = queryRequest.getJsonArray("samples"); - for (int i = 0; i < samples.size(); i++) { - String sample = samples.getString(i); - JsonObject stringQuery = OpensearchQueryBuilder.buildStringQuery(sample, "sample.name", - "sample.type.name"); - filterBuilder.add(OpensearchQueryBuilder.buildNestedQuery("sample", stringQuery)); - } - break; - case "parameters": - for (JsonObject parameterObject : queryRequest.getJsonArray("parameters") - .getValuesAs(JsonObject.class)) { - String path = index + "parameter"; - List parameterQueries = new ArrayList<>(); - if (parameterObject.containsKey("name")) { - String name = parameterObject.getString("name"); - parameterQueries.add(OpensearchQueryBuilder.buildMatchQuery(path + ".type.name", name)); - } - if (parameterObject.containsKey("units")) { - String units = parameterObject.getString("units"); - parameterQueries.add(OpensearchQueryBuilder.buildMatchQuery(path + ".type.units", units)); - } - if (parameterObject.containsKey("stringValue")) { - String stringValue = parameterObject.getString("stringValue"); - parameterQueries - .add(OpensearchQueryBuilder.buildMatchQuery(path + ".stringValue", stringValue)); - } else if (parameterObject.containsKey("lowerDateValue") - && parameterObject.containsKey("upperDateValue")) { - Long lower = parseDate(parameterObject, "lowerDateValue", 0, Long.MIN_VALUE); - Long upper = parseDate(parameterObject, "upperDateValue", 59999, Long.MAX_VALUE); - parameterQueries - .add(OpensearchQueryBuilder.buildLongRangeQuery(path + ".dateTimeValue", lower, - upper)); - } else if (parameterObject.containsKey("lowerNumericValue") - && parameterObject.containsKey("upperNumericValue")) { - JsonNumber lower = parameterObject.getJsonNumber("lowerNumericValue"); - JsonNumber upper = parameterObject.getJsonNumber("upperNumericValue"); - parameterQueries - .add(OpensearchQueryBuilder.buildRangeQuery(path + ".numericValue", lower, upper)); - } - filterBuilder.add( - OpensearchQueryBuilder.buildNestedQuery(path, - parameterQueries.toArray(new JsonObject[0]))); - } - break; - default: - // If the term doesn't require special logic, handle according to type - JsonObject defaultTermQuery; - String field = queryKey; - if (dimensionPrefix != null) { - field = dimensionPrefix + "." + field; - } - ValueType valueType = queryRequest.get(queryKey).getValueType(); - // if (queryKey.contains(".")) { - // int pathEnd = queryKey.indexOf("."); - // path = queryKey.substring(0, pathEnd); - // if (path.equals(index)) { - // // e.g. "dataset.id" should be interpretted as "id" iff we're searching - // Datasets - // field = queryKey.substring(pathEnd + 1); - // } - // } - switch (valueType) { - case STRING: - defaultTermQuery = OpensearchQueryBuilder.buildTermQuery(field + ".keyword", - queryRequest.getString(queryKey)); - break; - case NUMBER: - defaultTermQuery = OpensearchQueryBuilder.buildTermQuery(field, - queryRequest.getJsonNumber(queryKey)); - break; - case ARRAY: - // Only support array of String as list of ICAT ids is currently only use case - defaultTermQuery = OpensearchQueryBuilder.buildTermsQuery(field, - queryRequest.getJsonArray(queryKey)); - break; - default: - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Query values should be ARRAY, STRING or NUMBER, but had value of type " - + valueType); - } - if (dimensionPrefix != null) { - // e.g. "sample.id" should use a nested query as sample is nested on other - // entities - filterBuilder.add(OpensearchQueryBuilder.buildNestedQuery(dimensionPrefix, defaultTermQuery)); - } else { - // Otherwise, we can associate the query directly with the searched entity - filterBuilder.add(defaultTermQuery); - } - } - } - - if (lowerTime != Long.MIN_VALUE || upperTime != Long.MAX_VALUE) { - if (index.equals("datafile")) { - // datafile has only one date field - filterBuilder.add(OpensearchQueryBuilder.buildLongRangeQuery("date", lowerTime, upperTime)); - } else { - filterBuilder.add(OpensearchQueryBuilder.buildLongRangeQuery("startDate", lowerTime, upperTime)); - filterBuilder.add(OpensearchQueryBuilder.buildLongRangeQuery("endDate", lowerTime, upperTime)); - } - } - - JsonArray filterArray = filterBuilder.build(); - if (filterArray.size() > 0) { - boolBuilder.add("filter", filterArray); - } - return builder.add("query", queryBuilder.add("bool", boolBuilder)); - } - - /** - * Parses a filter object applied to a single field. Note that in the case that - * this field is actually a nested object, more complex logic will be applied to - * ensure that only object matching all nested filters are returned. - * - * @param filterBuilder Builder for the array of queries to filter by. - * @param field Field to apply the filter to. In the case of nested - * queries, this should only be the name of the top level - * field. For example "investigationparameter". - * @param value JsonValue representing the filter query. This can be a - * STRING for simple terms, or an OBJECT containing nested - * "value", "exact" or "range" filters. - * @throws IcatException - */ - private void parseFilter(JsonArrayBuilder filterBuilder, String field, JsonValue value) throws IcatException { - ValueType valueType = value.getValueType(); - switch (valueType) { - case STRING: - filterBuilder.add( - OpensearchQueryBuilder.buildTermQuery(field + ".keyword", ((JsonString) value).getString())); - return; - case OBJECT: - JsonObject valueObject = (JsonObject) value; - if (valueObject.containsKey("filter")) { - List queryObjectsList = new ArrayList<>(); - for (JsonObject nestedFilter : valueObject.getJsonArray("filter").getValuesAs(JsonObject.class)) { - String nestedField = nestedFilter.getString("field"); - if (nestedFilter.containsKey("value")) { - // String based term query - String stringValue = nestedFilter.getString("value"); - queryObjectsList.add( - OpensearchQueryBuilder.buildTermQuery(field + "." + nestedField + ".keyword", - stringValue)); - } else if (nestedFilter.containsKey("exact")) { - JsonNumber exact = nestedFilter.getJsonNumber("exact"); - String units = nestedFilter.getString("units", null); - if (units != null) { - SystemValue exactValue = icatUnits.new SystemValue(exact.doubleValue(), units); - if (exactValue.value != null) { - // If we were able to parse the units, apply query to the SI value - JsonObject bottomQuery = OpensearchQueryBuilder - .buildDoubleRangeQuery(field + ".rangeBottomSI", null, exactValue.value); - JsonObject topQuery = OpensearchQueryBuilder - .buildDoubleRangeQuery(field + ".rangeTopSI", exactValue.value, null); - JsonObject inRangeQuery = OpensearchQueryBuilder - .buildBoolQuery(Arrays.asList(bottomQuery, topQuery), null); - JsonObject exactQuery = OpensearchQueryBuilder - .buildTermQuery(field + "." + nestedField + "SI", exactValue.value); - queryObjectsList.add( - OpensearchQueryBuilder.buildBoolQuery(null, - Arrays.asList(inRangeQuery, exactQuery))); - } else { - // If units could not be parsed, make them part of the query on the raw data - JsonObject bottomQuery = OpensearchQueryBuilder - .buildRangeQuery(field + ".rangeBottom", null, exact); - JsonObject topQuery = OpensearchQueryBuilder.buildRangeQuery(field + ".rangeTop", - exact, null); - JsonObject inRangeQuery = OpensearchQueryBuilder - .buildBoolQuery(Arrays.asList(bottomQuery, topQuery), null); - JsonObject exactQuery = OpensearchQueryBuilder - .buildTermQuery(field + "." + nestedField, exact); - queryObjectsList.add( - OpensearchQueryBuilder.buildBoolQuery(null, - Arrays.asList(inRangeQuery, exactQuery))); - queryObjectsList.add( - OpensearchQueryBuilder.buildTermQuery(field + ".type.units.keyword", - units)); - } - } else { - // If units were not provided, just apply to the raw data - JsonObject bottomQuery = OpensearchQueryBuilder.buildRangeQuery(field + ".rangeBottom", - null, exact); - JsonObject topQuery = OpensearchQueryBuilder.buildRangeQuery(field + ".rangeTop", exact, - null); - JsonObject inRangeQuery = OpensearchQueryBuilder - .buildBoolQuery(Arrays.asList(bottomQuery, topQuery), null); - JsonObject exactQuery = OpensearchQueryBuilder.buildTermQuery(field + "." + nestedField, - exact); - queryObjectsList.add( - OpensearchQueryBuilder.buildBoolQuery(null, - Arrays.asList(inRangeQuery, exactQuery))); - } - } else { - JsonNumber from = nestedFilter.getJsonNumber("from"); - JsonNumber to = nestedFilter.getJsonNumber("to"); - String units = nestedFilter.getString("units", null); - if (units != null) { - SystemValue fromValue = icatUnits.new SystemValue(from.doubleValue(), units); - SystemValue toValue = icatUnits.new SystemValue(to.doubleValue(), units); - if (fromValue.value != null && toValue.value != null) { - // If we were able to parse the units, apply query to the SI value - queryObjectsList.add(OpensearchQueryBuilder.buildDoubleRangeQuery( - field + "." + nestedField + "SI", fromValue.value, toValue.value)); - } else { - // If units could not be parsed, make them part of the query on the raw data - queryObjectsList.add( - OpensearchQueryBuilder.buildRangeQuery(field + "." + nestedField, from, - to)); - queryObjectsList.add( - OpensearchQueryBuilder.buildTermQuery(field + ".type.units.keyword", - units)); - } - } else { - // If units were not provided, just apply to the raw data - queryObjectsList.add( - OpensearchQueryBuilder.buildRangeQuery(field + "." + nestedField, from, to)); - } - } - } - JsonObject[] queryObjects = queryObjectsList.toArray(new JsonObject[0]); - filterBuilder.add(OpensearchQueryBuilder.buildNestedQuery(field, queryObjects)); - } else { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "expected an ARRAY with the key 'filter', but received " + valueObject.toString()); - } - return; - - default: - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "filter values should be STRING, OBJECT or and ARRAY of the former, but were " + valueType); - } - - } - /** * Create mappings for indices that do not already have them. * @@ -1030,40 +518,50 @@ private void parseFilter(JsonArrayBuilder filterBuilder, String field, JsonValue */ public void initMappings() throws IcatException { for (String index : indices) { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath("/" + index).build(); - logger.debug("Making call {}", uri); - HttpHead httpHead = new HttpHead(uri); - try (CloseableHttpResponse response = httpclient.execute(httpHead)) { - int statusCode = response.getStatusLine().getStatusCode(); - // If the index isn't present, we should get 404 and create the index - if (statusCode == 200) { - // If the index already exists (200), do not attempt to create it - logger.debug("{} index already exists, continue", index); - continue; - } else if (statusCode != 404) { - // If the code isn't 200 or 404, something has gone wrong + if (!indexExists(index)) { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URI uri = new URIBuilder(server).setPath("/" + index).build(); + HttpPut httpPut = new HttpPut(uri); + JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); + bodyBuilder.add("settings", indexSettings).add("mappings", buildMappings(index)); + String body = bodyBuilder.build().toString(); + logger.debug("Making call {} with body {}", uri, body); + httpPut.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + try (CloseableHttpResponse response = httpclient.execute(httpPut)) { Rest.checkStatus(response, IcatExceptionType.INTERNAL); } + } catch (URISyntaxException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } - } catch (URISyntaxException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } + } + } - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath("/" + index).build(); - HttpPut httpPut = new HttpPut(uri); - JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); - bodyBuilder.add("settings", indexSettings).add("mappings", buildMappings(index)); - String body = bodyBuilder.build().toString(); - logger.debug("Making call {} with body {}", uri, body); - httpPut.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); - try (CloseableHttpResponse response = httpclient.execute(httpPut)) { + /** + * @param index Name of an index (entity) to check the existence of + * @return Whether index exists on the cluster or not + * @throws IcatException + */ + private boolean indexExists(String index) throws IcatException { + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + URI uri = new URIBuilder(server).setPath("/" + index).build(); + logger.debug("Making call {}", uri); + HttpHead httpHead = new HttpHead(uri); + try (CloseableHttpResponse response = httpclient.execute(httpHead)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 404) { + // If the index isn't present, we should get 404 + logger.debug("{} index does not exist", index); + return false; + } else { + // checkStatus will throw unless the code is 200 (index exists) Rest.checkStatus(response, IcatExceptionType.INTERNAL); + logger.debug("{} index already exists", index); + return true; } - } catch (URISyntaxException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } + } catch (URISyntaxException | IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } } @@ -1077,35 +575,39 @@ public void initScripts() throws IcatException { String key = entry.getKey(); ParentRelation relation = entry.getValue().get(0); // Special cases - if (key.equals("parametertype")) { - // ParameterType can apply to 4 different nested objects - post("/_scripts/update_parametertype", - OpensearchScriptBuilder.buildParameterTypeScript(ParameterType.docFields, true)); - post("/_scripts/delete_parametertype", - OpensearchScriptBuilder.buildParameterTypeScript(ParameterType.docFields, false)); - continue; - } else if (key.equals("sample")) { - // Sample is a child of Datafile and Dataset... - post("/_scripts/update_sample", OpensearchScriptBuilder.buildChildScript(Sample.docFields, true)); - post("/_scripts/delete_sample", OpensearchScriptBuilder.buildChildScript(Sample.docFields, false)); - // ...but a nested child of Investigations - post("/_scripts/update_nestedsample", OpensearchScriptBuilder.buildNestedChildScript(key, true)); - post("/_scripts/delete_nestedsample", OpensearchScriptBuilder.buildNestedChildScript(key, false)); - String createScript = OpensearchScriptBuilder.buildCreateNestedChildScript(key); - post("/_scripts/create_" + key, createScript); - continue; - } else if (key.equals("sampletype")) { - // SampleType is a child of Datafile and Dataset... - post("/_scripts/update_sampletype", - OpensearchScriptBuilder.buildChildScript(SampleType.docFields, true)); - post("/_scripts/delete_sampletype", - OpensearchScriptBuilder.buildChildScript(SampleType.docFields, false)); - // ...but a nested grandchild of Investigations - post("/_scripts/update_nestedsampletype", - OpensearchScriptBuilder.buildGrandchildScript("sample", SampleType.docFields, true)); - post("/_scripts/delete_nestedsampletype", - OpensearchScriptBuilder.buildGrandchildScript("sample", SampleType.docFields, false)); - continue; + switch (key) { + case "parametertype": + // ParameterType can apply to 4 different nested objects + post("/_scripts/update_parametertype", + OpensearchScriptBuilder.buildParameterTypesScript(ParameterType.docFields, true)); + post("/_scripts/delete_parametertype", + OpensearchScriptBuilder.buildParameterTypesScript(ParameterType.docFields, false)); + continue; + + case "sample": + // Sample is a child of Datafile and Dataset... + post("/_scripts/update_sample", OpensearchScriptBuilder.buildChildScript(Sample.docFields, true)); + post("/_scripts/delete_sample", OpensearchScriptBuilder.buildChildScript(Sample.docFields, false)); + // ...but a nested child of Investigations + post("/_scripts/update_nestedsample", OpensearchScriptBuilder.buildNestedChildScript(key, true)); + post("/_scripts/delete_nestedsample", OpensearchScriptBuilder.buildNestedChildScript(key, false)); + String createScript = OpensearchScriptBuilder.buildCreateNestedChildScript(key); + post("/_scripts/create_" + key, createScript); + continue; + + case "sampletype": + // SampleType is a child of Datafile and Dataset... + post("/_scripts/update_sampletype", + OpensearchScriptBuilder.buildChildScript(SampleType.docFields, true)); + post("/_scripts/delete_sampletype", + OpensearchScriptBuilder.buildChildScript(SampleType.docFields, false)); + // ...but a nested grandchild of Investigations + post("/_scripts/update_nestedsampletype", + OpensearchScriptBuilder.buildGrandchildScript("sample", SampleType.docFields, true)); + post("/_scripts/delete_nestedsampletype", + OpensearchScriptBuilder.buildGrandchildScript("sample", SampleType.docFields, false)); + continue; + } String updateScript = ""; String deleteScript = ""; @@ -1136,69 +638,29 @@ public void initScripts() throws IcatException { public void modify(String json) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - List updatesByQuery = new ArrayList<>(); - Set investigationIds = new HashSet<>(); - Map investigationAggregations = new HashMap<>(); - Map datasetAggregations = new HashMap<>(); - StringBuilder sb = new StringBuilder(); + OpensearchBulk bulk = new OpensearchBulk(); JsonReader jsonReader = Json.createReader(new StringReader(json)); JsonArray outerArray = jsonReader.readArray(); for (JsonObject operation : outerArray.getValuesAs(JsonObject.class)) { - Set operationKeys = operation.keySet(); - if (operationKeys.size() != 1) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, - "Operation should only have one key, but it had " + operationKeys); - } - String operationKey = operationKeys.toArray(new String[1])[0]; - ModificationType modificationType = ModificationType.valueOf(operationKey.toUpperCase()); - JsonObject innerOperation = operation.getJsonObject(modificationType.toString().toLowerCase()); - String index = innerOperation.getString("_index").toLowerCase(); - String id = innerOperation.getString("_id"); - JsonObject document = innerOperation.containsKey("doc") ? innerOperation.getJsonObject("doc") : null; - logger.trace("{} {} with id {}", operationKey, index, id); - - if (relations.containsKey(index)) { - // Related entities (with or without an index) will have one or more other - // indices that need to - // be updated with their information - for (ParentRelation relation : relations.get(index)) { - modifyNestedEntity(sb, updatesByQuery, id, index, document, modificationType, relation); - } - } - if (indices.contains(index)) { - // Also modify any main, indexable entities - modifyEntity(httpclient, sb, investigationIds, investigationAggregations, datasetAggregations, id, - index, document, modificationType); - } + parseModification(httpclient, bulk, operation); } - if (sb.toString().length() > 0) { - // Perform simple bulk modifications - URI uri = new URIBuilder(server).setPath("/_bulk").build(); - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(sb.toString(), ContentType.APPLICATION_JSON)); - logger.trace("Making call {} with body {}", uri, sb.toString()); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } + postModify("/_bulk", bulk.bulkBody()); - if (updatesByQuery.size() > 0) { - // Ensure bulk changes are committed before performing updatesByQuery - commit(); - for (HttpPost updateByQuery : updatesByQuery) { - try (CloseableHttpResponse response = httpclient.execute(updateByQuery)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); + if (bulk.updatesMap.size() > 0) { + for (String path : bulk.updatesMap.keySet()) { + for (String body : bulk.updatesMap.get(path)) { + postModify(path, body); } } } - if (investigationIds.size() > 0) { + if (bulk.investigationIds.size() > 0) { // Ensure bulk changes are committed before checking for InvestigationUsers commit(); - for (String investigationId : investigationIds) { - URI uriGet = new URIBuilder(server).setPath("/investigation/_source/" + investigationId) - .build(); + for (String investigationId : bulk.investigationIds) { + String path = "/investigation/_source/" + investigationId; + URI uriGet = new URIBuilder(server).setPath(path).build(); HttpGet httpGet = new HttpGet(uriGet); try (CloseableHttpResponse responseGet = httpclient.execute(httpGet)) { if (responseGet.getStatusLine().getStatusCode() == 200) { @@ -1208,24 +670,71 @@ public void modify(String json) throws IcatException { } } - StringBuilder fileSizeStringBuilder = new StringBuilder(); - buildFileSizeUpdates("investigation", investigationAggregations, fileSizeStringBuilder); - buildFileSizeUpdates("dataset", datasetAggregations, fileSizeStringBuilder); - if (fileSizeStringBuilder.toString().length() > 0) { - // Perform simple bulk modifications - URI uri = new URIBuilder(server).setPath("/_bulk").build(); - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(fileSizeStringBuilder.toString(), ContentType.APPLICATION_JSON)); - logger.trace("Making call {} with body {}", uri, fileSizeStringBuilder.toString()); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } + buildFileSizeUpdates("investigation", bulk.investigationAggregations, bulk.fileAggregationBuilder); + buildFileSizeUpdates("dataset", bulk.datasetAggregations, bulk.fileAggregationBuilder); + postModify("/_bulk", bulk.fileAggregationBody()); + + postModify("/_bulk", bulk.deletedBody()); } catch (IOException | URISyntaxException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); } } + /** + * Parses a modification from operation, and adds it to bulk. + * + * @param httpclient The client being used to send HTTP + * @param bulk OpensearchBulk object recording the requests for updates + * @param operation JsonObject representing the operation to be performed as + * part of the bulk modification + * @throws IcatException + * @throws URISyntaxException + * @throws ClientProtocolException + * @throws IOException + */ + private void parseModification(CloseableHttpClient httpclient, OpensearchBulk bulk, JsonObject operation) + throws IcatException, URISyntaxException, ClientProtocolException, IOException { + Set operationKeys = operation.keySet(); + if (operationKeys.size() != 1) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Operation should only have one key, but it had " + operationKeys); + } + String operationKey = operationKeys.toArray(new String[1])[0]; + ModificationType modificationType = ModificationType.valueOf(operationKey.toUpperCase()); + JsonObject innerOperation = operation.getJsonObject(modificationType.toString().toLowerCase()); + String index = innerOperation.getString("_index").toLowerCase(); + String id = innerOperation.getString("_id"); + JsonObject document = innerOperation.containsKey("doc") ? innerOperation.getJsonObject("doc") : null; + logger.trace("{} {} with id {}", operationKey, index, id); + + if (relations.containsKey(index)) { + // Related entities (with or without an index) will have one or more other + // indices that need to be updated with their information + for (ParentRelation relation : relations.get(index)) { + modifyNestedEntity(bulk, id, index, document, modificationType, relation); + } + } + if (indices.contains(index)) { + // Also modify any main, indexable entities + modifyEntity(httpclient, bulk, id, index, document, modificationType); + } + } + + /** + * Commits to ensure index is up to date, then sends a POST request for + * modification. This may be bulk, a single update, update by query etc. + * + * @param path Path on the search engine to POST to + * @param body String of Json to send as the request body + * @throws IcatException + */ + private void postModify(String path, String body) throws IcatException { + if (body.length() > 0) { + commit(); + post(path, body); + } + } + /** * Builds commands for updating the fileSizes of the entities keyed in * aggregations. @@ -1235,15 +744,15 @@ public void modify(String json) throws IcatException { * entity ids as keys. * @param fileSizeStringBuilder StringBuilder for constructing the bulk updates. */ - private void buildFileSizeUpdates(String entity, Map aggregations, + private void buildFileSizeUpdates(String entity, Map aggregations, StringBuilder fileSizeStringBuilder) { if (aggregations.size() > 0) { for (String id : aggregations.keySet()) { JsonObject targetObject = Json.createObjectBuilder().add("_id", new Long(id)).add("_index", entity) .build(); JsonObject update = Json.createObjectBuilder().add("update", targetObject).build(); - Long deltaFileSize = aggregations.get(id)[0]; - Long deltaFileCount = aggregations.get(id)[1]; + long deltaFileSize = aggregations.get(id)[0]; + long deltaFileCount = aggregations.get(id)[1]; JsonObjectBuilder paramsBuilder = Json.createObjectBuilder(); JsonObjectBuilder scriptBuilder = Json.createObjectBuilder(); paramsBuilder.add("deltaFileSize", deltaFileSize).add("deltaFileCount", deltaFileCount); @@ -1282,9 +791,10 @@ private JsonObject extractSource(CloseableHttpClient httpclient, String id) * For cases when Datasets and Datafiles are created after an Investigation, * some nested fields such as InvestigationUser and InvestigationInstrument may * have already been indexed on the Investigation but not the Dataset/file as - * the latter did not yet exist. This method retrieves these arrays from the - * Investigation index ensuring that all information is available on all indices - * at the time of creation. + * the latter did not yet exist. + * + * This method retrieves these arrays from the Investigation index ensuring that + * all information is available on all indices at the time of creation. * * @param httpclient The client being used to send HTTP * @param investigationId Id of an investigation which may contain relevant @@ -1302,77 +812,105 @@ private void extractFromInvestigation(CloseableHttpClient httpclient, String inv throws IOException, URISyntaxException, IcatException, ClientProtocolException { JsonObject responseObject = Json.createReader(responseGet.getEntity().getContent()).readObject(); if (responseObject.containsKey("investigationuser")) { - JsonArray jsonArray = responseObject.getJsonArray("investigationuser"); - for (String index : new String[] { "datafile", "dataset" }) { - URI uri = new URIBuilder(server).setPath("/" + index + "/_update_by_query").build(); - HttpPost httpPost = new HttpPost(uri); - JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("doc", jsonArray); - JsonObjectBuilder scriptBuilder = Json.createObjectBuilder(); - scriptBuilder.add("id", "create_investigationuser").add("params", paramsBuilder); - JsonObject queryObject = OpensearchQueryBuilder.buildTermQuery("investigation.id", investigationId); - JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); - String body = bodyBuilder.add("query", queryObject).add("script", scriptBuilder).build().toString(); - httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); - logger.trace("Making call {} with body {}", uri, body); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - commit(); - } - } + extractEntity(httpclient, investigationId, responseObject, "investigationuser", false); } if (responseObject.containsKey("investigationinstrument")) { - JsonArray jsonArray = responseObject.getJsonArray("investigationinstrument"); - for (String index : new String[] { "datafile", "dataset" }) { - URI uri = new URIBuilder(server).setPath("/" + index + "/_update_by_query").build(); - HttpPost httpPost = new HttpPost(uri); - JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("doc", jsonArray); - JsonObjectBuilder scriptBuilder = Json.createObjectBuilder(); - scriptBuilder.add("id", "create_investigationinstrument").add("params", paramsBuilder); - JsonObject queryObject = OpensearchQueryBuilder.buildTermQuery("investigation.id", investigationId); - JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); - String body = bodyBuilder.add("query", queryObject).add("script", scriptBuilder).build().toString(); - logger.trace("Making call {} with body {}", uri, body); - httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - commit(); - } - } + extractEntity(httpclient, investigationId, responseObject, "investigationinstrument", false); } if (responseObject.containsKey("sample")) { - JsonArray jsonArray = responseObject.getJsonArray("sample"); - for (String index : new String[] { "datafile", "dataset" }) { - URI uri = new URIBuilder(server).setPath("/" + index + "/_update_by_query").build(); - HttpPost httpPost = new HttpPost(uri); - for (JsonObject sampleObject : jsonArray.getValuesAs(JsonObject.class)) { + extractEntity(httpclient, investigationId, responseObject, "sample", true); + } + } + + /** + * For cases when Datasets and Datafiles are created after an Investigation, + * some nested fields such as InvestigationUser and InvestigationInstrument may + * have already been indexed on the Investigation but not the Dataset/file as + * the latter did not yet exist. + * + * This method extracts a single entity and uses it to update the + * dataset/datafile indices. + * + * @param httpclient The client being used to send HTTP + * @param investigationId Id of an investigation which may contain relevant + * information. + * @param responseObject JsonObject to extract the entity from + * @param entityName Name of the entity being extracted + * @param addFields Whether to add individual fields (true) or the entire + * entity as one "doc" (false) + * @throws URISyntaxException + * @throws IcatException + * @throws IOException + * @throws ClientProtocolException + */ + private void extractEntity(CloseableHttpClient httpclient, String investigationId, JsonObject responseObject, + String entityName, boolean addFields) + throws URISyntaxException, IcatException, IOException, ClientProtocolException { + JsonArray jsonArray = responseObject.getJsonArray(entityName); + for (String index : new String[] { "datafile", "dataset" }) { + URI uri = new URIBuilder(server).setPath("/" + index + "/_update_by_query").build(); + HttpPost httpPost = new HttpPost(uri); + if (addFields) { + for (JsonObject document : jsonArray.getValuesAs(JsonObject.class)) { + String documentId = document.getString("id"); + JsonObject queryObject = OpensearchQuery.buildTermQuery(entityName + ".id", documentId); JsonObjectBuilder paramsBuilder = Json.createObjectBuilder(); JsonObjectBuilder scriptBuilder = Json.createObjectBuilder(); - String sampleId = sampleObject.getString("id"); - for (String field : sampleObject.keySet()) { - paramsBuilder.add("sample." + field, sampleObject.get(field)); - } - scriptBuilder.add("id", "update_sample").add("params", paramsBuilder); - JsonObject queryObject = OpensearchQueryBuilder.buildTermQuery("sample.id", sampleId); - JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); - String body = bodyBuilder.add("query", queryObject).add("script", scriptBuilder).build().toString(); - logger.trace("Making call {} with body {}", uri, body); - httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - commit(); + for (String field : document.keySet()) { + paramsBuilder.add(entityName + "." + field, document.get(field)); } + scriptBuilder.add("id", "update_" + entityName).add("params", paramsBuilder); + + updateWithExtractedEntity(httpclient, uri, httpPost, queryObject, scriptBuilder); } + } else { + JsonObject queryObject = OpensearchQuery.buildTermQuery("investigation.id", investigationId); + JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("doc", jsonArray); + JsonObjectBuilder scriptBuilder = Json.createObjectBuilder(); + scriptBuilder.add("id", "create_" + entityName).add("params", paramsBuilder); + + updateWithExtractedEntity(httpclient, uri, httpPost, queryObject, scriptBuilder); } } } + /** + * For cases when Datasets and Datafiles are created after an Investigation, + * some nested fields such as InvestigationUser and InvestigationInstrument may + * have already been indexed on the Investigation but not the Dataset/file as + * the latter did not yet exist. + * + * This updates an index with the result of the extraction. + * + * @param httpclient The client being used to send HTTP + * @param uri URI for the relevant _update_by_query path + * @param httpPost HttpPost to be sent + * @param queryObject JsonObject determining which entities should be updated + * @param scriptBuilder JsonObjectBuilder for the script used to perform the + * update + * @throws IcatException + * @throws IOException + * @throws ClientProtocolException + */ + private void updateWithExtractedEntity(CloseableHttpClient httpclient, URI uri, HttpPost httpPost, + JsonObject queryObject, JsonObjectBuilder scriptBuilder) + throws IcatException, IOException, ClientProtocolException { + JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); + String body = bodyBuilder.add("query", queryObject).add("script", scriptBuilder).build().toString(); + httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + logger.trace("Making call {} with body {}", uri, body); + try (CloseableHttpResponse response = httpclient.execute(httpPost)) { + Rest.checkStatus(response, IcatExceptionType.INTERNAL); + commit(); + } + } + /** * Performs more complex update of an entity nested to a parent, for example * parameters. * - * @param sb StringBuilder used for bulk modifications. - * @param updatesByQuery List of HttpPost that cannot be bulked, and update - * existing documents based on a query. + * @param bulk OpensearchBulk object recording the requests for + * updates by query * @param id Id of the entity. * @param index Index of the entity. * @param document JsonObject containing the key value pairs of the @@ -1383,9 +921,8 @@ private void extractFromInvestigation(CloseableHttpClient httpclient, String inv * @throws URISyntaxException * @throws IcatException */ - private void modifyNestedEntity(StringBuilder sb, List updatesByQuery, String id, String index, - JsonObject document, ModificationType modificationType, ParentRelation relation) - throws URISyntaxException, IcatException { + private void modifyNestedEntity(OpensearchBulk bulk, String id, String index, JsonObject document, + ModificationType modificationType, ParentRelation relation) throws URISyntaxException, IcatException { switch (modificationType) { case CREATE: @@ -1403,9 +940,9 @@ private void modifyNestedEntity(StringBuilder sb, List updatesByQuery, for (Entry entry : document.entrySet()) { documentBuilder.add(entry.getKey().replace("sample.", ""), entry.getValue()); } - createNestedEntity(sb, id, index, documentBuilder.build(), relation); + createNestedEntity(bulk, id, index, documentBuilder.build(), relation); } else { - createNestedEntity(sb, id, index, document, relation); + createNestedEntity(bulk, id, index, document, relation); } } else if (index.equals("sampletype")) { // Otherwise, in most cases we don't need to update, as User and ParameterType @@ -1413,23 +950,21 @@ private void modifyNestedEntity(StringBuilder sb, List updatesByQuery, // when that parent is created so the information is captured. However, since // SampleType can be null upon creation of a Sample, need to account for the // creation of a SampleType at a later date. - updateNestedEntityByQuery(updatesByQuery, id, index, document, relation, true); + updateNestedEntityByQuery(bulk, id, index, document, relation, true); } else if (index.equals("sampleparameter")) { // SampleParameter requires specific logic, as the join is performed using the // Sample id rather than the SampleParameter id or the parent id. - logger.debug("index: {}, parent: {}, joinField: {}, doc: {}", index, relation.parentName, - relation.joinField, document.toString()); if (document.containsKey("sample.id")) { String sampleId = document.getString("sample.id"); - updateNestedEntityByQuery(updatesByQuery, sampleId, index, document, relation, true); + updateNestedEntityByQuery(bulk, sampleId, index, document, relation, true); } } break; case UPDATE: - updateNestedEntityByQuery(updatesByQuery, id, index, document, relation, true); + updateNestedEntityByQuery(bulk, id, index, document, relation, true); break; case DELETE: - updateNestedEntityByQuery(updatesByQuery, id, index, document, relation, false); + updateNestedEntityByQuery(bulk, id, index, document, relation, false); break; } } @@ -1437,23 +972,27 @@ private void modifyNestedEntity(StringBuilder sb, List updatesByQuery, /** * Create a new nested entity in an array on its parent. * - * @param sb StringBuilder used for bulk modifications. + * @param bulk OpensearchBulk object recording the requests for single + * updates * @param id Id of the entity. * @param index Index of the entity. * @param document JsonObject containing the key value pairs of the document * fields. * @param relation The relation between the nested entity and its parent. - * @throws IcatException If parentId is missing from document. + * @throws IcatException If parentId is missing from document. + * @throws URISyntaxException */ - private static void createNestedEntity(StringBuilder sb, String id, String index, JsonObject document, - ParentRelation relation) throws IcatException { + private void createNestedEntity(OpensearchBulk bulk, String id, String index, JsonObject document, + ParentRelation relation) throws IcatException, URISyntaxException { + if (!document.containsKey(relation.joinField + ".id")) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, relation.joinField + ".id not found in " + document.toString()); } + String parentId = document.getString(relation.joinField + ".id"); - JsonObjectBuilder innerBuilder = Json.createObjectBuilder() - .add("_id", parentId).add("_index", relation.parentName); + String path = "/" + relation.parentName + "/_update/" + parentId; + // For nested 0:* relationships, wrap single documents in an array JsonArray docArray = Json.createArrayBuilder().add(document).build(); JsonObjectBuilder paramsBuilder = Json.createObjectBuilder().add("id", id).add("doc", docArray); @@ -1463,34 +1002,33 @@ private static void createNestedEntity(StringBuilder sb, String id, String index } else { scriptId = "update_" + index; } + JsonObjectBuilder scriptBuilder = Json.createObjectBuilder().add("id", scriptId).add("params", paramsBuilder); JsonObjectBuilder upsertBuilder = Json.createObjectBuilder().add(index, docArray); JsonObjectBuilder payloadBuilder = Json.createObjectBuilder() .add("upsert", upsertBuilder).add("script", scriptBuilder); - sb.append(Json.createObjectBuilder().add("update", innerBuilder).build().toString()).append("\n"); - sb.append(payloadBuilder.build().toString()).append("\n"); + bulk.addUpdate(path, payloadBuilder.build().toString()); } /** * For existing nested objects, painless scripting must be used to update or * delete them. * - * @param updatesByQuery List of HttpPost that cannot be bulked, and update - * existing documents based on a query. - * @param id Id of the entity. - * @param index Index of the entity. - * @param document JsonObject containing the key value pairs of the - * document fields. - * @param relation The relation between the nested entity and its parent. - * @param update Whether to update, or if false delete nested entity - * with the specified id. + * @param bulk OpensearchBulk object recording the requests for updates by + * query + * @param id Id of the entity. + * @param index Index of the entity. + * @param document JsonObject containing the key value pairs of the + * document fields. + * @param relation The relation between the nested entity and its parent. + * @param update Whether to update, or if false delete nested entity + * with the specified id. * @throws URISyntaxException */ - private void updateNestedEntityByQuery(List updatesByQuery, String id, String index, JsonObject document, + private void updateNestedEntityByQuery(OpensearchBulk bulk, String id, String index, JsonObject document, ParentRelation relation, boolean update) throws URISyntaxException { + String path = "/" + relation.parentName + "/_update_by_query"; - URI uri = new URIBuilder(server).setPath(path).build(); - HttpPost httpPost = new HttpPost(uri); // Determine the Id of the painless script to use String scriptId = update ? "update_" : "delete_"; @@ -1509,25 +1047,20 @@ private void updateNestedEntityByQuery(List updatesByQuery, String id, paramsBuilder.add("doc", Json.createArrayBuilder().add(document)); } else { // Need to update individual nested fields - paramsBuilder = convertScriptUnits(paramsBuilder, document, relation.fields); + convertScriptUnits(paramsBuilder, document, relation.fields); } } JsonObjectBuilder scriptBuilder = Json.createObjectBuilder().add("id", scriptId).add("params", paramsBuilder); - JsonObject queryObject; String idField = relation.joinField.equals(relation.parentName) ? "id" : relation.joinField + ".id"; // sample.id is a nested field on investigations, so need a nested query to // successfully add sampleparameter + JsonObject queryObject = OpensearchQuery.buildTermQuery(idField, id); if (relation.relationType.equals(RelationType.NESTED_GRANDCHILD) || index.equals("sampleparameter") && relation.parentName.equals("investigation")) { - queryObject = OpensearchQueryBuilder.buildNestedQuery(relation.joinField, - OpensearchQueryBuilder.buildTermQuery(idField, id)); - } else { - queryObject = OpensearchQueryBuilder.buildTermQuery(idField, id); + queryObject = OpensearchQuery.buildNestedQuery(relation.joinField, queryObject); } JsonObject bodyJson = Json.createObjectBuilder().add("query", queryObject).add("script", scriptBuilder).build(); - logger.trace("Making call {} with body {}", path, bodyJson.toString()); - httpPost.setEntity(new StringEntity(bodyJson.toString(), ContentType.APPLICATION_JSON)); - updatesByQuery.add(httpPost); + bulk.addUpdate(path, bodyJson.toString()); } /** @@ -1591,9 +1124,8 @@ private JsonObject convertDocumentUnits(JsonObject document) { * @param paramsBuilder JsonObjectBuilder for the painless script parameters. * @param document JsonObject containing the field/values. * @param fields List of fields to be included in the parameters. - * @return paramsBuilder with fields added. */ - private JsonObjectBuilder convertScriptUnits(JsonObjectBuilder paramsBuilder, JsonObject document, + private void convertScriptUnits(JsonObjectBuilder paramsBuilder, JsonObject document, Set fields) { for (String field : fields) { if (document.containsKey(field)) { @@ -1606,36 +1138,28 @@ private JsonObjectBuilder convertScriptUnits(JsonObjectBuilder paramsBuilder, Js } } } - return paramsBuilder; } /** - * Adds modification command to sb. If relevant, also adds to the list of + * Adds modification command to bulk. If relevant, also adds to the list of * investigationIds which may contain relevant information (e.g. nested * InvestigationUsers). * - * @param httpclient The client being used to send HTTP - * @param sb StringBuilder used for bulk modifications. - * @param investigationIds List of investigationIds to check for - * relevant - * fields. - * @param investigationAggregations Map of aggregated fileSize changes with the - * Investigation ids as keys. - * @param datasetAggregations Map of aggregated fileSize changes with the - * Dataset ids as keys. - * @param id Id of the entity. - * @param index Index of the entity. - * @param document JsonObject containing the key value pairs of - * the - * document fields. - * @param modificationType The type of operation to be performed. + * @param httpclient The client being used to send HTTP + * @param bulk OpensearchBulk object recording the requests for + * updates and aggregations + * @param id Id of the entity. + * @param index Index of the entity. + * @param document JsonObject containing the key value pairs of + * the + * document fields. + * @param modificationType The type of operation to be performed. * @throws URISyntaxException * @throws IOException * @throws ClientProtocolException */ - private void modifyEntity(CloseableHttpClient httpclient, StringBuilder sb, Set investigationIds, - Map investigationAggregations, Map datasetAggregations, String id, - String index, JsonObject document, ModificationType modificationType) + private void modifyEntity(CloseableHttpClient httpclient, OpensearchBulk bulk, String id, String index, + JsonObject document, ModificationType modificationType) throws ClientProtocolException, IOException, URISyntaxException { JsonObject targetObject = Json.createObjectBuilder().add("_id", new Long(id)).add("_index", index).build(); @@ -1644,82 +1168,98 @@ private void modifyEntity(CloseableHttpClient httpclient, StringBuilder sb, Set< switch (modificationType) { case CREATE: docAsUpsert = Json.createObjectBuilder().add("doc", document).add("doc_as_upsert", true).build(); - sb.append(update.toString()).append("\n").append(docAsUpsert.toString()).append("\n"); + bulk.bulkBuilder.append(update.toString()).append("\n").append(docAsUpsert.toString()).append("\n"); if (document.containsKey("investigation.id")) { // In principle a Dataset/Datafile could be created after InvestigationUser // entities are attached to an Investigation, so need to check for those - investigationIds.add(document.getString("investigation.id")); - } - if (aggregateFiles && index.equals("datafile") && document.containsKey("fileSize")) { - long newFileSize = document.getJsonNumber("fileSize").longValueExact(); - if (document.containsKey("investigation.id")) { - String investigationId = document.getString("investigation.id"); - Long[] runningFileSize = investigationAggregations.getOrDefault(investigationId, - new Long[] { 0L, 0L }); - Long[] newValue = new Long[] { runningFileSize[0] + newFileSize, runningFileSize[1] + 1L }; - investigationAggregations.put(investigationId, newValue); - } - if (document.containsKey("dataset.id")) { - String datasetId = document.getString("dataset.id"); - Long[] runningFileSize = datasetAggregations.getOrDefault(datasetId, new Long[] { 0L, 0L }); - Long[] newValue = new Long[] { runningFileSize[0] + newFileSize, runningFileSize[1] + 1L }; - datasetAggregations.put(datasetId, newValue); - } + bulk.investigationIds.add(document.getString("investigation.id")); } break; case UPDATE: docAsUpsert = Json.createObjectBuilder().add("doc", document).add("doc_as_upsert", true).build(); - sb.append(update.toString()).append("\n").append(docAsUpsert.toString()).append("\n"); - if (aggregateFiles && index.equals("datafile") && document.containsKey("fileSize")) { - long newFileSize = document.getJsonNumber("fileSize").longValueExact(); - long oldFileSize; - JsonObject source = extractSource(httpclient, id); - if (source != null && source.containsKey("fileSize")) { - oldFileSize = source.getJsonNumber("fileSize").longValueExact(); - } else { - oldFileSize = 0; - } - if (newFileSize != oldFileSize) { - if (document.containsKey("investigation.id")) { - String investigationId = document.getString("investigation.id"); - Long[] runningFileSize = investigationAggregations.getOrDefault(investigationId, - new Long[] { 0L, 0L }); - Long[] newValue = new Long[] { runningFileSize[0] + newFileSize - oldFileSize, - runningFileSize[1] }; - investigationAggregations.put(investigationId, newValue); - } - if (document.containsKey("dataset.id")) { - String datasetId = document.getString("dataset.id"); - Long[] runningFileSize = datasetAggregations.getOrDefault(datasetId, new Long[] { 0L, 0L }); - Long[] newValue = new Long[] { runningFileSize[0] + newFileSize - oldFileSize, - runningFileSize[1] }; - datasetAggregations.put(datasetId, newValue); - } - } - } + bulk.bulkBuilder.append(update.toString()).append("\n").append(docAsUpsert.toString()).append("\n"); break; case DELETE: - sb.append(Json.createObjectBuilder().add("delete", targetObject).build().toString()).append("\n"); - if (aggregateFiles && index.equals("datafile")) { - JsonObject source = extractSource(httpclient, id); - if (source != null && source.containsKey("fileSize")) { - long oldFileSize = source.getJsonNumber("fileSize").longValueExact(); - if (source.containsKey("investigation.id")) { - String investigationId = source.getString("investigation.id"); - Long[] runningFileSize = investigationAggregations.getOrDefault(investigationId, - new Long[] { 0L, 0L }); - Long[] newValue = new Long[] { runningFileSize[0] - oldFileSize, runningFileSize[1] - 1 }; - investigationAggregations.put(investigationId, newValue); - } - if (source.containsKey("dataset.id")) { - String datasetId = source.getString("dataset.id"); - Long[] runningFileSize = datasetAggregations.getOrDefault(datasetId, new Long[] { 0L, 0L }); - Long[] newValue = new Long[] { runningFileSize[0] - oldFileSize, runningFileSize[1] - 1 }; - datasetAggregations.put(datasetId, newValue); - } - } - } + bulk.deletionBuilder.append(Json.createObjectBuilder().add("delete", targetObject).build().toString()) + .append("\n"); + break; + } + if (aggregateFiles && index.equals("datafile") && document.containsKey("fileSize")) { + aggregateFiles(modificationType, bulk, index, document, httpclient, id); + } + } + + /** + * Aggregates any change to file size to relevant paret entities. + * + * @param modificationType The type of operation to be performed + * @param bulk OpensearchBulk object recording the requests for + * updates and aggregations + * @param index Index of the entity + * @param document Document containing the parent entity ids + * @param httpclient CloseableHttpClient to use + * @param id Datafile id + * @throws ClientProtocolException + * @throws IOException + * @throws URISyntaxException + */ + private void aggregateFiles(ModificationType modificationType, OpensearchBulk bulk, String index, + JsonObject document, CloseableHttpClient httpclient, String id) + throws ClientProtocolException, IOException, URISyntaxException { + long deltaFileSize = 0; + long deltaFileCount = 0; + switch (modificationType) { + case CREATE: + deltaFileSize = document.getJsonNumber("fileSize").longValueExact(); + deltaFileCount = 1; + break; + case UPDATE: + deltaFileSize = document.getJsonNumber("fileSize").longValueExact() - extractFileSize(httpclient, id); break; + case DELETE: + deltaFileSize = -extractFileSize(httpclient, id); + deltaFileCount = -1; + break; + } + incrementEntity(bulk.investigationAggregations, document, deltaFileSize, deltaFileCount, "investigation.id"); + incrementEntity(bulk.datasetAggregations, document, deltaFileSize, deltaFileCount, "dataset.id"); + } + + /** + * Increments the changes to a parent entity by the values of deltaFileSize and + * deltaFileCount. + * + * @param aggregations Map of aggregated fileSize changes with the parent ids + * as keys. + * @param document Document containing the parent entity id + * @param deltaFileSize Change in file size + * @param deltaFileCount Change in file count + * @param idField The field of the id of parent entity to be incremented + */ + private void incrementEntity(Map aggregations, JsonObject document, long deltaFileSize, + long deltaFileCount, String idField) { + if (document.containsKey(idField)) { + String id = document.getString(idField); + long[] runningFileSize = aggregations.getOrDefault(id, new long[] { 0, 0 }); + long[] newValue = new long[] { runningFileSize[0] + deltaFileSize, runningFileSize[1] + deltaFileCount }; + aggregations.put(id, newValue); + } + } + + /** + * @param httpclient CloseableHttpClient to use + * @param id Datafile id + * @return Size of the Datafile in bytes + * @throws IOException + * @throws URISyntaxException + * @throws ClientProtocolException + */ + private long extractFileSize(CloseableHttpClient httpclient, String id) + throws IOException, URISyntaxException, ClientProtocolException { + JsonObject source = extractSource(httpclient, id); + if (source != null && source.containsKey("fileSize")) { + return source.getJsonNumber("fileSize").longValueExact(); } + return 0; } } diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchBulk.java b/src/main/java/org/icatproject/core/manager/search/OpensearchBulk.java new file mode 100644 index 00000000..26cbaac4 --- /dev/null +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchBulk.java @@ -0,0 +1,57 @@ +package org.icatproject.core.manager.search; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Holds information for the various types of request that need to be made as + * part of a bulk modification. + */ +public class OpensearchBulk { + + public Map> updatesMap = new HashMap<>(); + public Set investigationIds = new HashSet<>(); + public Map investigationAggregations = new HashMap<>(); + public Map datasetAggregations = new HashMap<>(); + public StringBuilder bulkBuilder = new StringBuilder(); + public StringBuilder deletionBuilder = new StringBuilder(); + public StringBuilder fileAggregationBuilder = new StringBuilder(); + + /** + * Adds a path and body for a single update to updatesMap, if not already + * present. + * + * @param path Path of request + * @param body Body of request + */ + public void addUpdate(String path, String body) { + Set bodies = updatesMap.getOrDefault(path, new HashSet<>()); + bodies.add(body); + updatesMap.put(path, bodies); + } + + /** + * @return String of updates that should be performed as a bulk request + */ + public String bulkBody() { + return bulkBuilder.toString(); + } + + /** + * @return String of deletes that should be performed as a bulk request + */ + public String deletedBody() { + return deletionBuilder.toString(); + } + + /** + * @return String of file aggregations that should be performed as a bulk + * request + */ + public String fileAggregationBody() { + return fileAggregationBuilder.toString(); + } + +} \ No newline at end of file diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchQuery.java b/src/main/java/org/icatproject/core/manager/search/OpensearchQuery.java new file mode 100644 index 00000000..be3296f2 --- /dev/null +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchQuery.java @@ -0,0 +1,827 @@ +package org.icatproject.core.manager.search; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonArrayBuilder; +import javax.json.JsonNumber; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import javax.json.JsonString; +import javax.json.JsonValue; +import javax.json.JsonValue.ValueType; + +import org.icatproject.core.IcatException; +import org.icatproject.core.IcatException.IcatExceptionType; +import org.icatproject.utils.IcatUnits.SystemValue; + +/** + * Utilities for building queries in Json understood by Opensearch. + */ +public class OpensearchQuery { + + private static JsonObject matchAll = build("match_all", Json.createObjectBuilder()); + public static JsonObject matchAllQuery = build("query", matchAll); + + private JsonObjectBuilder builder = Json.createObjectBuilder(); + private OpensearchApi opensearchApi; + + public OpensearchQuery(OpensearchApi opensearchApi) { + this.opensearchApi = opensearchApi; + } + + /** + * @param filter Path to nested Object. + * @param should Any number of pre-built queries. + * @return {"bool": {"filter": [...filter], "should": [...should]}} + */ + public static JsonObject buildBoolQuery(List filter, List should) { + JsonObjectBuilder boolBuilder = Json.createObjectBuilder(); + addToBoolArray("should", should, boolBuilder); + addToBoolArray("filter", filter, boolBuilder); + return build("bool", boolBuilder); + } + + /** + * @param occur String of an occurance keyword ("filter", "should", "must" + * etc.) + * @param queries List of JsonObjects representing the queries to occur. + * @param boolBuilder Builder of the main boolean query. + */ + private static void addToBoolArray(String occur, List queries, JsonObjectBuilder boolBuilder) { + if (queries != null && queries.size() > 0) { + JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); + for (JsonObject queryObject : queries) { + filterBuilder.add(queryObject); + } + boolBuilder.add(occur, filterBuilder); + } + } + + /** + * @param field Field containing the match. + * @param value Value to match. + * @return {"match": {"`field`.keyword": {"query": `value`, "operator": "and"}}} + */ + public static JsonObject buildMatchQuery(String field, String value) { + JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("query", value).add("operator", "and"); + JsonObject matchBuilder = build(field + ".keyword", fieldBuilder); + return build("match", matchBuilder); + } + + /** + * @param path Path to nested Object. + * @param queryObjects Any number of pre-built queries. + * @return {"nested": {"path": `path`, "query": {"bool": {"filter": [...queryObjects]}}}} + */ + public static JsonObject buildNestedQuery(String path, JsonObject... queryObjects) { + JsonObject builtQueries; + if (queryObjects.length == 0) { + builtQueries = matchAllQuery; + } else if (queryObjects.length == 1) { + builtQueries = queryObjects[0]; + } else { + JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); + for (JsonObject queryObject : queryObjects) { + filterBuilder.add(queryObject); + } + JsonObject boolObject = build("filter", filterBuilder.build()); + builtQueries = build("bool", boolObject); + } + JsonObjectBuilder nestedBuilder = Json.createObjectBuilder().add("path", path).add("query", builtQueries); + return build("nested", nestedBuilder); + } + + /** + * @param value String value to query for. + * @param fields List of fields to check for value. + * @return {"query_string": {"query": `value`, "fields": [...fields]}} + */ + public static JsonObject buildStringQuery(String value, String... fields) { + JsonObjectBuilder queryStringBuilder = Json.createObjectBuilder().add("query", value); + if (fields.length > 0) { + JsonArrayBuilder fieldsBuilder = Json.createArrayBuilder(); + for (String field : fields) { + fieldsBuilder.add(field); + } + queryStringBuilder.add("fields", fieldsBuilder); + } + return build("query_string", queryStringBuilder); + } + + /** + * @param field Field containing the term. + * @param value Term to match. + * @return {"term": {`field`: `value`}} + */ + public static JsonObject buildTermQuery(String field, String value) { + return build("term", Json.createObjectBuilder().add(field, value)); + } + + /** + * @param field Field containing the number. + * @param value Number to match. + * @return {"term": {`field`: `value`}} + */ + public static JsonObject buildTermQuery(String field, JsonNumber value) { + return build("term", build(field, value)); + } + + /** + * @param field Field containing the double value. + * @param value Double to match. + * @return {"term": {`field`: `value`}} + */ + public static JsonObject buildTermQuery(String field, double value) { + return build("term", Json.createObjectBuilder().add(field, value)); + } + + /** + * @param field Field containing on of the terms. + * @param values JsonArray of possible terms. + * @return {"terms": {`field`: `values`}} + */ + public static JsonObject buildTermsQuery(String field, JsonArray values) { + return build("terms", build(field, values)); + } + + /** + * @param field Field to apply the range to. + * @param lowerValue Lowest allowed value in the range. + * @param upperValue Highest allowed value in the range. + * @return {"range": {`field`: {"gte": `upperValue`, "lte": `lowerValue`}}} + */ + public static JsonObject buildDoubleRangeQuery(String field, Double lowerValue, Double upperValue) { + JsonObjectBuilder fieldBuilder = Json.createObjectBuilder(); + if (lowerValue != null) + fieldBuilder.add("gte", lowerValue); + if (upperValue != null) + fieldBuilder.add("lte", upperValue); + return buildRange(field, fieldBuilder); + } + + /** + * @param field Field to apply the range to. + * @param lowerValue Lowest allowed value in the range. + * @param upperValue Highest allowed value in the range. + * @return {"range": {`field`: {"gte": `upperValue`, "lte": `lowerValue`}}} + */ + public static JsonObject buildLongRangeQuery(String field, Long lowerValue, Long upperValue) { + JsonObjectBuilder fieldBuilder = Json.createObjectBuilder(); + if (lowerValue != null) + fieldBuilder.add("gte", lowerValue); + if (upperValue != null) + fieldBuilder.add("lte", upperValue); + return buildRange(field, fieldBuilder); + } + + /** + * @param field Field to apply the range to. + * @param lowerValue Lowest allowed value in the range. + * @param upperValue Highest allowed value in the range. + * @return {"range": {`field`: {"gte": `upperValue`, "lte": `lowerValue`}}} + */ + public static JsonObject buildRangeQuery(String field, JsonNumber lowerValue, JsonNumber upperValue) { + JsonObjectBuilder fieldBuilder = Json.createObjectBuilder(); + if (lowerValue != null) + fieldBuilder.add("gte", lowerValue); + if (upperValue != null) + fieldBuilder.add("lte", upperValue); + return buildRange(field, fieldBuilder); + } + + /** + * @param field Field to apply the range to + * @param fieldBuilder JsonObjectBuilder for the field + * @return {"range": {`field`: `fieldBuilder`}} + */ + private static JsonObject buildRange(String field, JsonObjectBuilder fieldBuilder) { + JsonObject rangeObject = build(field, fieldBuilder); + return build("range", rangeObject); + } + + /** + * @param field Field to facet. + * @param ranges JsonArray of ranges to allocate documents to. + * @return {"range": {"field": `field`, "keyed": true, "ranges": `ranges`}} + */ + public static JsonObject buildRangeFacet(String field, JsonArray ranges) { + JsonObjectBuilder rangeBuilder = Json.createObjectBuilder(); + rangeBuilder.add("field", field).add("keyed", true).add("ranges", ranges); + return build("range", rangeBuilder); + } + + /** + * @param field Field to facet. + * @param maxLabels Maximum number of labels per dimension. + * @return {"terms": {"field": `field`, "size": `maxLabels`}} + */ + public static JsonObject buildStringFacet(String field, int maxLabels) { + JsonObjectBuilder termsBuilder = Json.createObjectBuilder(); + termsBuilder.add("field", field).add("size", maxLabels); + return build("terms", termsBuilder); + } + + /** + * @param key Arbitrary key + * @param value Arbitrary JsonObjectBuilder + * @return {`key`: `builder`}} + */ + private static JsonObject build(String key, JsonObjectBuilder builder) { + return Json.createObjectBuilder().add(key, builder).build(); + } + + /** + * @param key Arbitrary key + * @param value Arbitrary JsonValue + * @return {`key`: `value`}} + */ + private static JsonObject build(String key, JsonValue value) { + return Json.createObjectBuilder().add(key, value).build(); + } + + /** + * Extracts and parses a date value from jsonObject. If the value is a NUMBER + * (ms since epoch), then it is taken as is. If it is a STRING, then it is + * expected in the yyyyMMddHHmm format. + * + * @param jsonObject JsonObject to extract the date from. + * @param key Key of the date field to extract. + * @param offset In the event of the date being a string, we do not have + * second or ms precision. To ensure ranges are successful, + * it may be necessary to add 59999 ms to the parsed value + * as an offset. + * @param defaultValue The value to return if key is not present in jsonObject. + * @return Time since epoch in ms. + * @throws IcatException + */ + private static long parseDate(JsonObject jsonObject, String key, int offset, long defaultValue) + throws IcatException { + if (jsonObject.containsKey(key)) { + ValueType valueType = jsonObject.get(key).getValueType(); + switch (valueType) { + case STRING: + String dateString = jsonObject.getString(key); + try { + return SearchApi.decodeTime(dateString) + offset; + } catch (Exception e) { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Could not parse date " + dateString + " using expected format yyyyMMddHHmm"); + } + case NUMBER: + return jsonObject.getJsonNumber(key).longValueExact(); + default: + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Dates should be represented by a NUMBER or STRING JsonValue, but got " + valueType); + } + } + return defaultValue; + } + + /** + * Parses incoming Json encoding the requested facets and uses bodyBuilder to + * construct Json that can be understood by Opensearch. + * + * @param bodyBuilder JsonObjectBuilder being used to build the body of the + * request. + * @param dimensions JsonArray of JsonObjects representing dimensions to be + * faceted. + * @param maxLabels The maximum number of labels to collect for each + * dimension. + * @param dimensionPrefix Optional prefix to apply to the dimension names. This + * is needed to distinguish between potentially ambiguous + * dimensions, such as "(investigation.)type.name" and + * "(investigationparameter.)type.name". + * @return The bodyBuilder originally passed with facet information added to it. + */ + public void parseFacets(JsonArray dimensions, int maxLabels, + String dimensionPrefix) { + JsonObjectBuilder aggsBuilder = Json.createObjectBuilder(); + for (JsonObject dimensionObject : dimensions.getValuesAs(JsonObject.class)) { + String dimensionString = dimensionObject.getString("dimension"); + String field = dimensionPrefix == null ? dimensionString : dimensionPrefix + "." + dimensionString; + if (dimensionObject.containsKey("ranges")) { + JsonArray ranges = dimensionObject.getJsonArray("ranges"); + aggsBuilder.add(dimensionString, buildRangeFacet(field, ranges)); + } else { + aggsBuilder.add(dimensionString, + buildStringFacet(field + ".keyword", maxLabels)); + } + } + buildFacetRequestJson(dimensionPrefix, aggsBuilder); + } + + /** + * Uses bodyBuilder to construct Json for faceting string fields. + * + * @param bodyBuilder JsonObjectBuilder being used to build the body of the + * request. + * @param dimensions List of dimensions to perform string based faceting + * on. + * @param maxLabels The maximum number of labels to collect for each + * dimension. + * @param dimensionPrefix Optional prefix to apply to the dimension names. This + * is needed to distinguish between potentially ambiguous + * dimensions, such as "(investigation.)type.name" and + * "(investigationparameter.)type.name". + * @return The bodyBuilder originally passed with facet information added to it. + */ + public void parseFacets(List dimensions, int maxLabels, + String dimensionPrefix) { + JsonObjectBuilder aggsBuilder = Json.createObjectBuilder(); + for (String dimensionString : dimensions) { + String field = dimensionPrefix == null ? dimensionString : dimensionPrefix + "." + dimensionString; + aggsBuilder.add(dimensionString, buildStringFacet(field + ".keyword", maxLabels)); + } + buildFacetRequestJson(dimensionPrefix, aggsBuilder); + } + + /** + * Finalises the construction of faceting Json by handling the possibility of + * faceting a nested object. + * + * @param bodyBuilder JsonObjectBuilder being used to build the body of the + * request. + * @param dimensionPrefix Optional prefix to apply to the dimension names. This + * is needed to distinguish between potentially ambiguous + * dimensions, such as "(investigation.)type.name" and + * "(investigationparameter.)type.name". + * @param aggsBuilder JsonObjectBuilder that has the faceting details. + * @return The bodyBuilder originally passed with facet information added to it. + */ + private void buildFacetRequestJson(String dimensionPrefix, JsonObjectBuilder aggsBuilder) { + if (dimensionPrefix == null) { + builder.add("aggs", aggsBuilder); + } else { + builder.add("aggs", Json.createObjectBuilder() + .add(dimensionPrefix, Json.createObjectBuilder() + .add("nested", Json.createObjectBuilder().add("path", dimensionPrefix)) + .add("aggs", aggsBuilder))); + } + } + + /** + * Parses a filter object applied to a single field. Note that in the case that + * this field is actually a nested object, more complex logic will be applied to + * ensure that only object matching all nested filters are returned. + * + * @param filterBuilder Builder for the array of queries to filter by. + * @param field Field to apply the filter to. In the case of nested + * queries, this should only be the name of the top level + * field. For example "investigationparameter". + * @param value JsonValue representing the filter query. This can be a + * STRING for simple terms, or an OBJECT containing nested + * "value", "exact" or "range" filters. + * @throws IcatException + */ + private void parseFilter(JsonArrayBuilder filterBuilder, String field, JsonValue value) throws IcatException { + ValueType valueType = value.getValueType(); + switch (valueType) { + case STRING: + filterBuilder.add(buildTermQuery(field + ".keyword", ((JsonString) value).getString())); + return; + case OBJECT: + JsonObject valueObject = (JsonObject) value; + if (valueObject.containsKey("filter")) { + List queryObjectsList = new ArrayList<>(); + for (JsonObject nestedFilter : valueObject.getJsonArray("filter").getValuesAs(JsonObject.class)) { + String nestedField = nestedFilter.getString("field"); + if (nestedFilter.containsKey("value")) { + // String based term query + String stringValue = nestedFilter.getString("value"); + queryObjectsList.add(buildTermQuery(field + "." + nestedField + ".keyword", stringValue)); + } else if (nestedFilter.containsKey("exact")) { + parseExactFilter(field, queryObjectsList, nestedFilter, nestedField); + } else { + parseRangeFilter(field, queryObjectsList, nestedFilter, nestedField); + } + } + JsonObject[] queryObjects = queryObjectsList.toArray(new JsonObject[0]); + filterBuilder.add(buildNestedQuery(field, queryObjects)); + } else { + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "expected an ARRAY with the key 'filter', but received " + valueObject); + } + return; + + default: + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "filter values should be STRING, OBJECT or and ARRAY of the former, but were " + valueType); + } + + } + + /** + * Parses a range based filter for a single field. + * + * @param field Field to apply the filter to. In the case of nested + * queries, this should only be the name of the top + * level + * field. For example "investigationparameter" + * @param queryObjectsList List of JsonObjects to add the filter to + * @param nestedFilter The nested JsonObject which contains the details of + * the filter + * @param nestedField The nested field on which to actually apply the + * filter + */ + private void parseRangeFilter(String field, List queryObjectsList, JsonObject nestedFilter, + String nestedField) { + JsonNumber from = nestedFilter.getJsonNumber("from"); + JsonNumber to = nestedFilter.getJsonNumber("to"); + String units = nestedFilter.getString("units", null); + if (units != null) { + SystemValue fromValue = opensearchApi.icatUnits.new SystemValue(from.doubleValue(), units); + SystemValue toValue = opensearchApi.icatUnits.new SystemValue(to.doubleValue(), units); + if (fromValue.value != null && toValue.value != null) { + // If we were able to parse the units, apply query to the SI value + String fieldSI = field + "." + nestedField + "SI"; + queryObjectsList.add(buildDoubleRangeQuery(fieldSI, fromValue.value, toValue.value)); + } else { + // If units could not be parsed, make them part of the query on the raw data + queryObjectsList.add(buildRangeQuery(field + "." + nestedField, from, to)); + queryObjectsList.add(buildTermQuery(field + ".type.units.keyword", units)); + } + } else { + // If units were not provided, just apply to the raw data + queryObjectsList.add(buildRangeQuery(field + "." + nestedField, from, to)); + } + } + + /** + * Parses an exact filter for a single field. + * + * @param field Field to apply the filter to. In the case of nested + * queries, this should only be the name of the top + * level + * field. For example "investigationparameter" + * @param queryObjectsList List of JsonObjects to add the filter to + * @param nestedFilter The nested JsonObject which contains the details of + * the filter + * @param nestedField The nested field on which to actually apply the + * filter + */ + private void parseExactFilter(String field, List queryObjectsList, JsonObject nestedFilter, + String nestedField) { + JsonNumber exact = nestedFilter.getJsonNumber("exact"); + String units = nestedFilter.getString("units", null); + if (units != null) { + SystemValue exactValue = opensearchApi.icatUnits.new SystemValue(exact.doubleValue(), units); + if (exactValue.value != null) { + // If we were able to parse the units, apply query to the SI value + JsonObject bottomQuery = buildDoubleRangeQuery(field + ".rangeBottomSI", null, exactValue.value); + JsonObject topQuery = buildDoubleRangeQuery(field + ".rangeTopSI", exactValue.value, null); + JsonObject inRangeQuery = buildBoolQuery(Arrays.asList(bottomQuery, topQuery), null); + JsonObject exactQuery = buildTermQuery(field + "." + nestedField + "SI", exactValue.value); + queryObjectsList.add(buildBoolQuery(null, Arrays.asList(inRangeQuery, exactQuery))); + } else { + // If units could not be parsed, make them part of the query on the raw data + JsonObject bottomQuery = buildRangeQuery(field + ".rangeBottom", null, exact); + JsonObject topQuery = buildRangeQuery(field + ".rangeTop", exact, null); + JsonObject inRangeQuery = buildBoolQuery(Arrays.asList(bottomQuery, topQuery), null); + JsonObject exactQuery = buildTermQuery(field + "." + nestedField, exact); + queryObjectsList.add(buildBoolQuery(null, Arrays.asList(inRangeQuery, exactQuery))); + queryObjectsList.add(buildTermQuery(field + ".type.units.keyword", units)); + } + } else { + // If units were not provided, just apply to the raw data + JsonObject bottomQuery = buildRangeQuery(field + ".rangeBottom", null, exact); + JsonObject topQuery = buildRangeQuery(field + ".rangeTop", exact, null); + JsonObject inRangeQuery = buildBoolQuery(Arrays.asList(bottomQuery, topQuery), null); + JsonObject exactQuery = buildTermQuery(field + "." + nestedField, exact); + queryObjectsList.add(buildBoolQuery(null, Arrays.asList(inRangeQuery, exactQuery))); + } + } + + /** + * Parses the search query from the incoming queryRequest into Json that the + * search cluster can understand. + * + * @param queryRequest The Json object containing the information on the + * requested query, NOT formatted for the search cluster. + * @param index The index to search. + * @param dimensionPrefix Used to build nested queries for arbitrary fields. + * @param defaultFields Default fields to apply parsed string queries to. + * @throws IcatException If the query cannot be parsed. + */ + public void parseQuery(JsonObject queryRequest, String index, String dimensionPrefix, List defaultFields) + throws IcatException { + // In general, we use a boolean query to compound queries on individual fields + JsonObjectBuilder queryBuilder = Json.createObjectBuilder(); + JsonObjectBuilder boolBuilder = Json.createObjectBuilder(); + + // Non-scored elements are added to the "filter" + JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); + + long lowerTime = Long.MIN_VALUE; + long upperTime = Long.MAX_VALUE; + for (String queryKey : queryRequest.keySet()) { + switch (queryKey) { + case "target": + case "facets": + break; // Avoid using the target index, or facet request as a term in the search + case "lower": + lowerTime = parseDate(queryRequest, "lower", 0, Long.MIN_VALUE); + break; + case "upper": + upperTime = parseDate(queryRequest, "upper", 59999, Long.MAX_VALUE); + break; + case "filter": + parseQueryFilter(queryRequest, index, filterBuilder); + break; + case "text": + parseQueryText(queryRequest, index, defaultFields, boolBuilder); + break; + case "user": + parseQueryUser(queryRequest, filterBuilder); + break; + case "userFullName": + parseQueryUserFullName(queryRequest, filterBuilder); + break; + case "samples": + parseQuerySamples(queryRequest, filterBuilder); + break; + case "parameters": + parseQueryParameters(queryRequest, index, filterBuilder); + break; + default: + parseQueryDefault(queryRequest, dimensionPrefix, filterBuilder, queryKey); + } + } + + if (lowerTime != Long.MIN_VALUE || upperTime != Long.MAX_VALUE) { + if (index.equals("datafile")) { + // datafile has only one date field + filterBuilder.add(buildLongRangeQuery("date", lowerTime, upperTime)); + } else { + filterBuilder.add(buildLongRangeQuery("startDate", lowerTime, upperTime)); + filterBuilder.add(buildLongRangeQuery("endDate", lowerTime, upperTime)); + } + } + + JsonArray filterArray = filterBuilder.build(); + if (filterArray.size() > 0) { + boolBuilder.add("filter", filterArray); + } + builder.add("query", queryBuilder.add("bool", boolBuilder)); + } + + /** + * Parses a generic field name from the queryRequest, and adds them to + * filterBuilder. + * + * @param queryRequest JsonObject with the requested query + * @param dimensionPrefix Used to build nested queries for arbitrary fields + * @param filterBuilder JsonArrayBuilder for adding criteria to filter on + * @param queryKey The key from the queryRequest to be treated as a + * Document field + * @throws IcatException + */ + private void parseQueryDefault(JsonObject queryRequest, String dimensionPrefix, JsonArrayBuilder filterBuilder, + String queryKey) throws IcatException { + // If the term doesn't require special logic, handle according to type + JsonObject defaultTermQuery; + String field = queryKey; + if (dimensionPrefix != null) { + field = dimensionPrefix + "." + field; + } + ValueType valueType = queryRequest.get(queryKey).getValueType(); + switch (valueType) { + case STRING: + defaultTermQuery = buildTermQuery(field + ".keyword", queryRequest.getString(queryKey)); + break; + case NUMBER: + defaultTermQuery = buildTermQuery(field, queryRequest.getJsonNumber(queryKey)); + break; + case ARRAY: + // Only support array of String as list of ICAT ids is currently only use case + defaultTermQuery = buildTermsQuery(field, queryRequest.getJsonArray(queryKey)); + break; + default: + throw new IcatException(IcatExceptionType.BAD_PARAMETER, + "Query values should be ARRAY, STRING or NUMBER, but had value of type " + valueType); + } + if (dimensionPrefix != null) { + // e.g. "sample.id" should use a nested query as sample is nested on other + // entities + filterBuilder.add(buildNestedQuery(dimensionPrefix, defaultTermQuery)); + } else { + // Otherwise, we can associate the query directly with the searched entity + filterBuilder.add(defaultTermQuery); + } + } + + /** + * Parses parameters from the queryRequest, and adds them to filterBuilder. + * + * @param queryRequest JsonObject with the requested query + * @param index The index to search + * @param filterBuilder JsonArrayBuilder for adding criteria to filter on + * @throws IcatException + */ + private void parseQueryParameters(JsonObject queryRequest, String index, JsonArrayBuilder filterBuilder) + throws IcatException { + for (JsonObject parameterObject : queryRequest.getJsonArray("parameters").getValuesAs(JsonObject.class)) { + String path = index + "parameter"; + List parameterQueries = new ArrayList<>(); + if (parameterObject.containsKey("name")) { + String name = parameterObject.getString("name"); + parameterQueries.add(buildMatchQuery(path + ".type.name", name)); + } + if (parameterObject.containsKey("units")) { + String units = parameterObject.getString("units"); + parameterQueries.add(buildMatchQuery(path + ".type.units", units)); + } + if (parameterObject.containsKey("stringValue")) { + String stringValue = parameterObject.getString("stringValue"); + parameterQueries.add(buildMatchQuery(path + ".stringValue", stringValue)); + } else if (parameterObject.containsKey("lowerDateValue") && parameterObject.containsKey("upperDateValue")) { + long lower = parseDate(parameterObject, "lowerDateValue", 0, Long.MIN_VALUE); + long upper = parseDate(parameterObject, "upperDateValue", 59999, Long.MAX_VALUE); + parameterQueries.add(buildLongRangeQuery(path + ".dateTimeValue", lower, upper)); + } else if (parameterObject.containsKey("lowerNumericValue") + && parameterObject.containsKey("upperNumericValue")) { + JsonNumber lower = parameterObject.getJsonNumber("lowerNumericValue"); + JsonNumber upper = parameterObject.getJsonNumber("upperNumericValue"); + parameterQueries.add(buildRangeQuery(path + ".numericValue", lower, upper)); + } + filterBuilder.add(buildNestedQuery(path, parameterQueries.toArray(new JsonObject[0]))); + } + } + + /** + * Parses samples from the queryRequest, and adds them to filterBuilder. + * + * @param queryRequest JsonObject with the requested query + * @param filterBuilder JsonArrayBuilder for adding criteria to filter on + */ + private void parseQuerySamples(JsonObject queryRequest, JsonArrayBuilder filterBuilder) { + JsonArray samples = queryRequest.getJsonArray("samples"); + for (int i = 0; i < samples.size(); i++) { + String sample = samples.getString(i); + JsonObject stringQuery = buildStringQuery(sample, "sample.name", + "sample.type.name"); + filterBuilder.add(buildNestedQuery("sample", stringQuery)); + } + } + + /** + * Parses the userFullName from the queryRequest, and adds it to filterBuilder. + * This uses joins to InvestigationUser and performs a non-exact string match. + * + * @param queryRequest JsonObject with the requested query + * @param filterBuilder JsonArrayBuilder for adding criteria to filter on + * @throws IcatException + */ + private void parseQueryUserFullName(JsonObject queryRequest, JsonArrayBuilder filterBuilder) { + String fullName = queryRequest.getString("userFullName"); + JsonObject fullNameQuery = buildStringQuery(fullName, "investigationuser.user.fullName"); + filterBuilder.add(buildNestedQuery("investigationuser", fullNameQuery)); + } + + /** + * Parses the user from the queryRequest, and adds it to filterBuilder. This + * uses joins to both InvestigationUser and InstrumentScientist entities to + * mimic common ICAT rules that only allow users to see their "own" data by + * using an exact term match. + * + * @param queryRequest JsonObject with the requested query + * @param filterBuilder JsonArrayBuilder for adding criteria to filter on + * @throws IcatException + */ + private void parseQueryUser(JsonObject queryRequest, JsonArrayBuilder filterBuilder) throws IcatException { + String user = queryRequest.getString("user"); + // Because InstrumentScientist is on a separate index, we need to explicitly + // perform a search here + JsonObject termQuery = buildTermQuery("user.name.keyword", user); + String body = Json.createObjectBuilder().add("query", termQuery).build().toString(); + Map parameterMap = new HashMap<>(); + parameterMap.put("_source", "instrument.id"); + JsonObject postResponse = opensearchApi.postResponse("/instrumentscientist/_search", body, parameterMap); + JsonArray hits = postResponse.getJsonObject("hits").getJsonArray("hits"); + JsonArrayBuilder instrumentIdsBuilder = Json.createArrayBuilder(); + for (JsonObject hit : hits.getValuesAs(JsonObject.class)) { + String instrumentId = hit.getJsonObject("_source").getString("instrument.id"); + instrumentIdsBuilder.add(instrumentId); + } + JsonObject instrumentQuery = buildTermsQuery("investigationinstrument.instrument.id", + instrumentIdsBuilder.build()); + JsonObject nestedInstrumentQuery = buildNestedQuery("investigationinstrument", instrumentQuery); + // InvestigationUser should be a nested field on the main Document + JsonObject investigationUserQuery = buildMatchQuery("investigationuser.user.name", user); + JsonObject nestedUserQuery = buildNestedQuery("investigationuser", investigationUserQuery); + // At least one of being an InstrumentScientist or an InvestigationUser is + // necessary + JsonArrayBuilder array = Json.createArrayBuilder().add(nestedInstrumentQuery).add(nestedUserQuery); + filterBuilder.add(Json.createObjectBuilder().add("bool", Json.createObjectBuilder().add("should", array))); + } + + /** + * Parses text for a single field from the queryRequest, and adds it to + * boolBuilder. + * + * @param queryRequest JsonObject with the requested query + * @param index Index (entity) to apply the query to + * @param defaultFields If text does not contain specific field targetting, then + * matches will be attempting against the defaultFields + * @param boolBuilder JsonObjectBuilder for adding criteria to + */ + private void parseQueryText(JsonObject queryRequest, String index, List defaultFields, + JsonObjectBuilder boolBuilder) { + // The free text is the only element we perform scoring on, so "must" occur + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + String text = queryRequest.getString("text"); + arrayBuilder.add(buildStringQuery(text, defaultFields.toArray(new String[0]))); + if (index.equals("investigation")) { + JsonObject stringQuery = buildStringQuery(text, "sample.name", "sample.type.name"); + arrayBuilder.add(buildNestedQuery("sample", stringQuery)); + JsonObjectBuilder textBoolBuilder = Json.createObjectBuilder().add("should", arrayBuilder); + JsonObjectBuilder textMustBuilder = Json.createObjectBuilder().add("bool", textBoolBuilder); + boolBuilder.add("must", Json.createArrayBuilder().add(textMustBuilder)); + } else { + boolBuilder.add("must", arrayBuilder); + } + } + + /** + * Parses a filter for a single field from the queryRequest, and adds it to + * filterBuilder. + * + * @param queryRequest JsonObject with the requested query + * @param index Index (entity) to apply the query to + * @param filterBuilder JsonArrayBuilder for adding criteria to filter on + * @throws IcatException + */ + private void parseQueryFilter(JsonObject queryRequest, String index, JsonArrayBuilder filterBuilder) + throws IcatException { + JsonObject filterObject = queryRequest.getJsonObject("filter"); + for (String fld : filterObject.keySet()) { + JsonValue value = filterObject.get(fld); + String field = fld.replace(index + ".", ""); + if (value.getValueType().equals(ValueType.ARRAY)) { + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + for (JsonValue arrayValue : ((JsonArray) value).getValuesAs(JsonString.class)) { + parseFilter(arrayBuilder, field, arrayValue); + } + // If the key was just a nested entity (no ".") then we should FILTER all of our + // queries on that entity. + String occur = fld.contains(".") ? "should" : "filter"; + filterBuilder.add(Json.createObjectBuilder().add("bool", + Json.createObjectBuilder().add(occur, arrayBuilder))); + } else { + parseFilter(filterBuilder, field, value); + } + } + } + + /** + * Parse sort criteria and add it to the request body. + * + * @param sort String of JsonObject containing the sort criteria. + */ + public void parseSort(String sort) { + if (sort == null || sort.equals("")) { + builder.add("sort", Json.createArrayBuilder() + .add(Json.createObjectBuilder().add("_score", "desc")) + .add(Json.createObjectBuilder().add("id", "asc")).build()); + } else { + JsonObject sortObject = Json.createReader(new StringReader(sort)).readObject(); + JsonArrayBuilder sortArrayBuilder = Json.createArrayBuilder(); + for (String key : sortObject.keySet()) { + if (key.toLowerCase().contains("date") || key.startsWith("file")) { + // Dates and fileSize/fileCount are numeric, so can be used as is + sortArrayBuilder.add(Json.createObjectBuilder().add(key, sortObject.getString(key))); + } else { + // Text fields should use the .keyword field for sorting + sortArrayBuilder.add(Json.createObjectBuilder().add(key + ".keyword", sortObject.getString(key))); + } + } + builder.add("sort", sortArrayBuilder.add(Json.createObjectBuilder().add("id", "asc")).build()); + } + } + + /** + * Add searchAfter to the request body. + * + * @param searchAfter Possibly null JsonValue representing the last document of + * a previous search. + */ + public void parseSearchAfter(JsonValue searchAfter) { + if (searchAfter != null) { + builder.add("search_after", searchAfter); + } + } + + /** + * @return The parsed query, as a String with Json formatting + */ + public String body() { + return builder.build().toString(); + } + +} diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java b/src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java deleted file mode 100644 index 0f262f75..00000000 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchQueryBuilder.java +++ /dev/null @@ -1,213 +0,0 @@ -package org.icatproject.core.manager.search; - -import java.util.List; - -import javax.json.Json; -import javax.json.JsonArray; -import javax.json.JsonArrayBuilder; -import javax.json.JsonNumber; -import javax.json.JsonObject; -import javax.json.JsonObjectBuilder; - -/** - * Utility for building queries in Json understood by Opensearch. - */ -public class OpensearchQueryBuilder { - - private static JsonObject matchAllQuery = Json.createObjectBuilder().add("match_all", Json.createObjectBuilder()) - .build(); - - /** - * @param query JsonObject representing an Opensearch query. - * @return JsonObjectBuilder with JsonObject {"query": {...query}} - */ - public static JsonObjectBuilder addQuery(JsonObject query) { - return Json.createObjectBuilder().add("query", query); - } - - /** - * @param filter Path to nested Object. - * @param should Any number of pre-built queries. - * @return {"bool": {"filter": [...filter], "should": [...should]}} - */ - public static JsonObject buildBoolQuery(List filter, List should) { - JsonObjectBuilder boolBuilder = Json.createObjectBuilder(); - buildBoolArray("should", should, boolBuilder); - buildBoolArray("filter", filter, boolBuilder); - return Json.createObjectBuilder().add("bool", boolBuilder).build(); - } - - /** - * @param occur String of an occurance keyword ("filter", "should", "must" etc.) - * @param queries List of JsonObjects representing the queries to occur. - * @param boolBuilder Builder of the main boolean query. - */ - private static void buildBoolArray(String occur, List queries, JsonObjectBuilder boolBuilder) { - if (queries != null && queries.size() > 0) { - JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); - for (JsonObject queryObject : queries) { - filterBuilder.add(queryObject); - } - boolBuilder.add(occur, filterBuilder); - } - } - - /** - * @return {"match_all": {}} - */ - public static JsonObject buildMatchAllQuery() { - return matchAllQuery; - } - - /** - * @param field Field containing the match. - * @param value Value to match. - * @return {"match": {"`field`.keyword": {"query": `value`, "operator": "and"}}} - */ - public static JsonObject buildMatchQuery(String field, String value) { - JsonObjectBuilder fieldBuilder = Json.createObjectBuilder().add("query", value).add("operator", "and"); - JsonObjectBuilder matchBuilder = Json.createObjectBuilder().add(field + ".keyword", fieldBuilder); - return Json.createObjectBuilder().add("match", matchBuilder).build(); - } - - /** - * @param path Path to nested Object. - * @param queryObjects Any number of pre-built queries. - * @return {"nested": {"path": `path`, "query": {"bool": {"filter": [...queryObjects]}}}} - */ - public static JsonObject buildNestedQuery(String path, JsonObject... queryObjects) { - JsonObject builtQueries = null; - if (queryObjects.length == 0) { - builtQueries = matchAllQuery; - } else if (queryObjects.length == 1) { - builtQueries = queryObjects[0]; - } else { - JsonArrayBuilder filterBuilder = Json.createArrayBuilder(); - for (JsonObject queryObject : queryObjects) { - filterBuilder.add(queryObject); - } - JsonObjectBuilder boolBuilder = Json.createObjectBuilder().add("filter", filterBuilder); - builtQueries = Json.createObjectBuilder().add("bool", boolBuilder).build(); - } - JsonObjectBuilder nestedBuilder = Json.createObjectBuilder().add("path", path).add("query", builtQueries); - return Json.createObjectBuilder().add("nested", nestedBuilder).build(); - } - - /** - * @param value String value to query for. - * @param fields List of fields to check for value. - * @return {"query_string": {"query": `value`, "fields": [...fields]}} - */ - public static JsonObject buildStringQuery(String value, String... fields) { - JsonObjectBuilder queryStringBuilder = Json.createObjectBuilder().add("query", value); - if (fields.length > 0) { - JsonArrayBuilder fieldsBuilder = Json.createArrayBuilder(); - for (String field : fields) { - fieldsBuilder.add(field); - } - queryStringBuilder.add("fields", fieldsBuilder); - } - return Json.createObjectBuilder().add("query_string", queryStringBuilder).build(); - } - - /** - * @param field Field containing the term. - * @param value Term to match. - * @return {"term": {`field`: `value`}} - */ - public static JsonObject buildTermQuery(String field, String value) { - return Json.createObjectBuilder().add("term", Json.createObjectBuilder().add(field, value)).build(); - } - - /** - * @param field Field containing the number. - * @param value Number to match. - * @return {"term": {`field`: `value`}} - */ - public static JsonObject buildTermQuery(String field, JsonNumber value) { - return Json.createObjectBuilder().add("term", Json.createObjectBuilder().add(field, value)).build(); - } - - /** - * @param field Field containing the double value. - * @param value Double to match. - * @return {"term": {`field`: `value`}} - */ - public static JsonObject buildTermQuery(String field, double value) { - return Json.createObjectBuilder().add("term", Json.createObjectBuilder().add(field, value)).build(); - } - - /** - * @param field Field containing on of the terms. - * @param values JsonArray of possible terms. - * @return {"terms": {`field`: `values`}} - */ - public static JsonObject buildTermsQuery(String field, JsonArray values) { - return Json.createObjectBuilder().add("terms", Json.createObjectBuilder().add(field, values)).build(); - } - - /** - * @param field Field to apply the range to. - * @param lowerValue Lowest allowed value in the range. - * @param upperValue Highest allowed value in the range. - * @return {"range": {`field`: {"gte": `upperValue`, "lte": `lowerValue`}}} - */ - public static JsonObject buildDoubleRangeQuery(String field, Double lowerValue, Double upperValue) { - JsonObjectBuilder fieldBuilder = Json.createObjectBuilder(); - if (lowerValue != null) fieldBuilder.add("gte", lowerValue); - if (upperValue != null) fieldBuilder.add("lte", upperValue); - JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); - return Json.createObjectBuilder().add("range", rangeBuilder).build(); - } - - /** - * @param field Field to apply the range to. - * @param lowerValue Lowest allowed value in the range. - * @param upperValue Highest allowed value in the range. - * @return {"range": {`field`: {"gte": `upperValue`, "lte": `lowerValue`}}} - */ - public static JsonObject buildLongRangeQuery(String field, Long lowerValue, Long upperValue) { - JsonObjectBuilder fieldBuilder = Json.createObjectBuilder(); - if (lowerValue != null) fieldBuilder.add("gte", lowerValue); - if (upperValue != null) fieldBuilder.add("lte", upperValue); - JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); - return Json.createObjectBuilder().add("range", rangeBuilder).build(); - } - - /** - * @param field Field to apply the range to. - * @param lowerValue Lowest allowed value in the range. - * @param upperValue Highest allowed value in the range. - * @return {"range": {`field`: {"gte": `upperValue`, "lte": `lowerValue`}}} - */ - public static JsonObject buildRangeQuery(String field, JsonNumber lowerValue, JsonNumber upperValue) { - JsonObjectBuilder fieldBuilder = Json.createObjectBuilder(); - if (lowerValue != null) fieldBuilder.add("gte", lowerValue); - if (upperValue != null) fieldBuilder.add("lte", upperValue); - JsonObjectBuilder rangeBuilder = Json.createObjectBuilder().add(field, fieldBuilder); - return Json.createObjectBuilder().add("range", rangeBuilder).build(); - } - - /** - * @param field Field to facet. - * @param ranges JsonArray of ranges to allocate documents to. - * @return {"range": {"field": `field`, "keyed": true, "ranges": `ranges`}} - */ - public static JsonObject buildRangeFacet(String field, JsonArray ranges) { - JsonObjectBuilder rangeBuilder = Json.createObjectBuilder(); - rangeBuilder.add("field", field).add("keyed", true).add("ranges", ranges); - return Json.createObjectBuilder().add("range", rangeBuilder).build(); - } - - /** - * @param field Field to facet. - * @param maxLabels Maximum number of labels per dimension. - * @return {"terms": {"field": `field`, "size": `maxLabels`}} - */ - public static JsonObject buildStringFacet(String field, int maxLabels) { - JsonObjectBuilder termsBuilder = Json.createObjectBuilder(); - termsBuilder.add("field", field).add("size", maxLabels); - return Json.createObjectBuilder().add("terms", termsBuilder).build(); - } - -} diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchScriptBuilder.java b/src/main/java/org/icatproject/core/manager/search/OpensearchScriptBuilder.java index 2e8468fa..bde81a13 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchScriptBuilder.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchScriptBuilder.java @@ -168,36 +168,32 @@ public static String buildFileSizeScript() { * nested entity will be removed from the array. * @return */ - public static String buildParameterTypeScript(Set docFields, boolean update) { - String source = findNestedChild("investigationparameter", true); - String ctxSource = "ctx._source.investigationparameter.get(childIndex)"; - if (docFields != null) { - source += "if (childIndex != -1) { "; - for (String field : docFields) { - source += updateField(field, ctxSource, update); - } - source += " } "; - } - source += findNestedChild("datasetparameter", false); - ctxSource = "ctx._source.datasetparameter.get(childIndex)"; - if (docFields != null) { - source += "if (childIndex != -1) { "; - for (String field : docFields) { - source += updateField(field, ctxSource, update); - } - source += " } "; - } - source += findNestedChild("datafileparameter", false); - ctxSource = "ctx._source.datafileparameter.get(childIndex)"; - if (docFields != null) { - source += "if (childIndex != -1) { "; - for (String field : docFields) { - source += updateField(field, ctxSource, update); - } - source += " } "; - } - source += findNestedChild("sampleparameter", false); - ctxSource = "ctx._source.sampleparameter.get(childIndex)"; + public static String buildParameterTypesScript(Set docFields, boolean update) { + String source = buildParameterTypeScript(docFields, update, "investigationparameter", true); + source += buildParameterTypeScript(docFields, update, "datasetparameter", false); + source += buildParameterTypeScript(docFields, update, "datafileparameter", false); + source += buildParameterTypeScript(docFields, update, "sampleparameter", false); + return buildScript(source); + } + + /** + * Modifies a single type of Parameter (Investigation, Dataset, Datafile, + * Sample) with changes to a ParameterType. + * + * @param update If true the script will replace a nested entity, else + * the nested entity will be removed from the array + * @param nestedChildName Name of the Parameter entity to modify + * @param declareChildId Whether the childId needs to be declared. This should + * only be true for the first parameter in the script. + * @param fields The fields belonging to the ParameterType to be + * modified + * + * @return The script to modify the Parameter as a String + */ + private static String buildParameterTypeScript(Set docFields, boolean update, String nestedChildName, + boolean declareChildId) { + String ctxSource = "ctx._source." + nestedChildName + ".get(childIndex)"; + String source = findNestedChild(nestedChildName, declareChildId); if (docFields != null) { source += "if (childIndex != -1) { "; for (String field : docFields) { @@ -205,6 +201,6 @@ public static String buildParameterTypeScript(Set docFields, boolean upd } source += " } "; } - return buildScript(source); + return source; } } diff --git a/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java b/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java index 6485e97b..ad25b566 100644 --- a/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java +++ b/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java @@ -36,10 +36,10 @@ public class ScoredEntityBaseBean { * a key-value pair in the source JsonObject. */ public ScoredEntityBaseBean(int engineDocId, int shardIndex, float score, JsonObject source) throws IcatException { - if (!source.keySet().contains("id")) { + if (!source.containsKey("id")) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, "Document source must have 'id' and the entityBaseBeanId as a key-value pair, but it was " - + source.toString()); + + source); } this.engineDocId = engineDocId; this.shardIndex = shardIndex; diff --git a/src/main/java/org/icatproject/core/manager/search/SearchApi.java b/src/main/java/org/icatproject/core/manager/search/SearchApi.java index 467ee0e8..1a8ec02e 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchApi.java @@ -26,6 +26,7 @@ import javax.json.stream.JsonGenerator; import javax.persistence.EntityManager; +import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.utils.URIBuilder; @@ -139,7 +140,7 @@ public static void encodeLong(JsonGenerator gen, String name, Long value) { public static String encodeOperation(String operation, EntityBaseBean bean) throws IcatException { Long icatId = bean.getId(); if (icatId == null) { - throw new IcatException(IcatExceptionType.BAD_PARAMETER, bean.toString() + " had null id"); + throw new IcatException(IcatExceptionType.BAD_PARAMETER, bean + " had null id"); } String entityName = bean.getClass().getSimpleName(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -433,16 +434,7 @@ public void unlock(String entityName) throws IcatException { * @throws IcatException */ protected void post(String path) throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(path).build(); - HttpPost httpPost = new HttpPost(uri); - logger.trace("Making call {}", uri); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } catch (URISyntaxException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } + postResponse(path, null, null); } /** @@ -453,17 +445,7 @@ protected void post(String path) throws IcatException { * @throws IcatException */ protected void post(String path, String body) throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(path).build(); - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); - logger.trace("Making call {} with body {}", uri, body); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - } - } catch (URISyntaxException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } + postResponse(path, body, null); } /** @@ -475,19 +457,7 @@ protected void post(String path, String body) throws IcatException { * @throws IcatException */ protected JsonObject postResponse(String path, String body) throws IcatException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - URI uri = new URIBuilder(server).setPath(path).build(); - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); - logger.trace("Making call {} with body {}", uri, body); - try (CloseableHttpResponse response = httpclient.execute(httpPost)) { - Rest.checkStatus(response, IcatExceptionType.INTERNAL); - JsonReader jsonReader = Json.createReader(response.getEntity().getContent()); - return jsonReader.readObject(); - } - } catch (URISyntaxException | IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } + return postResponse(path, body, null); } @@ -503,18 +473,26 @@ protected JsonObject postResponse(String path, String body) throws IcatException protected JsonObject postResponse(String path, String body, Map parameterMap) throws IcatException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { URIBuilder builder = new URIBuilder(server).setPath(path); - for (Entry entry : parameterMap.entrySet()) { - builder.addParameter(entry.getKey(), entry.getValue()); + if (parameterMap != null) { + for (Entry entry : parameterMap.entrySet()) { + builder.addParameter(entry.getKey(), entry.getValue()); + } } URI uri = builder.build(); HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + if (body != null) { + httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + } logger.trace("Making call {} with body {}", uri, body); try (CloseableHttpResponse response = httpclient.execute(httpPost)) { int code = response.getStatusLine().getStatusCode(); Rest.checkStatus(response, code == 400 ? IcatExceptionType.BAD_PARAMETER : IcatExceptionType.INTERNAL); - JsonReader jsonReader = Json.createReader(response.getEntity().getContent()); - return jsonReader.readObject(); + HttpEntity entity = response.getEntity(); + if (entity != null) { + JsonReader jsonReader = Json.createReader(entity.getContent()); + return jsonReader.readObject(); + } + return null; } } catch (URISyntaxException | IOException e) { throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); diff --git a/src/main/java/org/icatproject/core/manager/search/SearchManager.java b/src/main/java/org/icatproject/core/manager/search/SearchManager.java index bbc60f2b..a3a9c2d1 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchManager.java @@ -466,6 +466,7 @@ private void enqueue(String json) throws IcatException { private void synchronizedWrite(String line, Long fileLock, File file) throws IcatException { synchronized (fileLock) { try { + logger.trace("Writing {} to {}", line, file.getAbsolutePath()); FileWriter output = new FileWriter(file, true); output.write(line + "\n"); output.close(); diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 54d06ec4..5f8c162d 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -285,8 +285,8 @@ private void checkFacets(List facetDimensions, FacetDimension... FacetLabel expectedLabel = expectedLabels.get(j); FacetLabel actualLabel = actualLabels.get(j); String label = expectedLabel.getLabel(); - Long expectedValue = expectedLabel.getValue(); - Long actualValue = actualLabel.getValue(); + long expectedValue = expectedLabel.getValue(); + long actualValue = actualLabel.getValue(); assertEquals(label, actualLabel.getLabel()); message = "Label <" + label + ">: "; assertEquals(message, expectedValue, actualValue); @@ -345,8 +345,8 @@ private void checkOrder(SearchResult lsr, Long... n) { checkResults(lsr, n); } for (int i = 0; i < n.length; i++) { - Long resultId = results.get(i).getEntityBaseBeanId(); - Long expectedId = (Long) Array.get(n, i); + long resultId = results.get(i).getEntityBaseBeanId(); + long expectedId = (long) Array.get(n, i); if (resultId != expectedId) { fail("Expected id " + expectedId + " in position " + i + " but got " + resultId); } @@ -521,12 +521,12 @@ private void populateParameters(List queue, int i, EntityBaseBean parent */ private void populate() throws IcatException { List queue = new ArrayList<>(); - Long investigationUserId = 0L; + long investigationUserId = 0; - Instrument instrumentZero = populateInstrument(queue, 0L); - Instrument instrumentOne = populateInstrument(queue, 1L); - Technique techniqueZero = populateTechnique(queue, 0L); - Technique techniqueOne = populateTechnique(queue, 1L); + Instrument instrumentZero = populateInstrument(queue, 0); + Instrument instrumentOne = populateInstrument(queue, 1); + Technique techniqueZero = populateTechnique(queue, 0); + Technique techniqueOne = populateTechnique(queue, 1); for (int investigationId = 0; investigationId < NUMINV; investigationId++) { String word = word(investigationId % 26, (investigationId + 7) % 26, (investigationId + 17) % 26); @@ -668,14 +668,13 @@ private Technique populateTechnique(List queue, long techniqueId) throws * @return The DatasetTechnique entity created. * @throws IcatException */ - private DatasetTechnique populateDatasetTechnique(List queue, Technique technique, Dataset dataset) + private void populateDatasetTechnique(List queue, Technique technique, Dataset dataset) throws IcatException { DatasetTechnique datasetTechnique = new DatasetTechnique(); datasetTechnique.setId(technique.getId() * 100 + dataset.getId()); datasetTechnique.setTechnique(technique); datasetTechnique.setDataset(dataset); queue.add(SearchApi.encodeOperation("create", datasetTechnique)); - return datasetTechnique; } private String word(int j, int k, int l) { diff --git a/src/test/java/org/icatproject/integration/TestWS.java b/src/test/java/org/icatproject/integration/TestWS.java index 97cc7369..70e05686 100644 --- a/src/test/java/org/icatproject/integration/TestWS.java +++ b/src/test/java/org/icatproject/integration/TestWS.java @@ -71,7 +71,7 @@ */ public class TestWS { - private static final String version = "5.0."; + private static final String version = "5.1."; private static Random random; private static WSession session; From e91214fb09337f4628984a55904adddb4e1eeb1a Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 21 Oct 2022 11:54:58 +0100 Subject: [PATCH 39/51] Update OpensearchQuery docstrings after refactor #267 --- .../core/manager/search/OpensearchApi.java | 2 +- .../core/manager/search/OpensearchQuery.java | 19 ++++--------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index 757dcae0..848eebb5 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -987,7 +987,7 @@ private void createNestedEntity(OpensearchBulk bulk, String id, String index, Js if (!document.containsKey(relation.joinField + ".id")) { throw new IcatException(IcatExceptionType.BAD_PARAMETER, - relation.joinField + ".id not found in " + document.toString()); + relation.joinField + ".id not found in " + document); } String parentId = document.getString(relation.joinField + ".id"); diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchQuery.java b/src/main/java/org/icatproject/core/manager/search/OpensearchQuery.java index be3296f2..48c4d00c 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchQuery.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchQuery.java @@ -229,8 +229,8 @@ public static JsonObject buildStringFacet(String field, int maxLabels) { } /** - * @param key Arbitrary key - * @param value Arbitrary JsonObjectBuilder + * @param key Arbitrary key + * @param builder Arbitrary JsonObjectBuilder * @return {`key`: `builder`}} */ private static JsonObject build(String key, JsonObjectBuilder builder) { @@ -288,8 +288,6 @@ private static long parseDate(JsonObject jsonObject, String key, int offset, lon * Parses incoming Json encoding the requested facets and uses bodyBuilder to * construct Json that can be understood by Opensearch. * - * @param bodyBuilder JsonObjectBuilder being used to build the body of the - * request. * @param dimensions JsonArray of JsonObjects representing dimensions to be * faceted. * @param maxLabels The maximum number of labels to collect for each @@ -298,10 +296,8 @@ private static long parseDate(JsonObject jsonObject, String key, int offset, lon * is needed to distinguish between potentially ambiguous * dimensions, such as "(investigation.)type.name" and * "(investigationparameter.)type.name". - * @return The bodyBuilder originally passed with facet information added to it. */ - public void parseFacets(JsonArray dimensions, int maxLabels, - String dimensionPrefix) { + public void parseFacets(JsonArray dimensions, int maxLabels, String dimensionPrefix) { JsonObjectBuilder aggsBuilder = Json.createObjectBuilder(); for (JsonObject dimensionObject : dimensions.getValuesAs(JsonObject.class)) { String dimensionString = dimensionObject.getString("dimension"); @@ -320,8 +316,6 @@ public void parseFacets(JsonArray dimensions, int maxLabels, /** * Uses bodyBuilder to construct Json for faceting string fields. * - * @param bodyBuilder JsonObjectBuilder being used to build the body of the - * request. * @param dimensions List of dimensions to perform string based faceting * on. * @param maxLabels The maximum number of labels to collect for each @@ -330,10 +324,8 @@ public void parseFacets(JsonArray dimensions, int maxLabels, * is needed to distinguish between potentially ambiguous * dimensions, such as "(investigation.)type.name" and * "(investigationparameter.)type.name". - * @return The bodyBuilder originally passed with facet information added to it. */ - public void parseFacets(List dimensions, int maxLabels, - String dimensionPrefix) { + public void parseFacets(List dimensions, int maxLabels, String dimensionPrefix) { JsonObjectBuilder aggsBuilder = Json.createObjectBuilder(); for (String dimensionString : dimensions) { String field = dimensionPrefix == null ? dimensionString : dimensionPrefix + "." + dimensionString; @@ -346,14 +338,11 @@ public void parseFacets(List dimensions, int maxLabels, * Finalises the construction of faceting Json by handling the possibility of * faceting a nested object. * - * @param bodyBuilder JsonObjectBuilder being used to build the body of the - * request. * @param dimensionPrefix Optional prefix to apply to the dimension names. This * is needed to distinguish between potentially ambiguous * dimensions, such as "(investigation.)type.name" and * "(investigationparameter.)type.name". * @param aggsBuilder JsonObjectBuilder that has the faceting details. - * @return The bodyBuilder originally passed with facet information added to it. */ private void buildFacetRequestJson(String dimensionPrefix, JsonObjectBuilder aggsBuilder) { if (dimensionPrefix == null) { From 03213aaac476a860c38e813c0cb678ac49d57092 Mon Sep 17 00:00:00 2001 From: patrick-austin <61705287+patrick-austin@users.noreply.github.com> Date: Mon, 24 Oct 2022 17:26:25 +0100 Subject: [PATCH 40/51] Apply suggestions from code review Co-authored-by: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> --- src/test/java/org/icatproject/integration/TestRS.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 0350ff43..42f1ea73 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -95,8 +95,7 @@ private Session rootSession() throws URISyntaxException, IcatException { Map credentials = new HashMap<>(); credentials.put("username", "root"); credentials.put("password", "password"); - Session session = icat.login("db", credentials); - return session; + return icat.login("db", credentials); } private Session piOneSession() throws URISyntaxException, IcatException { @@ -104,8 +103,7 @@ private Session piOneSession() throws URISyntaxException, IcatException { Map credentials = new HashMap<>(); credentials.put("username", "piOne"); credentials.put("password", "piOne"); - Session session = icat.login("db", credentials); - return session; + return icat.login("db", credentials); } @Ignore("Test fails because of bug in eclipselink") From 906ba63b943fb0ffe8a9480168a79f13278cfee7 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Mon, 23 Jan 2023 16:20:05 +0000 Subject: [PATCH 41/51] InvestigationFacilityCycle search support --- .../org/icatproject/core/entity/Datafile.java | 5 ++ .../org/icatproject/core/entity/Dataset.java | 5 ++ .../core/entity/Investigation.java | 3 + .../entity/InvestigationFacilityCycle.java | 11 +++- .../core/manager/search/OpensearchApi.java | 12 +++- .../core/manager/TestEntityInfo.java | 6 +- .../core/manager/TestSearchApi.java | 57 ++++++++++++++----- 7 files changed, 79 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index 9d744288..81585276 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -271,6 +271,10 @@ public static Map getDocumentFields() throws IcatExcepti Relationship[] investigationRelationships = { eiHandler.getRelationshipsByName(Datafile.class).get("dataset"), eiHandler.getRelationshipsByName(Dataset.class).get("investigation") }; + Relationship[] investigationFacilityCyclesRelationships = { + eiHandler.getRelationshipsByName(Datafile.class).get("dataset"), + eiHandler.getRelationshipsByName(Dataset.class).get("investigation"), + eiHandler.getRelationshipsByName(Investigation.class).get("investigationFacilityCycles") }; Relationship[] instrumentRelationships = { eiHandler.getRelationshipsByName(Datafile.class).get("dataset"), eiHandler.getRelationshipsByName(Dataset.class).get("investigation"), @@ -304,6 +308,7 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("visitId", investigationRelationships); documentFields.put("datafileFormat.id", null); documentFields.put("datafileFormat.name", datafileFormatRelationships); + documentFields.put("InvestigationFacilityCycle facilityCycle.id", investigationFacilityCyclesRelationships); documentFields.put("InvestigationInstrument instrument.id", instrumentRelationships); } return documentFields; diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index c31f6a89..ade30dd3 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -294,7 +294,11 @@ public static Map getDocumentFields() throws IcatExcepti Relationship[] typeRelationships = { eiHandler.getRelationshipsByName(Dataset.class).get("type") }; Relationship[] investigationRelationships = { eiHandler.getRelationshipsByName(Dataset.class).get("investigation") }; + Relationship[] investigationFacilityCyclesRelationships = { + eiHandler.getRelationshipsByName(Dataset.class).get("investigation"), + eiHandler.getRelationshipsByName(Investigation.class).get("investigationFacilityCycles") }; Relationship[] instrumentRelationships = { + eiHandler.getRelationshipsByName(Dataset.class).get("investigation"), eiHandler.getRelationshipsByName(Investigation.class).get("investigationInstruments"), eiHandler.getRelationshipsByName(InvestigationInstrument.class).get("instrument") }; documentFields.put("name", null); @@ -318,6 +322,7 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("sample.type.name", sampleTypeRelationships); documentFields.put("type.id", null); documentFields.put("type.name", typeRelationships); + documentFields.put("InvestigationFacilityCycle facilityCycle.id", investigationFacilityCyclesRelationships); documentFields.put("InvestigationInstrument instrument.id", instrumentRelationships); } return documentFields; diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index b02168aa..7fbe335f 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -369,6 +369,8 @@ public static Map getDocumentFields() throws IcatExcepti Relationship[] typeRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("type") }; Relationship[] facilityRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("facility") }; + Relationship[] investigationFacilityCyclesRelationships = { + eiHandler.getRelationshipsByName(Investigation.class).get("investigationFacilityCycles") }; Relationship[] instrumentRelationships = { eiHandler.getRelationshipsByName(Investigation.class).get("investigationInstruments"), eiHandler.getRelationshipsByName(InvestigationInstrument.class).get("instrument") }; @@ -397,6 +399,7 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("facility.id", null); documentFields.put("type.name", typeRelationships); documentFields.put("type.id", null); + documentFields.put("InvestigationFacilityCycle facilityCycle.id", investigationFacilityCyclesRelationships); documentFields.put("InvestigationInstrument instrument.fullName", instrumentRelationships); documentFields.put("InvestigationInstrument instrument.id", instrumentRelationships); documentFields.put("InvestigationInstrument instrument.name", instrumentRelationships); diff --git a/src/main/java/org/icatproject/core/entity/InvestigationFacilityCycle.java b/src/main/java/org/icatproject/core/entity/InvestigationFacilityCycle.java index edc75c94..980b81af 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationFacilityCycle.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationFacilityCycle.java @@ -2,12 +2,15 @@ import java.io.Serializable; +import javax.json.stream.JsonGenerator; import javax.persistence.Entity; import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.icatproject.core.manager.search.SearchApi; + @Comment("Many to many relationship between investigation and facilityCycle. " + "Allows investigations to belong to multiple cycles at once.") @SuppressWarnings("serial") @@ -15,7 +18,6 @@ @Table(uniqueConstraints = { @UniqueConstraint(columnNames = { "FACILITYCYCLE_ID", "INVESTIGATION_ID" }) }) public class InvestigationFacilityCycle extends EntityBaseBean implements Serializable { - @JoinColumn(name = "FACILITYCYCLE_ID", nullable = false) @ManyToOne private FacilityCycle facilityCycle; @@ -44,4 +46,11 @@ public void setInvestigation(Investigation investigation) { this.investigation = investigation; } + @Override + public void getDoc(JsonGenerator gen) { + SearchApi.encodeString(gen, "facilityCycle.id", facilityCycle.id); + SearchApi.encodeString(gen, "investigation.id", investigation.id); + SearchApi.encodeString(gen, "id", id); + } + } diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index 848eebb5..4789b251 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -164,6 +164,10 @@ public ParentRelation(RelationType relationType, String parentName, String joinF new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null), new ParentRelation(RelationType.NESTED_CHILD, "dataset", "investigation", null), new ParentRelation(RelationType.NESTED_CHILD, "datafile", "investigation", null))); + relations.put("investigationfacilitycycle", Arrays.asList( + new ParentRelation(RelationType.NESTED_CHILD, "investigation", "investigation", null), + new ParentRelation(RelationType.NESTED_CHILD, "dataset", "investigation", null), + new ParentRelation(RelationType.NESTED_CHILD, "datafile", "investigation", null))); // Grandchildren are entities that are related to one of the nested // children, but do not have a direct reference to one of the indexed entities, @@ -244,7 +248,8 @@ private static JsonObject buildMappings(String index) { .add("sampleparameter", buildNestedMapping("sample.id", "type.id")) .add("investigationparameter", buildNestedMapping("investigation.id", "type.id")) .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) - .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")); + .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")) + .add("investigationfacilitycycle", buildNestedMapping("investigation.id", "facilityCycle.id")); break; case "dataset": @@ -260,6 +265,7 @@ private static JsonObject buildMappings(String index) { .add("datasettechnique", buildNestedMapping("dataset.id", "technique.id")) .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")) + .add("investigationfacilitycycle", buildNestedMapping("investigation.id", "facilityCycle.id")) .add("sampleparameter", buildNestedMapping("sample.id", "type.id")); break; @@ -274,6 +280,7 @@ private static JsonObject buildMappings(String index) { .add("datafileparameter", buildNestedMapping("datafile.id", "type.id")) .add("investigationuser", buildNestedMapping("investigation.id", "user.id")) .add("investigationinstrument", buildNestedMapping("investigation.id", "instrument.id")) + .add("investigationfacilitycycle", buildNestedMapping("investigation.id", "facilityCycle.id")) .add("sampleparameter", buildNestedMapping("sample.id", "type.id")); break; @@ -817,6 +824,9 @@ private void extractFromInvestigation(CloseableHttpClient httpclient, String inv if (responseObject.containsKey("investigationinstrument")) { extractEntity(httpclient, investigationId, responseObject, "investigationinstrument", false); } + if (responseObject.containsKey("investigationfacilitycycle")) { + extractEntity(httpclient, investigationId, responseObject, "investigationfacilitycycle", false); + } if (responseObject.containsKey("sample")) { extractEntity(httpclient, investigationId, responseObject, "sample", true); } diff --git a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java index 02815783..873557c0 100644 --- a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java +++ b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java @@ -48,9 +48,9 @@ public void testBadname() throws Exception { public void testHasSearchDoc() throws Exception { Set docdbeans = new HashSet<>(Arrays.asList("Datafile", "DatafileFormat", "DatafileParameter", "Dataset", "DatasetParameter", "DatasetTechnique", "DatasetType", "Facility", "Instrument", - "InstrumentScientist", "Investigation", "InvestigationInstrument", "InvestigationParameter", - "InvestigationType", "InvestigationUser", "ParameterType", "Sample", "SampleType", "SampleParameter", - "Technique", "User")); + "InstrumentScientist", "Investigation", "InvestigationFacilityCycle", "InvestigationInstrument", + "InvestigationParameter", "InvestigationType", "InvestigationUser", "ParameterType", "Sample", + "SampleType", "SampleParameter", "Technique", "User")); for (String beanName : EntityInfoHandler.getEntityNamesList()) { @SuppressWarnings("unchecked") Class bean = (Class) Class diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 5f8c162d..5f4580f0 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -33,9 +33,11 @@ import org.icatproject.core.entity.DatasetType; import org.icatproject.core.entity.EntityBaseBean; import org.icatproject.core.entity.Facility; +import org.icatproject.core.entity.FacilityCycle; import org.icatproject.core.entity.Instrument; import org.icatproject.core.entity.InstrumentScientist; import org.icatproject.core.entity.Investigation; +import org.icatproject.core.entity.InvestigationFacilityCycle; import org.icatproject.core.entity.InvestigationInstrument; import org.icatproject.core.entity.InvestigationParameter; import org.icatproject.core.entity.InvestigationType; @@ -95,13 +97,26 @@ public Filter(String fld, JsonObject... values) { private static final String SEARCH_AFTER_NOT_NULL = "Expected searchAfter to be set, but it was null"; private static final List datafileFields = Arrays.asList("id", "name", "location", "date", "dataset.id", - "dataset.name", "investigation.id", "investigation.name", "InvestigationInstrument instrument.id"); + "dataset.name", "investigation.id", "investigation.name", "InvestigationInstrument instrument.id", + "InvestigationFacilityCycle facilityCycle.id"); private static final List datasetFields = Arrays.asList("id", "name", "startDate", "endDate", "investigation.id", "investigation.name", "investigation.title", "investigation.startDate", - "InvestigationInstrument instrument.id"); + "InvestigationInstrument instrument.id", "InvestigationFacilityCycle facilityCycle.id"); private static final List investigationFields = Arrays.asList("id", "name", "title", "startDate", "endDate", "InvestigationInstrument instrument.id", "InvestigationInstrument instrument.name", - "InvestigationInstrument instrument.fullName"); + "InvestigationInstrument instrument.fullName", "InvestigationFacilityCycle facilityCycle.id"); + + private static Facility facility = new Facility(); + private static FacilityCycle facilityCycle = new FacilityCycle(); + private static InvestigationType investigationType = new InvestigationType(); + static { + facility.setName("facility"); + facility.setId(0L); + facilityCycle.setFacility(facility); + facilityCycle.setId(0L); + investigationType.setName("type"); + investigationType.setId(0L); + } final static Logger logger = LoggerFactory.getLogger(TestSearchApi.class); @@ -237,7 +252,8 @@ private void checkDatafile(ScoredEntityBaseBean datafile) { JsonObject source = datafile.getSource(); assertNotNull(source); Set expectedKeys = new HashSet<>(Arrays.asList("id", "name", "location", "date", "dataset.id", - "dataset.name", "investigation.id", "investigation.name", "investigationinstrument")); + "dataset.name", "investigation.id", "investigation.name", "investigationinstrument", + "investigationfacilitycycle")); assertEquals(expectedKeys, source.keySet()); assertEquals("0", source.getString("id")); assertEquals("DFaaa", source.getString("name")); @@ -250,13 +266,17 @@ private void checkDatafile(ScoredEntityBaseBean datafile) { JsonArray instruments = source.getJsonArray("investigationinstrument"); assertEquals(1, instruments.size()); assertEquals("0", instruments.getJsonObject(0).getString("instrument.id")); + JsonArray facilityCycles = source.getJsonArray("investigationfacilitycycle"); + assertEquals(1, facilityCycles.size()); + assertEquals("0", facilityCycles.getJsonObject(0).getString("facilityCycle.id")); } private void checkDataset(ScoredEntityBaseBean dataset) { JsonObject source = dataset.getSource(); assertNotNull(source); Set expectedKeys = new HashSet<>(Arrays.asList("id", "name", "startDate", "endDate", "investigation.id", - "investigation.name", "investigation.title", "investigation.startDate", "investigationinstrument")); + "investigation.name", "investigation.title", "investigation.startDate", "investigationinstrument", + "investigationfacilitycycle")); assertEquals(expectedKeys, source.keySet()); assertEquals("0", source.getString("id")); assertEquals("DSaaa", source.getString("name")); @@ -269,6 +289,9 @@ private void checkDataset(ScoredEntityBaseBean dataset) { JsonArray instruments = source.getJsonArray("investigationinstrument"); assertEquals(1, instruments.size()); assertEquals("0", instruments.getJsonObject(0).getString("instrument.id")); + JsonArray facilityCycles = source.getJsonArray("investigationfacilitycycle"); + assertEquals(1, facilityCycles.size()); + assertEquals("0", facilityCycles.getJsonObject(0).getString("facilityCycle.id")); } private void checkFacets(List facetDimensions, FacetDimension... dimensions) { @@ -297,8 +320,9 @@ private void checkFacets(List facetDimensions, FacetDimension... private void checkInvestigation(ScoredEntityBaseBean investigation) { JsonObject source = investigation.getSource(); assertNotNull(source); - Set expectedKeys = new HashSet<>( - Arrays.asList("id", "name", "title", "startDate", "endDate", "investigationinstrument")); + Set expectedKeys = new HashSet<>(Arrays.asList( + "id", "name", "title", "startDate", "endDate", "investigationinstrument", + "investigationfacilitycycle")); assertEquals(expectedKeys, source.keySet()); assertEquals("0", source.getString("id")); assertEquals("a h r", source.getString("name")); @@ -309,6 +333,9 @@ private void checkInvestigation(ScoredEntityBaseBean investigation) { assertEquals("0", instruments.getJsonObject(0).getString("instrument.id")); assertEquals("bl0", instruments.getJsonObject(0).getString("instrument.name")); assertEquals("Beamline 0", instruments.getJsonObject(0).getString("instrument.fullName")); + JsonArray facilityCycles = source.getJsonArray("investigationfacilitycycle"); + assertEquals(1, facilityCycles.size()); + assertEquals("0", facilityCycles.getJsonObject(0).getString("facilityCycle.id")); } private void checkResults(SearchResult lsr, Long... n) { @@ -385,12 +412,6 @@ private Dataset dataset(long id, String name, Date startDate, Date endDate, Inve } private Investigation investigation(long id, String name, Date startDate, Date endDate) { - InvestigationType type = new InvestigationType(); - type.setName("type"); - type.setId(0L); - Facility facility = new Facility(); - facility.setName("facility"); - facility.setId(0L); Investigation investigation = new Investigation(); investigation.setId(id); investigation.setName(name); @@ -399,7 +420,7 @@ private Investigation investigation(long id, String name, Date startDate, Date e investigation.setCreateTime(startDate); investigation.setModTime(endDate); investigation.setFacility(facility); - investigation.setType(type); + investigation.setType(investigationType); return investigation; } @@ -535,6 +556,12 @@ private void populate() throws IcatException { Investigation investigation = investigation(investigationId, word, startDate, endDate); queue.add(SearchApi.encodeOperation("create", investigation)); + InvestigationFacilityCycle investigationFacilityCycle = new InvestigationFacilityCycle(); + investigationFacilityCycle.setId(new Long(investigationId)); + investigationFacilityCycle.setFacilityCycle(facilityCycle); + investigationFacilityCycle.setInvestigation(investigation); + queue.add(SearchApi.encodeOperation("create", investigationFacilityCycle)); + InvestigationInstrument investigationInstrument = new InvestigationInstrument(); investigationInstrument.setId(new Long(investigationId)); if (investigationId % 2 == 0) { @@ -1264,7 +1291,7 @@ public void modifyDatafile() throws IcatException { JsonObject lowRange = buildFacetRangeObject("low", 0L, 2L); JsonObject highRange = buildFacetRangeObject("high", 2L, 4L); JsonObject facetIdQuery = buildFacetIdQuery("id", "42"); - JsonObject rangeFacetRequest = buildFacetRangeRequest(facetIdQuery, "date", lowRange,highRange); + JsonObject rangeFacetRequest = buildFacetRangeRequest(facetIdQuery, "date", lowRange, highRange); JsonObject stringFacetRequest = buildFacetStringRequest("id", "42", "datafileFormat.name"); JsonObject sparseFacetRequest = Json.createObjectBuilder().add("query", facetIdQuery).build(); FacetDimension lowFacet = new FacetDimension("", "date", new FacetLabel("low", 1L), new FacetLabel("high", 0L)); From 8abef9a6e2bc6051c3bc95aeb2af82c5a7566f3a Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Mon, 3 Jul 2023 09:15:31 +0000 Subject: [PATCH 42/51] Remove placeholder values for size and count #267 --- src/main/java/org/icatproject/core/entity/Datafile.java | 6 +----- src/main/java/org/icatproject/core/entity/Dataset.java | 4 ++-- .../java/org/icatproject/core/entity/Investigation.java | 4 ++-- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index 81585276..ae181fd9 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -212,11 +212,7 @@ public void getDoc(JsonGenerator gen) { if (doi != null) { SearchApi.encodeString(gen, "doi", doi); } - if (fileSize != null) { - SearchApi.encodeLong(gen, "fileSize", fileSize); - } else { - SearchApi.encodeLong(gen, "fileSize", 0L); - } + SearchApi.encodeLong(gen, "fileSize", fileSize); SearchApi.encodeLong(gen, "fileCount", 1L); // Always 1, but makes sorting on fields consistent if (datafileFormat != null) { datafileFormat.getDoc(gen); diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index ade30dd3..e5b43c84 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -253,8 +253,8 @@ public void getDoc(JsonGenerator gen) { } else { SearchApi.encodeLong(gen, "endDate", modTime); } - SearchApi.encodeLong(gen, "fileSize", 0L); // This is a placeholder to allow us to dynamically build size - SearchApi.encodeLong(gen, "fileCount", 0L); // This is a placeholder to allow us to dynamically build count + SearchApi.encodeLong(gen, "fileSize", fileSize); + SearchApi.encodeLong(gen, "fileCount", fileCount); SearchApi.encodeString(gen, "id", id); if (investigation != null) { SearchApi.encodeString(gen, "investigation.id", investigation.id); diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index 7fbe335f..67141c3b 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -344,8 +344,8 @@ public void getDoc(JsonGenerator gen) { } else { SearchApi.encodeLong(gen, "endDate", modTime); } - SearchApi.encodeLong(gen, "fileSize", 0L); // This is a placeholder to allow us to dynamically build size - SearchApi.encodeLong(gen, "fileCount", 0L); // This is a placeholder to allow us to dynamically build count + SearchApi.encodeLong(gen, "fileSize", fileSize); + SearchApi.encodeLong(gen, "fileCount", fileCount); SearchApi.encodeString(gen, "id", id); facility.getDoc(gen); From 2589b5235bc75d9b4261e14be5dbbb661c8153df Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 8 Sep 2023 16:14:49 +0000 Subject: [PATCH 43/51] 6.1.0 release notes and icatadmin update --- src/main/config/run.properties.example | 6 +- src/main/resources/run.properties | 4 +- src/main/scripts/icatadmin | 85 +++++++++++++------- src/site/xhtml/installation.xhtml.vm | 105 ++++++++++++++++--------- src/site/xhtml/release-notes.xhtml | 12 +++ 5 files changed, 145 insertions(+), 67 deletions(-) diff --git a/src/main/config/run.properties.example b/src/main/config/run.properties.example index 1a0af133..d64fba5e 100644 --- a/src/main/config/run.properties.example +++ b/src/main/config/run.properties.example @@ -47,8 +47,8 @@ log.list = SESSION WRITE READ INFO search.engine = LUCENE search.urls = https://localhost:8181 search.populateBlockSize = 10000 -# Recommend setting lucene.searchBlockSize equal to maxIdsInQuery, so that all Lucene results can be authorised at once -# If lucene.searchBlockSize > maxIdsInQuery, then multiple auth checks may be needed for a single search to Lucene +# Recommend setting search.searchBlockSize equal to maxIdsInQuery, so that all results can be authorised at once +# If search.searchBlockSize > maxIdsInQuery, then multiple auth checks may be needed for a single search # The optimal value depends on how likely a user's auth request fails: larger values are more efficient when rejection is more likely search.searchBlockSize = 1000 search.directory = ${HOME}/data/icat/search @@ -56,7 +56,7 @@ search.backlogHandlerIntervalSeconds = 60 search.enqueuedRequestIntervalSeconds = 5 search.aggregateFilesIntervalSeconds = 3600 search.maxSearchTimeSeconds = 5 -# The entities to index with the search engine. For example, remove 'Datafile' and 'DatafileParameter' if the number of datafiles exceeds lucene's limit of 2^32 entries in an index +# The entities to index with the search engine. !search.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample # List members of cluster diff --git a/src/main/resources/run.properties b/src/main/resources/run.properties index db1d7f13..e78b0740 100644 --- a/src/main/resources/run.properties +++ b/src/main/resources/run.properties @@ -20,8 +20,8 @@ log.list = SESSION WRITE READ INFO search.engine = lucene search.urls = https://localhost.localdomain:8181 search.populateBlockSize = 10000 -# Recommend setting lucene.searchBlockSize equal to maxIdsInQuery, so that all Lucene results can be authorised at once -# If lucene.searchBlockSize > maxIdsInQuery, then multiple auth checks may be needed for a single search to Lucene +# Recommend setting search.searchBlockSize equal to maxIdsInQuery, so that all results can be authorised at once +# If search.searchBlockSize > maxIdsInQuery, then multiple auth checks may be needed for a single search # The optimal value depends on how likely a user's auth request fails: larger values are more efficient when rejection is more likely search.searchBlockSize = 1000 search.directory = ${HOME}/data/search diff --git a/src/main/scripts/icatadmin b/src/main/scripts/icatadmin index 9e8b3685..21f2c690 100755 --- a/src/main/scripts/icatadmin +++ b/src/main/scripts/icatadmin @@ -159,37 +159,68 @@ def getPopulating(args): def populate(args): parser.set_usage(usagebase + "populate []") parser.set_description("Populate lucene (for that entry name)") + parser.add_option( + "-e", + "--entity-name", + action="append", + dest="entityName", + help="Name of entity to populate.", + ) + parser.add_option( + "--min-id", + dest="minId", + help="Minimum (exclusive) ICAT entity id to populate", + type="int", + ) + parser.add_option( + "--max-id", + dest="maxId", + help="Maximum (inclusive) ICAT entity id to populate", + type="int", + ) + parser.add_option( + "-d", + "--delete", + dest="delete", + action="store_true", + help="Whether to delete all existing documents for this index", + ) options, args = parser.parse_args(args) - - if len(args) == 0: - try: - sessionId = getService() - parameters = {"sessionId": sessionId} - for entity in "Datafile", "DatafileParameter", "Dataset", "DatasetParameter", "Investigation", "InvestigationParameter", "InvestigationUser", "Sample": - print(entity) - _process("lucene/db/" + entity + "/-1", parameters, "POST") - except Exception as e: - fatal(e) - return - - if len(args) == 1: - try: - sessionId = getService() - parameters = {"sessionId": sessionId} - entity = args[0] - _process("lucene/db/" + entity + "/-1", parameters, "POST") - except Exception as e: - fatal(e) - return - - if len(args) > 2: - fatal("Must have zero arguments after the operation 'populate' or one - the name of the entity or two with the name of the entity and minid") - + entities = options.entityName or [] + entities += args + if not entities: + # This does not need to include "nested" entities such as ParameterType, as this + # will be included in the READ operation on the DB implicitly + entities = [ + "Datafile", + "Dataset", + "Investigation", + "DatafileParameter", + "DatasetParameter", + "DatasetTechnique", + "InstrumentScientist", + "InvestigationFacilityCycle", + "InvestigationInstrument", + "InvestigationParameter", + "InvestigationUser", + "Sample", + "SampleParameter", + ] + try: sessionId = getService() parameters = {"sessionId": sessionId} - entity = args[0] - _process("lucene/db/" + entity + "/" + args[1], parameters, "POST") + if options.minId: + parameters["minId"] = options.minId + if options.maxId: + parameters["maxId"] = options.maxId + if options.delete: + parameters["delete"] = True + else: + parameters["delete"] = False + + for entity in entities: + _process("lucene/db/" + entity, parameters, "POST") except Exception as e: fatal(e) diff --git a/src/site/xhtml/installation.xhtml.vm b/src/site/xhtml/installation.xhtml.vm index 70912e5f..ebae7ca6 100644 --- a/src/site/xhtml/installation.xhtml.vm +++ b/src/site/xhtml/installation.xhtml.vm @@ -27,7 +27,7 @@ installation instructions installed on the server
  • Deployed ICAT authenticators.
    • -
  • A deployed icat.lucene server it you plan to use free-text search.
    • +
  • A deployed icat.lucene server of at least version 3.0.0 or Open/Elasticsearch cluster if you plan to use free-text search.
  • Python 3.6+ and the suds-community package installed on the server.
    • @@ -86,11 +86,12 @@

    Schema upgrade

    -

    Lucene database

    +

    Lucene indices

    - Any existing lucene database should be removed. The location of + Any existing lucene indices should be removed. The location of this would have been specified in the previous icat.properties file. - Ensure that the directory specified there is empty. + Ensure that the directory specified there is empty. Indices generated by + icat.lucene versions before 3 are no longer compatible.

    Database schema

    @@ -262,29 +263,61 @@ log via JMS calls. The types are specified by a space separated list of values taken from READ, WRITE, SESSION, INFO. -

    lucene.url
    -
    This is optional. It is the machine url of the icat.lucene - server if needed. It is needed for TopCAT to work.
    - -
    lucene.populateBlockSize
    -
    This is ignored if lucene.url is not set. The number of - entries to batch off to the lucene server when using lucenePopulate.
    - -
    lucene.directory
    -
    This is ignored if lucene.url is not set. Path of a directory - holding files for requests that are queued to go the icat.lucene - server.
    - -
    lucene.backlogHandlerIntervalSeconds
    -
    This is ignored if lucene.url is not set. How often to check - the backlog file.
    - -
    lucene.enqueuedRequestIntervalSecond
    -
    This is ignored if lucene.url is not set. How often to +
    search.engine
    +
    This is optional. Specifies the engine used for free-text searches. + Value should be one of LUCENE, OPENSEARCH and ELASTICSEARCH.
    + +
    search.urls
    +
    This is optional. It is the machine url of the search engine + server if needed.
    + +
    search.populateBlockSize
    +
    This is ignored if search.engine and search.urls are not set. The number of + entries to batch off to the lucene server when populating the index.
    + +
    search.searchBlockSize
    +
    This is ignored if search.engine and search.urls are not set. Recommend + setting search.searchBlockSize equal to maxIdsInQuery, so that all results + can be authorised at once. If search.searchBlockSize > maxIdsInQuery, then + multiple auth checks may be needed for a single search. The optimal value + depends on how likely a user's auth request fails: larger values are more + efficient when rejection is more likely.
    + +
    search.directory
    +
    This is ignored if search.engine and search.urls are not set. Path of a + directoryholding files for requests that are queued to go the search engine. +
    + +
    search.backlogHandlerIntervalSeconds
    +
    This is ignored if search.engine and search.urls are not set. How often to + check the backlog file.
    + +
    search.enqueuedRequestIntervalSecond
    +
    This is ignored if search.engine and search.urls are not set. How often to transmit lucene requests to the icat.lucene server.
    -
    lucene.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample
    -
    The entities to index with Lucene. For example, remove 'Datafile' and 'DatafileParameter' if the number of datafiles exceeds lucene's limit of 2^32 entries in an index
    +
    search.aggregateFilesIntervalSeconds
    +
    This is ignored if search.engine and search.urls are not set. How often to + update file size and counts for Datasets and Investigations containing + recently modified Datafiles. If 0, then rather than being performed on timer + will update the parent documents in real time. Note that this can have a + significant performance impact.
    + +
    search.maxSearchTimeSeconds
    +
    This is ignored if search.engine and search.urls are not set. How long to + wait before cancelling a long-running search. This can prevent badly formed + queries from blocking other searches from completing.
    + +
    search.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample
    +
    The entities to index with the search engine.
    + +
    search.units
    +
    This is optional. Recognised unit names/symbols. Each symbol recognised by + indriya's SimpleUnitFormat should be followed by a colon, and then a comma + separated list of units measuring the same property. If the unit is simply + an alias (e.g. "K: kelvin") this is sufficient. If a conversion is required, + it should be followed by this factor (e.g. "J: eV 1.602176634e-19"). + Different units can be separated by a semi-colon.
    jms.topicConnectionFactory
    This is optional and may be used to override the default @@ -400,18 +433,20 @@
    -
    populate [<entity name>]
    +
    populate [--min-id 0 --max-id 1 --delete] [<entity names>...]
    re-populates lucene for the specified entity name. This is useful if the database has been modified directly rather than by - using the ICAT API. This call is asynchronous and simply places the - request in a set of entity types to be populated. When the request is - processed all lucene entries of the specified entity type are first - cleared then the corresponding icat entries are scanned to - re-populate lucene. To find what it is doing please use the - "populating" operation described below. It may also be run without an - entity name in which case it will process all entities. The new - lucene index will not be seen until it is completely rebuilt. While - the index is being rebuilt ICAT can be used as normal as any lucene + using the ICAT API, or to backpopulate from the database after a breaking + change to the search engine. This call is asynchronous and simply places the + request in a set of entity types to be populated. By default runs over all + relevant entities, or names can be provided as arguments. Also has the + options "min-id" to specify a non-inclusive lower limit, and "max-id" for an + inclusive upper limit on the operation. If documents are found in this + range, then the operation will not proceed, unless "delete" is also + specified - in which case all existing documents are cleared first. + To find what it is doing please use the "populating" operation described + below. The new lucene index will not be seen until it is completely rebuilt. + While the index is being rebuilt ICAT can be used as normal as any lucene updates are stored to be applied later.
    populating
    diff --git a/src/site/xhtml/release-notes.xhtml b/src/site/xhtml/release-notes.xhtml index b0113ef3..aed82dcb 100644 --- a/src/site/xhtml/release-notes.xhtml +++ b/src/site/xhtml/release-notes.xhtml @@ -6,6 +6,18 @@

    ICAT Server Release Notes

    +

    6.1.0

    +

    Add support for Open/Elasticsearch engine backends for free text searches. Adds to REST endpoints for free-text searches, and deprecates old functionality. Significant changes to the functionality and performance of searches:

    +
      +
    • Ability to search on over 2 billion documents
      • +
    • Enable sorting on specific entity fields
      • +
    • "Infinitely" search the data by using the searchAfter parameter
      • +
    • Faceted searches
      • +
    • Replace single "text" field with specific fields that reflect the ICAT schema to allow field targeting
      • +
    • Support for unit conversion on numeric Parameters
      • +
    • Support for synonym injection
      • +
    +

    6.0.0

    Upgrade from JavaEE to JakartaEE 10. Requires Java 11+ and an application server that supports JakartaEE 10 such as Payara 6.

    From bfcb7b0f403ae082dbb370c6a4421633ca3bc1e9 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Tue, 26 Sep 2023 10:07:17 +0000 Subject: [PATCH 44/51] Index id as long instead of String #267 --- .../org/icatproject/core/entity/Datafile.java | 6 +- .../core/entity/DatafileFormat.java | 2 +- .../core/entity/DatafileParameter.java | 2 +- .../org/icatproject/core/entity/Dataset.java | 4 +- .../core/entity/DatasetParameter.java | 2 +- .../core/entity/DatasetTechnique.java | 4 +- .../icatproject/core/entity/DatasetType.java | 2 +- .../org/icatproject/core/entity/Facility.java | 2 +- .../icatproject/core/entity/Instrument.java | 2 +- .../core/entity/InstrumentScientist.java | 4 +- .../core/entity/Investigation.java | 2 +- .../entity/InvestigationFacilityCycle.java | 6 +- .../core/entity/InvestigationInstrument.java | 4 +- .../core/entity/InvestigationParameter.java | 2 +- .../core/entity/InvestigationType.java | 2 +- .../core/entity/InvestigationUser.java | 4 +- .../icatproject/core/entity/Parameter.java | 2 +- .../core/entity/ParameterType.java | 5 +- .../org/icatproject/core/entity/Sample.java | 4 +- .../core/entity/SampleParameter.java | 2 +- .../icatproject/core/entity/SampleType.java | 2 +- .../icatproject/core/entity/Technique.java | 2 +- .../org/icatproject/core/entity/User.java | 2 +- .../core/manager/PropertyHandler.java | 7 +- .../core/manager/search/OpensearchApi.java | 24 +++--- .../manager/search/ScoredEntityBaseBean.java | 2 +- .../core/manager/search/SearchApi.java | 20 +---- .../core/manager/search/SearchManager.java | 4 +- .../core/manager/TestSearchApi.java | 76 +++++++++---------- .../org/icatproject/integration/TestRS.java | 19 +++-- 30 files changed, 107 insertions(+), 114 deletions(-) diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java index b4cd3a16..5e55fa4e 100644 --- a/src/main/java/org/icatproject/core/entity/Datafile.java +++ b/src/main/java/org/icatproject/core/entity/Datafile.java @@ -224,9 +224,9 @@ public void getDoc(JsonGenerator gen) { } else { SearchApi.encodeLong(gen, "date", modTime); } - SearchApi.encodeString(gen, "id", id); + SearchApi.encodeLong(gen, "id", id); if (dataset != null) { - SearchApi.encodeString(gen, "dataset.id", dataset.id); + SearchApi.encodeLong(gen, "dataset.id", dataset.id); SearchApi.encodeString(gen, "dataset.name", dataset.getName()); Sample sample = dataset.getSample(); if (sample != null) { @@ -234,7 +234,7 @@ public void getDoc(JsonGenerator gen) { } Investigation investigation = dataset.getInvestigation(); if (investigation != null) { - SearchApi.encodeString(gen, "investigation.id", investigation.id); + SearchApi.encodeLong(gen, "investigation.id", investigation.id); SearchApi.encodeString(gen, "investigation.name", investigation.getName()); SearchApi.encodeString(gen, "visitId", investigation.getVisitId()); if (investigation.getStartDate() != null) { diff --git a/src/main/java/org/icatproject/core/entity/DatafileFormat.java b/src/main/java/org/icatproject/core/entity/DatafileFormat.java index d0998db4..b6fca402 100644 --- a/src/main/java/org/icatproject/core/entity/DatafileFormat.java +++ b/src/main/java/org/icatproject/core/entity/DatafileFormat.java @@ -106,7 +106,7 @@ public void setVersion(String version) { @Override public void getDoc(JsonGenerator gen) { SearchApi.encodeString(gen, "datafileFormat.name", name); - SearchApi.encodeString(gen, "datafileFormat.id", id); + SearchApi.encodeLong(gen, "datafileFormat.id", id); } } diff --git a/src/main/java/org/icatproject/core/entity/DatafileParameter.java b/src/main/java/org/icatproject/core/entity/DatafileParameter.java index ff50a361..8d996dd7 100644 --- a/src/main/java/org/icatproject/core/entity/DatafileParameter.java +++ b/src/main/java/org/icatproject/core/entity/DatafileParameter.java @@ -56,7 +56,7 @@ public void setDatafile(Datafile datafile) { @Override public void getDoc(JsonGenerator gen) { super.getDoc(gen); - SearchApi.encodeString(gen, "datafile.id", datafile.id); + SearchApi.encodeLong(gen, "datafile.id", datafile.id); } } diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java index 8a0cd5aa..71ce09c6 100644 --- a/src/main/java/org/icatproject/core/entity/Dataset.java +++ b/src/main/java/org/icatproject/core/entity/Dataset.java @@ -255,9 +255,9 @@ public void getDoc(JsonGenerator gen) { } SearchApi.encodeLong(gen, "fileSize", fileSize); SearchApi.encodeLong(gen, "fileCount", fileCount); - SearchApi.encodeString(gen, "id", id); + SearchApi.encodeLong(gen, "id", id); if (investigation != null) { - SearchApi.encodeString(gen, "investigation.id", investigation.id); + SearchApi.encodeLong(gen, "investigation.id", investigation.id); SearchApi.encodeString(gen, "investigation.name", investigation.getName()); SearchApi.encodeString(gen, "investigation.title", investigation.getTitle()); SearchApi.encodeString(gen, "visitId", investigation.getVisitId()); diff --git a/src/main/java/org/icatproject/core/entity/DatasetParameter.java b/src/main/java/org/icatproject/core/entity/DatasetParameter.java index da6a2e67..0723be1d 100644 --- a/src/main/java/org/icatproject/core/entity/DatasetParameter.java +++ b/src/main/java/org/icatproject/core/entity/DatasetParameter.java @@ -56,6 +56,6 @@ public void setDataset(Dataset dataset) { @Override public void getDoc(JsonGenerator gen) { super.getDoc(gen); - SearchApi.encodeString(gen, "dataset.id", dataset.id); + SearchApi.encodeLong(gen, "dataset.id", dataset.id); } } \ No newline at end of file diff --git a/src/main/java/org/icatproject/core/entity/DatasetTechnique.java b/src/main/java/org/icatproject/core/entity/DatasetTechnique.java index cf14674f..6dea3d77 100644 --- a/src/main/java/org/icatproject/core/entity/DatasetTechnique.java +++ b/src/main/java/org/icatproject/core/entity/DatasetTechnique.java @@ -44,8 +44,8 @@ public void setTechnique(Technique technique) { @Override public void getDoc(JsonGenerator gen) { - SearchApi.encodeString(gen, "id", id); - SearchApi.encodeString(gen, "dataset.id", dataset.id); + SearchApi.encodeLong(gen, "id", id); + SearchApi.encodeLong(gen, "dataset.id", dataset.id); technique.getDoc(gen); } } diff --git a/src/main/java/org/icatproject/core/entity/DatasetType.java b/src/main/java/org/icatproject/core/entity/DatasetType.java index 556dbc2e..e17867e3 100644 --- a/src/main/java/org/icatproject/core/entity/DatasetType.java +++ b/src/main/java/org/icatproject/core/entity/DatasetType.java @@ -82,7 +82,7 @@ public void setName(String name) { @Override public void getDoc(JsonGenerator gen) { SearchApi.encodeString(gen, "type.name", name); - SearchApi.encodeString(gen, "type.id", id); + SearchApi.encodeLong(gen, "type.id", id); } } diff --git a/src/main/java/org/icatproject/core/entity/Facility.java b/src/main/java/org/icatproject/core/entity/Facility.java index a5d9fa2c..7259b0c8 100644 --- a/src/main/java/org/icatproject/core/entity/Facility.java +++ b/src/main/java/org/icatproject/core/entity/Facility.java @@ -202,7 +202,7 @@ public void setUrl(String url) { @Override public void getDoc(JsonGenerator gen) { SearchApi.encodeString(gen, "facility.name", name); - SearchApi.encodeString(gen, "facility.id", id); + SearchApi.encodeLong(gen, "facility.id", id); } } diff --git a/src/main/java/org/icatproject/core/entity/Instrument.java b/src/main/java/org/icatproject/core/entity/Instrument.java index daf50372..5a18b23c 100644 --- a/src/main/java/org/icatproject/core/entity/Instrument.java +++ b/src/main/java/org/icatproject/core/entity/Instrument.java @@ -158,7 +158,7 @@ public void getDoc(JsonGenerator gen) { SearchApi.encodeText(gen, "instrument.fullName", fullName); } SearchApi.encodeString(gen, "instrument.name", name); - SearchApi.encodeString(gen, "instrument.id", id); + SearchApi.encodeLong(gen, "instrument.id", id); } } diff --git a/src/main/java/org/icatproject/core/entity/InstrumentScientist.java b/src/main/java/org/icatproject/core/entity/InstrumentScientist.java index 9472c99b..692781a9 100644 --- a/src/main/java/org/icatproject/core/entity/InstrumentScientist.java +++ b/src/main/java/org/icatproject/core/entity/InstrumentScientist.java @@ -49,8 +49,8 @@ public InstrumentScientist() { @Override public void getDoc(JsonGenerator gen) { user.getDoc(gen); - SearchApi.encodeString(gen, "instrument.id", instrument.id); - SearchApi.encodeString(gen, "id", id); + SearchApi.encodeLong(gen, "instrument.id", instrument.id); + SearchApi.encodeLong(gen, "id", id); } } diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index 084a7119..dd81afda 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -347,7 +347,7 @@ public void getDoc(JsonGenerator gen) { SearchApi.encodeLong(gen, "fileSize", fileSize); SearchApi.encodeLong(gen, "fileCount", fileCount); - SearchApi.encodeString(gen, "id", id); + SearchApi.encodeLong(gen, "id", id); facility.getDoc(gen); type.getDoc(gen); } diff --git a/src/main/java/org/icatproject/core/entity/InvestigationFacilityCycle.java b/src/main/java/org/icatproject/core/entity/InvestigationFacilityCycle.java index 5cc3a761..5f9163e7 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationFacilityCycle.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationFacilityCycle.java @@ -48,9 +48,9 @@ public void setInvestigation(Investigation investigation) { @Override public void getDoc(JsonGenerator gen) { - SearchApi.encodeString(gen, "facilityCycle.id", facilityCycle.id); - SearchApi.encodeString(gen, "investigation.id", investigation.id); - SearchApi.encodeString(gen, "id", id); + SearchApi.encodeLong(gen, "facilityCycle.id", facilityCycle.id); + SearchApi.encodeLong(gen, "investigation.id", investigation.id); + SearchApi.encodeLong(gen, "id", id); } } diff --git a/src/main/java/org/icatproject/core/entity/InvestigationInstrument.java b/src/main/java/org/icatproject/core/entity/InvestigationInstrument.java index 71776b59..a9208fb3 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationInstrument.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationInstrument.java @@ -45,8 +45,8 @@ public void setInvestigation(Investigation investigation) { @Override public void getDoc(JsonGenerator gen) { instrument.getDoc(gen); - SearchApi.encodeString(gen, "investigation.id", investigation.id); - SearchApi.encodeString(gen, "id", id); + SearchApi.encodeLong(gen, "investigation.id", investigation.id); + SearchApi.encodeLong(gen, "id", id); } } diff --git a/src/main/java/org/icatproject/core/entity/InvestigationParameter.java b/src/main/java/org/icatproject/core/entity/InvestigationParameter.java index 46a8ba7d..9fcbdb78 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationParameter.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationParameter.java @@ -57,6 +57,6 @@ public void setInvestigation(Investigation investigation) { @Override public void getDoc(JsonGenerator gen) { super.getDoc(gen); - SearchApi.encodeString(gen, "investigation.id", investigation.id); + SearchApi.encodeLong(gen, "investigation.id", investigation.id); } } \ No newline at end of file diff --git a/src/main/java/org/icatproject/core/entity/InvestigationType.java b/src/main/java/org/icatproject/core/entity/InvestigationType.java index 56dc17c4..1fdca619 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationType.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationType.java @@ -82,7 +82,7 @@ public void setDescription(String description) { @Override public void getDoc(JsonGenerator gen) { SearchApi.encodeString(gen, "type.name", name); - SearchApi.encodeString(gen, "type.id", id); + SearchApi.encodeLong(gen, "type.id", id); } } diff --git a/src/main/java/org/icatproject/core/entity/InvestigationUser.java b/src/main/java/org/icatproject/core/entity/InvestigationUser.java index 6f63307a..b53a8ac0 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationUser.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationUser.java @@ -40,8 +40,8 @@ public InvestigationUser() { @Override public void getDoc(JsonGenerator gen) { user.getDoc(gen); - SearchApi.encodeString(gen, "investigation.id", investigation.id); - SearchApi.encodeString(gen, "id", id); + SearchApi.encodeLong(gen, "investigation.id", investigation.id); + SearchApi.encodeLong(gen, "id", id); } public String getRole() { diff --git a/src/main/java/org/icatproject/core/entity/Parameter.java b/src/main/java/org/icatproject/core/entity/Parameter.java index e9e6d4cd..d1ebc7f2 100644 --- a/src/main/java/org/icatproject/core/entity/Parameter.java +++ b/src/main/java/org/icatproject/core/entity/Parameter.java @@ -177,7 +177,7 @@ public void getDoc(JsonGenerator gen) { SearchApi.encodeDouble(gen, "rangeBottom", rangeBottom); } type.getDoc(gen); - SearchApi.encodeString(gen, "id", id); + SearchApi.encodeLong(gen, "id", id); } } diff --git a/src/main/java/org/icatproject/core/entity/ParameterType.java b/src/main/java/org/icatproject/core/entity/ParameterType.java index 98e8c341..e9aef525 100644 --- a/src/main/java/org/icatproject/core/entity/ParameterType.java +++ b/src/main/java/org/icatproject/core/entity/ParameterType.java @@ -97,7 +97,8 @@ public class ParameterType extends EntityBaseBean implements Serializable { @Comment("If ordinary users are allowed to create their own parameter types this indicates that this one has been approved") private boolean verified; - public static Set docFields = new HashSet<>(Arrays.asList("type.name", "type.units", "type.unitsSI", "numericValueSI", "type.id")); + public static Set docFields = new HashSet<>( + Arrays.asList("type.name", "type.units", "type.unitsSI", "numericValueSI", "type.id")); /* Needed for JPA */ public ParameterType() { @@ -283,7 +284,7 @@ public void setVerified(boolean verified) { public void getDoc(JsonGenerator gen) { SearchApi.encodeString(gen, "type.name", name); SearchApi.encodeString(gen, "type.units", units); - SearchApi.encodeString(gen, "type.id", id); + SearchApi.encodeLong(gen, "type.id", id); } } diff --git a/src/main/java/org/icatproject/core/entity/Sample.java b/src/main/java/org/icatproject/core/entity/Sample.java index 0a0ffbc8..47970900 100644 --- a/src/main/java/org/icatproject/core/entity/Sample.java +++ b/src/main/java/org/icatproject/core/entity/Sample.java @@ -104,8 +104,8 @@ public void setType(SampleType type) { @Override public void getDoc(JsonGenerator gen) { SearchApi.encodeString(gen, "sample.name", name); - SearchApi.encodeString(gen, "sample.id", id); - SearchApi.encodeString(gen, "sample.investigation.id", investigation.id); + SearchApi.encodeLong(gen, "sample.id", id); + SearchApi.encodeLong(gen, "sample.investigation.id", investigation.id); if (type != null) { type.getDoc(gen); } diff --git a/src/main/java/org/icatproject/core/entity/SampleParameter.java b/src/main/java/org/icatproject/core/entity/SampleParameter.java index bb1fc7b1..62f6914d 100644 --- a/src/main/java/org/icatproject/core/entity/SampleParameter.java +++ b/src/main/java/org/icatproject/core/entity/SampleParameter.java @@ -56,7 +56,7 @@ public void setSample(Sample sample) { @Override public void getDoc(JsonGenerator gen) { super.getDoc(gen); - SearchApi.encodeString(gen, "sample.id", sample.id); + SearchApi.encodeLong(gen, "sample.id", sample.id); } } \ No newline at end of file diff --git a/src/main/java/org/icatproject/core/entity/SampleType.java b/src/main/java/org/icatproject/core/entity/SampleType.java index 3b8d011f..9f0edc44 100644 --- a/src/main/java/org/icatproject/core/entity/SampleType.java +++ b/src/main/java/org/icatproject/core/entity/SampleType.java @@ -95,7 +95,7 @@ public void setSamples(List samples) { @Override public void getDoc(JsonGenerator gen) { SearchApi.encodeString(gen, "sample.type.name", name); - SearchApi.encodeString(gen, "sample.type.id", id); + SearchApi.encodeLong(gen, "sample.type.id", id); } } diff --git a/src/main/java/org/icatproject/core/entity/Technique.java b/src/main/java/org/icatproject/core/entity/Technique.java index e2c656fd..beb8db22 100644 --- a/src/main/java/org/icatproject/core/entity/Technique.java +++ b/src/main/java/org/icatproject/core/entity/Technique.java @@ -73,7 +73,7 @@ public void setDatasetTechniques(List datasetTechniques) { @Override public void getDoc(JsonGenerator gen) { - SearchApi.encodeString(gen, "technique.id", id); + SearchApi.encodeLong(gen, "technique.id", id); SearchApi.encodeString(gen, "technique.name", name); SearchApi.encodeString(gen, "technique.description", description); SearchApi.encodeString(gen, "technique.pid", pid); diff --git a/src/main/java/org/icatproject/core/entity/User.java b/src/main/java/org/icatproject/core/entity/User.java index b8b05f7a..78b939ba 100644 --- a/src/main/java/org/icatproject/core/entity/User.java +++ b/src/main/java/org/icatproject/core/entity/User.java @@ -172,7 +172,7 @@ public void getDoc(JsonGenerator gen) { SearchApi.encodeText(gen, "user.fullName", fullName); } SearchApi.encodeString(gen, "user.name", name); - SearchApi.encodeString(gen, "user.id", id); + SearchApi.encodeLong(gen, "user.id", id); } } diff --git a/src/main/java/org/icatproject/core/manager/PropertyHandler.java b/src/main/java/org/icatproject/core/manager/PropertyHandler.java index 62fadac2..31169972 100644 --- a/src/main/java/org/icatproject/core/manager/PropertyHandler.java +++ b/src/main/java/org/icatproject/core/manager/PropertyHandler.java @@ -401,9 +401,10 @@ private void init() { * result in no change to behaviour if the property is not specified. */ entitiesToIndex.addAll(Arrays.asList("Datafile", "DatafileFormat", "DatafileParameter", - "Dataset", "DatasetParameter", "DatasetType", "Facility", "Instrument", "InstrumentScientist", - "Investigation", "InvestigationInstrument", "InvestigationParameter", "InvestigationType", - "InvestigationUser", "ParameterType", "Sample", "SampleType", "SampleParameter", "User")); + "Dataset", "DatasetParameter", "DatasetType", "DatasetTechnique", "Facility", "Instrument", + "InstrumentScientist", "Investigation", "InvestigationInstrument", "InvestigationParameter", + "InvestigationType", "InvestigationUser", "ParameterType", "Sample", "SampleType", + "SampleParameter", "User")); logger.info("search.entitiesToIndex not set. Defaulting to: {}", entitiesToIndex.toString()); } formattedProps.add("search.entitiesToIndex " + entitiesToIndex.toString()); diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index c23930f7..4e1509f4 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -332,7 +332,7 @@ public void addNow(String entityName, List ids, EntityManager manager, EntityBaseBean bean = (EntityBaseBean) manager.find(klass, id); if (bean != null) { gen.writeStartObject().writeStartObject("create"); - gen.write("_index", entityName).write("_id", bean.getId().toString()); + gen.write("_index", entityName).write("_id", bean.getId()); gen.writeStartObject("doc"); bean.getDoc(gen); gen.writeEnd().writeEnd().writeEnd(); @@ -710,7 +710,7 @@ private void parseModification(CloseableHttpClient httpclient, OpensearchBulk bu ModificationType modificationType = ModificationType.valueOf(operationKey.toUpperCase()); JsonObject innerOperation = operation.getJsonObject(modificationType.toString().toLowerCase()); String index = innerOperation.getString("_index").toLowerCase(); - String id = innerOperation.getString("_id"); + long id = innerOperation.getJsonNumber("_id").longValueExact(); JsonObject document = innerOperation.containsKey("doc") ? innerOperation.getJsonObject("doc") : null; logger.trace("{} {} with id {}", operationKey, index, id); @@ -755,7 +755,7 @@ private void buildFileSizeUpdates(String entity, Map aggregation StringBuilder fileSizeStringBuilder) { if (aggregations.size() > 0) { for (String id : aggregations.keySet()) { - JsonObject targetObject = Json.createObjectBuilder().add("_id", new Long(id)).add("_index", entity) + JsonObject targetObject = Json.createObjectBuilder().add("_id", Long.valueOf(id)).add("_index", entity) .build(); JsonObject update = Json.createObjectBuilder().add("update", targetObject).build(); long deltaFileSize = aggregations.get(id)[0]; @@ -781,7 +781,7 @@ private void buildFileSizeUpdates(String entity, Map aggregation * @throws URISyntaxException * @throws ClientProtocolException */ - private JsonObject extractSource(CloseableHttpClient httpclient, String id) + private JsonObject extractSource(CloseableHttpClient httpclient, long id) throws IOException, URISyntaxException, ClientProtocolException { URI uriGet = new URIBuilder(server).setPath("/datafile/_source/" + id) .build(); @@ -931,7 +931,7 @@ private void updateWithExtractedEntity(CloseableHttpClient httpclient, URI uri, * @throws URISyntaxException * @throws IcatException */ - private void modifyNestedEntity(OpensearchBulk bulk, String id, String index, JsonObject document, + private void modifyNestedEntity(OpensearchBulk bulk, long id, String index, JsonObject document, ModificationType modificationType, ParentRelation relation) throws URISyntaxException, IcatException { switch (modificationType) { @@ -965,7 +965,7 @@ private void modifyNestedEntity(OpensearchBulk bulk, String id, String index, Js // SampleParameter requires specific logic, as the join is performed using the // Sample id rather than the SampleParameter id or the parent id. if (document.containsKey("sample.id")) { - String sampleId = document.getString("sample.id"); + long sampleId = document.getJsonNumber("sample.id").longValueExact(); updateNestedEntityByQuery(bulk, sampleId, index, document, relation, true); } } @@ -992,7 +992,7 @@ private void modifyNestedEntity(OpensearchBulk bulk, String id, String index, Js * @throws IcatException If parentId is missing from document. * @throws URISyntaxException */ - private void createNestedEntity(OpensearchBulk bulk, String id, String index, JsonObject document, + private void createNestedEntity(OpensearchBulk bulk, long id, String index, JsonObject document, ParentRelation relation) throws IcatException, URISyntaxException { if (!document.containsKey(relation.joinField + ".id")) { @@ -1035,7 +1035,7 @@ private void createNestedEntity(OpensearchBulk bulk, String id, String index, Js * with the specified id. * @throws URISyntaxException */ - private void updateNestedEntityByQuery(OpensearchBulk bulk, String id, String index, JsonObject document, + private void updateNestedEntityByQuery(OpensearchBulk bulk, long id, String index, JsonObject document, ParentRelation relation, boolean update) throws URISyntaxException { String path = "/" + relation.parentName + "/_update_by_query"; @@ -1168,11 +1168,11 @@ private void convertScriptUnits(JsonObjectBuilder paramsBuilder, JsonObject docu * @throws IOException * @throws ClientProtocolException */ - private void modifyEntity(CloseableHttpClient httpclient, OpensearchBulk bulk, String id, String index, + private void modifyEntity(CloseableHttpClient httpclient, OpensearchBulk bulk, long id, String index, JsonObject document, ModificationType modificationType) throws ClientProtocolException, IOException, URISyntaxException { - JsonObject targetObject = Json.createObjectBuilder().add("_id", new Long(id)).add("_index", index).build(); + JsonObject targetObject = Json.createObjectBuilder().add("_id", id).add("_index", index).build(); JsonObject update = Json.createObjectBuilder().add("update", targetObject).build(); JsonObject docAsUpsert; switch (modificationType) { @@ -1214,7 +1214,7 @@ private void modifyEntity(CloseableHttpClient httpclient, OpensearchBulk bulk, S * @throws URISyntaxException */ private void aggregateFiles(ModificationType modificationType, OpensearchBulk bulk, String index, - JsonObject document, CloseableHttpClient httpclient, String id) + JsonObject document, CloseableHttpClient httpclient, long id) throws ClientProtocolException, IOException, URISyntaxException { long deltaFileSize = 0; long deltaFileCount = 0; @@ -1264,7 +1264,7 @@ private void incrementEntity(Map aggregations, JsonObject docume * @throws URISyntaxException * @throws ClientProtocolException */ - private long extractFileSize(CloseableHttpClient httpclient, String id) + private long extractFileSize(CloseableHttpClient httpclient, long id) throws IOException, URISyntaxException, ClientProtocolException { JsonObject source = extractSource(httpclient, id); if (source != null && source.containsKey("fileSize")) { diff --git a/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java b/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java index cc48f0ca..2f79c2de 100644 --- a/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java +++ b/src/main/java/org/icatproject/core/manager/search/ScoredEntityBaseBean.java @@ -46,7 +46,7 @@ public ScoredEntityBaseBean(int engineDocId, int shardIndex, float score, JsonOb this.shardIndex = shardIndex; this.score = score; this.source = source; - this.id = new Long(source.getString("id")); + this.id = source.getJsonNumber("id").longValueExact(); } public Long getId() { diff --git a/src/main/java/org/icatproject/core/manager/search/SearchApi.java b/src/main/java/org/icatproject/core/manager/search/SearchApi.java index 9edd743d..d8225e8e 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchApi.java @@ -88,7 +88,7 @@ public static String encodeDeletion(EntityBaseBean bean) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartObject().writeStartObject("delete"); - gen.write("_index", entityName).write("_id", bean.getId().toString()); + gen.write("_index", entityName).write("_id", bean.getId()); gen.writeEnd().writeEnd(); } return baos.toString(); @@ -146,7 +146,7 @@ public static String encodeOperation(String operation, EntityBaseBean bean) thro ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartObject().writeStartObject(operation); - gen.write("_index", entityName).write("_id", icatId.toString()); + gen.write("_index", entityName).write("_id", icatId); gen.writeStartObject("doc"); bean.getDoc(gen); gen.writeEnd().writeEnd().writeEnd(); @@ -154,17 +154,6 @@ public static String encodeOperation(String operation, EntityBaseBean bean) thro return baos.toString(); } - /** - * Writes a key value pair to the JsonGenerator being used to encode an entity. - * - * @param gen JsonGenerator being used to encode. - * @param name Name of the field. - * @param value Long value to encode as a string. - */ - public static void encodeString(JsonGenerator gen, String name, Long value) { - gen.write(name, Long.toString(value)); - } - /** * Writes a key value pair to the JsonGenerator being used to encode an entity. * @@ -451,8 +440,8 @@ protected void post(String path, String body) throws IcatException { /** * POST to path with a body and response handling. * - * @param path Path on the search engine to POST to. - * @param body String of Json to send as the request body. + * @param path Path on the search engine to POST to. + * @param body String of Json to send as the request body. * @return JsonObject returned by the search engine. * @throws IcatException */ @@ -460,7 +449,6 @@ protected JsonObject postResponse(String path, String body) throws IcatException return postResponse(path, body, null); } - /** * POST to path with a body and response handling. * diff --git a/src/main/java/org/icatproject/core/manager/search/SearchManager.java b/src/main/java/org/icatproject/core/manager/search/SearchManager.java index 16c61f08..9b893ba6 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchManager.java @@ -441,7 +441,7 @@ public static List getPublicSearchFields(GateKeeper gateKeeper, String s gateKeeper.markPublicSearchFieldsFresh(); } List requestedFields = publicSearchFields.get(simpleName); - logger.trace("{} has public fields {}", simpleName, requestedFields); + logger.debug("{} has public fields {}", simpleName, requestedFields); return requestedFields; } @@ -537,7 +537,7 @@ public void deleteDocument(EntityBaseBean bean) throws IcatException { */ public static JsonObject buildFacetQuery(List results, String idField, JsonObject facetJson) { JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); - results.forEach(r -> arrayBuilder.add(Long.toString(r.getId()))); + results.forEach(r -> arrayBuilder.add(r.getId())); JsonObject terms = Json.createObjectBuilder().add(idField, arrayBuilder.build()).build(); return buildFacetQuery(terms, facetJson); } diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index edf4d43a..60ea61f4 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -96,9 +96,9 @@ public Filter(String fld, JsonObject... values) { } private static final String SEARCH_AFTER_NOT_NULL = "Expected searchAfter to be set, but it was null"; - private static final List datafileFields = Arrays.asList("id", "name", "location", "date", "dataset.id", - "dataset.name", "investigation.id", "investigation.name", "InvestigationInstrument instrument.id", - "InvestigationFacilityCycle facilityCycle.id"); + private static final List datafileFields = Arrays.asList("id", "name", "location", "datafileFormat.name", + "date", "dataset.id", "dataset.name", "investigation.id", "investigation.name", + "InvestigationInstrument instrument.id", "InvestigationFacilityCycle facilityCycle.id"); private static final List datasetFields = Arrays.asList("id", "name", "startDate", "endDate", "investigation.id", "investigation.name", "investigation.title", "investigation.startDate", "InvestigationInstrument instrument.id", "InvestigationFacilityCycle facilityCycle.id"); @@ -217,7 +217,7 @@ public static JsonObject buildQuery(String target, String user, String text, Dat return builder.build(); } - private static JsonObject buildFacetIdQuery(String idField, String idValue) { + private static JsonObject buildFacetIdQuery(String idField, long idValue) { return Json.createObjectBuilder().add(idField, Json.createArrayBuilder().add(idValue)).build(); } @@ -241,7 +241,7 @@ private static JsonObject buildFacetRangeRequest(JsonObject queryObject, String return Json.createObjectBuilder().add("query", queryObject).add("dimensions", rangedDimensionsBuilder).build(); } - private static JsonObject buildFacetStringRequest(String idField, String idValue, String dimension) { + private static JsonObject buildFacetStringRequest(String idField, long idValue, String dimension) { JsonObject idQuery = buildFacetIdQuery(idField, idValue); JsonObjectBuilder stringDimensionBuilder = Json.createObjectBuilder().add("dimension", dimension); JsonArrayBuilder stringDimensionsBuilder = Json.createArrayBuilder().add(stringDimensionBuilder); @@ -255,20 +255,20 @@ private void checkDatafile(ScoredEntityBaseBean datafile) { "dataset.name", "investigation.id", "investigation.name", "investigationinstrument", "investigationfacilitycycle")); assertEquals(expectedKeys, source.keySet()); - assertEquals("0", source.getString("id")); + assertEquals(0, source.getJsonNumber("id").longValueExact()); assertEquals("DFaaa", source.getString("name")); assertEquals("/dir/DFaaa", source.getString("location")); assertNotNull(source.getJsonNumber("date")); - assertEquals("0", source.getString("dataset.id")); + assertEquals(0, source.getJsonNumber("dataset.id").longValueExact()); assertEquals("DSaaa", source.getString("dataset.name")); - assertEquals("0", source.getString("investigation.id")); + assertEquals(0, source.getJsonNumber("investigation.id").longValueExact()); assertEquals("a h r", source.getString("investigation.name")); JsonArray instruments = source.getJsonArray("investigationinstrument"); assertEquals(1, instruments.size()); - assertEquals("0", instruments.getJsonObject(0).getString("instrument.id")); + assertEquals(0, instruments.getJsonObject(0).getJsonNumber("instrument.id").longValueExact()); JsonArray facilityCycles = source.getJsonArray("investigationfacilitycycle"); assertEquals(1, facilityCycles.size()); - assertEquals("0", facilityCycles.getJsonObject(0).getString("facilityCycle.id")); + assertEquals(0, facilityCycles.getJsonObject(0).getJsonNumber("facilityCycle.id").longValueExact()); } private void checkDataset(ScoredEntityBaseBean dataset) { @@ -278,20 +278,20 @@ private void checkDataset(ScoredEntityBaseBean dataset) { "investigation.name", "investigation.title", "investigation.startDate", "investigationinstrument", "investigationfacilitycycle")); assertEquals(expectedKeys, source.keySet()); - assertEquals("0", source.getString("id")); + assertEquals(0, source.getJsonNumber("id").longValueExact()); assertEquals("DSaaa", source.getString("name")); assertNotNull(source.getJsonNumber("startDate")); assertNotNull(source.getJsonNumber("endDate")); - assertEquals("0", source.getString("investigation.id")); + assertEquals(0, source.getJsonNumber("investigation.id").longValueExact()); assertEquals("a h r", source.getString("investigation.name")); assertEquals("title", source.getString("investigation.title")); assertNotNull(source.getJsonNumber("investigation.startDate")); JsonArray instruments = source.getJsonArray("investigationinstrument"); assertEquals(1, instruments.size()); - assertEquals("0", instruments.getJsonObject(0).getString("instrument.id")); + assertEquals(0, instruments.getJsonObject(0).getJsonNumber("instrument.id").longValueExact()); JsonArray facilityCycles = source.getJsonArray("investigationfacilitycycle"); assertEquals(1, facilityCycles.size()); - assertEquals("0", facilityCycles.getJsonObject(0).getString("facilityCycle.id")); + assertEquals(0, facilityCycles.getJsonObject(0).getJsonNumber("facilityCycle.id").longValueExact()); } private void checkFacets(List facetDimensions, FacetDimension... dimensions) { @@ -324,18 +324,18 @@ private void checkInvestigation(ScoredEntityBaseBean investigation) { "id", "name", "title", "startDate", "endDate", "investigationinstrument", "investigationfacilitycycle")); assertEquals(expectedKeys, source.keySet()); - assertEquals("0", source.getString("id")); + assertEquals(0, source.getJsonNumber("id").longValueExact()); assertEquals("a h r", source.getString("name")); assertNotNull(source.getJsonNumber("startDate")); assertNotNull(source.getJsonNumber("endDate")); JsonArray instruments = source.getJsonArray("investigationinstrument"); assertEquals(1, instruments.size()); - assertEquals("0", instruments.getJsonObject(0).getString("instrument.id")); + assertEquals(0, instruments.getJsonObject(0).getJsonNumber("instrument.id").longValueExact()); assertEquals("bl0", instruments.getJsonObject(0).getString("instrument.name")); assertEquals("Beamline 0", instruments.getJsonObject(0).getString("instrument.fullName")); JsonArray facilityCycles = source.getJsonArray("investigationfacilitycycle"); assertEquals(1, facilityCycles.size()); - assertEquals("0", facilityCycles.getJsonObject(0).getString("facilityCycle.id")); + assertEquals(0, facilityCycles.getJsonObject(0).getJsonNumber("facilityCycle.id").longValueExact()); } private void checkResults(SearchResult lsr, Long... n) { @@ -996,8 +996,8 @@ public void datasets() throws Exception { checkResults(lsr); // Test DatasetTechnique Facets - JsonObject stringFacetRequestZero = buildFacetStringRequest("dataset.id", "0", "technique.name"); - JsonObject stringFacetRequestOne = buildFacetStringRequest("dataset.id", "1", "technique.name"); + JsonObject stringFacetRequestZero = buildFacetStringRequest("dataset.id", 0, "technique.name"); + JsonObject stringFacetRequestOne = buildFacetStringRequest("dataset.id", 1, "technique.name"); FacetDimension facetZero = new FacetDimension("", "technique.name", new FacetLabel("technique0", 1L)); FacetDimension facetOne = new FacetDimension("", "technique.name", new FacetLabel("technique1", 1L)); checkFacets(searchApi.facetSearch("DatasetTechnique", stringFacetRequestZero, 5, 5), facetZero); @@ -1290,9 +1290,9 @@ public void modifyDatafile() throws IcatException { JsonObject pngQuery = buildQuery("Datafile", null, "datafileFormat.name:png", null, null, null, null); JsonObject lowRange = buildFacetRangeObject("low", 0L, 2L); JsonObject highRange = buildFacetRangeObject("high", 2L, 4L); - JsonObject facetIdQuery = buildFacetIdQuery("id", "42"); + JsonObject facetIdQuery = buildFacetIdQuery("id", 42); JsonObject rangeFacetRequest = buildFacetRangeRequest(facetIdQuery, "date", lowRange, highRange); - JsonObject stringFacetRequest = buildFacetStringRequest("id", "42", "datafileFormat.name"); + JsonObject stringFacetRequest = buildFacetStringRequest("id", 42, "datafileFormat.name"); JsonObject sparseFacetRequest = Json.createObjectBuilder().add("query", facetIdQuery).build(); FacetDimension lowFacet = new FacetDimension("", "date", new FacetLabel("low", 1L), new FacetLabel("high", 0L)); FacetDimension highFacet = new FacetDimension("", "date", new FacetLabel("low", 0L), @@ -1302,40 +1302,40 @@ public void modifyDatafile() throws IcatException { // Original modify(SearchApi.encodeOperation("create", elephantDatafile)); - checkResults(searchApi.getResults(elephantQuery, 5), 42L); - checkResults(searchApi.getResults(rhinoQuery, 5)); - checkResults(searchApi.getResults(pdfQuery, 5)); - checkResults(searchApi.getResults(pngQuery, 5)); + checkResults(searchApi.getResults(elephantQuery, null, 5, null, datafileFields), 42L); + checkResults(searchApi.getResults(rhinoQuery, null, 5, null, datafileFields)); + checkResults(searchApi.getResults(pdfQuery, null, 5, null, datafileFields)); + checkResults(searchApi.getResults(pngQuery, null, 5, null, datafileFields)); checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5)); checkFacets(searchApi.facetSearch("Datafile", sparseFacetRequest, 5, 5)); checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), lowFacet); // Change name and add a format modify(SearchApi.encodeOperation("update", rhinoDatafile)); - checkResults(searchApi.getResults(elephantQuery, 5)); - checkResults(searchApi.getResults(rhinoQuery, 5), 42L); - checkResults(searchApi.getResults(pdfQuery, 5), 42L); - checkResults(searchApi.getResults(pngQuery, 5)); + checkResults(searchApi.getResults(elephantQuery, null, 5, null, datafileFields)); + checkResults(searchApi.getResults(rhinoQuery, null, 5, null, datafileFields), 42L); + checkResults(searchApi.getResults(pdfQuery, null, 5, null, datafileFields), 42L); + checkResults(searchApi.getResults(pngQuery, null, 5, null, datafileFields)); checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5), pdfFacet); checkFacets(searchApi.facetSearch("Datafile", sparseFacetRequest, 5, 5), pdfFacet); checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), highFacet); // Change just the format modify(SearchApi.encodeOperation("update", pngFormat)); - checkResults(searchApi.getResults(elephantQuery, 5)); - checkResults(searchApi.getResults(rhinoQuery, 5), 42L); - checkResults(searchApi.getResults(pdfQuery, 5)); - checkResults(searchApi.getResults(pngQuery, 5), 42L); + checkResults(searchApi.getResults(elephantQuery, null, 5, null, datafileFields)); + checkResults(searchApi.getResults(rhinoQuery, null, 5, null, datafileFields), 42L); + checkResults(searchApi.getResults(pdfQuery, null, 5, null, datafileFields)); + checkResults(searchApi.getResults(pngQuery, null, 5, null, datafileFields), 42L); checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5), pngFacet); checkFacets(searchApi.facetSearch("Datafile", sparseFacetRequest, 5, 5), pngFacet); checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), highFacet); // Remove the format modify(SearchApi.encodeOperation("delete", pngFormat)); - checkResults(searchApi.getResults(elephantQuery, 5)); - checkResults(searchApi.getResults(rhinoQuery, 5), 42L); - checkResults(searchApi.getResults(pdfQuery, 5)); - checkResults(searchApi.getResults(pngQuery, 5)); + checkResults(searchApi.getResults(elephantQuery, null, 5, null, datafileFields)); + checkResults(searchApi.getResults(rhinoQuery, null, 5, null, datafileFields), 42L); + checkResults(searchApi.getResults(pdfQuery, null, 5, null, datafileFields)); + checkResults(searchApi.getResults(pngQuery, null, 5, null, datafileFields)); checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5)); checkFacets(searchApi.facetSearch("Datafile", sparseFacetRequest, 5, 5)); checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), highFacet); @@ -1474,7 +1474,7 @@ public void sampleParameters() throws IcatException { dataset.setSample(sample); // Queries and expected responses - JsonObjectBuilder query = Json.createObjectBuilder().add("sample.id", Json.createArrayBuilder().add("3")); + JsonObjectBuilder query = Json.createObjectBuilder().add("sample.id", Json.createArrayBuilder().add(3)); JsonObjectBuilder dimension = Json.createObjectBuilder().add("dimension", "type.name"); JsonArrayBuilder dimensions = Json.createArrayBuilder().add(dimension); JsonObject facet = Json.createObjectBuilder().add("query", query).add("dimensions", dimensions).build(); diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 2f35afda..e1c5272e 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -1055,7 +1055,7 @@ public void testSearchParameterValidation() throws Exception { badParameters = Arrays.asList(new ParameterForLucene("color", null, null)); try { - searchInvestigations(session, null, null, null, null, badParameters, null, null,10, null, null, 0); + searchInvestigations(session, null, null, null, null, badParameters, null, null, 10, null, null, 0); fail("BAD_PARAMETER exception not caught"); } catch (IcatException e) { assertEquals(IcatExceptionType.BAD_PARAMETER, e.getType()); @@ -1082,13 +1082,15 @@ private JsonArray buildFacetRequest(String target) { private void checkFacets(JsonObject responseObject, String dimension, List expectedLabels, List expectedCounts) { - String dimensionsMessage = "Expected responseObject to contain 'dimensions', but it had keys " - + responseObject.keySet(); + Set responseKeys = responseObject.keySet(); + String dimensionsMessage = "Expected responseObject to contain 'dimensions', but it had keys " + responseKeys; assertTrue(dimensionsMessage, responseObject.containsKey("dimensions")); + JsonObject dimensions = responseObject.getJsonObject("dimensions"); - String dimensionMessage = "Expected 'dimensions' to contain " + dimension + " but keys were " - + dimensions.keySet(); + Set dimensionKeys = dimensions.keySet(); + String dimensionMessage = "Expected 'dimensions' to contain " + dimension + " but keys were " + dimensionKeys; assertTrue(dimensionMessage, dimensions.containsKey(dimension)); + JsonObject labelsObject = dimensions.getJsonObject(dimension); assertEquals(expectedLabels.size(), labelsObject.size()); for (int i = 0; i < expectedLabels.size(); i++) { @@ -1503,14 +1505,15 @@ public void testSearchLists() throws Exception { wSession.addRule(null, "Facility", "R"); search(notrootSession, query, 3); // notroot is in user group giving CRUD to all, so should see all 3 search(piOneSession, query, 0); // piOne should pass for Facility, but not for any Investigation - + wSession.addRule(null, "SELECT i FROM Investigation i WHERE i.visitId = 'zero'", "R"); search(notrootSession, query, 3); // notroot is in user group giving CRUD to all, so should see all 3 JsonArray results = search(piOneSession, query, 1); // piOne should pass for Facility, one Investigation JsonObject result = results.getJsonObject(0); JsonObject investigation = result.getJsonObject("Investigation"); - assertEquals("Wrong visitId in "+ investigation.toString(), "zero", investigation.getString("visitId", null)); - + String visitId = investigation.getString("visitId", null); + assertEquals("Wrong visitId in " + investigation.toString(), "zero", visitId); + query = "SELECT f.investigationTypes FROM Facility f"; wSession.addRule(null, "InvestigationType", "R"); search(notrootSession, query, 2); // notroot is in user group giving CRUD to all, so should see both From f8a977bed5833303d265ef8254281798a34df414 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 28 Sep 2023 10:04:41 +0000 Subject: [PATCH 45/51] Special handling for InvestigationInstrument facets #267 --- .../core/manager/EntityBeanManager.java | 45 ++++++++++++++++--- .../core/manager/search/SearchManager.java | 44 +++++++++--------- src/main/resources/run.properties | 5 ++- src/site/xhtml/installation.xhtml.vm | 33 +++++++++++++- .../core/manager/TestSearchApi.java | 44 +++++++++++++++--- .../org/icatproject/integration/TestRS.java | 23 +++++++++- 6 files changed, 155 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 53bfd1c0..100a8c56 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -63,6 +63,8 @@ import org.icatproject.core.entity.Datafile; import org.icatproject.core.entity.Dataset; import org.icatproject.core.entity.EntityBaseBean; +import org.icatproject.core.entity.Investigation; +import org.icatproject.core.entity.InvestigationInstrument; import org.icatproject.core.entity.ParameterValueType; import org.icatproject.core.entity.Sample; import org.icatproject.core.entity.Session; @@ -1687,18 +1689,21 @@ private JsonObject buildFacetQuery(Class klass, String */ private JsonObject buildFacetQuery(Class klass, String target, List results, JsonObject jsonFacet) throws IcatException { - if (target.equals(klass.getSimpleName())) { + String parentName = klass.getSimpleName(); + if (target.equals(parentName)) { return SearchManager.buildFacetQuery(results, "id", jsonFacet); } else { Relationship relationship; if (target.equals("SampleParameter")) { Relationship sampleRelationship; - if (klass.getSimpleName().equals("Investigation")) { + if (parentName.equals("Investigation")) { sampleRelationship = eiHandler.getRelationshipsByName(klass).get("samples"); } else { - if (klass.getSimpleName().equals("Datafile")) { + if (parentName.equals("Datafile")) { Relationship datasetRelationship = eiHandler.getRelationshipsByName(klass).get("dataset"); if (!gateKeeper.allowed(datasetRelationship)) { + logger.debug("Cannot collect facets for {} as Relationship with parent {} is not allowed", target, + parentName); return null; } } @@ -1706,9 +1711,32 @@ private JsonObject buildFacetQuery(Class klass, String } Relationship parameterRelationship = eiHandler.getRelationshipsByName(Sample.class).get("parameters"); if (!gateKeeper.allowed(sampleRelationship) || !gateKeeper.allowed(parameterRelationship)) { + logger.debug("Cannot collect facets for {} as Relationship with parent {} is not allowed", target, + parentName); return null; } - return SearchManager.buildSampleFacetQuery(results, jsonFacet); + return SearchManager.buildFacetQuery(results, "sample.id", "sample.id", jsonFacet); + } else if (target.equals("InvestigationInstrument")) { + List relationships = new ArrayList<>(); + String resultIdField = "id"; + if (parentName.equals("Datafile")) { + resultIdField = "investigation.id"; + relationships.add(eiHandler.getRelationshipsByName(Datafile.class).get("dataset")); + relationships.add(eiHandler.getRelationshipsByName(Dataset.class).get("investigation")); + } else if (parentName.equals("Dataset")) { + resultIdField = "investigation.id"; + relationships.add(eiHandler.getRelationshipsByName(Dataset.class).get("investigation")); + } + relationships.add(eiHandler.getRelationshipsByName(Investigation.class).get("investigationInstruments")); + relationships.add(eiHandler.getRelationshipsByName(InvestigationInstrument.class).get("instrument")); + for (Relationship r : relationships) { + if (!gateKeeper.allowed(r)) { + logger.debug("Cannot collect facets for {} as Relationship with parent {} is not allowed", target, + parentName); + return null; + } + } + return SearchManager.buildFacetQuery(results, resultIdField, "investigation.id", jsonFacet); } else if (target.contains("Parameter")) { relationship = eiHandler.getRelationshipsByName(klass).get("parameters"); } else { @@ -1716,10 +1744,15 @@ private JsonObject buildFacetQuery(Class klass, String } if (gateKeeper.allowed(relationship)) { - return SearchManager.buildFacetQuery(results, klass.getSimpleName().toLowerCase() + ".id", jsonFacet); + if (target.equals("Sample") && parentName.equals("Investigation")) { + // As samples can be one to many on Investigations or one to one on Datasets, they do not follow + // usual naming conventions in the document mapping + return SearchManager.buildFacetQuery(results, "sample.investigation.id", jsonFacet); + } + return SearchManager.buildFacetQuery(results, parentName.toLowerCase() + ".id", jsonFacet); } else { logger.debug("Cannot collect facets for {} as Relationship with parent {} is not allowed", - target, klass.getSimpleName()); + target, parentName); return null; } } diff --git a/src/main/java/org/icatproject/core/manager/search/SearchManager.java b/src/main/java/org/icatproject/core/manager/search/SearchManager.java index 9b893ba6..094d129b 100644 --- a/src/main/java/org/icatproject/core/manager/search/SearchManager.java +++ b/src/main/java/org/icatproject/core/manager/search/SearchManager.java @@ -34,9 +34,9 @@ import jakarta.ejb.Startup; import jakarta.json.Json; import jakarta.json.JsonArrayBuilder; +import jakarta.json.JsonNumber; import jakarta.json.JsonObject; import jakarta.json.JsonObjectBuilder; -import jakarta.json.JsonString; import jakarta.json.JsonValue; import jakarta.json.JsonValue.ValueType; import jakarta.persistence.EntityManager; @@ -529,44 +529,48 @@ public void deleteDocument(EntityBaseBean bean) throws IcatException { * Builds a JsonObject for performing faceting against results from a previous * search. * - * @param results List of results from a previous search, containing entity - * ids. - * @param idField The field to perform id querying against. - * @param facetJson JsonObject containing the dimensions to facet. + * @param results List of results from a previous search, containing entity + * ids. + * @param queryIdField The field to perform id querying against. + * @param facetJson JsonObject containing the dimensions to facet. * @return {"query": {`idField`: [...]}, "dimensions": [...]} */ - public static JsonObject buildFacetQuery(List results, String idField, JsonObject facetJson) { + public static JsonObject buildFacetQuery(List results, String queryIdField, + JsonObject facetJson) { JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); results.forEach(r -> arrayBuilder.add(r.getId())); - JsonObject terms = Json.createObjectBuilder().add(idField, arrayBuilder.build()).build(); + JsonObject terms = Json.createObjectBuilder().add(queryIdField, arrayBuilder.build()).build(); return buildFacetQuery(terms, facetJson); } /** * Builds a JsonObject for performing faceting against results from a previous - * search. Has specific logic for handling the nesting of Samples. + * search. * - * @param results List of results from a previous search, containing sample - * ids. - * @param facetJson JsonObject containing the dimensions to facet. - * @return + * @param results List of results from a previous search, containing + * entity ids. + * @param resultIdField The id(s) to extract from the results. + * @param queryIdField The id field to target with the query. + * @param facetJson JsonObject containing the dimensions to facet. + * @return {"query": {`idField`: [...]}, "dimensions": [...]} */ - public static JsonObject buildSampleFacetQuery(List results, JsonObject facetJson) { + public static JsonObject buildFacetQuery(List results, String resultIdField, + String queryIdField, JsonObject facetJson) { JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); results.forEach(r -> { JsonObject source = r.getSource(); - if (source.containsKey("sample.id")) { - ValueType valueType = source.get("sample.id").getValueType(); - if (valueType.equals(ValueType.STRING)) { - arrayBuilder.add(source.getString("sample.id")); + if (source.containsKey(resultIdField)) { + ValueType valueType = source.get(resultIdField).getValueType(); + if (valueType.equals(ValueType.NUMBER)) { + arrayBuilder.add(source.getJsonNumber(resultIdField)); } else if (valueType.equals(ValueType.ARRAY)) { - source.getJsonArray("sample.id").getValuesAs(JsonString.class).forEach(sampleId -> { - arrayBuilder.add(sampleId); + source.getJsonArray(resultIdField).getValuesAs(JsonNumber.class).forEach(id -> { + arrayBuilder.add(id); }); } } }); - JsonObject terms = Json.createObjectBuilder().add("sample.id", arrayBuilder.build()).build(); + JsonObject terms = Json.createObjectBuilder().add(queryIdField, arrayBuilder.build()).build(); return buildFacetQuery(terms, facetJson); } diff --git a/src/main/resources/run.properties b/src/main/resources/run.properties index e78b0740..2b59216f 100644 --- a/src/main/resources/run.properties +++ b/src/main/resources/run.properties @@ -30,8 +30,9 @@ search.enqueuedRequestIntervalSeconds = 3 search.aggregateFilesIntervalSeconds = 3600 search.maxSearchTimeSeconds = 5 # Configure this option to prevent certain entities being indexed -# For example, remove Datafile and DatafileParameter -!search.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample +# For example, remove Datafile and DatafileParameter if these are not of interest +# Note then when commented out, the full set of all possible entities will be indexed - to disable all search functionality, instead comment out search.engine or search.urls +!search.entitiesToIndex = Datafile DatafileFormat DatafileParameter Dataset DatasetParameter DatasetType DatasetTechnique Facility Instrument InstrumentScientist Investigation InvestigationInstrument InvestigationParameter InvestigationType InvestigationUser ParameterType Sample SampleType SampleParameter User units = \u2103: celsius degC, K: kelvin !cluster = https://smfisher:8181 diff --git a/src/site/xhtml/installation.xhtml.vm b/src/site/xhtml/installation.xhtml.vm index ebae7ca6..b701e5d1 100644 --- a/src/site/xhtml/installation.xhtml.vm +++ b/src/site/xhtml/installation.xhtml.vm @@ -93,6 +93,35 @@ Ensure that the directory specified there is empty. Indices generated by icat.lucene versions before 3 are no longer compatible.

    +

    + An additional consequence of these changes to icat.lucene is that the Rules and + PublicSteps set in ICAT directly affect what metadata is returned from searches + against the search component. While the normal authorization process is applied + to the Investigation, Dataset and Datafiles that are returned as results, it is + also possible to include metadata from related entities with each result. For + example, the Instrument(s) used in an Investigation. In order to provide results + in a reasonable amount of time, the full authorization process cannot be + followed for these related entities. Instead, a related field will only be + returned if: +

      +
    • + The entity in question is a "public table" - that is, there is a Rule + providing READ access to all users for that entity. +
      • +
    • + There are one or more PublicSteps from the + Investigation/Dataset/Datafile entity to the entity of interest. +
      • +
    + It is entirely reasonable to decide that a PublicStep or public table Rule is + not appropriate for the entitiy in question, however be aware that this will + limit the metadata returned with the search results. +

    +

    + The same principle applies to the post-search faceting enabled in this release. + Only fields that are allowed to all users via one of the above methods will be + returned, to avoid exposing any unauthorized metadata. +

    Database schema

    The direct upgrade of the database schema is supported for icat.server @@ -273,7 +302,7 @@

    search.populateBlockSize
    This is ignored if search.engine and search.urls are not set. The number of - entries to batch off to the lucene server when populating the index.
    + entries to batch off to the search engine when populating the index.
    search.searchBlockSize
    This is ignored if search.engine and search.urls are not set. Recommend @@ -294,7 +323,7 @@
    search.enqueuedRequestIntervalSecond
    This is ignored if search.engine and search.urls are not set. How often to - transmit lucene requests to the icat.lucene server.
    + transmit requests to the search engine.
    search.aggregateFilesIntervalSeconds
    This is ignored if search.engine and search.urls are not set. How often to diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 60ea61f4..d23de537 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -217,8 +217,12 @@ public static JsonObject buildQuery(String target, String user, String text, Dat return builder.build(); } - private static JsonObject buildFacetIdQuery(String idField, long idValue) { - return Json.createObjectBuilder().add(idField, Json.createArrayBuilder().add(idValue)).build(); + private static JsonObject buildFacetIdQuery(String idField, long... idValues) { + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + for (long id : idValues) { + arrayBuilder.add(id); + } + return Json.createObjectBuilder().add(idField, arrayBuilder).build(); } private static JsonObject buildFacetRangeObject(String key, double from, double to) { @@ -248,6 +252,10 @@ private static JsonObject buildFacetStringRequest(String idField, long idValue, return Json.createObjectBuilder().add("query", idQuery).add("dimensions", stringDimensionsBuilder).build(); } + private JsonObject buildFacetSparseRequest(JsonObject facetIdQuery) { + return Json.createObjectBuilder().add("query", facetIdQuery).build(); + } + private void checkDatafile(ScoredEntityBaseBean datafile) { JsonObject source = datafile.getSource(); assertNotNull(source); @@ -503,6 +511,7 @@ private Sample sample(long id, String name, Investigation investigation) { sample.setId(id); sample.setName(name); sample.setInvestigation(investigation); + sample.setType(sampleType); return sample; } @@ -1002,6 +1011,14 @@ public void datasets() throws Exception { FacetDimension facetOne = new FacetDimension("", "technique.name", new FacetLabel("technique1", 1L)); checkFacets(searchApi.facetSearch("DatasetTechnique", stringFacetRequestZero, 5, 5), facetZero); checkFacets(searchApi.facetSearch("DatasetTechnique", stringFacetRequestOne, 5, 5), facetOne); + + // Test instrument.name Facets + JsonObject instrumentFacetRequestZero = buildFacetStringRequest("investigation.id", 0, "instrument.name"); + JsonObject instrumentFacetRequestOne = buildFacetStringRequest("investigation.id", 1, "instrument.name"); + FacetDimension instrumentFacetZero = new FacetDimension("", "instrument.name", new FacetLabel("bl0", 1L)); + FacetDimension instrumentFacetOne = new FacetDimension("", "instrument.name", new FacetLabel("bl1", 1L)); + checkFacets(searchApi.facetSearch("InvestigationInstrument", instrumentFacetRequestZero, 5, 5), instrumentFacetZero); + checkFacets(searchApi.facetSearch("InvestigationInstrument", instrumentFacetRequestOne, 5, 5), instrumentFacetOne); } @Test @@ -1293,7 +1310,7 @@ public void modifyDatafile() throws IcatException { JsonObject facetIdQuery = buildFacetIdQuery("id", 42); JsonObject rangeFacetRequest = buildFacetRangeRequest(facetIdQuery, "date", lowRange, highRange); JsonObject stringFacetRequest = buildFacetStringRequest("id", 42, "datafileFormat.name"); - JsonObject sparseFacetRequest = Json.createObjectBuilder().add("query", facetIdQuery).build(); + JsonObject sparseFacetRequest = buildFacetSparseRequest(facetIdQuery); FacetDimension lowFacet = new FacetDimension("", "date", new FacetLabel("low", 1L), new FacetLabel("high", 0L)); FacetDimension highFacet = new FacetDimension("", "date", new FacetLabel("low", 0L), new FacetLabel("high", 1L)); @@ -1474,10 +1491,18 @@ public void sampleParameters() throws IcatException { dataset.setSample(sample); // Queries and expected responses - JsonObjectBuilder query = Json.createObjectBuilder().add("sample.id", Json.createArrayBuilder().add(3)); + JsonObjectBuilder sampleQuery = Json.createObjectBuilder().add("sample.id", Json.createArrayBuilder().add(3)); JsonObjectBuilder dimension = Json.createObjectBuilder().add("dimension", "type.name"); JsonArrayBuilder dimensions = Json.createArrayBuilder().add(dimension); - JsonObject facet = Json.createObjectBuilder().add("query", query).add("dimensions", dimensions).build(); + JsonObject sampleParameterFacetQuery = Json.createObjectBuilder().add("query", sampleQuery).add("dimensions", dimensions).build(); + + JsonObjectBuilder sampleInvestigationQuery = Json.createObjectBuilder().add("sample.investigation.id", Json.createArrayBuilder().add(0)); + JsonObjectBuilder sampleTypeDimension = Json.createObjectBuilder().add("dimension", "sample.type.name"); + JsonArrayBuilder sampleTypeDimensions = Json.createArrayBuilder().add(sampleTypeDimension); + JsonObject sampleTypeFacetQuery = Json.createObjectBuilder().add("query", sampleInvestigationQuery).add("dimensions", sampleTypeDimensions).build(); + + JsonObject facetIdQuery = buildFacetIdQuery("id", 1, 2); + JsonObject sparseRequest = buildFacetSparseRequest(facetIdQuery); JsonObjectBuilder filterBuilder = Json.createObjectBuilder(); JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); @@ -1487,7 +1512,9 @@ public void sampleParameters() throws IcatException { filterBuilder.add("key", "key").add("label", "label").add("filter", arrayBuilder); JsonObject filter = filterBuilder.build(); - FacetDimension expectedFacet = new FacetDimension("", "type.name", new FacetLabel("parameter", 1L)); + FacetDimension sampleParemeterFacet = new FacetDimension("", "type.name", new FacetLabel("parameter", 1L)); + FacetDimension sampleTypeFacet = new FacetDimension("", "sample.type.name", new FacetLabel("test", 1L)); + FacetDimension datasetTypeFacet = new FacetDimension("", "type.name", new FacetLabel("type", 1L)); JsonObject investigationQuery = buildQuery("Investigation", null, null, null, null, null, null, new Filter("sampleparameter", filter)); JsonObject datasetQuery = buildQuery("Dataset", null, null, null, null, null, null, @@ -1504,7 +1531,10 @@ public void sampleParameters() throws IcatException { SearchApi.encodeOperation("create", parameter)); // Test - checkFacets(searchApi.facetSearch("SampleParameter", facet, 5, 5), expectedFacet); + checkFacets(searchApi.facetSearch("SampleParameter", sampleParameterFacetQuery, 5, 5), sampleParemeterFacet); + checkFacets(searchApi.facetSearch("Sample", sampleTypeFacetQuery, 5, 5), sampleTypeFacet); + checkFacets(searchApi.facetSearch("Dataset", sparseRequest, 5, 5), datasetTypeFacet, sampleTypeFacet); + checkFacets(searchApi.facetSearch("Datafile", sparseRequest, 5, 5), sampleTypeFacet); SearchResult lsr = searchApi.getResults(investigationQuery, null, 5, null, investigationFields); checkResults(lsr, 0L); diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index e1c5272e..ba4dd30a 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -1037,6 +1037,21 @@ public void testSearchInvestigations() throws Exception { assertFalse(responseObject.containsKey("search_after")); checkFacets(responseObject, "InvestigationParameter.type.name", Arrays.asList("colour"), Arrays.asList(1L)); + + // Test no facets match on Sample due to lack of READ access + facets = buildFacetRequest("Sample", "sample.type.name"); + responseObject = searchInvestigations(session, null, null, null, null, null, null, null, 10, null, facets, + 3); + assertFalse(responseObject.containsKey("search_after")); + assertFalse(NO_DIMENSIONS, responseObject.containsKey("dimensions")); + + // Test facets match on Sample + wSession.addRule(null, "Sample", "R"); + responseObject = searchInvestigations(session, null, null, null, null, null, null, null, 10, null, facets, + 3); + assertFalse(responseObject.containsKey("search_after")); + checkFacets(responseObject, "Sample.sample.type.name", Arrays.asList("diamond", "rust"), + Arrays.asList(1L, 1L)); } @Test @@ -1073,9 +1088,13 @@ public void testSearchParameterValidation() throws Exception { } private JsonArray buildFacetRequest(String target) { + return buildFacetRequest(target, "type.name"); + } + + private JsonArray buildFacetRequest(String target, String dimension) { JsonObjectBuilder builder = Json.createObjectBuilder(); - JsonObjectBuilder dimension = Json.createObjectBuilder().add("dimension", "type.name"); - JsonArrayBuilder dimensions = Json.createArrayBuilder().add(dimension); + JsonObjectBuilder dimensionBuilder = Json.createObjectBuilder().add("dimension", dimension); + JsonArrayBuilder dimensions = Json.createArrayBuilder().add(dimensionBuilder); builder.add("target", target).add("dimensions", dimensions); return Json.createArrayBuilder().add(builder).build(); } From 3dac87b60277feefc6f423409ebcb31cdd724a07 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 5 Oct 2023 14:46:35 +0000 Subject: [PATCH 46/51] Revert accidental deletion of Facility setters #267 --- src/main/config/run.properties.example | 6 ++++-- src/main/java/org/icatproject/core/entity/Facility.java | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/config/run.properties.example b/src/main/config/run.properties.example index d64fba5e..8b4fdc8b 100644 --- a/src/main/config/run.properties.example +++ b/src/main/config/run.properties.example @@ -56,8 +56,10 @@ search.backlogHandlerIntervalSeconds = 60 search.enqueuedRequestIntervalSeconds = 5 search.aggregateFilesIntervalSeconds = 3600 search.maxSearchTimeSeconds = 5 -# The entities to index with the search engine. -!search.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample +# Configure this option to prevent certain entities being indexed +# For example, remove Datafile and DatafileParameter if these are not of interest +# Note then when commented out, the full set of all possible entities will be indexed - to disable all search functionality, instead comment out search.engine or search.urls +!search.entitiesToIndex = Datafile DatafileFormat DatafileParameter Dataset DatasetParameter DatasetType DatasetTechnique Facility Instrument InstrumentScientist Investigation InvestigationInstrument InvestigationParameter InvestigationType InvestigationUser ParameterType Sample SampleType SampleParameter User # List members of cluster !cluster = http://vm200.nubes.stfc.ac.uk:8080 https://smfisher:8181 diff --git a/src/main/java/org/icatproject/core/entity/Facility.java b/src/main/java/org/icatproject/core/entity/Facility.java index 7259b0c8..c111535c 100644 --- a/src/main/java/org/icatproject/core/entity/Facility.java +++ b/src/main/java/org/icatproject/core/entity/Facility.java @@ -199,6 +199,14 @@ public void setUrl(String url) { this.url = url; } + public void setDataPublications(List dataPublications) { + this.dataPublications = dataPublications; + } + + public void setDataPublicationTypes(List dataPublicationTypes) { + this.dataPublicationTypes = dataPublicationTypes; + } + @Override public void getDoc(JsonGenerator gen) { SearchApi.encodeString(gen, "facility.name", name); From efd1261aecf1b66ff63752c15eb4a1f0bdfc767f Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 6 Oct 2023 16:12:12 +0000 Subject: [PATCH 47/51] Add investigation null check in Sample.getDoc #267 --- src/main/java/org/icatproject/core/entity/Sample.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/icatproject/core/entity/Sample.java b/src/main/java/org/icatproject/core/entity/Sample.java index 47970900..871c297c 100644 --- a/src/main/java/org/icatproject/core/entity/Sample.java +++ b/src/main/java/org/icatproject/core/entity/Sample.java @@ -105,7 +105,12 @@ public void setType(SampleType type) { public void getDoc(JsonGenerator gen) { SearchApi.encodeString(gen, "sample.name", name); SearchApi.encodeLong(gen, "sample.id", id); - SearchApi.encodeLong(gen, "sample.investigation.id", investigation.id); + if (investigation != null) { + // Investigation is not nullable, but it is possible to pass Samples without their Investigation + // relationship populated when creating Datasets, where this field is not needed anyway - so guard against + // null pointers + SearchApi.encodeLong(gen, "sample.investigation.id", investigation.id); + } if (type != null) { type.getDoc(gen); } From f2e59e3d07fb2fa4ef359c9766f109e18b96270e Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Tue, 10 Oct 2023 09:44:05 +0000 Subject: [PATCH 48/51] Tests for Investigation Sample filtering #267 --- .../org/icatproject/core/manager/TestSearchApi.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index d23de537..bbc8f2de 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -1209,6 +1209,16 @@ public void investigations() throws Exception { pojos, "b"); lsr = searchApi.getResults(query, 100, null); checkResults(lsr, 3L); + + // Sample filtering + query = buildQuery("Investigation", null, null, null, null, null, null, new Filter("sample.sample.type.name", "test")); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L); + + query = buildQuery("Investigation", null, null, null, null, null, null, new Filter("sample.sample.type.name", "fail")); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr); + } @Test From aff1fc99e722a3bb0e8fe0658839309541af5394 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 14 Mar 2024 16:31:45 +0000 Subject: [PATCH 49/51] Return searchAfter when searching as root --- .../java/org/icatproject/core/manager/EntityBeanManager.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 100a8c56..55495263 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -822,6 +822,7 @@ private ScoredEntityBaseBean filterReadAccess(List accepte int needed = maxCount - acceptedResults.size(); if (newResults.size() > needed) { acceptedResults.addAll(newResults.subList(0, needed)); + return newResults.get(needed - 1); } else { acceptedResults.addAll(newResults); } From d4bd8f9d8f0beb37bb1304dc0acb56bb6d36fa04 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 22 Mar 2024 11:48:35 +0000 Subject: [PATCH 50/51] Account for changes to IcatUnits --- .../core/manager/search/OpensearchApi.java | 40 +++++++++---------- .../core/manager/search/OpensearchQuery.java | 20 +++++----- 2 files changed, 29 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java index 4e1509f4..d3cef95e 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchApi.java @@ -50,7 +50,7 @@ import org.icatproject.core.entity.User; import org.icatproject.core.manager.Rest; import org.icatproject.utils.IcatUnits; -import org.icatproject.utils.IcatUnits.SystemValue; +import org.icatproject.utils.IcatUnits.Value; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1081,17 +1081,15 @@ private void updateNestedEntityByQuery(OpensearchBulk bulk, long id, String inde * @param rebuilder JsonObjectBuilder being used to create a new document * with converted units. * @param valueString Field name of the numeric value. - * @param numericValue Value to possibly be converted. + * @param numericalValue Value to possibly be converted. */ private void convertUnits(JsonObject document, JsonObjectBuilder rebuilder, String valueString, - Double numericValue) { + double numericalValue) { String unitString = document.getString("type.units"); - SystemValue systemValue = icatUnits.new SystemValue(numericValue, unitString); - if (systemValue.units != null) { - rebuilder.add("type.unitsSI", systemValue.units); - } - if (systemValue.value != null) { - rebuilder.add(valueString, systemValue.value); + Value value = icatUnits.convertValueToSiUnits(numericalValue, unitString); + if (value != null) { + rebuilder.add("type.unitsSI", value.units); + rebuilder.add(valueString + "SI", value.numericalValue); } } @@ -1111,18 +1109,18 @@ private JsonObject convertDocumentUnits(JsonObject document) { for (String key : document.keySet()) { rebuilder.add(key, document.get(key)); } - Double numericValue = document.containsKey("numericValue") - ? document.getJsonNumber("numericValue").doubleValue() - : null; - Double rangeBottom = document.containsKey("rangeBottom") - ? document.getJsonNumber("rangeBottom").doubleValue() - : null; - Double rangeTop = document.containsKey("rangeTop") - ? document.getJsonNumber("rangeTop").doubleValue() - : null; - convertUnits(document, rebuilder, "numericValueSI", numericValue); - convertUnits(document, rebuilder, "rangeBottomSI", rangeBottom); - convertUnits(document, rebuilder, "rangeTopSI", rangeTop); + if (document.containsKey("numericValue")) { + double numericValue = document.getJsonNumber("numericValue").doubleValue(); + convertUnits(document, rebuilder, "numericValueSI", numericValue); + } + if (document.containsKey("rangeBottom")) { + double rangeBottom = document.getJsonNumber("rangeBottom").doubleValue(); + convertUnits(document, rebuilder, "rangeBottomSI", rangeBottom); + } + if (document.containsKey("rangeTop")) { + double rangeTop = document.getJsonNumber("rangeTop").doubleValue(); + convertUnits(document, rebuilder, "rangeTopSI", rangeTop); + } document = rebuilder.build(); return document; } diff --git a/src/main/java/org/icatproject/core/manager/search/OpensearchQuery.java b/src/main/java/org/icatproject/core/manager/search/OpensearchQuery.java index 3d929fde..1324ac04 100644 --- a/src/main/java/org/icatproject/core/manager/search/OpensearchQuery.java +++ b/src/main/java/org/icatproject/core/manager/search/OpensearchQuery.java @@ -19,7 +19,7 @@ import org.icatproject.core.IcatException; import org.icatproject.core.IcatException.IcatExceptionType; -import org.icatproject.utils.IcatUnits.SystemValue; +import org.icatproject.utils.IcatUnits.Value; /** * Utilities for building queries in Json understood by Opensearch. @@ -425,12 +425,12 @@ private void parseRangeFilter(String field, List queryObjectsList, J JsonNumber to = nestedFilter.getJsonNumber("to"); String units = nestedFilter.getString("units", null); if (units != null) { - SystemValue fromValue = opensearchApi.icatUnits.new SystemValue(from.doubleValue(), units); - SystemValue toValue = opensearchApi.icatUnits.new SystemValue(to.doubleValue(), units); - if (fromValue.value != null && toValue.value != null) { + Value fromValue = opensearchApi.icatUnits.convertValueToSiUnits(from.doubleValue(), units); + Value toValue = opensearchApi.icatUnits.convertValueToSiUnits(to.doubleValue(), units); + if (fromValue != null && toValue != null) { // If we were able to parse the units, apply query to the SI value String fieldSI = field + "." + nestedField + "SI"; - queryObjectsList.add(buildDoubleRangeQuery(fieldSI, fromValue.value, toValue.value)); + queryObjectsList.add(buildDoubleRangeQuery(fieldSI, fromValue.numericalValue, toValue.numericalValue)); } else { // If units could not be parsed, make them part of the query on the raw data queryObjectsList.add(buildRangeQuery(field + "." + nestedField, from, to)); @@ -460,13 +460,13 @@ private void parseExactFilter(String field, List queryObjectsList, J JsonNumber exact = nestedFilter.getJsonNumber("exact"); String units = nestedFilter.getString("units", null); if (units != null) { - SystemValue exactValue = opensearchApi.icatUnits.new SystemValue(exact.doubleValue(), units); - if (exactValue.value != null) { + Value exactValue = opensearchApi.icatUnits.convertValueToSiUnits(exact.doubleValue(), units); + if (exactValue != null) { // If we were able to parse the units, apply query to the SI value - JsonObject bottomQuery = buildDoubleRangeQuery(field + ".rangeBottomSI", null, exactValue.value); - JsonObject topQuery = buildDoubleRangeQuery(field + ".rangeTopSI", exactValue.value, null); + JsonObject bottomQuery = buildDoubleRangeQuery(field + ".rangeBottomSI", null, exactValue.numericalValue); + JsonObject topQuery = buildDoubleRangeQuery(field + ".rangeTopSI", exactValue.numericalValue, null); JsonObject inRangeQuery = buildBoolQuery(Arrays.asList(bottomQuery, topQuery), null); - JsonObject exactQuery = buildTermQuery(field + "." + nestedField + "SI", exactValue.value); + JsonObject exactQuery = buildTermQuery(field + "." + nestedField + "SI", exactValue.numericalValue); queryObjectsList.add(buildBoolQuery(null, Arrays.asList(inRangeQuery, exactQuery))); } else { // If units could not be parsed, make them part of the query on the raw data From 519c05de8df445422b485d7019231ea6fc8fc0a1 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Wed, 30 Oct 2024 17:39:07 +0000 Subject: [PATCH 51/51] Use icat-6.1 branch of icat-ansible in CI --- .github/workflows/ci-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 928aa60e..1136b69f 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -38,6 +38,7 @@ jobs: with: repository: icatproject-contrib/icat-ansible path: icat-ansible + ref: icat-6.1 - name: Install Ansible run: pip install -r icat-ansible/requirements.txt