Search Improvements (OS/ES support)

pom.xml

@@ -3,7 +3,7 @@
 
 	<groupId>org.icatproject</groupId>
 	<artifactId>icat.server</artifactId>
-	<version>5.0.1-SNAPSHOT</version>
+	<version>5.1.0-SNAPSHOT</version>
 	<packaging>war</packaging>
 	<name>ICAT Server</name>
 	<description>A metadata catalogue to support Large Facility experimental data, 
@@ -113,13 +113,13 @@
 		<dependency>
 			<groupId>org.icatproject</groupId>
 			<artifactId>icat.utils</artifactId>
-			<version>4.16.1</version>
+			<version>4.17.0-SNAPSHOT</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.icatproject</groupId>
 			<artifactId>icat.client</artifactId>
-			<version>5.0.0</version>
+			<version>5.1.0-SNAPSHOT</version>
 			<scope>test</scope>
 		</dependency>
 
@@ -226,7 +226,8 @@
 					</excludes>
 					<systemPropertyVariables>
 						<javax.net.ssl.trustStore>${javax.net.ssl.trustStore}</javax.net.ssl.trustStore>
-						<luceneUrl>${luceneUrl}</luceneUrl>
+						<searchEngine>${searchEngine}</searchEngine>
+						<searchUrls>${searchUrls}</searchUrls>
 					</systemPropertyVariables>
 					<testFailureIgnore>false</testFailureIgnore>
 				</configuration>
@@ -244,7 +245,8 @@
 					<systemPropertyVariables>
 						<javax.net.ssl.trustStore>${javax.net.ssl.trustStore}</javax.net.ssl.trustStore>
 						<serverUrl>${serverUrl}</serverUrl>
-						<luceneUrl>${luceneUrl}</luceneUrl>
+						<searchEngine>${searchEngine}</searchEngine>
+						<searchUrls>${searchUrls}</searchUrls>
 					</systemPropertyVariables>
 				</configuration>
 				<executions>
@@ -324,7 +326,8 @@
 								<argument>src/test/scripts/prepare_test.py</argument>
 								<argument>${containerHome}</argument>
 								<argument>${serverUrl}</argument>
-								<argument>${luceneUrl}</argument>
+								<argument>${searchEngine}</argument>
+								<argument>${searchUrls}</argument>
 							</arguments>
 						</configuration>
 						<goals>
@@ -402,6 +405,3 @@
 	</reporting>
 
 </project>
-
-
-

src/main/config/run.properties.example

@@ -42,14 +42,16 @@ notification.Datafile = CU
 # Call logging setup
 log.list = SESSION WRITE READ INFO
 
-# Lucene
-lucene.url = https://localhost:8181
-lucene.populateBlockSize = 10000
-lucene.directory = ${HOME}/data/icat/lucene
-lucene.backlogHandlerIntervalSeconds = 60
-lucene.enqueuedRequestIntervalSeconds = 5
-# The entities to index with Lucene. For example, remove 'Datafile' and 'DatafileParameter' if the number of datafiles exceeds lucene's limit of 2^32 entries in an index
-!lucene.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample
+# Search Engine
+# LUCENE, OPENSEARCH and ELASTICSEARCH engines are supported, however the latter two are considered experimental
+search.engine = LUCENE
+search.urls = https://localhost:8181
+search.populateBlockSize = 10000
+search.directory = ${HOME}/data/icat/search
+search.backlogHandlerIntervalSeconds = 60
+search.enqueuedRequestIntervalSeconds = 5
+# The entities to index with the search engine. For example, remove 'Datafile' and 'DatafileParameter' if the number of datafiles exceeds lucene's limit of 2^32 entries in an index
+!search.entitiesToIndex = Datafile Dataset Investigation InvestigationUser DatafileParameter DatasetParameter InvestigationParameter Sample
 
 # List members of cluster
 !cluster = http://vm200.nubes.stfc.ac.uk:8080 https://smfisher:8181				

src/main/java/org/icatproject/core/entity/Datafile.java

@@ -3,7 +3,9 @@
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import javax.json.stream.JsonGenerator;
 import javax.persistence.CascadeType;
@@ -20,7 +22,10 @@
 import javax.persistence.UniqueConstraint;
 import javax.xml.bind.annotation.XmlRootElement;
 
-import org.icatproject.core.manager.LuceneApi;
+import org.icatproject.core.IcatException;
+import org.icatproject.core.manager.EntityInfoHandler;
+import org.icatproject.core.manager.EntityInfoHandler.Relationship;
+import org.icatproject.core.manager.search.SearchApi;
 
 @Comment("A data file")
 @SuppressWarnings("serial")
@@ -77,6 +82,8 @@ public class Datafile extends EntityBaseBean implements Serializable {
 	@OneToMany(cascade = CascadeType.ALL, mappedBy = "sourceDatafile")
 	private List<RelatedDatafile> sourceDatafiles = new ArrayList<RelatedDatafile>();
 
+	private static final Map<String, Relationship[]> documentFields = new HashMap<>();
+
 	/* Needed for JPA */
 	public Datafile() {
 	}
@@ -195,25 +202,111 @@ public void setSourceDatafiles(List<RelatedDatafile> sourceDatafiles) {
 
 	@Override
 	public void getDoc(JsonGenerator gen) {
-		StringBuilder sb = new StringBuilder(name);
+		SearchApi.encodeString(gen, "name", name);
 		if (description != null) {
-			sb.append(" " + description);
+			SearchApi.encodeString(gen, "description", description);
+		}
+		if (location != null) {
+			SearchApi.encodeString(gen, "location", location);
 		}
 		if (doi != null) {
-			sb.append(" " + doi);
+			SearchApi.encodeString(gen, "doi", doi);
 		}
+		if (fileSize != null) {
+			SearchApi.encodeLong(gen, "fileSize", fileSize);
+		} else {
+			SearchApi.encodeLong(gen, "fileSize", 0L);
+		}
+		SearchApi.encodeLong(gen, "fileCount", 1L); // Always 1, but makes sorting on fields consistent
 		if (datafileFormat != null) {
-			sb.append(" " + datafileFormat.getName());
+			datafileFormat.getDoc(gen);
 		}
-		LuceneApi.encodeTextfield(gen, "text", sb.toString());
 		if (datafileModTime != null) {
-			LuceneApi.encodeStringField(gen, "date", datafileModTime);
+			SearchApi.encodeLong(gen, "date", datafileModTime);
 		} else if (datafileCreateTime != null) {
-			LuceneApi.encodeStringField(gen, "date", datafileCreateTime);
+			SearchApi.encodeLong(gen, "date", datafileCreateTime);
 		} else {
-			LuceneApi.encodeStringField(gen, "date", modTime);
+			SearchApi.encodeLong(gen, "date", modTime);
+		}
+		SearchApi.encodeString(gen, "id", id);
+		if (dataset != null) {
+			SearchApi.encodeString(gen, "dataset.id", dataset.id);
+			SearchApi.encodeString(gen, "dataset.name", dataset.getName());
+			Sample sample = dataset.getSample();
+			if (sample != null) {
+				sample.getDoc(gen);
+			}
+			Investigation investigation = dataset.getInvestigation();
+			if (investigation != null) {
+				SearchApi.encodeString(gen, "investigation.id", investigation.id);
+				SearchApi.encodeString(gen, "investigation.name", investigation.getName());
+				SearchApi.encodeString(gen, "visitId", investigation.getVisitId());
+				if (investigation.getStartDate() != null) {
+					SearchApi.encodeLong(gen, "investigation.startDate", investigation.getStartDate());
+				} else if (investigation.getCreateTime() != null) {
+					SearchApi.encodeLong(gen, "investigation.startDate", investigation.getCreateTime());
+				}
+			}
 		}
-		LuceneApi.encodeStoredId(gen, id);
-		LuceneApi.encodeStringField(gen, "dataset", dataset.id);
 	}
+
+	/**
+	 * Gets the fields used in the search component for this entity, and the
+	 * relationships that would restrict the content of those fields.
+	 * 
+	 * @return Map of field names (as they appear on the search document) against
+	 *         the Relationships that need to be allowed for that field to be
+	 *         viewable. If there are no restrictive relationships, then the value
+	 *         will be null.
+	 * @throws IcatException If the EntityInfoHandler cannot find one of the
+	 *                       Relationships.
+	 */
+	public static Map<String, Relationship[]> getDocumentFields() throws IcatException {
+		if (documentFields.size() == 0) {
+			EntityInfoHandler eiHandler = EntityInfoHandler.getInstance();
+			Relationship[] datafileFormatRelationships = {
+					eiHandler.getRelationshipsByName(Datafile.class).get("datafileFormat") };
+			Relationship[] datasetRelationships = {
+					eiHandler.getRelationshipsByName(Datafile.class).get("dataset") };
+			Relationship[] investigationRelationships = {
+					eiHandler.getRelationshipsByName(Datafile.class).get("dataset"),
+					eiHandler.getRelationshipsByName(Dataset.class).get("investigation") };
+			Relationship[] instrumentRelationships = {
+					eiHandler.getRelationshipsByName(Datafile.class).get("dataset"),
+					eiHandler.getRelationshipsByName(Dataset.class).get("investigation"),
+					eiHandler.getRelationshipsByName(Investigation.class).get("investigationInstruments"),
+					eiHandler.getRelationshipsByName(InvestigationInstrument.class).get("instrument") };
+			Relationship[] sampleRelationships = {
+					eiHandler.getRelationshipsByName(Datafile.class).get("dataset"),
+					eiHandler.getRelationshipsByName(Dataset.class).get("sample"),
+					eiHandler.getRelationshipsByName(Sample.class).get("type") };
+			Relationship[] sampleTypeRelationships = {
+					eiHandler.getRelationshipsByName(Datafile.class).get("dataset"),
+					eiHandler.getRelationshipsByName(Dataset.class).get("sample") };
+			documentFields.put("name", null);
+			documentFields.put("description", null);
+			documentFields.put("location", null);
+			documentFields.put("doi", null);
+			documentFields.put("date", null);
+			documentFields.put("fileSize", null);
+			documentFields.put("fileCount", null);
+			documentFields.put("id", null);
+			documentFields.put("dataset.id", null);
+			documentFields.put("dataset.name", datasetRelationships);
+			documentFields.put("sample.id", datasetRelationships);
+			documentFields.put("sample.name", sampleRelationships);
+			documentFields.put("sample.investigation.id", sampleRelationships);
+			documentFields.put("sample.type.id", sampleRelationships);
+			documentFields.put("sample.type.name", sampleTypeRelationships);
+			documentFields.put("investigation.id", datasetRelationships);
+			documentFields.put("investigation.name", investigationRelationships);
+			documentFields.put("investigation.startDate", investigationRelationships);
+			documentFields.put("visitId", investigationRelationships);
+			documentFields.put("datafileFormat.id", null);
+			documentFields.put("datafileFormat.name", datafileFormatRelationships);
+			documentFields.put("InvestigationInstrument instrument.id", instrumentRelationships);
+		}
+		return documentFields;
+	}
+
 }

src/main/java/org/icatproject/core/entity/DatafileFormat.java

@@ -2,8 +2,12 @@
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
+import javax.json.stream.JsonGenerator;
 import javax.persistence.CascadeType;
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -14,6 +18,8 @@
 import javax.persistence.Table;
 import javax.persistence.UniqueConstraint;
 
+import org.icatproject.core.manager.search.SearchApi;
+
 @Comment("A data file format")
 @SuppressWarnings("serial")
 @Entity
@@ -51,6 +57,8 @@ public void setFacility(Facility facility) {
 	@Column(name = "VERSION", nullable = false)
 	private String version;
 
+	public static Set<String> docFields = new HashSet<>(Arrays.asList("datafileFormat.name", "datafileFormat.id"));
+
 	/* Needed for JPA */
 	public DatafileFormat() {
 	}
@@ -95,4 +103,10 @@ public void setVersion(String version) {
 		this.version = version;
 	}
 
+	@Override
+	public void getDoc(JsonGenerator gen) {
+		SearchApi.encodeString(gen, "datafileFormat.name", name);
+		SearchApi.encodeString(gen, "datafileFormat.id", id);
+	}
+
 }

src/main/java/org/icatproject/core/entity/DatafileParameter.java

@@ -13,8 +13,8 @@
 
 import org.icatproject.core.IcatException;
 import org.icatproject.core.manager.EntityBeanManager.PersistMode;
+import org.icatproject.core.manager.search.SearchApi;
 import org.icatproject.core.manager.GateKeeper;
-import org.icatproject.core.manager.LuceneApi;
 
 @Comment("A parameter associated with a data file")
 @SuppressWarnings("serial")
@@ -56,7 +56,7 @@ public void setDatafile(Datafile datafile) {
 	@Override
 	public void getDoc(JsonGenerator gen) {
 		super.getDoc(gen);
-		LuceneApi.encodeSortedDocValuesField(gen, "datafile", datafile.id);
+		SearchApi.encodeString(gen, "datafile.id", datafile.id);
 	}
 
 }