Skip to content

Commit

Permalink
SOLR-15576: Allow filtering on ISO-8601 formatted timestamp literals …
Browse files Browse the repository at this point in the history 
…in SQL WHERE clause (apache#247)
  • Loading branch information
@thelabdude
thelabdude authored Aug 6, 2021
1 parent dd655f7 commit 072c9c8
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 28 deletions.
2 changes: 2 additions & 0 deletions solr/CHANGES.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,8 @@ Improvements

* SOLR-15564: Improve filtering expressions in /admin/metrics. (ab)

* SOLR-15576: Allow filtering on ISO-8601 formatted timestamp literals in SQL WHERE clause (Timothy Potter)

Optimizations
---------------------
* SOLR-15433: Replace transient core cache LRU by Caffeine cache. (Bruno Roustant)
Expand Down
67 changes: 45 additions & 22 deletions solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.Pair;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -218,7 +219,8 @@ protected String translateLike(RexNode like, boolean isNegativeQuery) {
terms = terms.replace("'", "").replace('%', '*').replace('_', '?');
boolean wrappedQuotes = false;
if (!terms.startsWith("(") && !terms.startsWith("[") && !terms.startsWith("{")) {
terms = "\"" + terms + "\"";
// restore the * and ? after escaping
terms = "\"" + ClientUtils.escapeQueryChars(terms).replace("\\*", "*").replace("\\?", "?") + "\"";
wrappedQuotes = true;
}

Expand All @@ -239,23 +241,10 @@ protected String translateComparison(RexNode node) {
RexLiteral value = binaryTranslated.getValue();
switch (kind) {
case EQUALS:
SqlTypeName fieldTypeName = ((RexCall) node).getOperands().get(0).getType().getSqlTypeName();
String terms = toSolrLiteralForEquals(value, fieldTypeName).trim();

boolean wrappedQuotes = false;
if (!terms.startsWith("(") && !terms.startsWith("[") && !terms.startsWith("{")) {
terms = "\"" + terms + "\"";
wrappedQuotes = true;
}

String clause = key + ":" + terms;
if (terms.contains("*") && wrappedQuotes) {
clause = "{!complexphrase}" + clause;
}
this.negativeQuery = false;
return clause;
return toEqualsClause(key, value, node);
case NOT_EQUALS:
return "-(" + key + ":" + toSolrLiteral(value) + ")";
return "-(" + toEqualsClause(key, value, node) + ")";
case LESS_THAN:
this.negativeQuery = false;
return "(" + key + ": [ * TO " + toSolrLiteral(value) + " })";
Expand All @@ -278,6 +267,24 @@ protected String translateComparison(RexNode node) {
}
}

private String toEqualsClause(String key, RexLiteral value, RexNode node) {
SqlTypeName fieldTypeName = ((RexCall) node).getOperands().get(0).getType().getSqlTypeName();
String terms = toSolrLiteralForEquals(value, fieldTypeName).trim();

boolean wrappedQuotes = false;
if (!terms.startsWith("(") && !terms.startsWith("[") && !terms.startsWith("{")) {
terms = "\"" + ClientUtils.escapeQueryChars(terms) + "\"";
wrappedQuotes = true;
}

String clause = key + ":" + terms;
if (terms.contains("*") && wrappedQuotes) {
clause = "{!complexphrase}" + clause;
}

return clause;
}

// translate to a literal string value for Solr queries, such as translating a
// Calcite timestamp value into an ISO-8601 formatted timestamp that Solr likes
private String toSolrLiteral(RexLiteral literal) {
Expand Down Expand Up @@ -339,15 +346,27 @@ protected Pair<String, RexLiteral> translateBinary(RexCall call) {
if (a != null) {
return a;
}
final Pair<String, RexLiteral> b = translateBinary2(right, left);
if (b != null) {
return b;

// we can swap these if doing an equals / not equals
if (call.op.kind == SqlKind.EQUALS || call.op.kind == SqlKind.NOT_EQUALS) {
final Pair<String, RexLiteral> b = translateBinary2(right, left);
if (b != null) {
return b;
}
}

if (left.getKind() == SqlKind.CAST && right.getKind() == SqlKind.CAST) {
return translateBinary2(((RexCall)left).operands.get(0), ((RexCall)right).operands.get(0));
}

// for WHERE clause like: pdatex >= '2021-07-13T15:12:10.037Z'
if (left.getKind() == SqlKind.INPUT_REF && right.getKind() == SqlKind.CAST) {
final RexCall cast = ((RexCall)right);
if (cast.operands.size() == 1 && cast.operands.get(0).getKind() == SqlKind.LITERAL) {
return translateBinary2(left, cast.operands.get(0));
}
}

throw new AssertionError("cannot translate call " + call);
}

Expand Down Expand Up @@ -493,10 +512,14 @@ protected Pair<String, RexLiteral> translateBinary(RexCall call) {
}
return a;
}
final Pair<String, RexLiteral> b = translateBinary2(right, left);
if (b != null) {
return b;

if (call.op.kind == SqlKind.EQUALS || call.op.kind == SqlKind.NOT_EQUALS) {
final Pair<String, RexLiteral> b = translateBinary2(right, left);
if (b != null) {
return b;
}
}

throw new AssertionError("cannot translate call " + call);
}

Expand Down
45 changes: 41 additions & 4 deletions solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -338,10 +338,10 @@ public void testWhere() throws Exception {
.add("id", "2", "text_t", "XXXX XXXX", "str_s", "b", "field_i", "8")
.add("id", "3", "text_t", "XXXX XXXX", "str_s", "a", "field_i", "20")
.add("id", "4", "text_t", "XXXX XXXX", "str_s", "b", "field_i", "11")
.add("id", "5", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "30")
.add("id", "6", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "40")
.add("id", "7", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "50")
.add("id", "8", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "60")
.add("id", "5", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "30", "specialchars_s", "witha|pipe")
.add("id", "6", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "40", "specialchars_s", "witha\\slash")
.add("id", "7", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "50", "specialchars_s", "witha!bang")
.add("id", "8", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "60", "specialchars_s", "witha\"quote")
.commit(cluster.getSolrClient(), COLLECTIONORALIAS);

String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/" + COLLECTIONORALIAS;
Expand Down Expand Up @@ -452,6 +452,14 @@ public void testWhere() throws Exception {
tuple = tuples.get(1);
assertEquals("8", tuple.get("id"));

expectResults("SELECT id FROM $ALIAS WHERE str_s = 'a'", 2);
expectResults("SELECT id FROM $ALIAS WHERE 'a' = str_s", 2);
expectResults("SELECT id FROM $ALIAS WHERE str_s <> 'c'", 4);
expectResults("SELECT id FROM $ALIAS WHERE 'c' <> str_s", 4);
expectResults("SELECT id FROM $ALIAS WHERE specialchars_s = 'witha\"quote'", 1);
expectResults("SELECT id FROM $ALIAS WHERE specialchars_s = 'witha|pipe'", 1);
expectResults("SELECT id FROM $ALIAS WHERE specialchars_s LIKE 'with%'", 4);
expectResults("SELECT id FROM $ALIAS WHERE specialchars_s LIKE 'witha|%'", 1);
}

@Test
Expand Down Expand Up @@ -1641,6 +1649,17 @@ public void testSQLException() throws Exception {
assert (tuple.EOF);
assert (tuple.EXCEPTION);
assert (tuple.getException().contains("No match found for function signature blah"));

// verify exception message formatting with wildcard query
sParams = mapParams(CommonParams.QT, "/sql", "aggregationMode", "map_reduce",
"stmt",
"select str_s from collection1 where not_a_field LIKE 'foo%'");

solrStream = new SolrStream(baseUrl, sParams);
tuple = getTuple(new ExceptionStream(solrStream));
assert (tuple.EOF);
assert (tuple.EXCEPTION);
assert (tuple.getException().contains("Column 'not_a_field' not found in any table"));
}

@Test
Expand Down Expand Up @@ -2058,6 +2077,24 @@ public void testDateHandling() throws Exception {
expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex BETWEEN '2021-06-03' AND '2021-06-05'", 4);
}

@Test
public void testISO8601TimestampFiltering() throws Exception {
new UpdateRequest()
.add("id", "1", "pdatex", "2021-07-13T15:12:09.037Z")
.add("id", "2", "pdatex", "2021-07-13T15:12:10.037Z")
.add("id", "3", "pdatex", "2021-07-13T15:12:11.037Z")
.commit(cluster.getSolrClient(), COLLECTIONORALIAS);

expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex >= CAST('2021-07-13 15:12:10.037' as TIMESTAMP)", 2);
expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex >= '2021-07-13T15:12:10.037Z'", 2);
expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex < '2021-07-13T15:12:10.037Z'", 1);
expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex = '2021-07-13T15:12:10.037Z'", 1);
expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex <> '2021-07-13T15:12:10.037Z'", 2);
expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex BETWEEN '2021-07-13T15:12:09.037Z' AND '2021-07-13T15:12:10.037Z' ORDER BY pdatex ASC", 2);
expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex >= '2021-07-13T15:12:10.037Z'", 2);
expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex >= '2021-07-13T15:12:10.037Z' ORDER BY pdatex ASC LIMIT 10", 2);
}

@Test
public void testAggsOnCustomFieldType() throws Exception {
new UpdateRequest()
Expand Down
3 changes: 1 addition & 2 deletions solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,8 +283,7 @@ public void open() throws IOException {
resultSet = statement.executeQuery(sqlQuery);
resultSet.setFetchSize(fetchSize);
} catch (SQLException e) {
throw new IOException(String.format(Locale.ROOT, "Failed to execute sqlQuery '%s' against JDBC connection '%s'.\n"
+ e.getMessage(), sqlQuery, connectionUrl), e);
throw new IOException(String.format(Locale.ROOT, "Failed to execute sqlQuery '%s' against JDBC connection '%s'.%nCaused by: %s", sqlQuery, connectionUrl, e.getMessage()), e);
}

try{
Expand Down

0 comments on commit 072c9c8

Please sign in to comment.