Skip to content

Commit

Permalink
DataSync Support distinctOn
Browse files Browse the repository at this point in the history
  • Loading branch information
@s0kil
s0kil committed Mar 16, 2022
1 parent 0736b4f commit e39c3f0
Show file tree
Hide file tree
Showing 5 changed files with 65 additions and 24 deletions.
3 changes: 2 additions & 1 deletion IHP/DataSync/DynamicQuery.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ data DynamicSQLQuery = DynamicSQLQuery
, selectedColumns :: SelectedColumns
, whereCondition :: !(Maybe ConditionExpression)
, orderByClause :: ![OrderByClause]
, distinctOnColumn :: !(Maybe ByteString)
, limit :: !(Maybe Int)
, offset :: !(Maybe Int)
} deriving (Show, Eq)
Expand Down Expand Up @@ -178,6 +179,7 @@ instance FromJSON DynamicSQLQuery where
<*> v .: "selectedColumns"
<*> v .: "whereCondition"
<*> v .: "orderByClause"
<*> v .:? "distinctOnColumn" -- distinctOnColumn can be absent in older versions of ihp-datasync.js
<*> v .:? "limit" -- Limit can be absent in older versions of ihp-datasync.js
<*> v .:? "offset" -- Offset can be absent in older versions of ihp-datasync.js

Expand All @@ -194,4 +196,3 @@ instance FromJSON OrderByClause where
"OrderByTSRank" -> OrderByTSRank <$> v .: "tsvector" <*> v .: "tsquery"
otherwise -> error ("Invalid tag: " <> otherwise)
tagged <|> oldFormat

19 changes: 12 additions & 7 deletions IHP/DataSync/DynamicQueryCompiler.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,21 @@ import qualified Data.List as List
compileQuery :: DynamicSQLQuery -> (PG.Query, [PG.Action])
compileQuery DynamicSQLQuery { .. } = (sql, args)
where
sql = "SELECT ? FROM ?" <> whereSql <> orderBySql <> limitSql <> offsetSql
args = catMaybes
[ Just (compileSelectedColumns selectedColumns)
, Just (PG.toField (PG.Identifier table))
]
sql = "SELECT" <> distinctOnSql <> "? FROM ?" <> whereSql <> orderBySql <> limitSql <> offsetSql
args = distinctOnArgs
<> catMaybes
[ Just (compileSelectedColumns selectedColumns)
, Just (PG.toField (PG.Identifier table))
]
<> whereArgs
<> orderByArgs
<> limitArgs
<> offsetArgs

(distinctOnSql, distinctOnArgs) = case distinctOnColumn of
Just column -> (" DISTINCT ON (?) ", [PG.toField $ PG.Identifier (fieldNameToColumnName $ cs column)])
Nothing -> (" ", [])

(orderBySql, orderByArgs) = case orderByClause of
[] -> ("", [])
orderByClauses ->
Expand Down Expand Up @@ -56,7 +61,7 @@ compileQuery DynamicSQLQuery { .. } = (sql, args)
(limitSql, limitArgs) = case limit of
Just limit -> (" LIMIT ?", [PG.toField limit])
Nothing -> ("", [])

(offsetSql, offsetArgs) = case offset of
Just offset -> (" OFFSET ?", [PG.toField offset])
Nothing -> ("", [])
Expand Down Expand Up @@ -115,4 +120,4 @@ compileOperator OpAnd = "AND"
compileOperator OpOr = "OR"
compileOperator OpIs = "IS"
compileOperator OpIsNot = "IS NOT"
compileOperator OpTSMatch = "@@"
compileOperator OpTSMatch = "@@"
13 changes: 7 additions & 6 deletions IHP/DataSync/REST/Controller.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ instance (
|> map aesonValueToPostgresValue

let params = (PG.Identifier table, PG.In (map PG.Identifier columns), PG.In values)

result :: Either EnhancedSqlError [[Field]] <- Exception.try do
sqlQueryWithRLS query params

Expand Down Expand Up @@ -87,14 +87,14 @@ instance (
|> HashMap.elems
|> map aesonValueToPostgresValue
)


let params = (PG.Identifier table, PG.In (map PG.Identifier columns), PG.Values [] values)

result :: [[Field]] <- sqlQueryWithRLS query params
renderJson result



action UpdateRecordAction { table, id } = do
ensureRLSEnabled table
Expand Down Expand Up @@ -170,6 +170,7 @@ buildDynamicQueryFromRequest table = DynamicSQLQuery
, selectedColumns = paramOrDefault SelectAll "fields"
, whereCondition = Nothing
, orderByClause = paramList "orderBy"
, distinctOnColumn = paramOrNothing "distinctOnColumn"
, limit = paramOrNothing "limit"
, offset = paramOrNothing "offset"
}
Expand Down Expand Up @@ -220,18 +221,18 @@ aesonValueToPostgresValue (Number value) = case Scientific.floatingOrInteger val
Left (floating :: Double) -> PG.toField floating
Right (integer :: Integer) -> PG.toField integer
aesonValueToPostgresValue Data.Aeson.Null = PG.toField PG.Null
aesonValueToPostgresValue object@(Object values) =
aesonValueToPostgresValue object@(Object values) =
let
tryDecodeAsPoint :: Maybe Point
tryDecodeAsPoint = do
xValue <- HashMap.lookup "x" values
yValue <- HashMap.lookup "y" values
x <- case xValue of
Number number -> pure (Scientific.toRealFloat number)
otherwise -> Nothing
otherwise -> Nothing
y <- case yValue of
Number number -> pure (Scientific.toRealFloat number)
otherwise -> Nothing
otherwise -> Nothing
pure Point { x, y }
in
-- This is really hacky and is mostly duck typing. We should refactor this in the future to
Expand Down
45 changes: 36 additions & 9 deletions Test/DataSync/DynamicQueryCompiler.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ tests = do
, selectedColumns = SelectAll
, whereCondition = Nothing
, orderByClause = []
, distinctOnColumn = Nothing
, limit = Nothing
, offset = Nothing
}
Expand All @@ -27,12 +28,13 @@ tests = do
( "SELECT ? FROM ?"
, [PG.Plain "*", PG.EscapeIdentifier "posts"]
)

it "compile a select query with order by" do
let query = DynamicSQLQuery
{ table = "posts"
, selectedColumns = SelectAll
, whereCondition = Nothing
, distinctOnColumn = Nothing
, orderByClause = [OrderByClause { orderByColumn = "title", orderByDirection = Desc }]
, limit = Nothing
, offset = Nothing
Expand All @@ -42,7 +44,7 @@ tests = do
( "SELECT ? FROM ? ORDER BY ? ?"
, [PG.Plain "*", PG.EscapeIdentifier "posts", PG.EscapeIdentifier "title", PG.Plain "DESC"]
)

it "compile a select query with multiple order bys" do
let query = DynamicSQLQuery
{ table = "posts"
Expand All @@ -52,6 +54,7 @@ tests = do
OrderByClause { orderByColumn = "createdAt", orderByDirection = Desc },
OrderByClause { orderByColumn = "title", orderByDirection = Asc }
]
, distinctOnColumn = Nothing
, limit = Nothing
, offset = Nothing
}
Expand All @@ -60,13 +63,14 @@ tests = do
( "SELECT ? FROM ? ORDER BY ? ?, ? ?"
, [PG.Plain "*", PG.EscapeIdentifier "posts", PG.EscapeIdentifier "created_at", PG.Plain "DESC", PG.EscapeIdentifier "title", PG.Plain ""]
)

it "compile a basic select query with a where condition" do
let query = DynamicSQLQuery
{ table = "posts"
, selectedColumns = SelectAll
, whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpEqual (LiteralExpression (TextValue "b8553ce9-6a42-4a68-b5fc-259be3e2acdc"))
, orderByClause = []
, distinctOnColumn = Nothing
, limit = Nothing
, offset = Nothing
}
Expand All @@ -82,6 +86,7 @@ tests = do
, selectedColumns = SelectAll
, whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpEqual (LiteralExpression (TextValue "b8553ce9-6a42-4a68-b5fc-259be3e2acdc"))
, orderByClause = [ OrderByClause { orderByColumn = "createdAt", orderByDirection = Desc } ]
, distinctOnColumn = Nothing
, limit = Nothing
, offset = Nothing
}
Expand All @@ -96,6 +101,7 @@ tests = do
{ table = "posts"
, selectedColumns = SelectAll
, whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpEqual (LiteralExpression (TextValue "b8553ce9-6a42-4a68-b5fc-259be3e2acdc"))
, distinctOnColumn = Nothing
, orderByClause = []
, limit = Just 50
, offset = Nothing
Expand All @@ -112,6 +118,7 @@ tests = do
, selectedColumns = SelectAll
, whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpEqual (LiteralExpression (TextValue "b8553ce9-6a42-4a68-b5fc-259be3e2acdc"))
, orderByClause = []
, distinctOnColumn = Nothing
, limit = Nothing
, offset = Just 50
}
Expand All @@ -127,25 +134,27 @@ tests = do
, selectedColumns = SelectAll
, whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpEqual (LiteralExpression (TextValue "b8553ce9-6a42-4a68-b5fc-259be3e2acdc"))
, orderByClause = []
, distinctOnColumn = Nothing
, limit = Just 25
, offset = Just 50
}

compileQuery query `shouldBe`
( "SELECT ? FROM ? WHERE (?) = (?) LIMIT ? OFFSET ?"
, [PG.Plain "*", PG.EscapeIdentifier "posts", PG.EscapeIdentifier "user_id", PG.Escape "b8553ce9-6a42-4a68-b5fc-259be3e2acdc", PG.Plain "25", PG.Plain "50"]
)

it "compile 'field = NULL' conditions to 'field IS NULL'" do
let query = DynamicSQLQuery
{ table = "posts"
, selectedColumns = SelectAll
, whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpEqual NullExpression
, orderByClause = []
, distinctOnColumn = Nothing
, limit = Nothing
, offset = Nothing
}

compileQuery query `shouldBe`
( "SELECT ? FROM ? WHERE (?) IS NULL"
, [PG.Plain "*", PG.EscapeIdentifier "posts", PG.EscapeIdentifier "user_id"]
Expand All @@ -157,10 +166,11 @@ tests = do
, selectedColumns = SelectAll
, whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpNotEqual NullExpression
, orderByClause = []
, distinctOnColumn = Nothing
, limit = Nothing
, offset = Nothing
}

compileQuery query `shouldBe`
( "SELECT ? FROM ? WHERE (?) IS NOT NULL"
, [PG.Plain "*", PG.EscapeIdentifier "posts", PG.EscapeIdentifier "user_id"]
Expand All @@ -172,11 +182,28 @@ tests = do
, selectedColumns = SelectAll
, whereCondition = Just $ InfixOperatorExpression (ColumnExpression "ts") OpTSMatch (CallExpression { functionCall = ToTSQuery { text = "test" }})
, orderByClause = [ OrderByTSRank { tsvector = "ts", tsquery = "test" } ]
, distinctOnColumn = Nothing
, limit = Nothing
, offset = Nothing
}

compileQuery query `shouldBe`
( "SELECT ? FROM ? WHERE (?) @@ (to_tsquery('english', ?)) ORDER BY ts_rank(?, to_tsquery('english', ?))"
, [PG.Plain "*", PG.EscapeIdentifier "products", PG.EscapeIdentifier "ts", PG.Escape "test", PG.EscapeIdentifier "ts", PG.Escape "test"]
)
)

it "compile a basic select query with distinctOn" do
let query = DynamicSQLQuery
{ table = "posts"
, selectedColumns = SelectAll
, whereCondition = Nothing
, orderByClause = []
, distinctOnColumn = Just "groupId"
, limit = Nothing
, offset = Nothing
}

compileQuery query `shouldBe`
( "SELECT DISTINCT ON (?) ? FROM ?"
, [PG.EscapeIdentifier "group_id", PG.Plain "*", PG.EscapeIdentifier "posts"]
)
9 changes: 8 additions & 1 deletion lib/IHP/DataSync/ihp-querybuilder.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ class QueryBuilder extends ConditionBuildable {
selectedColumns: { tag: 'SelectAll' },
whereCondition: null,
orderByClause: [],
distinctOnColumn: null,
limit: null,
offset: null
};
Expand Down Expand Up @@ -308,6 +309,12 @@ class QueryBuilder extends ConditionBuildable {
return this;
}

distinctOn(column) {
this.query.distinctOnColumn = column

return this
}

limit(limit) {
if (limit !== null && !Number.isInteger(limit)) {
throw new Error('limit needs to be an integer, or null if no limit should be used');
Expand Down Expand Up @@ -339,7 +346,7 @@ class QueryBuilder extends ConditionBuildable {
const result = await this.limit(1).fetch();
return result.length > 0 ? result[0] : null;
}

subscribe(callback) {
const dataSubscription = new DataSubscription(this.query);
dataSubscription.createOnServer();
Expand Down

0 comments on commit e39c3f0

Please sign in to comment.