Skip to content

Commit

Permalink
Merge branch 'master' of github.com:agentm/project-m36 into sqlegacy
Browse files Browse the repository at this point in the history
# Conflicts:
#	project-m36.cabal
  • Loading branch information
agentm committed May 9, 2024
2 parents bed20c6 + 79b3a66 commit 7c5070c
Show file tree
Hide file tree
Showing 11 changed files with 195 additions and 28 deletions.
3 changes: 2 additions & 1 deletion Changelog.markdown
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Future
# 2024-01-12 (v0.9.9)

* revert to using streamly 0.9.0 due to over-the-wire corruption bug in 0.10.0
* fixed toAtom/fromAtom for NonEmpty lists (#363)

# 2023-12-30 (v0.9.8)
Expand Down
2 changes: 1 addition & 1 deletion cabal.project
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ package *
split-sections: True

-- allow fast-builder to build with GHC 9.2.2 (currently pegged at 9.0.1)
allow-newer: fast-builder:base, streamly-bytestring:bytestring
allow-newer: fast-builder:base
5 changes: 3 additions & 2 deletions project-m36.cabal
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Cabal-Version: 2.2
Name: project-m36
Version: 0.9.8
Version: 0.9.9
License: MIT
--note that this license specification is erroneous and only labeled MIT to appease hackage which does not recognize public domain packages in cabal >2.2- Project:M36 is dedicated to the public domain
Build-Type: Simple
Expand Down Expand Up @@ -35,7 +35,7 @@ Flag haskell-scripting
Default: True

Library
Build-Depends: base>=4.16 && < 4.19, ghc-paths, mtl, containers, unordered-containers, hashable, haskeline, directory, MonadRandom, random-shuffle, uuid >= 1.3.12, cassava >= 0.4.5.1 && < 0.6, text, bytestring, deepseq, deepseq-generics, vector, parallel, monad-parallel, exceptions, transformers, gnuplot, filepath, zlib, directory, temporary, stm, time, old-locale, rset, attoparsec, either, base64-bytestring, data-interval, extended-reals, aeson >= 1.1, path-pieces, conduit, resourcet, http-api-data, semigroups, QuickCheck, quickcheck-instances, list-t, stm-containers >= 0.2.15, foldl, optparse-applicative, Glob, cryptohash-sha256, text-manipulate >= 0.2.0.1 && < 0.4, winery >= 1.4, curryer-rpc>=0.3.2, network, async, vector-instances, recursion-schemes, streamly == 0.9.0, convertible, fast-builder, scientific
Build-Depends: base>=4.16 && < 4.19, ghc-paths, mtl, containers, unordered-containers, hashable, haskeline, directory, MonadRandom, random-shuffle, uuid >= 1.3.12, cassava >= 0.4.5.1 && < 0.6, text, bytestring, deepseq, deepseq-generics, vector, parallel, monad-parallel, exceptions, transformers, gnuplot, filepath, zlib, directory, temporary, stm, time, old-locale, rset, attoparsec, either, base64-bytestring, data-interval, extended-reals, aeson >= 1.1, path-pieces, conduit, resourcet, http-api-data, semigroups, QuickCheck, quickcheck-instances, list-t, stm-containers >= 0.2.15, foldl, optparse-applicative, Glob, cryptohash-sha256, text-manipulate >= 0.2.0.1 && < 0.4, winery >= 1.4, curryer-rpc>=0.3.5, network, async, vector-instances, recursion-schemes, streamly == 0.9.0, convertible, fast-builder, scientific
if flag(haskell-scripting)
Build-Depends: ghc >= 9.0 && < 9.5
CPP-Options: -DPM36_HASKELL_SCRIPTING
Expand Down Expand Up @@ -145,6 +145,7 @@ Library
ProjectM36.SQL.DBUpdate,
ProjectM36.SQL.CreateTable,
ProjectM36.SQL.DropTable
ProjectM36.ReferencedTransactionIds
GHC-Options: -Wall -rdynamic
if os(windows)
Build-Depends: Win32 >= 2.12
Expand Down
9 changes: 7 additions & 2 deletions release.nix
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ let
overrides = self: super: {
curryer-rpc = self.callHackageDirect {
pkg = "curryer-rpc";
ver = "0.3.2";
sha256 = "sha256-QiKsaFcIzOrtCpgVrgArnj7Hd09JVjF67huam+0aZSc="; } {};
ver = "0.3.5";
sha256 = "sha256-7mEJOBKzA2rTnLxZme8E6zFv0VkiXBo5L/jUJSNPaNE="; } {};

streamly = self.callHackageDirect {
pkg = "streamly";
Expand All @@ -27,6 +27,11 @@ let
pkg = "streamly-core";
ver = "0.1.0";
sha256 = "sha256-hoSV6Q2+X5a7hFnJAArqNPjcMaCVyX9Vz4FcxeJ+jgI="; } {};

streamly-bytestring = self.callHackageDirect {
pkg = "streamly-bytestring";
ver = "0.2.1";
sha256 = "sha256-EcH6qq4nRjea3xQ66Zlqgjjg7lF/grkKJI0+tTO4B84="; } {};

lockfree-queue = self.callHackageDirect {
pkg = "lockfree-queue";
Expand Down
2 changes: 1 addition & 1 deletion src/lib/ProjectM36/Atomable.hs
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ instance Atomable a => Atomable [a] where

instance Atomable a => Atomable (NE.NonEmpty a) where
toAtom (x NE.:| []) = ConstructedAtom "NECons" (nonEmptyListAtomType (toAtomType (Proxy :: Proxy a))) [toAtom x]
toAtom (x NE.:| xs) = ConstructedAtom "NECons" (nonEmptyListAtomType (toAtomType (Proxy :: Proxy a))) (toAtom x : toAtom xs : [])
toAtom (x NE.:| xs) = ConstructedAtom "NECons" (nonEmptyListAtomType (toAtomType (Proxy :: Proxy a))) [toAtom x, toAtom xs]
fromAtom (ConstructedAtom "NECons" _ [x]) = fromAtom x NE.:| []
fromAtom (ConstructedAtom "NECons" _ [x,y] ) = fromAtom x NE.:| fromAtom y
fromAtom _x = error "improper fromAtom (NonEmptyList a)"
Expand Down
8 changes: 3 additions & 5 deletions src/lib/ProjectM36/Base.hs
Original file line number Diff line number Diff line change
Expand Up @@ -459,6 +459,9 @@ transactionHeadsForGraph (TransactionGraph hs _) = hs
transactionsForGraph :: TransactionGraph -> S.Set Transaction
transactionsForGraph (TransactionGraph _ ts) = ts

transactionIdsForGraph :: TransactionGraph -> S.Set TransactionId
transactionIdsForGraph = S.map transactionId . transactionsForGraph

-- | Every transaction has context-specific information attached to it.
-- The `TransactionDiff`s represent child/edge relationships to previous transactions (branches or continuations of the same branch).
data TransactionInfo = TransactionInfo {
Expand All @@ -468,11 +471,6 @@ data TransactionInfo = TransactionInfo {
} deriving (Show, Generic)

type TransactionParents = NE.NonEmpty TransactionId
{-
data TransactionInfo = TransactionInfo TransactionId TransactionDiffs UTCTime | -- 1 parent + n children
MergeTransactionInfo TransactionId TransactionId TransactionDiffs UTCTime -- 2 parents, n children
deriving (Show, Generic)
-}

-- | Every set of modifications made to the database are atomically committed to the transaction graph as a transaction.
type TransactionId = UUID
Expand Down
3 changes: 2 additions & 1 deletion src/lib/ProjectM36/Error.hs
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,8 @@ data MergeError = SelectedHeadMismatchMergeError |
InvalidMergeStrategyError MergeStrategy | -- this is an internal coding error
DisconnectedTransactionNotAMergeHeadError TransactionId |
StrategyViolatesComponentMergeError | --failed merge in inc deps, relvars, etc.
StrategyViolatesRelationVariableMergeError |
StrategyViolatesRelationVariableMergeError RelationalError |
StrategyWithoutPreferredBranchResolutionMergeError |
StrategyViolatesTypeConstructorMergeError |
StrategyViolatesRegisteredQueryMergeError [RegisteredQueryName]
deriving (Show, Eq, Generic, Typeable)
Expand Down
139 changes: 139 additions & 0 deletions src/lib/ProjectM36/ReferencedTransactionIds.hs
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
{-# LANGUAGE FlexibleInstances #-}
module ProjectM36.ReferencedTransactionIds where
import ProjectM36.Base
import ProjectM36.Error
import qualified ProjectM36.Transaction as T
import ProjectM36.RelationalExpression
import qualified Data.Map as M
import qualified Data.Set as S
import Control.Monad (foldM)

type TransactionIds = S.Set TransactionId

-- return all transactionIds referenced recursively- can be used to create subgraph of transaction dependencies
class ReferencedTransactionIds a where
referencedTransactionIds :: a -> TransactionIds

instance ReferencedTransactionIds a => ReferencedTransactionIds (RelationalExprBase a) where
referencedTransactionIds x = case x of
MakeRelationFromExprs (Just attrExprs) tupleExprs ->
S.unions (referencedTransactionIds tupleExprs : map referencedTransactionIds attrExprs)
MakeRelationFromExprs Nothing tupleExprs ->
referencedTransactionIds tupleExprs
MakeStaticRelation{} -> S.empty
ExistingRelation{} -> S.empty
RelationVariable _ marker -> referencedTransactionIds marker
Project attrNames expr -> S.union (referencedTransactionIds attrNames) (referencedTransactionIds expr)
Union exprA exprB -> S.union (referencedTransactionIds exprA) (referencedTransactionIds exprB)
Join exprA exprB -> S.union (referencedTransactionIds exprA) (referencedTransactionIds exprB)
Rename _ _ expr -> referencedTransactionIds expr
Difference exprA exprB -> S.union (referencedTransactionIds exprA) (referencedTransactionIds exprB)
Group attrNames _ expr -> S.union (referencedTransactionIds attrNames) (referencedTransactionIds expr)
Ungroup _ expr -> referencedTransactionIds expr
Restrict pred' expr -> S.union (referencedTransactionIds pred') (referencedTransactionIds expr)
Equals exprA exprB -> S.union (referencedTransactionIds exprA) (referencedTransactionIds exprB)
NotEquals exprA exprB -> S.union (referencedTransactionIds exprA) (referencedTransactionIds exprB)
Extend extendTupleExpr expr -> S.union (referencedTransactionIds extendTupleExpr) (referencedTransactionIds expr)
With assocs expr -> S.unions (referencedTransactionIds expr : map tAssocs assocs)
where
tAssocs (withNameExpr, rExpr) = S.union (referencedTransactionIds withNameExpr) (referencedTransactionIds rExpr)

instance ReferencedTransactionIds a => ReferencedTransactionIds (AttributeExprBase a) where
referencedTransactionIds NakedAttributeExpr{} = S.empty
referencedTransactionIds (AttributeAndTypeNameExpr _ _ marker) = referencedTransactionIds marker

instance ReferencedTransactionIds a => ReferencedTransactionIds (TupleExprBase a) where
referencedTransactionIds (TupleExpr tMap) =
S.unions (referencedTransactionIds <$> M.elems tMap)

instance ReferencedTransactionIds a => ReferencedTransactionIds (TupleExprsBase a) where
referencedTransactionIds (TupleExprs marker tupleExprs) =
S.unions (referencedTransactionIds marker : (referencedTransactionIds <$> tupleExprs))

instance ReferencedTransactionIds GraphRefTransactionMarker where
referencedTransactionIds (TransactionMarker tid) = S.singleton tid
referencedTransactionIds UncommittedContextMarker = S.empty -- we have other methods to determine if there is an uncommitted transaction marker in the expr

instance ReferencedTransactionIds a => ReferencedTransactionIds (AttributeNamesBase a) where
referencedTransactionIds names =
case names of
AttributeNames{} -> S.empty
InvertedAttributeNames{} -> S.empty
UnionAttributeNames exprA exprB ->
S.union (referencedTransactionIds exprA) (referencedTransactionIds exprB)
IntersectAttributeNames exprA exprB ->
S.union (referencedTransactionIds exprA) (referencedTransactionIds exprB)
RelationalExprAttributeNames rExpr ->
referencedTransactionIds rExpr

instance ReferencedTransactionIds a => ReferencedTransactionIds (RestrictionPredicateExprBase a) where
referencedTransactionIds expr =
case expr of
TruePredicate -> mempty
AndPredicate exprA exprB ->
S.union (referencedTransactionIds exprA) (referencedTransactionIds exprB)
OrPredicate exprA exprB ->
S.union (referencedTransactionIds exprA) (referencedTransactionIds exprB)
NotPredicate exprA ->
referencedTransactionIds exprA
RelationalExprPredicate rExpr ->
referencedTransactionIds rExpr
AtomExprPredicate aExpr ->
referencedTransactionIds aExpr
AttributeEqualityPredicate _ aExpr ->
referencedTransactionIds aExpr

instance ReferencedTransactionIds a => ReferencedTransactionIds (ExtendTupleExprBase a) where
referencedTransactionIds (AttributeExtendTupleExpr _ aExpr) =
referencedTransactionIds aExpr

instance ReferencedTransactionIds a => ReferencedTransactionIds (WithNameExprBase a) where
referencedTransactionIds (WithNameExpr _ marker) = referencedTransactionIds marker

instance ReferencedTransactionIds a => ReferencedTransactionIds (AtomExprBase a) where
referencedTransactionIds expr =
case expr of
AttributeAtomExpr{} -> mempty
NakedAtomExpr{} -> mempty
FunctionAtomExpr _ args marker ->
S.unions (referencedTransactionIds marker : (referencedTransactionIds <$> args))
RelationAtomExpr rExpr ->
referencedTransactionIds rExpr
ConstructedAtomExpr _ args marker ->
S.unions (referencedTransactionIds marker : (referencedTransactionIds <$> args))

-- only the relvars can reference other transactions
instance ReferencedTransactionIds DatabaseContext where
referencedTransactionIds dbc =
S.unions [
--referencedTransactionIds (inclusionDependencies dbc),
referencedTransactionIds (relationVariables dbc)
--referencedTransactionIds (atomFunctions dbc),
--referencedTransactionIds (dbcFunctions dbc),
--referencedTransactionIds (notifications dbc),
--referencedTransactionIds (typeConstructorMapping dbc),
--referencedTransactionIds (registeredQueries dbc)
]

instance ReferencedTransactionIds RelationVariables where
referencedTransactionIds relVars =
S.unions (referencedTransactionIds <$> M.elems relVars)

-- | Recurse relvars references and transaction parents to extract a subset of relevant transactions.
-- probably could do some trimming of transactions that are not referenced by relvars, but that is rare, so probably of not much benefit
-- should be trim merge parents that don't contribute to the relvars? maybe
referencedTransactionIdsForTransaction :: Transaction -> TransactionGraph -> Either RelationalError (S.Set Transaction)
referencedTransactionIdsForTransaction trans graph
| parentIds == T.rootParent = pure (S.singleton trans)
| otherwise =
foldM folder (S.singleton trans) parentIds
where
parentIds = parents (transactionInfo trans)
folder acc transId' = do
trans' <- transactionForId transId' graph
transSet <- referencedTransactionIdsForTransaction trans' graph
pure (S.union acc transSet)




40 changes: 31 additions & 9 deletions src/lib/ProjectM36/TransactionGraph.hs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import ProjectM36.MerkleHash
import qualified ProjectM36.DisconnectedTransaction as Discon
import qualified ProjectM36.Attribute as A
import ProjectM36.HashSecurely
import ProjectM36.ReferencedTransactionIds

import Codec.Winery
import Control.Monad.Except hiding (join)
Expand Down Expand Up @@ -373,7 +374,7 @@ validateHeadName headName graph (t1, t2) =
else
pure trans

-- Algorithm: start at one transaction and work backwards up the parents. If there is a node we have not yet visited as a child, then walk that up to its head. If that branch contains the goal transaction, then we have completed a valid subgraph traversal.
-- Algorithm: start at one transaction and work backwards up the parents. If there is a node we have not yet visited as a child, then walk that up to its head. If that branch contains the goal transaction, then we have completed a valid subgraph traversal. The subgraph must also include any transactions which are referenced by other transactions.
subGraphOfFirstCommonAncestor :: TransactionGraph -> TransactionHeads -> Transaction -> Transaction -> S.Set Transaction -> Either RelationalError TransactionGraph
subGraphOfFirstCommonAncestor origGraph resultHeads currentTrans' goalTrans traverseSet = do
let currentid = transactionId currentTrans'
Expand All @@ -398,8 +399,14 @@ subGraphOfFirstCommonAncestor origGraph resultHeads currentTrans' goalTrans trav
Left err -> Left err
Right currentTransParent ->
subGraphOfFirstCommonAncestor origGraph resultHeads currentTransParent goalTrans (S.insert currentTrans' traverseSet)
else -- we found a path
Right (TransactionGraph resultHeads (S.unions (traverseSet : pathsFound)))
else do -- we found a path
-- we union all the relevant path transactions together, but we are missing any transactions which these transaction may reference. To make a valid transaction graph, we must include these referenced transactions.
let openSet = S.unions (traverseSet : pathsFound)
transactionIncluder acc trans = do
allTrans <- referencedTransactionIdsForTransaction trans origGraph
pure $ S.union allTrans acc
closedTransactionSet <- foldM transactionIncluder mempty (S.toList openSet)
Right (TransactionGraph resultHeads closedTransactionSet)
where
oneParent (Transaction _ tinfo _) = transactionForId (NE.head (parents tinfo)) origGraph

Expand Down Expand Up @@ -437,9 +444,14 @@ mergeTransactions stamp' newId parentId mergeStrategy (headNameA, headNameB) = d
transB <- transactionForHeadErr headNameB
disconParent <- gfTransForId parentId
let subHeads = M.filterWithKey (\k _ -> k `elem` [headNameA, headNameB]) (transactionHeadsForGraph graph)
-- is this an optimization???
subGraph <- runE $ subGraphOfFirstCommonAncestor graph subHeads transA transB S.empty
_ <- runE $ validateConnectivity subGraph

subGraph' <- runE $ filterSubGraph subGraph subHeads
mergedTrans <- local (const (freshGraphRefRelationalExprEnv Nothing subGraph')) $ createMergeTransaction stamp' newId mergeStrategy (transA, transB)
-- we cannot cut the transaction graph away only to "relevant" transactions because transactions can reference other transactions via relvar expressions
mergedTrans <- local (const (freshGraphRefRelationalExprEnv Nothing subGraph')) $
createMergeTransaction stamp' newId mergeStrategy (transA, transB)
case headNameForTransaction disconParent graph of
Nothing -> throwError (TransactionIsNotAHeadError parentId)
Just headName -> do
Expand All @@ -452,18 +464,20 @@ mergeTransactions stamp' newId parentId mergeStrategy (headNameA, headNameB) = d
pure (newDiscon, newGraph')

--TEMPORARY COPY/PASTE
showTransactionStructureX :: Transaction -> TransactionGraph -> String
showTransactionStructureX trans graph = headInfo ++ " " ++ show (transactionId trans) ++ " " ++ parentTransactionsInfo
showTransactionStructureX :: Bool -> Transaction -> TransactionGraph -> String
showTransactionStructureX showRelVars trans graph = headInfo ++ " " ++ show (transactionId trans) ++ " " ++ parentTransactionsInfo ++ relVarsInfo
where
relVarsInfo | showRelVars == False = ""
| otherwise = "\n" <> concatMap show (M.toList (relationVariables (concreteDatabaseContext trans)))
headInfo = maybe "" show (headNameForTransaction trans graph)
parentTransactionsInfo = if isRootTransaction trans then "root" else case parentTransactions trans graph of
Left err -> show err
Right parentTransSet -> concat $ S.toList $ S.map (show . transactionId) parentTransSet

showGraphStructureX :: TransactionGraph -> String
showGraphStructureX graph@(TransactionGraph heads transSet) = headsInfo ++ S.foldr folder "" transSet
showGraphStructureX :: Bool -> TransactionGraph -> String
showGraphStructureX showRelVars graph@(TransactionGraph heads transSet) = headsInfo ++ S.foldr folder "" transSet
where
folder trans acc = acc ++ showTransactionStructureX trans graph ++ "\n"
folder trans acc = acc ++ showTransactionStructureX showRelVars trans graph ++ "\n"
headsInfo = show $ M.map transactionId heads

-- | After splicing out a subgraph, run it through this function to remove references to transactions which are not in the subgraph.
Expand Down Expand Up @@ -625,3 +639,11 @@ validateMerkleHashes graph =
case validateMerkleHash trans graph of
Left err -> err : acc
_ -> acc

-- | Ensure that referenced transactions remain in the graph.
validateConnectivity :: TransactionGraph -> Either RelationalError TransactionGraph
validateConnectivity graph = do
let validateTrans trans =
mapM_ (`transactionForId` graph) (referencedTransactionIds (concreteDatabaseContext trans))
mapM_ validateTrans (transactionsForGraph graph)
pure graph
Loading

0 comments on commit 7c5070c

Please sign in to comment.