Add support for postgres instead of mysql. Change sql accourdingly. b…

…lob + varbinary -> bytea. No support for UNSIGNED or CREATE INDEX IF NOT EXISTS.

synapse/app/homeserver.py

@@ -373,6 +373,8 @@ def setup(config_options):
             "use_unicode": True,
             "collation": "utf8mb4_bin",
         })
+    elif name == "psycopg2":
+        pass
     elif name == "sqlite3":
         db_config.setdefault("args", {}).update({
             "cp_min": 1,

synapse/storage/__init__.py

@@ -236,7 +236,7 @@ def _setup_new_database(cur, database_engine):
 
     cur.execute(
         database_engine.convert_param_style(
-            "REPLACE INTO schema_version (version, upgraded)"
+            "INSERT INTO schema_version (version, upgraded)"
             " VALUES (?,?)"
         ),
         (max_current_ver, False,)
@@ -432,14 +432,11 @@ def executescript(txn, schema_path):
 
 
 def _get_or_create_schema_state(txn, database_engine):
-    try:
-        # Bluntly try creating the schema_version tables.
-        schema_path = os.path.join(
-            dir_path, "schema", "schema_version.sql",
-        )
-        executescript(txn, schema_path)
-    except:
-        pass
+    # Bluntly try creating the schema_version tables.
+    schema_path = os.path.join(
+        dir_path, "schema", "schema_version.sql",
+    )
+    executescript(txn, schema_path)
 
     txn.execute("SELECT version, upgraded FROM schema_version")
     row = txn.fetchone()

synapse/storage/_base.py

@@ -330,7 +330,7 @@ def inner_func(conn, *args, **kwargs):
                                     continue
                             raise
                 except Exception as e:
-                    logger.debug("[TXN FAIL] {%s}", name, e)
+                    logger.debug("[TXN FAIL] {%s} %s", name, e)
                     raise
                 finally:
                     end = time.time() * 1000

synapse/storage/engines/__init__.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from .maria import MariaEngine
+from .postgres import PostgresEngine
 from .sqlite3 import Sqlite3Engine
 
 import importlib
@@ -22,6 +23,7 @@
 SUPPORTED_MODULE = {
     "sqlite3": Sqlite3Engine,
     "mysql.connector": MariaEngine,
+    "psycopg2": PostgresEngine,
 }
 
 

synapse/storage/event_federation.py

@@ -153,7 +153,7 @@ def _get_prev_state(self, txn, event_id):
         results = self._get_prev_events_and_state(
             txn,
             event_id,
-            is_state=1,
+            is_state=True,
         )
 
         return [(e_id, h, ) for e_id, h, _ in results]
@@ -164,7 +164,7 @@ def _get_prev_events_and_state(self, txn, event_id, is_state=None):
         }
 
         if is_state is not None:
-            keyvalues["is_state"] = is_state
+            keyvalues["is_state"] = bool(is_state)
 
         res = self._simple_select_list_txn(
             txn,
@@ -259,7 +259,7 @@ def _handle_prev_events(self, txn, outlier, event_id, prev_events,
                     "event_id": event_id,
                     "prev_event_id": e_id,
                     "room_id": room_id,
-                    "is_state": 0,
+                    "is_state": False,
                 },
             )
 
@@ -397,7 +397,7 @@ def _get_missing_events(self, txn, room_id, earliest_events, latest_events,
 
         query = (
             "SELECT prev_event_id FROM event_edges "
-            "WHERE room_id = ? AND event_id = ? AND is_state = 0 "
+            "WHERE room_id = ? AND event_id = ? AND is_state = ? "
             "LIMIT ?"
         )
 
@@ -406,7 +406,7 @@ def _get_missing_events(self, txn, room_id, earliest_events, latest_events,
             for event_id in front:
                 txn.execute(
                     query,
-                    (room_id, event_id, limit - len(event_results))
+                    (room_id, event_id, False, limit - len(event_results))
                 )
 
                 for e_id, in txn.fetchall():

synapse/storage/events.py

@@ -188,12 +188,12 @@ def _persist_event_txn(self, txn, event, context, backfilled,
                 )
 
                 sql = (
-                    "UPDATE events SET outlier = 0"
+                    "UPDATE events SET outlier = ?"
                     " WHERE event_id = ?"
                 )
                 txn.execute(
                     sql,
-                    (event.event_id,)
+                    (False, event.event_id,)
                 )
             return
 

synapse/storage/room.py

@@ -105,22 +105,35 @@ def f(txn):
 
             # We use non printing ascii character US (\x1F) as a separator
             sql = (
-                "SELECT r.room_id, n.name, t.topic, "
-                "group_concat(a.room_alias, '\x1F') "
-                "FROM rooms AS r "
-                "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id "
-                "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id "
-                "INNER JOIN room_aliases AS a ON a.room_id = r.room_id "
-                "WHERE r.is_public = ? "
-                "GROUP BY r.room_id "
+                "SELECT r.room_id, max(n.name), max(t.topic)"
+                " FROM rooms AS r"
+                " LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id"
+                " LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id"
+                " WHERE r.is_public = ?"
+                " GROUP BY r.room_id"
             ) % {
                 "topic": topic_subquery,
                 "name": name_subquery,
             }
 
             txn.execute(sql, (is_public,))
 
-            return txn.fetchall()
+            rows = txn.fetchall()
+
+            for i, row in enumerate(rows):
+                room_id = row[0]
+                aliases = self._simple_select_onecol_txn(
+                    txn,
+                    table="room_aliases",
+                    keyvalues={
+                        "room_id": room_id
+                    },
+                    retcol="room_alias",
+                )
+
+                rows[i] = list(row) + [aliases]
+
+            return rows
 
         rows = yield self.runInteraction(
             "get_rooms", f
@@ -131,9 +144,10 @@ def f(txn):
                 "room_id": r[0],
                 "name": r[1],
                 "topic": r[2],
-                "aliases": r[3].split("\x1F"),
+                "aliases": r[3],
             }
             for r in rows
+            if r[3]  # We only return rooms that have at least one alias.
         ]
 
         defer.returnValue(ret)

synapse/storage/schema/full_schemas/16/application_services.sql

@@ -14,7 +14,7 @@
  */
 
 CREATE TABLE IF NOT EXISTS application_services(
-    id BIGINT UNSIGNED PRIMARY KEY,
+    id BIGINT PRIMARY KEY,
     url VARCHAR(150),
     token VARCHAR(150),
     hs_token VARCHAR(150),
@@ -23,8 +23,8 @@ CREATE TABLE IF NOT EXISTS application_services(
 );
 
 CREATE TABLE IF NOT EXISTS application_services_regex(
-    id BIGINT UNSIGNED PRIMARY KEY,
-    as_id BIGINT UNSIGNED NOT NULL,
+    id BIGINT PRIMARY KEY,
+    as_id BIGINT NOT NULL,
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
     regex VARCHAR(150),
     FOREIGN KEY(as_id) REFERENCES application_services(id)
@@ -39,10 +39,10 @@ CREATE TABLE IF NOT EXISTS application_services_state(
 CREATE TABLE IF NOT EXISTS application_services_txns(
     as_id VARCHAR(150) NOT NULL,
     txn_id INTEGER NOT NULL,
-    event_ids LONGBLOB NOT NULL,
+    event_ids bytea NOT NULL,
     UNIQUE(as_id, txn_id)
 );
 
-CREATE INDEX IF NOT EXISTS application_services_txns_id ON application_services_txns (
+CREATE INDEX application_services_txns_id ON application_services_txns (
     as_id
 );

synapse/storage/schema/full_schemas/16/event_edges.sql

@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities(
     UNIQUE (event_id, room_id)
 );
 
-CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id);
-CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id);
+CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id);
+CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_backward_extremities(
@@ -29,8 +29,8 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities(
     UNIQUE (event_id, room_id)
 );
 
-CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id);
-CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id);
+CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id);
+CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_edges(
@@ -41,8 +41,8 @@ CREATE TABLE IF NOT EXISTS event_edges(
     UNIQUE (event_id, prev_event_id, room_id, is_state)
 );
 
-CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id);
-CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id);
+CREATE INDEX ev_edges_id ON event_edges(event_id);
+CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id);
 
 
 CREATE TABLE IF NOT EXISTS room_depth(
@@ -51,17 +51,17 @@ CREATE TABLE IF NOT EXISTS room_depth(
     UNIQUE (room_id)
 );
 
-CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
+CREATE INDEX room_depth_room ON room_depth(room_id);
 
 
 create TABLE IF NOT EXISTS event_destinations(
     event_id VARCHAR(150) NOT NULL,
     destination VARCHAR(150) NOT NULL,
-    delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered
+    delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
     UNIQUE (event_id, destination)
 );
 
-CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id);
+CREATE INDEX event_destinations_id ON event_destinations(event_id);
 
 
 CREATE TABLE IF NOT EXISTS state_forward_extremities(
@@ -72,10 +72,10 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities(
     UNIQUE (event_id, room_id)
 );
 
-CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities(
+CREATE INDEX st_extrem_keys ON state_forward_extremities(
     room_id, type, state_key
 );
-CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id);
+CREATE INDEX st_extrem_id ON state_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_auth(
@@ -85,5 +85,5 @@ CREATE TABLE IF NOT EXISTS event_auth(
     UNIQUE (event_id, auth_id, room_id)
 );
 
-CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id);
-CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id);
+CREATE INDEX evauth_edges_id ON event_auth(event_id);
+CREATE INDEX evauth_edges_auth_id ON event_auth(auth_id);

synapse/storage/schema/full_schemas/16/event_signatures.sql

@@ -16,40 +16,40 @@
 CREATE TABLE IF NOT EXISTS event_content_hashes (
     event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash LONGBLOB,
+    hash bytea,
     UNIQUE (event_id, algorithm)
 );
 
-CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id);
+CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_reference_hashes (
     event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash LONGBLOB,
+    hash bytea,
     UNIQUE (event_id, algorithm)
 );
 
-CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id);
+CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_signatures (
     event_id VARCHAR(150),
     signature_name VARCHAR(150),
     key_id VARCHAR(150),
-    signature LONGBLOB,
+    signature bytea,
     UNIQUE (event_id, signature_name, key_id)
 );
 
-CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id);
+CREATE INDEX event_signatures_id ON event_signatures(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_edge_hashes(
     event_id VARCHAR(150),
     prev_event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash LONGBLOB,
+    hash bytea,
     UNIQUE (event_id, prev_event_id, algorithm)
 );
 
-CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id);
+CREATE INDEX event_edge_hashes_id ON event_edge_hashes(event_id);