core: remove msg_delivery_events unique constraint (recreates table); cleanup old messages (#2376)

2023-05-05 13:49:09 +04:00
parent 62bac800af
commit 54fc052e47
9 changed files with 127 additions and 72 deletions
--- a/cabal.project
+++ b/cabal.project
@@ -7,7 +7,7 @@ constraints: zip +disable-bzip2 +disable-zstd
 source-repository-package
    type: git
    location: https://github.com/simplex-chat/simplexmq.git
-    tag: af3f70829dca2483425eb8702cd9aeac2c026e14
+    tag: 8954f39425d971025e5e3df1a1628281dab61a3c

 source-repository-package
    type: git
--- a/scripts/nix/sha256map.nix
+++ b/scripts/nix/sha256map.nix
@@ -1,5 +1,5 @@
 {
-  "https://github.com/simplex-chat/simplexmq.git"."af3f70829dca2483425eb8702cd9aeac2c026e14" = "1ngngzqz6fjr11dk2v3d1wrfkyyac954a0fswhq27pfhapqdhlw0";
+  "https://github.com/simplex-chat/simplexmq.git"."8954f39425d971025e5e3df1a1628281dab61a3c" = "0m670s43m1ym51firdzxj77k49bi8qq5gwxc7w50nv8r2yl62ckd";
  "https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38";
  "https://github.com/kazu-yamamoto/http2.git"."b5a1b7200cf5bc7044af34ba325284271f6dff25" = "0dqb50j57an64nf4qcf5vcz4xkd1vzvghvf8bk529c1k30r9nfzb";
  "https://github.com/simplex-chat/direct-sqlcipher.git"."34309410eb2069b029b8fc1872deb1e0db123294" = "0kwkmhyfsn2lixdlgl15smgr1h5gjk7fky6abzh8rng2h5ymnffd";
--- a/simplex-chat.cabal
+++ b/simplex-chat.cabal
@@ -93,6 +93,7 @@ library
      Simplex.Chat.Migrations.M20230411_extra_xftp_file_descriptions
      Simplex.Chat.Migrations.M20230420_rcv_files_to_receive
      Simplex.Chat.Migrations.M20230422_profile_contact_links
+      Simplex.Chat.Migrations.M20230504_recreate_msg_delivery_events_cleanup_messages
      Simplex.Chat.Mobile
      Simplex.Chat.Mobile.WebRTC
      Simplex.Chat.Options
--- a/src/Simplex/Chat.hs
+++ b/src/Simplex/Chat.hs
@@ -44,7 +44,7 @@ import Data.Text (Text)
 import qualified Data.Text as T
 import Data.Text.Encoding (encodeUtf8)
 import Data.Time (NominalDiffTime, addUTCTime, defaultTimeLocale, formatTime)
-import Data.Time.Clock (UTCTime, diffUTCTime, getCurrentTime, nominalDiffTimeToSeconds)
+import Data.Time.Clock (UTCTime, diffUTCTime, getCurrentTime, nominalDay, nominalDiffTimeToSeconds)
 import Data.Time.Clock.System (SystemTime, systemToUTCTime)
 import Data.Time.LocalTime (getCurrentTimeZone, getZonedTime)
 import Data.Word (Word32)
@@ -2261,6 +2261,7 @@ cleanupManager = do
      let (us, us') = partition activeUser users
      forM_ us cleanupUser
      forM_ us' cleanupUser
+      cleanupMessages `catchError` (toView . CRChatError Nothing)
    liftIO $ threadDelay' $ cleanupManagerInterval * 1000000
  where
    cleanupUser user =
@@ -2269,7 +2270,11 @@ cleanupManager = do
      ts <- liftIO getCurrentTime
      let startTimedThreadCutoff = addUTCTime (realToFrac cleanupManagerInterval) ts
      timedItems <- withStore' $ \db -> getTimedItems db user startTimedThreadCutoff
-      forM_ timedItems $ uncurry (startTimedItemThread user)
+      forM_ timedItems $ \(itemRef, deleteAt) -> startTimedItemThread user itemRef deleteAt `catchError` const (pure ())
+    cleanupMessages = do
+      ts <- liftIO getCurrentTime
+      let cutoffTs = addUTCTime (- (30 * nominalDay)) ts
+      withStore' (`deleteOldMessages` cutoffTs)

 startProximateTimedItemThread :: ChatMonad m => User -> (ChatRef, ChatItemId) -> UTCTime -> m ()
 startProximateTimedItemThread user itemRef deleteAt = do
--- a/src/Simplex/Chat/Migrations/M20230504_recreate_msg_delivery_events_cleanup_messages.hs
+++ b/src/Simplex/Chat/Migrations/M20230504_recreate_msg_delivery_events_cleanup_messages.hs
@@ -0,0 +1,37 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Chat.Migrations.M20230504_recreate_msg_delivery_events_cleanup_messages where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20230504_recreate_msg_delivery_events_cleanup_messages :: Query
+m20230504_recreate_msg_delivery_events_cleanup_messages =
+  [sql|
+DROP TABLE msg_delivery_events;
+
+CREATE TABLE msg_delivery_events (
+  msg_delivery_event_id INTEGER PRIMARY KEY,
+  msg_delivery_id INTEGER NOT NULL REFERENCES msg_deliveries ON DELETE CASCADE,
+  delivery_status TEXT NOT NULL,
+  created_at TEXT NOT NULL DEFAULT (datetime('now')),
+  updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+DELETE FROM messages WHERE created_at < datetime('now', '-30 days');
+|]
+
+down_m20230504_recreate_msg_delivery_events_cleanup_messages :: Query
+down_m20230504_recreate_msg_delivery_events_cleanup_messages =
+  [sql|
+DROP TABLE msg_delivery_events;
+
+CREATE TABLE msg_delivery_events (
+  msg_delivery_event_id INTEGER PRIMARY KEY,
+  msg_delivery_id INTEGER NOT NULL REFERENCES msg_deliveries ON DELETE CASCADE, -- non UNIQUE for multiple events per msg delivery
+  delivery_status TEXT NOT NULL, -- see MsgDeliveryStatus for allowed values
+  created_at TEXT NOT NULL DEFAULT (datetime('now')),
+  updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+  UNIQUE (msg_delivery_id, delivery_status)
+);
+|]
--- a/src/Simplex/Chat/Migrations/chat_schema.sql
+++ b/src/Simplex/Chat/Migrations/chat_schema.sql
@@ -20,10 +20,6 @@ CREATE TABLE contact_profiles(
  preferences TEXT,
  contact_link BLOB
 );
-CREATE INDEX contact_profiles_index ON contact_profiles(
-  display_name,
-  full_name
-);
 CREATE TABLE users(
  user_id INTEGER PRIMARY KEY,
  contact_id INTEGER NOT NULL UNIQUE REFERENCES contacts ON DELETE CASCADE
@@ -146,7 +142,6 @@ CREATE TABLE groups(
  UNIQUE(user_id, local_display_name),
  UNIQUE(user_id, group_profile_id)
 );
-CREATE INDEX idx_groups_inv_queue_info ON groups(inv_queue_info);
 CREATE TABLE group_members(
  -- group members, excluding the local user
  group_member_id INTEGER PRIMARY KEY,
@@ -347,14 +342,6 @@ CREATE TABLE msg_deliveries(
  agent_ack_cmd_id INTEGER, -- broker_ts for received, created_at for sent
  UNIQUE(connection_id, agent_msg_id)
 );
-CREATE TABLE msg_delivery_events(
-  msg_delivery_event_id INTEGER PRIMARY KEY,
-  msg_delivery_id INTEGER NOT NULL REFERENCES msg_deliveries ON DELETE CASCADE, -- non UNIQUE for multiple events per msg delivery
-  delivery_status TEXT NOT NULL, -- see MsgDeliveryStatus for allowed values
-  created_at TEXT NOT NULL DEFAULT(datetime('now')),
-  updated_at TEXT CHECK(updated_at NOT NULL),
-  UNIQUE(msg_delivery_id, delivery_status)
-);
 CREATE TABLE pending_group_messages(
  pending_group_message_id INTEGER PRIMARY KEY,
  group_member_id INTEGER NOT NULL REFERENCES group_members ON DELETE CASCADE,
@@ -399,13 +386,6 @@ CREATE TABLE chat_item_messages(
  updated_at TEXT NOT NULL DEFAULT(datetime('now')),
  UNIQUE(chat_item_id, message_id)
 );
-CREATE INDEX idx_connections_via_contact_uri_hash ON connections(
-  via_contact_uri_hash
-);
-CREATE INDEX idx_contact_requests_xcontact_id ON contact_requests(xcontact_id);
-CREATE INDEX idx_contacts_xcontact_id ON contacts(xcontact_id);
-CREATE INDEX idx_messages_shared_msg_id ON messages(shared_msg_id);
-CREATE INDEX idx_chat_items_shared_msg_id ON chat_items(shared_msg_id);
 CREATE TABLE calls(
  -- stores call invitations state for communicating state between NSE and app when call notification comes
  call_id INTEGER PRIMARY KEY,
@@ -418,17 +398,6 @@ CREATE TABLE calls(
  created_at TEXT NOT NULL DEFAULT(datetime('now')),
  updated_at TEXT NOT NULL DEFAULT(datetime('now'))
 );
-CREATE INDEX idx_chat_items_groups ON chat_items(
-  user_id,
-  group_id,
-  item_ts,
-  chat_item_id
-);
-CREATE INDEX idx_chat_items_contacts ON chat_items(
-  user_id,
-  contact_id,
-  chat_item_id
-);
 CREATE TABLE commands(
  command_id INTEGER PRIMARY KEY AUTOINCREMENT, -- used as ACorrId
  connection_id INTEGER REFERENCES connections ON DELETE CASCADE,
@@ -446,6 +415,68 @@ CREATE TABLE settings(
  created_at TEXT NOT NULL DEFAULT(datetime('now')),
  updated_at TEXT NOT NULL DEFAULT(datetime('now'))
 );
+CREATE TABLE IF NOT EXISTS "protocol_servers"(
+  smp_server_id INTEGER PRIMARY KEY,
+  host TEXT NOT NULL,
+  port TEXT NOT NULL,
+  key_hash BLOB NOT NULL,
+  basic_auth TEXT,
+  preset INTEGER NOT NULL DEFAULT 0,
+  tested INTEGER,
+  enabled INTEGER NOT NULL DEFAULT 1,
+  user_id INTEGER NOT NULL REFERENCES users ON DELETE CASCADE,
+  created_at TEXT NOT NULL DEFAULT(datetime('now')),
+  updated_at TEXT NOT NULL DEFAULT(datetime('now')),
+  protocol TEXT NOT NULL DEFAULT 'smp',
+  UNIQUE(user_id, host, port)
+);
+CREATE TABLE xftp_file_descriptions(
+  file_descr_id INTEGER PRIMARY KEY AUTOINCREMENT,
+  user_id INTEGER NOT NULL REFERENCES users ON DELETE CASCADE,
+  file_descr_text TEXT NOT NULL,
+  file_descr_part_no INTEGER NOT NULL DEFAULT(0),
+  file_descr_complete INTEGER NOT NULL DEFAULT(0),
+  created_at TEXT NOT NULL DEFAULT(datetime('now')),
+  updated_at TEXT NOT NULL DEFAULT(datetime('now'))
+);
+CREATE TABLE extra_xftp_file_descriptions(
+  extra_file_descr_id INTEGER PRIMARY KEY,
+  file_id INTEGER NOT NULL REFERENCES files ON DELETE CASCADE,
+  user_id INTEGER NOT NULL REFERENCES users ON DELETE CASCADE,
+  file_descr_text TEXT NOT NULL,
+  created_at TEXT NOT NULL DEFAULT(datetime('now')),
+  updated_at TEXT NOT NULL DEFAULT(datetime('now'))
+);
+CREATE TABLE msg_delivery_events(
+  msg_delivery_event_id INTEGER PRIMARY KEY,
+  msg_delivery_id INTEGER NOT NULL REFERENCES msg_deliveries ON DELETE CASCADE,
+  delivery_status TEXT NOT NULL,
+  created_at TEXT NOT NULL DEFAULT(datetime('now')),
+  updated_at TEXT NOT NULL DEFAULT(datetime('now'))
+);
+CREATE INDEX contact_profiles_index ON contact_profiles(
+  display_name,
+  full_name
+);
+CREATE INDEX idx_groups_inv_queue_info ON groups(inv_queue_info);
+CREATE INDEX idx_connections_via_contact_uri_hash ON connections(
+  via_contact_uri_hash
+);
+CREATE INDEX idx_contact_requests_xcontact_id ON contact_requests(xcontact_id);
+CREATE INDEX idx_contacts_xcontact_id ON contacts(xcontact_id);
+CREATE INDEX idx_messages_shared_msg_id ON messages(shared_msg_id);
+CREATE INDEX idx_chat_items_shared_msg_id ON chat_items(shared_msg_id);
+CREATE INDEX idx_chat_items_groups ON chat_items(
+  user_id,
+  group_id,
+  item_ts,
+  chat_item_id
+);
+CREATE INDEX idx_chat_items_contacts ON chat_items(
+  user_id,
+  contact_id,
+  chat_item_id
+);
 CREATE UNIQUE INDEX idx_chat_items_direct_shared_msg_id ON chat_items(
  user_id,
  contact_id,
@@ -549,44 +580,12 @@ CREATE INDEX idx_snd_file_chunks_file_id_connection_id ON snd_file_chunks(
 CREATE INDEX idx_snd_files_group_member_id ON snd_files(group_member_id);
 CREATE INDEX idx_snd_files_connection_id ON snd_files(connection_id);
 CREATE INDEX idx_snd_files_file_id ON snd_files(file_id);
-CREATE TABLE IF NOT EXISTS "protocol_servers"(
-  smp_server_id INTEGER PRIMARY KEY,
-  host TEXT NOT NULL,
-  port TEXT NOT NULL,
-  key_hash BLOB NOT NULL,
-  basic_auth TEXT,
-  preset INTEGER NOT NULL DEFAULT 0,
-  tested INTEGER,
-  enabled INTEGER NOT NULL DEFAULT 1,
-  user_id INTEGER NOT NULL REFERENCES users ON DELETE CASCADE,
-  created_at TEXT NOT NULL DEFAULT(datetime('now')),
-  updated_at TEXT NOT NULL DEFAULT(datetime('now')),
-  protocol TEXT NOT NULL DEFAULT 'smp',
-  UNIQUE(user_id, host, port)
-);
 CREATE INDEX idx_smp_servers_user_id ON "protocol_servers"(user_id);
 CREATE INDEX idx_chat_items_item_deleted_by_group_member_id ON chat_items(
  item_deleted_by_group_member_id
 );
-CREATE TABLE xftp_file_descriptions(
-  file_descr_id INTEGER PRIMARY KEY AUTOINCREMENT,
-  user_id INTEGER NOT NULL REFERENCES users ON DELETE CASCADE,
-  file_descr_text TEXT NOT NULL,
-  file_descr_part_no INTEGER NOT NULL DEFAULT(0),
-  file_descr_complete INTEGER NOT NULL DEFAULT(0),
-  created_at TEXT NOT NULL DEFAULT(datetime('now')),
-  updated_at TEXT NOT NULL DEFAULT(datetime('now'))
-);
 CREATE INDEX idx_snd_files_file_descr_id ON snd_files(file_descr_id);
 CREATE INDEX idx_rcv_files_file_descr_id ON rcv_files(file_descr_id);
-CREATE TABLE extra_xftp_file_descriptions(
-  extra_file_descr_id INTEGER PRIMARY KEY,
-  file_id INTEGER NOT NULL REFERENCES files ON DELETE CASCADE,
-  user_id INTEGER NOT NULL REFERENCES users ON DELETE CASCADE,
-  file_descr_text TEXT NOT NULL,
-  created_at TEXT NOT NULL DEFAULT(datetime('now')),
-  updated_at TEXT NOT NULL DEFAULT(datetime('now'))
-);
 CREATE INDEX idx_extra_xftp_file_descriptions_file_id ON extra_xftp_file_descriptions(
  file_id
 );
--- a/src/Simplex/Chat/Store.hs
+++ b/src/Simplex/Chat/Store.hs
@@ -216,6 +216,7 @@ module Simplex.Chat.Store
    createPendingGroupMessage,
    getPendingGroupMessages,
    deletePendingGroupMessage,
+    deleteOldMessages,
    updateChatTs,
    createNewSndChatItem,
    createNewRcvChatItem,
@@ -375,6 +376,7 @@ import Simplex.Chat.Migrations.M20230402_protocol_servers
 import Simplex.Chat.Migrations.M20230411_extra_xftp_file_descriptions
 import Simplex.Chat.Migrations.M20230420_rcv_files_to_receive
 import Simplex.Chat.Migrations.M20230422_profile_contact_links
+import Simplex.Chat.Migrations.M20230504_recreate_msg_delivery_events_cleanup_messages
 import Simplex.Chat.Protocol
 import Simplex.Chat.Types
 import Simplex.Chat.Util (week)
@@ -450,7 +452,8 @@ schemaMigrations =
    ("20230402_protocol_servers", m20230402_protocol_servers, Just down_m20230402_protocol_servers),
    ("20230411_extra_xftp_file_descriptions", m20230411_extra_xftp_file_descriptions, Just down_m20230411_extra_xftp_file_descriptions),
    ("20230420_rcv_files_to_receive", m20230420_rcv_files_to_receive, Just down_m20230420_rcv_files_to_receive),
-    ("20230422_profile_contact_links", m20230422_profile_contact_links, Just down_m20230422_profile_contact_links)
+    ("20230422_profile_contact_links", m20230422_profile_contact_links, Just down_m20230422_profile_contact_links),
+    ("20230504_recreate_msg_delivery_events_cleanup_messages", m20230504_recreate_msg_delivery_events_cleanup_messages, Just down_m20230504_recreate_msg_delivery_events_cleanup_messages)
  ]

 -- | The list of migrations in ascending order by date
@@ -3582,6 +3585,10 @@ deletePendingGroupMessage :: DB.Connection -> Int64 -> MessageId -> IO ()
 deletePendingGroupMessage db groupMemberId messageId =
  DB.execute db "DELETE FROM pending_group_messages WHERE group_member_id = ? AND message_id = ?" (groupMemberId, messageId)

+deleteOldMessages :: DB.Connection -> UTCTime -> IO ()
+deleteOldMessages db createdAtCutoff = do
+  DB.execute db "DELETE FROM messages WHERE created_at <= ?" (Only createdAtCutoff)
+
 type NewQuoteRow = (Maybe SharedMsgId, Maybe UTCTime, Maybe MsgContent, Maybe Bool, Maybe MemberId)

 updateChatTs :: DB.Connection -> User -> ChatDirection c d -> UTCTime -> IO ()
--- a/stack.yaml
+++ b/stack.yaml
@@ -49,7 +49,7 @@ extra-deps:
  # - simplexmq-1.0.0@sha256:34b2004728ae396e3ae449cd090ba7410781e2b3cefc59259915f4ca5daa9ea8,8561
  # - ../simplexmq
  - github: simplex-chat/simplexmq
-    commit: af3f70829dca2483425eb8702cd9aeac2c026e14
+    commit: 8954f39425d971025e5e3df1a1628281dab61a3c
  - github: kazu-yamamoto/http2
    commit: b5a1b7200cf5bc7044af34ba325284271f6dff25
  # - ../direct-sqlcipher
--- a/tests/SchemaDump.hs
+++ b/tests/SchemaDump.hs
@@ -5,7 +5,7 @@ module SchemaDump where

 import ChatClient (withTmpFiles)
 import Control.DeepSeq
-import Control.Monad (void)
+import Control.Monad (unless, void)
 import Data.List (dropWhileEnd)
 import Data.Maybe (fromJust, isJust)
 import Simplex.Chat.Store (createChatStore)
@@ -57,9 +57,15 @@ testSchemaMigrations = withTmpFiles $ do
      schema' <- getSchema testDB testSchema
      schema' `shouldNotBe` schema
      withConnection st (`Migrations.run` MTRDown [downMigr])
-      schema'' <- getSchema testDB testSchema
-      schema'' `shouldBe` schema
+      unless (name m `elem` skipComparisonForDownMigrations) $ do
+        schema'' <- getSchema testDB testSchema
+        schema'' `shouldBe` schema
      withConnection st (`Migrations.run` MTRUp [m])
+      schema''' <- getSchema testDB testSchema
+      schema''' `shouldBe` schema'
+
+skipComparisonForDownMigrations :: [String]
+skipComparisonForDownMigrations = ["20230504_recreate_msg_delivery_events_cleanup_messages"]

 getSchema :: FilePath -> FilePath -> IO String
 getSchema dpPath schemaPath = do