core: support closing/re-opening store on chat stop/start (#3132)

* core: support closing/re-opening store on chat stop/start

* update .nix refs

* kotlin: types

* add test

* update simplexmq

apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt

@@ -516,8 +516,8 @@ object ChatController {
     throw Exception("failed to delete the user ${r.responseType} ${r.details}")
   }
 
-  suspend fun apiStartChat(): Boolean {
+  suspend fun apiStartChat(openDBWithKey: String? = null): Boolean {
-    val r = sendCmd(CC.StartChat(expire = true))
+    val r = sendCmd(CC.StartChat(ChatCtrlCfg(subConns = true, enableExpireCIs = true, startXFTPWorkers = true, openDBWithKey = openDBWithKey)))
     when (r) {
       is CR.ChatStarted -> return true
       is CR.ChatRunning -> return false
@@ -525,8 +525,8 @@ object ChatController {
     }
   }
 
-  suspend fun apiStopChat(): Boolean {
+  suspend fun apiStopChat(closeStore: Boolean): Boolean {
-    val r = sendCmd(CC.ApiStopChat())
+    val r = sendCmd(CC.ApiStopChat(closeStore))
     when (r) {
       is CR.ChatStopped -> return true
       else -> throw Error("failed stopping chat: ${r.responseType} ${r.details}")
@@ -1829,8 +1829,8 @@ sealed class CC {
   class ApiMuteUser(val userId: Long): CC()
   class ApiUnmuteUser(val userId: Long): CC()
   class ApiDeleteUser(val userId: Long, val delSMPQueues: Boolean, val viewPwd: String?): CC()
-  class StartChat(val expire: Boolean): CC()
+  class StartChat(val cfg: ChatCtrlCfg): CC()
-  class ApiStopChat: CC()
+  class ApiStopChat(val closeStore: Boolean): CC()
   class SetTempFolder(val tempFolder: String): CC()
   class SetFilesFolder(val filesFolder: String): CC()
   class ApiSetXFTPConfig(val config: XFTPFileConfig?): CC()
@@ -1933,8 +1933,9 @@ sealed class CC {
     is ApiMuteUser -> "/_mute user $userId"
     is ApiUnmuteUser -> "/_unmute user $userId"
     is ApiDeleteUser -> "/_delete user $userId del_smp=${onOff(delSMPQueues)}${maybePwd(viewPwd)}"
-    is StartChat -> "/_start subscribe=on expire=${onOff(expire)} xftp=on"
+//    is StartChat -> "/_start ${json.encodeToString(cfg)}" // this can be used with the new core
-    is ApiStopChat -> "/_stop"
+    is StartChat -> "/_start subscribe=on expire=${onOff(cfg.enableExpireCIs)} xftp=on"
+    is ApiStopChat -> if (closeStore) "/_stop close" else "/_stop"
     is SetTempFolder -> "/_temp_folder $tempFolder"
     is SetFilesFolder -> "/_files_folder $filesFolder"
     is ApiSetXFTPConfig -> if (config != null) "/_xftp on ${json.encodeToString(config)}" else "/_xftp off"
@@ -2151,6 +2152,14 @@ sealed class CC {
   }
 }
 
+@Serializable
+data class ChatCtrlCfg (
+  val subConns: Boolean,
+  val enableExpireCIs: Boolean,
+  val startXFTPWorkers: Boolean,
+  val openDBWithKey: String?
+)
+
 @Serializable
 data class NewUser(
   val profile: Profile?,

apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/database/DatabaseView.kt

@@ -419,7 +419,7 @@ private fun stopChat(m: ChatModel) {
 }
 
 suspend fun stopChatAsync(m: ChatModel) {
-  m.controller.apiStopChat()
+  m.controller.apiStopChat(false)
   m.chatRunning.value = false
 }
 

cabal.project

@@ -9,7 +9,7 @@ constraints: zip +disable-bzip2 +disable-zstd
 source-repository-package
     type: git
     location: https://github.com/simplex-chat/simplexmq.git
-    tag: 8d47f690838371bc848e4b31a4b09ef6bf67ccc5
+    tag: fda1284ae4b7c33cae2eb8ed0376a511aecc1d51
 
 source-repository-package
     type: git

scripts/nix/sha256map.nix

@@ -1,5 +1,5 @@
 {
-  "https://github.com/simplex-chat/simplexmq.git"."8d47f690838371bc848e4b31a4b09ef6bf67ccc5" = "1pwasv22ii3wy4xchaknlwczmy5ws7adx7gg2g58lxzrgdjm3650";
+  "https://github.com/simplex-chat/simplexmq.git"."fda1284ae4b7c33cae2eb8ed0376a511aecc1d51" = "1gq7scv9z8x3xhzl914xr46na0kkrqd1i743xbw69lyx33kj9xb5";
   "https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38";
   "https://github.com/kazu-yamamoto/http2.git"."b5a1b7200cf5bc7044af34ba325284271f6dff25" = "0dqb50j57an64nf4qcf5vcz4xkd1vzvghvf8bk529c1k30r9nfzb";
   "https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "0kiwhvml42g9anw4d2v0zd1fpc790pj9syg5x3ik4l97fnkbbwpp";

src/Simplex/Chat.hs

@@ -83,7 +83,7 @@ import Simplex.Messaging.Agent.Client (AgentStatsKey (..), SubInfo (..), agentCl
 import Simplex.Messaging.Agent.Env.SQLite (AgentConfig (..), InitialAgentServers (..), createAgentStore, defaultAgentConfig)
 import Simplex.Messaging.Agent.Lock
 import Simplex.Messaging.Agent.Protocol
-import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..), MigrationError, SQLiteStore (dbNew), execSQL, upMigration, withConnection)
+import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..), MigrationError, SQLiteStore (dbNew), execSQL, upMigration, withConnection, closeSQLiteStore, openSQLiteStore)
 import Simplex.Messaging.Agent.Store.SQLite.DB (SlowQueryStats (..))
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import qualified Simplex.Messaging.Agent.Store.SQLite.Migrations as Migrations
@@ -249,9 +249,13 @@ cfgServers p s = case p of
   SPSMP -> s.smp
   SPXFTP -> s.xftp
 
-startChatController :: forall m. ChatMonad' m => Bool -> Bool -> Bool -> m (Async ())
+startChatController :: forall m. ChatMonad' m => ChatCtrlCfg -> m (Async ())
-startChatController subConns enableExpireCIs startXFTPWorkers = do
+startChatController ChatCtrlCfg {subConns, enableExpireCIs, startXFTPWorkers, openDBWithKey} = do
-  asks smpAgent >>= resumeAgentClient
+  ChatController {chatStore, smpAgent} <- ask
+  forM_ openDBWithKey $ \(DBEncryptionKey dbKey) -> liftIO $ do
+    openSQLiteStore chatStore dbKey
+    openSQLiteStore (agentClientStore smpAgent) dbKey
+  resumeAgentClient smpAgent
   unless subConns $
     chatWriteVar subscriptionMode SMOnlyCreate
   users <- fromRight [] <$> runExceptT (withStoreCtx' (Just "startChatController, getUsers") getUsers)
@@ -323,8 +327,8 @@ restoreCalls = do
   calls <- asks currentCalls
   atomically $ writeTVar calls callsMap
 
-stopChatController :: forall m. MonadUnliftIO m => ChatController -> m ()
+stopChatController :: forall m. MonadUnliftIO m => ChatController -> Bool -> m ()
-stopChatController ChatController {smpAgent, agentAsync = s, sndFiles, rcvFiles, expireCIFlags} = do
+stopChatController ChatController {chatStore, smpAgent, agentAsync = s, sndFiles, rcvFiles, expireCIFlags} closeStore = do
   disconnectAgentClient smpAgent
   readTVarIO s >>= mapM_ (\(a1, a2) -> uninterruptibleCancel a1 >> mapM_ uninterruptibleCancel a2)
   closeFiles sndFiles
@@ -333,6 +337,9 @@ stopChatController ChatController {smpAgent, agentAsync = s, sndFiles, rcvFiles,
     keys <- M.keys <$> readTVar expireCIFlags
     forM_ keys $ \k -> TM.insert k False expireCIFlags
     writeTVar s Nothing
+  when closeStore $ liftIO $ do
+    closeSQLiteStore chatStore
+    closeSQLiteStore $ agentClientStore smpAgent
   where
     closeFiles :: TVar (Map Int64 Handle) -> m ()
     closeFiles files = do
@@ -462,12 +469,12 @@ processChatCommand = \case
     checkDeleteChatUser user'
     withChatLock "deleteUser" . procCmd $ deleteChatUser user' delSMPQueues
   DeleteUser uName delSMPQueues viewPwd_ -> withUserName uName $ \userId -> APIDeleteUser userId delSMPQueues viewPwd_
-  StartChat subConns enableExpireCIs startXFTPWorkers -> withUser' $ \_ ->
+  APIStartChat cfg -> withUser' $ \_ ->
     asks agentAsync >>= readTVarIO >>= \case
       Just _ -> pure CRChatRunning
-      _ -> checkStoreNotChanged $ startChatController subConns enableExpireCIs startXFTPWorkers $> CRChatStarted
+      _ -> checkStoreNotChanged $ startChatController cfg $> CRChatStarted
-  APIStopChat -> do
+  APIStopChat closeStore -> do
-    ask >>= stopChatController
+    ask >>= (`stopChatController` closeStore)
     pure CRChatStopped
   APIActivateChat -> withUser $ \_ -> do
     restoreCalls
@@ -5379,9 +5386,9 @@ chatCommandP =
       "/_delete user " *> (APIDeleteUser <$> A.decimal <* " del_smp=" <*> onOffP <*> optional (A.space *> jsonP)),
       "/delete user " *> (DeleteUser <$> displayName <*> pure True <*> optional (A.space *> pwdP)),
       ("/user" <|> "/u") $> ShowActiveUser,
-      "/_start subscribe=" *> (StartChat <$> onOffP <* " expire=" <*> onOffP <* " xftp=" <*> onOffP),
+      "/_start" *> (APIStartChat <$> ((A.space *> jsonP) <|> chatCtrlCfgP)),
-      "/_start" $> StartChat True True True,
+      "/_stop close" $> APIStopChat {closeStore = True},
-      "/_stop" $> APIStopChat,
+      "/_stop" $> APIStopChat False,
       "/_app activate" $> APIActivateChat,
       "/_app suspend " *> (APISuspendChat <$> A.decimal),
       "/_resubscribe all" $> ResubscribeAllConnections,
@@ -5609,6 +5616,12 @@ chatCommandP =
     ]
   where
     choice = A.choice . map (\p -> p <* A.takeWhile (== ' ') <* A.endOfInput)
+    chatCtrlCfgP = do
+      subConns <- (" subscribe=" *> onOffP) <|> pure True
+      enableExpireCIs <- (" expire=" *> onOffP) <|> pure True
+      startXFTPWorkers <- (" xftp=" *> onOffP) <|> pure True
+      openDBWithKey <- optional $ " key=" *> dbKeyP
+      pure ChatCtrlCfg {subConns, enableExpireCIs, startXFTPWorkers, openDBWithKey}
     incognitoP = (A.space *> ("incognito" <|> "i")) $> True <|> pure False
     incognitoOnOffP = (A.space *> "incognito=" *> onOffP) <|> pure False
     imagePrefix = (<>) <$> "data:" <*> ("image/png;base64," <|> "image/jpg;base64,")

src/Simplex/Chat/Archive.hs

@@ -124,7 +124,7 @@ sqlCipherExport DBEncryptionConfig {currentKey = DBEncryptionKey key, newKey = D
     checkFile `with` fs
     backup `with` fs
     (export chatDb chatEncrypted >> export agentDb agentEncrypted)
-      `catchChatError` \e -> (restore `with` fs) >> throwError e
+      `catchChatError` \e -> tryChatError (restore `with` fs) >> throwError e
   where
     action `with` StorageFiles {chatDb, agentDb} = action chatDb >> action agentDb
     backup f = copyFile f (f <> ".bak")

src/Simplex/Chat/Controller.hs

@@ -221,8 +221,8 @@ data ChatCommand
   | UnmuteUser
   | APIDeleteUser UserId Bool (Maybe UserPwd)
   | DeleteUser UserName Bool (Maybe UserPwd)
-  | StartChat {subscribeConnections :: Bool, enableExpireChatItems :: Bool, startXFTPWorkers :: Bool}
+  | APIStartChat ChatCtrlCfg
-  | APIStopChat
+  | APIStopChat {closeStore :: Bool}
   | APIActivateChat
   | APISuspendChat {suspendTimeout :: Int}
   | ResubscribeAllConnections
@@ -620,6 +620,17 @@ instance ToJSON ChatResponse where
   toJSON = J.genericToJSON . sumTypeJSON $ dropPrefix "CR"
   toEncoding = J.genericToEncoding . sumTypeJSON $ dropPrefix "CR"
 
+data ChatCtrlCfg = ChatCtrlCfg
+  { subConns :: Bool,
+    enableExpireCIs :: Bool,
+    startXFTPWorkers :: Bool,
+    openDBWithKey :: Maybe DBEncryptionKey
+  }
+  deriving (Show, Generic, FromJSON)
+
+defChatCtrlCfg :: ChatCtrlCfg
+defChatCtrlCfg = ChatCtrlCfg True True True Nothing
+
 newtype UserPwd = UserPwd {unUserPwd :: Text}
   deriving (Eq, Show)
 

src/Simplex/Chat/Core.hs

@@ -35,7 +35,7 @@ runSimplexChat :: ChatOpts -> User -> ChatController -> (User -> ChatController
 runSimplexChat ChatOpts {maintenance} u cc chat
   | maintenance = wait =<< async (chat u cc)
   | otherwise = do
-    a1 <- runReaderT (startChatController True True True) cc
+    a1 <- runReaderT (startChatController defChatCtrlCfg) cc
     a2 <- async $ chat u cc
     waitEither_ a1 a2
 

stack.yaml

@@ -49,7 +49,7 @@ extra-deps:
   # - simplexmq-1.0.0@sha256:34b2004728ae396e3ae449cd090ba7410781e2b3cefc59259915f4ca5daa9ea8,8561
   # - ../simplexmq
   - github: simplex-chat/simplexmq
-    commit: 8d47f690838371bc848e4b31a4b09ef6bf67ccc5
+    commit: fda1284ae4b7c33cae2eb8ed0376a511aecc1d51
   - github: kazu-yamamoto/http2
     commit: b5a1b7200cf5bc7044af34ba325284271f6dff25
   # - ../direct-sqlcipher

tests/ChatClient.hs

@@ -171,7 +171,7 @@ startTestChat_ db cfg opts user = do
 
 stopTestChat :: TestCC -> IO ()
 stopTestChat TestCC {chatController = cc, chatAsync, termAsync} = do
-  stopChatController cc
+  stopChatController cc False
   uninterruptibleCancel termAsync
   uninterruptibleCancel chatAsync
   threadDelay 200000

tests/ChatTests/Direct.hs

@@ -953,10 +953,15 @@ testDatabaseEncryption tmp = do
       alice ##> "/_start"
       alice <## "chat started"
       testChatWorking alice bob
-      alice ##> "/_stop"
+      alice ##> "/_stop close"
       alice <## "chat stopped"
       alice ##> "/db key wrongkey nextkey"
       alice <## "error encrypting database: wrong passphrase or invalid database file"
+      alice ##> "/_start key=mykey"
+      alice <## "chat started"
+      testChatWorking alice bob
+      alice ##> "/_stop close"
+      alice <## "chat stopped"
       alice ##> "/db key mykey nextkey"
       alice <## "ok"
       alice ##> "/_db encryption {\"currentKey\":\"nextkey\",\"newKey\":\"anotherkey\"}"