test simplexmq with mlock

cabal.project

@@ -12,7 +12,7 @@ constraints: zip +disable-bzip2 +disable-zstd
 source-repository-package
     type: git
     location: https://github.com/simplex-chat/simplexmq.git
-    tag: caeeb2df9ccca29a6bb504886736502d081fba0e
+    tag: e6c444f5d1e94f057ac776b8c6c6c8663236831f
 
 source-repository-package
     type: git

package.yaml

@@ -49,7 +49,6 @@ dependencies:
   - unliftio == 0.2.*
   - unliftio-core == 0.2.*
   - zip == 2.0.*
-  - zstd
 
 flags:
   swift:

scripts/nix/sha256map.nix

@@ -1,5 +1,5 @@
 {
-  "https://github.com/simplex-chat/simplexmq.git"."caeeb2df9ccca29a6bb504886736502d081fba0e" = "187avx8h014fhik76qv1l0nifv6db6nrg9kjk2azqia21n4s2m38";
+  "https://github.com/simplex-chat/simplexmq.git"."e6c444f5d1e94f057ac776b8c6c6c8663236831f" = "0r66s7q9l8ccpmg4gnk8z1yby9zp9p0c4gjsgx54cnc0rdl7nr4w";
   "https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38";
   "https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "1ql13f4kfwkbaq7nygkxgw84213i0zm7c1a8hwvramayxl38dq5d";
   "https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl";

simplex-chat.cabal

@@ -211,7 +211,6 @@ library
     , unliftio ==0.2.*
     , unliftio-core ==0.2.*
     , zip ==2.0.*
-    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON
@@ -273,7 +272,6 @@ executable simplex-bot
     , unliftio ==0.2.*
     , unliftio-core ==0.2.*
     , zip ==2.0.*
-    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON
@@ -335,7 +333,6 @@ executable simplex-bot-advanced
     , unliftio ==0.2.*
     , unliftio-core ==0.2.*
     , zip ==2.0.*
-    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON
@@ -399,7 +396,6 @@ executable simplex-broadcast-bot
     , unliftio ==0.2.*
     , unliftio-core ==0.2.*
     , zip ==2.0.*
-    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON
@@ -463,7 +459,6 @@ executable simplex-chat
     , unliftio-core ==0.2.*
     , websockets ==0.12.*
     , zip ==2.0.*
-    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON
@@ -530,7 +525,6 @@ executable simplex-directory-service
     , unliftio ==0.2.*
     , unliftio-core ==0.2.*
     , zip ==2.0.*
-    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON
@@ -627,7 +621,6 @@ test-suite simplex-chat-test
     , unliftio ==0.2.*
     , unliftio-core ==0.2.*
     , zip ==2.0.*
-    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON

src/Simplex/Chat.hs

@@ -26,7 +26,6 @@ import qualified Data.Aeson as J
 import Data.Attoparsec.ByteString.Char8 (Parser)
 import qualified Data.Attoparsec.ByteString.Char8 as A
 import Data.Bifunctor (bimap, first, second)
-import Data.ByteArray (ScrubbedBytes)
 import qualified Data.ByteArray as BA
 import qualified Data.ByteString.Base64 as B64
 import Data.ByteString.Char8 (ByteString)
@@ -99,6 +98,7 @@ import Simplex.Messaging.Client (defaultNetworkConfig)
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.File (CryptoFile (..), CryptoFileArgs (..))
 import qualified Simplex.Messaging.Crypto.File as CF
+import Simplex.Messaging.Crypto.Memory (LockedBytes)
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers (base64P)
@@ -197,7 +197,7 @@ smallGroupsRcptsMemLimit = 20
 logCfg :: LogConfig
 logCfg = LogConfig {lc_file = Nothing, lc_stderr = True}
 
-createChatDatabase :: FilePath -> ScrubbedBytes -> Bool -> MigrationConfirmation -> IO (Either MigrationError ChatDatabase)
+createChatDatabase :: FilePath -> LockedBytes -> Bool -> MigrationConfirmation -> IO (Either MigrationError ChatDatabase)
 createChatDatabase filePrefix key keepKey confirmMigrations = runExceptT $ do
   chatStore <- ExceptT $ createChatStore (chatStoreFile filePrefix) key keepKey confirmMigrations
   agentStore <- ExceptT $ createAgentStore (agentStoreFile filePrefix) key keepKey confirmMigrations
@@ -451,14 +451,6 @@ processChatCommand cmd = chatVersionRange >>= (`processChatCommand'` cmd)
 
 processChatCommand' :: forall m. ChatMonad m => VersionRange -> ChatCommand -> m ChatResponse
 processChatCommand' vr = \case
-  TestZstd outfile_ -> do
-    rows <- withStore' testZstd
-    case outfile_ of
-      Nothing -> pure $ CRZstdTest rows
-      Just path -> do
-        liftIO $ LB.writeFile path $ LB.unlines $
-          map (\ZstdRow {raw, z1, z3, z6, z9, z} -> LB.fromStrict . B.unwords $ map bshow [raw, z1, z3, z6, z9, z]) rows
-        ok_
   ShowActiveUser -> withUser' $ pure . CRActiveUser
   CreateActiveUser NewUser {profile, sameServers, pastTimestamp} -> do
     forM_ profile $ \Profile {displayName} -> checkValidName displayName
@@ -6504,8 +6496,7 @@ chatVersionRange = do
 chatCommandP :: Parser ChatCommand
 chatCommandP =
   choice
-    [ "/zstd" *> (TestZstd <$> optional (A.space *> filePath)),
-      "/mute " *> ((`SetShowMessages` MFNone) <$> chatNameP),
+    [ "/mute " *> ((`SetShowMessages` MFNone) <$> chatNameP),
       "/unmute " *> ((`SetShowMessages` MFAll) <$> chatNameP),
       "/unmute mentions " *> ((`SetShowMessages` MFMentions) <$> chatNameP),
       "/receipts " *> (SetSendReceipts <$> chatNameP <* " " <*> ((Just <$> onOffP) <|> ("default" $> Nothing))),

src/Simplex/Chat/Archive.hs

@@ -27,6 +27,7 @@ import qualified Database.SQLite3 as SQL
 import Simplex.Chat.Controller
 import Simplex.Messaging.Agent.Client (agentClientStore)
 import Simplex.Messaging.Agent.Store.SQLite (SQLiteStore (..), closeSQLiteStore, keyString, sqlString, storeKey)
+import Simplex.Messaging.Crypto.Memory (LockedBytes)
 import Simplex.Messaging.Util
 import System.FilePath
 import UnliftIO.Directory
@@ -172,7 +173,7 @@ withDB f' a err =
     sqliteError' :: Show e => e -> m (Maybe SQLiteError)
     sqliteError' = pure . Just . SQLiteError . show
 
-testSQL :: BA.ScrubbedBytes -> Text
+testSQL :: LockedBytes -> Text
 testSQL k =
   T.unlines $
     keySQL k
@@ -181,7 +182,7 @@ testSQL k =
             "SELECT count(*) FROM sqlite_master;"
           ]
 
-keySQL :: BA.ScrubbedBytes -> [Text]
+keySQL :: LockedBytes -> [Text]
 keySQL k = ["PRAGMA key = " <> keyString k <> ";" | not (BA.null k)]
 
 sqlCipherTestKey :: forall m. ChatMonad m => DBEncryptionKey -> m ()

src/Simplex/Chat/Controller.hs

@@ -29,7 +29,6 @@ import qualified Data.Aeson.TH as JQ
 import qualified Data.Aeson.Types as JT
 import qualified Data.Attoparsec.ByteString.Char8 as A
 import Data.Bifunctor (first)
-import Data.ByteArray (ScrubbedBytes)
 import qualified Data.ByteArray as BA
 import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Char8 as B
@@ -71,6 +70,7 @@ import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.File (CryptoFile (..))
 import qualified Simplex.Messaging.Crypto.File as CF
+import Simplex.Messaging.Crypto.Memory (LockedBytes)
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Notifications.Protocol (DeviceToken (..), NtfTknStatus)
 import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON, parseAll, parseString, sumTypeJSON)
@@ -215,8 +215,7 @@ data HelpSection = HSMain | HSFiles | HSGroups | HSContacts | HSMyAddress | HSIn
   deriving (Show)
 
 data ChatCommand
-  = TestZstd (Maybe FilePath)
-  | ShowActiveUser
+  = ShowActiveUser
   | CreateActiveUser NewUser
   | ListUsers
   | APISetActiveUser UserId (Maybe UserPwd)
@@ -718,12 +717,6 @@ data ChatResponse
   | CRChatErrors {user_ :: Maybe User, chatErrors :: [ChatError]}
   | CRArchiveImported {archiveErrors :: [ArchiveError]}
   | CRTimedAction {action :: String, durationMilliseconds :: Int64}
-  | CRZstdTest {zstdRows :: [ZstdRow]}
-  deriving (Show)
-
-data ZstdRow = ZstdRow
-  { raw, z1, z3, z6, z9, z :: !Int
-  }
   deriving (Show)
 
 -- some of these can only be used as command responses
@@ -887,7 +880,7 @@ data ArchiveConfig = ArchiveConfig {archivePath :: FilePath, disableCompression
 data DBEncryptionConfig = DBEncryptionConfig {currentKey :: DBEncryptionKey, newKey :: DBEncryptionKey, keepKey :: Maybe Bool}
   deriving (Show)
 
-newtype DBEncryptionKey = DBEncryptionKey ScrubbedBytes
+newtype DBEncryptionKey = DBEncryptionKey LockedBytes
   deriving (Show)
 
 instance IsString DBEncryptionKey where fromString = parseString $ parseAll strP
@@ -1419,8 +1412,6 @@ $(JQ.deriveJSON (sumTypeJSON $ dropPrefix "RCSR") ''RemoteCtrlStopReason)
 
 $(JQ.deriveJSON (sumTypeJSON $ dropPrefix "RHSR") ''RemoteHostStopReason)
 
-$(JQ.deriveJSON defaultJSON ''ZstdRow)
-
 $(JQ.deriveJSON (sumTypeJSON $ dropPrefix "CR") ''ChatResponse)
 
 $(JQ.deriveFromJSON defaultJSON ''ArchiveConfig)

src/Simplex/Chat/Mobile.hs

@@ -15,7 +15,6 @@ import Control.Monad.Reader
 import qualified Data.Aeson as J
 import qualified Data.Aeson.TH as JQ
 import Data.Bifunctor (first)
-import Data.ByteArray (ScrubbedBytes)
 import qualified Data.ByteArray as BA
 import qualified Data.ByteString.Base64.URL as U
 import Data.ByteString.Char8 (ByteString)
@@ -50,6 +49,7 @@ import Simplex.Messaging.Agent.Env.SQLite (createAgentStore)
 import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..), MigrationError, closeSQLiteStore, reopenSQLiteStore)
 import Simplex.Messaging.Client (defaultNetworkConfig)
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.Memory (LockedBytes)
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, sumTypeJSON)
 import Simplex.Messaging.Protocol (AProtoServerWithAuth (..), AProtocolType (..), BasicAuth (..), CorrId (..), ProtoServerWithAuth (..), ProtocolServer (..))
@@ -227,10 +227,10 @@ defaultMobileConfig =
 getActiveUser_ :: SQLiteStore -> IO (Maybe User)
 getActiveUser_ st = find activeUser <$> withTransaction st getUsers
 
-chatMigrateInit :: String -> ScrubbedBytes -> String -> IO (Either DBMigrationResult ChatController)
+chatMigrateInit :: String -> LockedBytes -> String -> IO (Either DBMigrationResult ChatController)
 chatMigrateInit dbFilePrefix dbKey confirm = chatMigrateInitKey dbFilePrefix dbKey False confirm False
 
-chatMigrateInitKey :: String -> ScrubbedBytes -> Bool -> String -> Bool -> IO (Either DBMigrationResult ChatController)
+chatMigrateInitKey :: String -> LockedBytes -> Bool -> String -> Bool -> IO (Either DBMigrationResult ChatController)
 chatMigrateInitKey dbFilePrefix dbKey keepKey confirm backgroundMode = runExceptT $ do
   confirmMigrations <- liftEitherWith (const DBMInvalidConfirmation) $ strDecode $ B.pack confirm
   chatStore <- migrate createChatStore (chatStoreFile dbFilePrefix) confirmMigrations

src/Simplex/Chat/Options.hs

@@ -19,7 +19,6 @@ where
 
 import Control.Logger.Simple (LogLevel (..))
 import qualified Data.Attoparsec.ByteString.Char8 as A
-import Data.ByteArray (ScrubbedBytes)
 import qualified Data.ByteString.Char8 as B
 import Data.Text (Text)
 import Numeric.Natural (Natural)
@@ -27,6 +26,7 @@ import Options.Applicative
 import Simplex.Chat.Controller (ChatLogLevel (..), updateStr, versionNumber, versionString)
 import Simplex.FileTransfer.Description (mb)
 import Simplex.Messaging.Client (NetworkConfig (..), defaultNetworkConfig)
+import Simplex.Messaging.Crypto.Memory (LockedBytes)
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers (parseAll)
 import Simplex.Messaging.Protocol (ProtoServerWithAuth, ProtocolTypeI, SMPServerWithAuth, XFTPServerWithAuth)
@@ -51,7 +51,7 @@ data ChatOpts = ChatOpts
 
 data CoreChatOpts = CoreChatOpts
   { dbFilePrefix :: String,
-    dbKey :: ScrubbedBytes,
+    dbKey :: LockedBytes,
     smpServers :: [SMPServerWithAuth],
     xftpServers :: [XFTPServerWithAuth],
     networkConfig :: NetworkConfig,

src/Simplex/Chat/Store.hs

@@ -12,13 +12,13 @@ module Simplex.Chat.Store
   )
 where
 
-import Data.ByteArray (ScrubbedBytes)
 import Simplex.Chat.Store.Migrations
 import Simplex.Chat.Store.Profiles
 import Simplex.Chat.Store.Shared
 import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation, MigrationError, SQLiteStore (..), createSQLiteStore, withTransaction)
+import Simplex.Messaging.Crypto.Memory (LockedBytes)
 
-createChatStore :: FilePath -> ScrubbedBytes -> Bool -> MigrationConfirmation -> IO (Either MigrationError SQLiteStore)
+createChatStore :: FilePath -> LockedBytes -> Bool -> MigrationConfirmation -> IO (Either MigrationError SQLiteStore)
 createChatStore dbPath key keepKey = createSQLiteStore dbPath key keepKey migrations
 
 chatStoreFile :: FilePath -> FilePath

src/Simplex/Chat/Store/Messages.hs

@@ -111,18 +111,15 @@ module Simplex.Chat.Store.Messages
     getGroupSndStatuses,
     getGroupSndStatusCounts,
     getGroupHistoryItems,
-    testZstd,
   )
 where
 
-import qualified Codec.Compression.Zstd as Zstd
 import Control.Monad
 import Control.Monad.Except
 import Control.Monad.IO.Class
 import Crypto.Random (ChaChaDRG)
 import Data.Bifunctor (first)
 import Data.ByteString.Char8 (ByteString)
-import qualified Data.ByteString.Char8 as B
 import Data.Either (fromRight, rights)
 import Data.Int (Int64)
 import Data.List (sortBy)
@@ -134,7 +131,7 @@ import Data.Time (addUTCTime)
 import Data.Time.Clock (UTCTime (..), getCurrentTime)
 import Database.SQLite.Simple (NamedParam (..), Only (..), Query, (:.) (..))
 import Database.SQLite.Simple.QQ (sql)
-import Simplex.Chat.Controller (ChatListQuery (..), ChatPagination (..), PaginationByTime (..), ZstdRow (..))
+import Simplex.Chat.Controller (ChatListQuery (..), ChatPagination (..), PaginationByTime (..))
 import Simplex.Chat.Markdown
 import Simplex.Chat.Messages
 import Simplex.Chat.Messages.CIContent
@@ -149,7 +146,7 @@ import Simplex.Messaging.Agent.Store.SQLite (firstRow, firstRow', maybeFirstRow)
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.File (CryptoFile (..), CryptoFileArgs (..))
-import Simplex.Messaging.Util (eitherToMaybe, (<$$>))
+import Simplex.Messaging.Util (eitherToMaybe)
 import Simplex.Messaging.Version (VersionRange)
 import UnliftIO.STM
 
@@ -2546,16 +2543,3 @@ getGroupHistoryItems db user@User {userId} GroupInfo {groupId} count = do
             LIMIT ?
           |]
           (userId, groupId, rcvMsgContentTag, sndMsgContentTag, count)
-
-testZstd :: DB.Connection -> IO [ZstdRow]
-testZstd db = process <$$> DB.query_ db "SELECT msg_body FROM messages"
-  where
-    process (Only msg_body) =
-      ZstdRow
-        { raw = B.length msg_body,
-          z1 = B.length $ Zstd.compress 1 msg_body,
-          z3 = B.length $ Zstd.compress 3 msg_body,
-          z6 = B.length $ Zstd.compress 6 msg_body,
-          z9 = B.length $ Zstd.compress 9 msg_body,
-          z = B.length $ Zstd.compress Zstd.maxCLevel msg_body
-        }

src/Simplex/Chat/View.hs

@@ -386,7 +386,6 @@ responseToView hu@(currentRH, user_) ChatConfig {logLevel, showReactions, showRe
   CRChatErrors u errs -> ttyUser' u $ concatMap (viewChatError logLevel testView) errs
   CRArchiveImported archiveErrs -> if null archiveErrs then ["ok"] else ["archive import errors: " <> plain (show archiveErrs)]
   CRTimedAction _ _ -> []
-  CRZstdTest {zstdRows} -> map (\ZstdRow {raw, z1, z3, z6, z9, z} -> plain . T.unwords $ map tshow [raw, z1, z3, z6, z9, z]) zstdRows
   where
     ttyUser :: User -> [StyledString] -> [StyledString]
     ttyUser user@User {showNtfs, activeUser} ss

tests/ChatClient.hs

@@ -15,7 +15,6 @@ import Control.Concurrent.STM
 import Control.Exception (bracket, bracket_)
 import Control.Monad
 import Control.Monad.Except
-import Data.ByteArray (ScrubbedBytes)
 import Data.Functor (($>))
 import Data.List (dropWhileEnd, find)
 import Data.Maybe (isNothing)
@@ -38,6 +37,7 @@ import Simplex.Messaging.Agent.RetryInterval
 import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..))
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import Simplex.Messaging.Client (ProtocolClientConfig (..), defaultNetworkConfig)
+import Simplex.Messaging.Crypto.Memory (LockedBytes)
 import Simplex.Messaging.Server (runSMPServerBlocking)
 import Simplex.Messaging.Server.Env.STM
 import Simplex.Messaging.Transport
@@ -92,7 +92,7 @@ testCoreOpts =
       highlyAvailable = False
     }
 
-getTestOpts :: Bool -> ScrubbedBytes -> ChatOpts
+getTestOpts :: Bool -> LockedBytes -> ChatOpts
 getTestOpts maintenance dbKey = testOpts {maintenance, coreOptions = testCoreOpts {dbKey}}
 
 termSettings :: VirtualTerminalSettings