Merge remote-tracking branch 'upstream/skynet' into SEARCH-438

# Conflicts:
#	js/search/search.js

demo/search.html

@@ -103,7 +103,7 @@
 </div>
 
 <script>
-    var search = new ssf.Search("7078106230763", "k2rbHBd5CgU3/PKLFUKUYkmuhCiL2Gw2bnsD7MdaZrU=");
+    var search = new ssf.Search("16149077034436", "RSDFOiuVqF3WmY1pC4MP145qPOpUP6A4if27WG9y3fY=");
     var buttonEl = document.getElementById('search');
     var merge = document.getElementById('merge');
     var buttonIndex = document.getElementById('index');

installer/win/Symphony-x64.aip

@@ -57,6 +57,7 @@
     <ROW Directory="config_Dir" Directory_Parent="APPDIR" DefaultDir="config"/>
     <ROW Directory="jobber_Dir" Directory_Parent="vendor_Dir" DefaultDir="jobber"/>
     <ROW Directory="lib_Dir" Directory_Parent="spawnrx_Dir" DefaultDir="lib"/>
+    <ROW Directory="library_Dir" Directory_Parent="APPDIR" DefaultDir="library"/>
     <ROW Directory="locales_Dir" Directory_Parent="APPDIR" DefaultDir="locales"/>
     <ROW Directory="node_modules_Dir" Directory_Parent="app.asar.unpacked_Dir" DefaultDir="NODE_M~1|node_modules"/>
     <ROW Directory="paulcbetts_Dir" Directory_Parent="node_modules_Dir" DefaultDir="@PAULC~1|@paulcbetts"/>
@@ -125,18 +126,25 @@
     <ROW Component="ffmpeg.dll" ComponentId="{A1C4A332-3490-44D8-A5C9-9523889B488B}" Directory_="APPDIR" Attributes="256" KeyPath="ffmpeg.dll"/>
     <ROW Component="index.d.ts" ComponentId="{74D261F1-A6C5-49DD-8554-E48337CC04AF}" Directory_="src_Dir" Attributes="0" KeyPath="index.d.ts" Type="0"/>
     <ROW Component="index.js" ComponentId="{77C48E00-B684-4E72-ACF4-15DD0253EF43}" Directory_="lib_Dir" Attributes="0" KeyPath="index.js" Type="0"/>
+    <ROW Component="indexvalidatorx64.exe" ComponentId="{0266F3CF-5462-4381-9971-2353034D7E1D}" Directory_="library_Dir" Attributes="256" KeyPath="indexvalidatorx64.exe"/>
+    <ROW Component="indexvalidatorx86.exe" ComponentId="{DE7AEDE9-CF30-4DB1-BC38-6CA6F1CDCAAC}" Directory_="library_Dir" Attributes="0" KeyPath="indexvalidatorx86.exe"/>
     <ROW Component="libEGL.dll" ComponentId="{8EEC76AB-3601-4D11-B13E-32EC2A38C539}" Directory_="APPDIR" Attributes="256" KeyPath="libEGL.dll"/>
     <ROW Component="libGLESv2.dll" ComponentId="{0E8B8B21-B4C0-45C9-95D3-637FD93A4EC0}" Directory_="APPDIR" Attributes="256" KeyPath="libGLESv2.dll"/>
+    <ROW Component="libsymphonysearchx64.dll" ComponentId="{A8C99D17-FA62-4996-8FAE-52D1DCF9BF26}" Directory_="library_Dir" Attributes="256" KeyPath="libsymphonysearchx64.dll"/>
+    <ROW Component="libsymphonysearchx86.dll" ComponentId="{AB1E061B-1558-4A57-B4C3-C2952E7471B2}" Directory_="library_Dir" Attributes="0" KeyPath="libsymphonysearchx86.dll"/>
+    <ROW Component="lz4winx64.exe" ComponentId="{8B78B313-EAE9-4533-AFEB-56F9E0CA73A1}" Directory_="library_Dir" Attributes="256" KeyPath="lz4winx64.exe"/>
+    <ROW Component="lz4winx86.exe" ComponentId="{907532E8-1E4B-48A5-B48B-7B05BE80D1A1}" Directory_="library_Dir" Attributes="0" KeyPath="lz4winx86.exe"/>
     <ROW Component="msvcp140.dll" ComponentId="{93A6289C-CF23-4BB8-A579-7FDDD1D15591}" Directory_="APPDIR" Attributes="256" KeyPath="msvcp140.dll"/>
     <ROW Component="node.dll" ComponentId="{C0972355-339E-438C-94A3-74174DE4C6B6}" Directory_="APPDIR" Attributes="256" KeyPath="node.dll"/>
     <ROW Component="node_modules" ComponentId="{A4EB33A8-FEA8-40A5-94EF-705EBE64DDC1}" Directory_="node_modules_Dir" Attributes="0"/>
     <ROW Component="npmignore" ComponentId="{849CFFE2-EBC3-4430-AC8B-DEE1B66DF589}" Directory_="spawnrx_Dir" Attributes="0" KeyPath="npmignore" Type="0"/>
+    <ROW Component="tarwin.exe" ComponentId="{4C98F3B1-1A73-4761-86C0-DE0FC18A8800}" Directory_="library_Dir" Attributes="0" KeyPath="tarwin.exe"/>
     <ROW Component="ucrtbase.dll" ComponentId="{16D802A3-DAD4-4BF4-AD64-88D6F63F5D1E}" Directory_="APPDIR" Attributes="256" KeyPath="ucrtbase.dll"/>
     <ROW Component="vcruntime140.dll" ComponentId="{2542FC82-8D71-4351-8514-2C0D12772ED5}" Directory_="APPDIR" Attributes="256" KeyPath="vcruntime140.dll"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.MsiFeatsComponent">
     <ROW Feature="D564007E3BBE4F85950A09B470A7CA65" Title="Visual C++ Redistributable for Visual Studio 2013 x86" Description="Visual C++ Redistributable for Visual Studio 2013 x86" Display="3" Level="1" Attributes="0"/>
-    <ROW Feature="MainFeature" Title="MainFeature" Description="Description" Display="1" Level="1" Directory_="APPDIR" Attributes="0" Components="AI_CustomARPName AI_DisableModify Jobber.exe PodUrl ProductInformation ScreenSnippet.exe Symphony Symphony.config Symphony.exe am.pak ambient.d.ts apimswincoreconsolel110.dll apimswincoredatetimel110.dll apimswincoredebugl110.dll apimswincoreerrorhandlingl110.dll apimswincorefilel110.dll apimswincorefilel120.dll apimswincorefilel210.dll apimswincorehandlel110.dll apimswincoreheapl110.dll apimswincoreinterlockedl110.dll apimswincorelibraryloaderl110.dll apimswincorelocalizationl120.dll apimswincorememoryl110.dll apimswincorenamedpipel110.dll apimswincoreprocessenvironmentl110.dll apimswincoreprocessthreadsl110.dll apimswincoreprocessthreadsl111.dll apimswincoreprofilel110.dll apimswincorertlsupportl110.dll apimswincorestringl110.dll apimswincoresynchl110.dll apimswincoresynchl120.dll apimswincoresysinfol110.dll apimswincoretimezonel110.dll apimswincoreutill110.dll apimswincrtconiol110.dll apimswincrtconvertl110.dll apimswincrtenvironmentl110.dll apimswincrtfilesysteml110.dll apimswincrtheapl110.dll apimswincrtlocalel110.dll apimswincrtmathl110.dll apimswincrtmultibytel110.dll apimswincrtprivatel110.dll apimswincrtprocessl110.dll apimswincrtruntimel110.dll apimswincrtstdiol110.dll apimswincrtstringl110.dll apimswincrttimel110.dll apimswincrtutilityl110.dll appupdate.yml blink_image_resources_200_percent.pak cld.node d3dcompiler_47.dll ffmpeg.dll index.d.ts index.js libEGL.dll libGLESv2.dll msvcp140.dll node.dll node_modules npmignore ucrtbase.dll vcruntime140.dll"/>
+    <ROW Feature="MainFeature" Title="MainFeature" Description="Description" Display="1" Level="1" Directory_="APPDIR" Attributes="0" Components="AI_CustomARPName AI_DisableModify Jobber.exe PodUrl ProductInformation ScreenSnippet.exe Symphony Symphony.config Symphony.exe am.pak ambient.d.ts apimswincoreconsolel110.dll apimswincoredatetimel110.dll apimswincoredebugl110.dll apimswincoreerrorhandlingl110.dll apimswincorefilel110.dll apimswincorefilel120.dll apimswincorefilel210.dll apimswincorehandlel110.dll apimswincoreheapl110.dll apimswincoreinterlockedl110.dll apimswincorelibraryloaderl110.dll apimswincorelocalizationl120.dll apimswincorememoryl110.dll apimswincorenamedpipel110.dll apimswincoreprocessenvironmentl110.dll apimswincoreprocessthreadsl110.dll apimswincoreprocessthreadsl111.dll apimswincoreprofilel110.dll apimswincorertlsupportl110.dll apimswincorestringl110.dll apimswincoresynchl110.dll apimswincoresynchl120.dll apimswincoresysinfol110.dll apimswincoretimezonel110.dll apimswincoreutill110.dll apimswincrtconiol110.dll apimswincrtconvertl110.dll apimswincrtenvironmentl110.dll apimswincrtfilesysteml110.dll apimswincrtheapl110.dll apimswincrtlocalel110.dll apimswincrtmathl110.dll apimswincrtmultibytel110.dll apimswincrtprivatel110.dll apimswincrtprocessl110.dll apimswincrtruntimel110.dll apimswincrtstdiol110.dll apimswincrtstringl110.dll apimswincrttimel110.dll apimswincrtutilityl110.dll appupdate.yml blink_image_resources_200_percent.pak cld.node d3dcompiler_47.dll ffmpeg.dll index.d.ts index.js indexvalidatorx64.exe indexvalidatorx86.exe libEGL.dll libGLESv2.dll libsymphonysearchx64.dll libsymphonysearchx86.dll lz4winx64.exe lz4winx86.exe msvcp140.dll node.dll node_modules npmignore tarwin.exe ucrtbase.dll vcruntime140.dll"/>
     <ATTRIBUTE name="CurrentFeature" value="MainFeature"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.MsiFilesComponent">
@@ -199,7 +207,7 @@
     <ROW File="build.cmd" Component_="npmignore" FileName="build.cmd" Attributes="0" SourcePath="..\..\dist\win-unpacked\resources\app.asar.unpacked\node_modules\spawn-rx\build.cmd" SelfReg="false" NextFile="build.sh"/>
     <ROW File="build.sh" Component_="npmignore" FileName="build.sh" Attributes="0" SourcePath="..\..\dist\win-unpacked\resources\app.asar.unpacked\node_modules\spawn-rx\build.sh" SelfReg="false" NextFile="CODE_OF_CONDUCT.md"/>
     <ROW File="ca.pak" Component_="am.pak" FileName="ca.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\ca.pak" SelfReg="false" NextFile="cs.pak"/>
-    <ROW File="cld.node" Component_="cld.node" FileName="CLD~1.NOD|cld.node" Attributes="0" SourcePath="..\..\dist\win-unpacked\resources\app.asar.unpacked\node_modules\@paulcbetts\cld\build\Release\cld.node" SelfReg="false"/>
+    <ROW File="cld.node" Component_="cld.node" FileName="CLD~1.NOD|cld.node" Attributes="0" SourcePath="..\..\dist\win-unpacked\resources\app.asar.unpacked\node_modules\@paulcbetts\cld\build\Release\cld.node" SelfReg="false" NextFile="indexvalidatorx64.exe"/>
     <ROW File="content_resources_200_percent.pak" Component_="blink_image_resources_200_percent.pak" FileName="CONTEN~1.PAK|content_resources_200_percent.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\content_resources_200_percent.pak" SelfReg="false" NextFile="content_shell.pak"/>
     <ROW File="content_shell.pak" Component_="blink_image_resources_200_percent.pak" FileName="CONTEN~2.PAK|content_shell.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\content_shell.pak" SelfReg="false" NextFile="d3dcompiler_47.dll"/>
     <ROW File="cs.pak" Component_="am.pak" FileName="cs.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\cs.pak" SelfReg="false" NextFile="da.pak"/>
@@ -232,14 +240,20 @@
     <ROW File="index.js.map" Component_="index.d.ts" FileName="INDEXJ~1.MAP|index.js.map" Attributes="0" SourcePath="..\..\dist\win-unpacked\resources\app.asar.unpacked\node_modules\spawn-rx\lib\src\index.js.map" SelfReg="false" NextFile="package.json"/>
     <ROW File="index.js_1" Component_="index.d.ts" FileName="index.js" Attributes="0" SourcePath="..\..\dist\win-unpacked\resources\app.asar.unpacked\node_modules\spawn-rx\lib\src\index.js" SelfReg="false" NextFile="index.js.map"/>
     <ROW File="index.ts" Component_="ambient.d.ts" FileName="index.ts" Attributes="0" SourcePath="..\..\dist\win-unpacked\resources\app.asar.unpacked\node_modules\spawn-rx\src\index.ts" SelfReg="false" NextFile="tsconfig.json"/>
+    <ROW File="indexvalidatorx64.exe" Component_="indexvalidatorx64.exe" FileName="INDEXV~1.EXE|indexvalidator-x64.exe" Attributes="0" SourcePath="..\..\library\indexvalidator-x64.exe" SelfReg="false" NextFile="indexvalidatorx86.exe" DigSign="true"/>
+    <ROW File="indexvalidatorx86.exe" Component_="indexvalidatorx86.exe" FileName="INDEXV~2.EXE|indexvalidator-x86.exe" Attributes="0" SourcePath="..\..\library\indexvalidator-x86.exe" SelfReg="false" NextFile="libsymphonysearchx64.dll" DigSign="true"/>
     <ROW File="it.pak" Component_="am.pak" FileName="it.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\it.pak" SelfReg="false" NextFile="ja.pak"/>
     <ROW File="ja.pak" Component_="am.pak" FileName="ja.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\ja.pak" SelfReg="false" NextFile="kn.pak"/>
     <ROW File="kn.pak" Component_="am.pak" FileName="kn.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\kn.pak" SelfReg="false" NextFile="ko.pak"/>
     <ROW File="ko.pak" Component_="am.pak" FileName="ko.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\ko.pak" SelfReg="false" NextFile="lt.pak"/>
     <ROW File="libEGL.dll" Component_="libEGL.dll" FileName="libEGL.dll" Attributes="0" SourcePath="..\..\dist\win-unpacked\libEGL.dll" SelfReg="false" NextFile="libGLESv2.dll"/>
     <ROW File="libGLESv2.dll" Component_="libGLESv2.dll" FileName="LIBGLE~1.DLL|libGLESv2.dll" Attributes="0" SourcePath="..\..\dist\win-unpacked\libGLESv2.dll" SelfReg="false" NextFile="LICENSE.electron.txt"/>
+    <ROW File="libsymphonysearchx64.dll" Component_="libsymphonysearchx64.dll" FileName="LIBSYM~1.DLL|libsymphonysearch-x64.dll" Attributes="0" SourcePath="..\..\library\libsymphonysearch-x64.dll" SelfReg="false" NextFile="libsymphonysearchx86.dll"/>
+    <ROW File="libsymphonysearchx86.dll" Component_="libsymphonysearchx86.dll" FileName="LIBSYM~2.DLL|libsymphonysearch-x86.dll" Attributes="0" SourcePath="..\..\library\libsymphonysearch-x86.dll" SelfReg="false" NextFile="lz4winx64.exe"/>
     <ROW File="lt.pak" Component_="am.pak" FileName="lt.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\lt.pak" SelfReg="false" NextFile="lv.pak"/>
     <ROW File="lv.pak" Component_="am.pak" FileName="lv.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\lv.pak" SelfReg="false" NextFile="ml.pak"/>
+    <ROW File="lz4winx64.exe" Component_="lz4winx64.exe" FileName="LZ4-WI~1.EXE|lz4-win-x64.exe" Attributes="0" SourcePath="..\..\library\lz4-win-x64.exe" SelfReg="false" NextFile="lz4winx86.exe" DigSign="true"/>
+    <ROW File="lz4winx86.exe" Component_="lz4winx86.exe" FileName="LZ4-WI~2.EXE|lz4-win-x86.exe" Attributes="0" SourcePath="..\..\library\lz4-win-x86.exe" SelfReg="false" NextFile="tarwin.exe" DigSign="true"/>
     <ROW File="ml.pak" Component_="am.pak" FileName="ml.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\ml.pak" SelfReg="false" NextFile="mr.pak"/>
     <ROW File="mr.pak" Component_="am.pak" FileName="mr.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\mr.pak" SelfReg="false" NextFile="ms.pak"/>
     <ROW File="ms.pak" Component_="am.pak" FileName="ms.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\ms.pak" SelfReg="false" NextFile="nb.pak"/>
@@ -262,6 +276,7 @@
     <ROW File="sv.pak" Component_="am.pak" FileName="sv.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\sv.pak" SelfReg="false" NextFile="sw.pak"/>
     <ROW File="sw.pak" Component_="am.pak" FileName="sw.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\sw.pak" SelfReg="false" NextFile="ta.pak"/>
     <ROW File="ta.pak" Component_="am.pak" FileName="ta.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\ta.pak" SelfReg="false" NextFile="te.pak"/>
+    <ROW File="tarwin.exe" Component_="tarwin.exe" FileName="tar-win.exe" Attributes="0" SourcePath="..\..\library\tar-win.exe" SelfReg="false" DigSign="true"/>
     <ROW File="te.pak" Component_="am.pak" FileName="te.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\te.pak" SelfReg="false" NextFile="th.pak"/>
     <ROW File="th.pak" Component_="am.pak" FileName="th.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\th.pak" SelfReg="false" NextFile="tr.pak"/>
     <ROW File="tr.pak" Component_="am.pak" FileName="tr.pak" Attributes="0" SourcePath="..\..\dist\win-unpacked\locales\tr.pak" SelfReg="false" NextFile="uk.pak"/>

js/compressionLib/index.js

@@ -0,0 +1,85 @@
+const electron = require('electron');
+const child = require('child_process');
+const app = electron.app;
+const path = require('path');
+const userData = path.join(app.getPath('userData'));
+const isMac = require('../utils/misc.js').isMac;
+const isDevEnv = require('../utils/misc.js').isDevEnv;
+const DATA_FOLDER_PATH = isDevEnv ? path.join(__dirname, '..', '..') : userData;
+const execPath = path.dirname(app.getPath('exe'));
+
+// lz4 library path
+const libraryFolderPath = isDevEnv ? path.join(__dirname, '..', '..', 'library') : path.join(execPath, 'library');
+const winArchPath = process.arch === 'ia32' ? 'lz4-win-x86.exe' : 'lz4-win-x64.exe';
+const productionPath = path.join(libraryFolderPath, winArchPath);
+const devPath = path.join(__dirname, '..', '..', 'library', winArchPath);
+const macLibraryPath = isDevEnv ? path.join(__dirname, '..', '..', 'library') : path.join(execPath, '..', 'library');
+const lz4Path = isDevEnv ? devPath : productionPath;
+
+/**
+ * Using the child process to execute the tar and lz4
+ * compression and the final output of this function
+ * will be compressed file with ext: .tar.lz4
+ * @param pathToFolder
+ * @param outputPath
+ * @param callback
+ */
+function compression(pathToFolder, outputPath, callback) {
+    if (isMac) {
+        child.exec(`cd "${DATA_FOLDER_PATH}" && tar cf - "${pathToFolder}" | "${macLibraryPath}/lz4.exec" > "${outputPath}.tar.lz4"`, (error, stdout, stderr) => {
+            if (error) {
+                return callback(new Error(error), null);
+            }
+            return callback(null, {
+                stderr: stderr.toString().trim(),
+                stdout: stdout.toString().trim()
+            });
+        })
+    } else {
+        child.exec(`cd "${DATA_FOLDER_PATH}" && "${libraryFolderPath}\\tar-win.exe" cf - "${pathToFolder}" | "${lz4Path}" > "${outputPath}.tar.lz4"`, (error, stdout, stderr) => {
+            if (error) {
+                return callback(new Error(error), null);
+            }
+            return callback(null, {
+                stderr: stderr.toString().trim(),
+                stdout: stdout.toString().trim()
+            });
+        })
+    }
+}
+
+/**
+ * This function decompress the file
+ * and the ext should be .tar.lz4
+ * the output will be the user index folder
+ * @param pathName
+ * @param callback
+ */
+function deCompression(pathName, callback) {
+    if (isMac) {
+        child.exec(`cd "${DATA_FOLDER_PATH}" && "${macLibraryPath}/lz4.exec" -d "${pathName}" | tar -xf - `, (error, stdout, stderr) => {
+            if (error) {
+                return callback(new Error(error), null);
+            }
+            return callback(null, {
+                stderr: stderr.toString().trim(),
+                stdout: stdout.toString().trim()
+            });
+        })
+    } else {
+        child.exec(`cd "${DATA_FOLDER_PATH}" && "${lz4Path}" -d "${pathName}" | "${libraryFolderPath}\\tar-win.exe" xf - `, (error, stdout, stderr) => {
+            if (error) {
+                return callback(new Error(error), null);
+            }
+            return callback(null, {
+                stderr: stderr.toString().trim(),
+                stdout: stdout.toString().trim()
+            });
+        })
+    }
+}
+
+module.exports = {
+    compression,
+    deCompression
+};

js/cryptoLib/index.js

@@ -3,10 +3,11 @@ const electron = require('electron');
 const app = electron.app;
 const path = require('path');
 const fs = require('fs');
-const archiver = require('archiver');
+const lz4 = require('../compressionLib');
-const extract = require('extract-zip');
 const isDevEnv = require('../utils/misc.js').isDevEnv;
 const crypto = require('./crypto');
+const log = require('../log.js');
+const logLevels = require('../enums/logLevels.js');
 
 const userData = path.join(app.getPath('userData'));
 const DATA_FOLDER = isDevEnv ? './data' : path.join(userData, 'data');
@@ -21,66 +22,53 @@ class Crypto {
         this.permanentIndexFolderName = 'search_index_' + userId + '_' + INDEX_VERSION;
         this.dump = TEMPORARY_PATH;
         this.key = key;
-        this.extractToPath = `${TEMPORARY_PATH}/data/${this.permanentIndexFolderName}`;
         this.encryptedIndex = `${TEMPORARY_PATH}/${this.permanentIndexFolderName}.enc`;
         this.dataFolder = DATA_FOLDER;
-        this.zipErrored = false;
     }
 
     /**
-     * Creates a zip of the data folder and encrypting
+     * Compressing the user index folder and
-     * removing the data folder and the dump files
+     * encrypting it
      * @returns {Promise}
      */
     encryption() {
         return new Promise((resolve, reject) => {
 
             if (!fs.existsSync(this.indexDataFolder)){
-                // will be handling after implementing in client app
+                log.send(logLevels.ERROR, 'user index folder not found');
                 reject();
                 return;
             }
 
-            const zipArchive = archiver('zip');
+            lz4.compression(`data/${this.permanentIndexFolderName}`, `${this.permanentIndexFolderName}`, (error, response) => {
-            let output = fs.createWriteStream(`${this.dump}/${this.permanentIndexFolderName}.zip`);
+                if (error) {
-
+                    log.send(logLevels.ERROR, 'lz4 compression error: ' + error);
-
+                    reject(error);
-            zipArchive.on('end', () => {
-
-                if (!fs.existsSync(`${this.dump}/${this.permanentIndexFolderName}.zip`)){
-                    // will be handling after implementing in client app
-                    reject();
                     return;
                 }
 
-                const input = fs.createReadStream(`${this.dump}/${this.permanentIndexFolderName}.zip`);
+                if (response && response.stderr) {
+                    log.send(logLevels.WARN, 'compression stderr, ' + response.stderr);
+                }
+                const input = fs.createReadStream(`${this.dump}/${this.permanentIndexFolderName}.tar.lz4`);
                 const outputEncryption = fs.createWriteStream(this.encryptedIndex);
                 let config = {
                     key: this.key
                 };
                 const encrypt = crypto.encrypt(config);
 
-                input.pipe(encrypt).pipe(outputEncryption).on('finish', (err) => {
+                let encryptionProcess = input.pipe(encrypt).pipe(outputEncryption);
+
+                encryptionProcess.on('finish', (err) => {
                     if (err) {
+                        log.send(logLevels.ERROR, 'encryption error: ' + err);
                         reject(new Error(err));
+                        return;
                     }
-                    if (!this.zipErrored) {
+                    fs.unlinkSync(`${this.dump}/${this.permanentIndexFolderName}.tar.lz4`);
-                        fs.unlinkSync(`${this.dump}/${this.permanentIndexFolderName}.zip`);
                     resolve('Success');
-                    }
                 });
             });
-
-            zipArchive.pipe(output);
-
-            zipArchive.directory(this.indexDataFolder + '/', false);
-
-            zipArchive.finalize((err) => {
-                if (err) {
-                    this.zipErrored = true;
-                    reject(new Error(err));
-                }
-            });
         });
     }
 
@@ -93,49 +81,42 @@ class Crypto {
         return new Promise((resolve, reject) => {
 
             if (!fs.existsSync(this.encryptedIndex)){
-                // will be handling after implementing in client app
+                log.send(logLevels.ERROR, 'encrypted file not found');
                 reject();
                 return;
             }
 
             const input = fs.createReadStream(this.encryptedIndex);
-            const output = fs.createWriteStream(`${this.dump}/decrypted.zip`);
+            const output = fs.createWriteStream(`${this.dump}/decrypted.tar.lz4`);
             let config = {
                 key: this.key
             };
             const decrypt = crypto.decrypt(config);
 
-            input.pipe(decrypt).pipe(output).on('finish', () => {
+            let decryptionProcess = input.pipe(decrypt).pipe(output);
 
-                if (!fs.existsSync(`${this.dump}/decrypted.zip`)){
+            decryptionProcess.on('finish', () => {
-                    // will be handling after implementing in client app
+
+                if (!fs.existsSync(`${this.dump}/decrypted.tar.lz4`)){
+                    log.send(logLevels.ERROR, 'decrypted.tar.lz4 file not found');
                     reject();
                     return;
                 }
 
-                let readStream = fs.createReadStream(`${this.dump}/decrypted.zip`);
+                lz4.deCompression(`${this.dump}/decrypted.tar.lz4`,(error, response) => {
-                readStream
+                    if (error) {
-                    .on('data', (data) => {
+                        log.send(logLevels.ERROR, 'lz4 deCompression error, ' + error);
-                        if (!data) {
+                        // no return, need to unlink if error
-                            reject(new Error("error reading zip"));
                     }
-                        extractZip();
-                    })
-                    .on('error', (error) => {
-                        reject(new Error(error.message));
-                    });
-            });
 
-            let extractZip = () => {
+                    if (response && response.stderr) {
-                extract(`${this.dump}/decrypted.zip`, {dir: `${this.extractToPath}`}, (err) => {
+                        log.send(logLevels.WARN, 'deCompression stderr, ' + response.stderr);
-                    if (err) {
-                        reject(new Error(err));
                     }
-                    fs.unlink(`${this.dump}/decrypted.zip`, () => {
+                    fs.unlink(`${this.dump}/decrypted.tar.lz4`, () => {
                         resolve('success');
                     });
                 })
-            }
+            });
         });
     }
 

js/main.js

@@ -5,6 +5,7 @@ const electron = require('electron');
 const app = electron.app;
 const crashReporter = electron.crashReporter;
 const nodeURL = require('url');
+const shellPath = require('shell-path');
 const squirrelStartup = require('electron-squirrel-startup');
 const AutoLaunch = require('auto-launch');
 const urlParser = require('url');
@@ -22,6 +23,19 @@ const crypto = new Crypto();
 
 require('electron-dl')();
 
+//setting the env path child_process issue https://github.com/electron/electron/issues/7688
+shellPath()
+    .then((path) => {
+        process.env.PATH = path
+    })
+    .catch(() => {
+        process.env.PATH = [
+            './node_modules/.bin',
+            '/usr/local/bin',
+            process.env.PATH
+        ].join(':');
+    });
+
 // used to check if a url was opened when the app was already open
 let isAppAlreadyOpen = false;
 

js/search/search.js

@@ -66,6 +66,10 @@ class Search {
             searchConfig.REAL_TIME_INDEXING_TIME, this.realTimeIndexing.bind(this));
     }
 
+    /**
+     * Decrypting the existing user .enc file
+     * and initialing the library
+     */
     decryptAndInit() {
         this.crypto.decryption().then(() => {
             console.timeEnd('Decrypting');
@@ -304,18 +308,21 @@ class Search {
                 return;
             }
 
-            let sd = new Date().getTime() - searchConfig.SEARCH_PERIOD_SUBTRACTOR;
+            let searchPeriod = new Date().getTime() - SEARCH_PERIOD_SUBTRACTOR;
-            let sd_time = searchConfig.MINIMUM_DATE;
+            let startDateTime = searchPeriod;
-            if (startDate && startDate !== "" && typeof startDate === 'object') {
+            if (startDate) {
-                sd_time = new Date(startDate).getTime();
+                startDateTime = new Date(parseInt(startDate, 10)).getTime();
-                if (sd_time >= sd) {
+                if (!startDateTime || startDateTime < searchPeriod) {
-                    sd_time = sd;
+                    startDateTime = searchPeriod;
                 }
             }
 
-            let ed_time = searchConfig.MAXIMUM_DATE;
+            let endDateTime = searchConfig.MAXIMUM_DATE;
-            if (endDate && endDate !== "" && typeof endDate === 'object') {
+            if (endDate) {
-                ed_time = new Date(endDate).getTime();
+                let eTime = new Date(parseInt(endDate, 10)).getTime();
+                if (eTime) {
+                    endDateTime = eTime;
+                }
             }
 
             if (!_limit && _limit === "" && typeof _limit !== 'number' && Math.round(_limit) !== _limit) {
@@ -330,7 +337,7 @@ class Search {
                 _sortOrder = searchConfig.SORT_BY_SCORE;
             }
 
-            const returnedResult = libSymphonySearch.symSESearch(this.indexFolderName, this.realTimeIndex, q, sd_time.toString(), ed_time.toString(), _offset, _limit, _sortOrder);
+            const returnedResult = libSymphonySearch.symSESearch(this.indexFolderName, this.realTimeIndex, q, startDateTime.toString(), endDateTime.toString(), _offset, _limit, _sortOrder);
             try {
                 let ret = returnedResult.readCString();
                 resolve(JSON.parse(ret));

package.json

@@ -43,7 +43,8 @@
     "extraFiles": [
       "config/Symphony.config",
       "library/libsymphonysearch.dylib",
-      "library/indexvalidator.exec"
+      "library/indexvalidator.exec",
+      "library/lz4.exec"
     ],
     "appId": "symphony-electron-desktop",
     "mac": {
@@ -101,7 +102,6 @@
   "dependencies": {
     "@paulcbetts/system-idle-time": "^1.0.4",
     "appdirectory": "^0.1.0",
-    "archiver": "^2.0.0",
     "async.map": "^0.5.2",
     "async.mapseries": "^0.5.2",
     "auto-launch": "^5.0.1",
@@ -109,7 +109,6 @@
     "electron-log": "^2.2.7",
     "electron-spellchecker": "^1.1.2",
     "electron-squirrel-startup": "^1.0.0",
-    "extract-zip": "^1.6.5",
     "ffi": "^2.2.0",
     "filesize": "^3.5.10",
     "keymirror": "0.1.1",
@@ -118,6 +117,7 @@
     "lodash.difference": "^4.5.0",
     "randomstring": "^1.1.5",
     "ref": "^1.3.4",
+    "shell-path": "^2.1.0",
     "winreg": "^1.2.3"
   },
   "optionalDependencies": {