diff --git a/demo/search.html b/demo/search.html index dc92aae1..4b6eda04 100644 --- a/demo/search.html +++ b/demo/search.html @@ -115,11 +115,6 @@

Results:

- - - - -
@@ -189,24 +184,38 @@ if (result.messages.length > 0) { out = result; var th = document.createElement('tr'); + var avatarTh = document.createElement('td'); + avatarTh.innerText = "Avatar"; + var senderNameTh = document.createElement('td'); + senderNameTh.innerText = "Sender Name"; var th1 = document.createElement('td'); th1.innerText = "ThreadId"; var th2 = document.createElement('td'); th2.innerText = 'SenderId'; var th3 = document.createElement('td'); th3.innerText = 'Text'; + th.appendChild(avatarTh); + th.appendChild(senderNameTh); th.appendChild(th1); th.appendChild(th2); th.appendChild(th3); table.appendChild(th); out.messages.forEach(function (msg) { var tr = document.createElement('tr'); + var avatar = document.createElement('img'); + avatar.src = msg.senderAvatar; + avatar.style.maxWidth = '75px'; + avatar.style.maxHeight = '75px'; + var senderName = document.createElement('td'); + senderName.innerText = msg.senderName; var t1 = document.createElement('td'); t1.innerText = msg.threadId; var t2 = document.createElement('td'); t2.innerText = msg.senderId; var t3 = document.createElement('td'); t3.innerText = msg.text; + tr.appendChild(avatar); + tr.appendChild(senderName); tr.appendChild(t1); tr.appendChild(t2); tr.appendChild(t3); @@ -280,4 +289,4 @@ }); - + \ No newline at end of file diff --git a/js/cryptoLib/index.js b/js/cryptoLib/index.js index 6b14b22a..355fb653 100644 --- a/js/cryptoLib/index.js +++ b/js/cryptoLib/index.js @@ -18,13 +18,8 @@ class Crypto { * @param key */ constructor(userId, key) { - this.indexDataFolder = `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${userId}`; - this.permanentIndexName = `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${userId}`; + this.userId = userId; this.key = key; - this.encryptedIndex = `${DUMP_PATH}/${this.permanentIndexName}.enc`; - this.dataFolder = searchConfig.FOLDERS_CONSTANTS.INDEX_PATH; - this.lz4Temp = `${DUMP_PATH}/${this.permanentIndexName}${searchConfig.TAR_LZ4_EXT}`; - this.decryptedTemp = `${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`; } /** @@ -35,14 +30,14 @@ class Crypto { encryption(key) { return new Promise((resolve, reject) => { - if (!fs.existsSync(this.indexDataFolder)){ + if (!fs.existsSync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`)){ log.send(logLevels.ERROR, 'Crypto: User index folder not found'); reject(); return; } - lz4.compression(`${searchConfig.FOLDERS_CONSTANTS.INDEX_FOLDER_NAME}/${this.permanentIndexName}`, - `${this.permanentIndexName}`, (error, response) => { + lz4.compression(`${searchConfig.FOLDERS_CONSTANTS.INDEX_FOLDER_NAME}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}`, + `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}`, (error, response) => { if (error) { log.send(logLevels.ERROR, 'Crypto: Error while compressing to lz4: ' + error); reject(error); @@ -52,8 +47,8 @@ class Crypto { if (response && response.stderr) { log.send(logLevels.WARN, 'Crypto: Child process stderr while compression, ' + response.stderr); } - const input = fs.createReadStream(this.lz4Temp); - const outputEncryption = fs.createWriteStream(this.encryptedIndex); + const input = fs.createReadStream(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`); + const outputEncryption = fs.createWriteStream(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}.enc`); let config = { key: key }; @@ -62,8 +57,8 @@ class Crypto { encrypt = crypto.encrypt(config); } catch (e) { log.send(logLevels.ERROR, 'Error encrypting : ' + e); - if (fs.existsSync(this.lz4Temp)) { - fs.unlinkSync(this.lz4Temp); + if (fs.existsSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`)) { + fs.unlinkSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`); } reject(); return; @@ -74,14 +69,14 @@ class Crypto { encryptionProcess.on('finish', (err) => { if (err) { log.send(logLevels.ERROR, 'Crypto: Error while encrypting the compressed file: ' + err); - if (fs.existsSync(this.lz4Temp)) { - fs.unlinkSync(this.lz4Temp); + if (fs.existsSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`)) { + fs.unlinkSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`); } reject(new Error(err)); return; } - if (fs.existsSync(this.lz4Temp)) { - fs.unlinkSync(this.lz4Temp); + if (fs.existsSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`)) { + fs.unlinkSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`); } resolve('Success'); }); @@ -97,14 +92,14 @@ class Crypto { decryption() { return new Promise((resolve, reject) => { - if (!fs.existsSync(this.encryptedIndex)){ + if (!fs.existsSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}.enc`)){ log.send(logLevels.ERROR, 'Crypto: Encrypted file not found'); reject(); return; } - const input = fs.createReadStream(this.encryptedIndex); - const output = fs.createWriteStream(this.decryptedTemp); + const input = fs.createReadStream(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}.enc`); + const output = fs.createWriteStream(`${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`); let config = { key: this.key }; @@ -113,8 +108,8 @@ class Crypto { decrypt = crypto.decrypt(config); } catch (e) { log.send(logLevels.ERROR, 'Error decrypting : ' + e); - if (fs.existsSync(this.decryptedTemp)) { - fs.unlinkSync(this.decryptedTemp); + if (fs.existsSync(`${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`)) { + fs.unlinkSync(`${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`); } reject(); return; @@ -124,13 +119,13 @@ class Crypto { decryptionProcess.on('finish', () => { - if (!fs.existsSync(this.decryptedTemp)){ + if (!fs.existsSync(`${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`)){ log.send(logLevels.ERROR, 'decrypted.tar.lz4 file not found'); reject(); return; } - lz4.deCompression(this.decryptedTemp,(error, response) => { + lz4.deCompression(`${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`,(error, response) => { if (error) { log.send(logLevels.ERROR, 'Crypto: Error while deCompression, ' + error); // no return, need to unlink if error @@ -139,7 +134,7 @@ class Crypto { if (response && response.stderr) { log.send(logLevels.WARN, 'Crypto: Child process stderr while deCompression, ' + response.stderr); } - fs.unlink(this.decryptedTemp, () => { + fs.unlink(`${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`, () => { resolve('success'); }); }) diff --git a/js/search/search.js b/js/search/search.js index 01886f04..19156c26 100644 --- a/js/search/search.js +++ b/js/search/search.js @@ -1,7 +1,7 @@ 'use strict'; const fs = require('fs'); -const randomString = require('randomstring'); +const { randomString } = require('../search/utils/randomString.js'); const childProcess = require('child_process'); const path = require('path'); const isDevEnv = require('../utils/misc.js').isDevEnv; @@ -31,10 +31,6 @@ class Search { this.isInitialized = false; this.userId = userId; this.key = key; - this.indexFolderName = `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`; - this.dataFolder = searchConfig.FOLDERS_CONSTANTS.INDEX_PATH; - this.realTimeIndex = searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX; - this.batchIndex = searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER; this.messageData = []; this.isRealTimeIndexing = false; this.crypto = new Crypto(userId, key); @@ -70,14 +66,14 @@ class Search { */ init() { libSymphonySearch.symSEInit(); - libSymphonySearch.symSEEnsureFolderExists(this.dataFolder); - Search.deleteIndexFolders(this.realTimeIndex); - Search.deleteIndexFolders(this.batchIndex); - Search.indexValidator(this.indexFolderName); - Search.indexValidator(this.realTimeIndex); + libSymphonySearch.symSEEnsureFolderExists(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH); + Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX); + Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER); + Search.indexValidator(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`); + Search.indexValidator(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX); let indexDateStartFrom = new Date().getTime() - searchConfig.SEARCH_PERIOD_SUBTRACTOR; // Deleting all the messages except 3 Months from now - libSymphonySearch.symSEDeleteMessages(this.indexFolderName, null, + libSymphonySearch.symSEDeleteMessages(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, null, searchConfig.MINIMUM_DATE, indexDateStartFrom.toString()); this.isInitialized = true; } @@ -115,14 +111,14 @@ class Search { return; } - if (!fs.existsSync(this.dataFolder)) { + if (!fs.existsSync(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH)) { log.send(logLevels.ERROR, 'User index folder not found'); reject(new Error('User index folder not found')); return; } - const indexId = randomString.generate(searchConfig.BATCH_RANDOM_INDEX_PATH_LENGTH); - libSymphonySearch.symSECreatePartialIndexAsync(this.batchIndex, indexId, messages, (err, res) => { + const indexId = randomString(); + libSymphonySearch.symSECreatePartialIndexAsync(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER, indexId, messages, (err, res) => { if (err) { log.send(logLevels.ERROR, 'Batch Indexing: error ->' + err); reject(new Error(err)); @@ -140,19 +136,19 @@ class Search { mergeIndexBatches() { return new Promise((resolve, reject) => { - if (!fs.existsSync(this.dataFolder)) { + if (!fs.existsSync(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH)) { log.send(logLevels.ERROR, 'User index folder not found'); reject(new Error('User index folder not found')); return; } - libSymphonySearch.symSEMergePartialIndexAsync(this.indexFolderName, this.batchIndex, (err, res) => { + libSymphonySearch.symSEMergePartialIndexAsync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER, (err, res) => { if (err) { log.send(logLevels.ERROR, 'Error merging the index ->' + err); reject(new Error(err)); return; } - Search.deleteIndexFolders(this.batchIndex); + Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER); resolve(res); }); }); @@ -199,13 +195,13 @@ class Search { throw new Error('Library not initialized'); } - if (!fs.existsSync(this.dataFolder)) { + if (!fs.existsSync(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH)) { log.send(logLevels.ERROR, 'User index folder not found'); throw new Error('User index folder not found'); } this.isRealTimeIndexing = true; - return libSymphonySearch.symSEIndexRealTimeAsync(this.realTimeIndex, message, (err, result) => { + return libSymphonySearch.symSEIndexRealTimeAsync(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX, message, (err, result) => { this.isRealTimeIndexing = false; if (err) { log.send(logLevels.ERROR, 'RealTime Indexing: error -> ' + err); @@ -222,26 +218,7 @@ class Search { * @returns {Promise} */ readJson(batch) { - return new Promise((resolve, reject) => { - let dirPath = path.join(searchConfig.FOLDERS_CONSTANTS.EXEC_PATH, isMac ? '..' : '', 'msgsjson', batch); - let messageFolderPath = isDevEnv ? path.join('./msgsjson', batch) : dirPath; - let files = fs.readdirSync(messageFolderPath); - this.messageData = []; - files.forEach((file) => { - let tempPath = path.join(messageFolderPath, file); - let data = fs.readFileSync(tempPath, "utf8"); - if (data) { - try { - this.messageData.push(JSON.parse(data)); - } catch (err) { - reject(new Error(err)) - } - } else { - reject(new Error('Error reading batch')) - } - }); - resolve(this.messageData); - }); + return readFile.call(this, batch); } /** @@ -280,7 +257,7 @@ class Search { return; } - if (!fs.existsSync(this.indexFolderName) || !fs.existsSync(this.realTimeIndex)) { + if (!fs.existsSync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`) || !fs.existsSync(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX)) { log.send(logLevels.ERROR, 'Index folder does not exist.'); reject(new Error('Index folder does not exist.')); return; @@ -322,7 +299,7 @@ class Search { _sortOrder = searchConfig.SORT_BY_SCORE; } - const returnedResult = libSymphonySearch.symSESearch(this.indexFolderName, this.realTimeIndex, q, startDateTime.toString(), endDateTime.toString(), _offset, _limit, _sortOrder); + const returnedResult = libSymphonySearch.symSESearch(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX, q, startDateTime.toString(), endDateTime.toString(), _offset, _limit, _sortOrder); try { let ret = returnedResult.readCString(); resolve(JSON.parse(ret)); @@ -345,13 +322,13 @@ class Search { return; } - if (!fs.existsSync(this.indexFolderName)) { + if (!fs.existsSync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`)) { log.send(logLevels.ERROR, 'Index folder does not exist.'); reject(new Error('Index folder does not exist.')); return; } - libSymphonySearch.symSEGetLastMessageTimestampAsync(this.indexFolderName, (err, res) => { + libSymphonySearch.symSEGetLastMessageTimestampAsync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, (err, res) => { if (err) { log.send(logLevels.ERROR, 'Error getting the index timestamp ->' + err); reject(new Error(err)); @@ -367,9 +344,10 @@ class Search { }); } + /*eslint class-methods-use-this: ["error", { "exceptMethods": ["deleteRealTimeFolder"] }] */ deleteRealTimeFolder() { - Search.deleteIndexFolders(this.realTimeIndex); - Search.indexValidator(this.realTimeIndex); + Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX); + Search.indexValidator(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX); } /** @@ -587,6 +565,35 @@ function deleteIndexFolder() { Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH); } +/** + * Reads the file from the msgjson + * this is only for the demo page + * @param batch + * @returns {Promise} + */ +function readFile(batch) { + return new Promise((resolve, reject) => { + let dirPath = path.join(searchConfig.FOLDERS_CONSTANTS.EXEC_PATH, isMac ? '..' : '', 'msgsjson', batch); + let messageFolderPath = isDevEnv ? path.join('./msgsjson', batch) : dirPath; + let files = fs.readdirSync(messageFolderPath); + this.messageData = []; + files.forEach((file) => { + let tempPath = path.join(messageFolderPath, file); + let data = fs.readFileSync(tempPath, "utf8"); + if (data) { + try { + this.messageData.push(JSON.parse(data)); + } catch (err) { + reject(new Error(err)) + } + } else { + reject(new Error('Error reading batch')) + } + }); + resolve(this.messageData); + }); +} + /** * Exporting the search library * @type {{Search: Search}} diff --git a/js/search/searchConfig.js b/js/search/searchConfig.js index 7d424bdc..fa11ee61 100644 --- a/js/search/searchConfig.js +++ b/js/search/searchConfig.js @@ -57,6 +57,7 @@ const searchConfig = { LIBRARY_CONSTANTS: libraryPaths, FOLDERS_CONSTANTS: folderPaths, TAR_LZ4_EXT: '.tar.lz4', + RANDOM_STRING: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", MINIMUM_DISK_SPACE: 300000000 // in bytes }; diff --git a/js/search/utils/randomString.js b/js/search/utils/randomString.js new file mode 100644 index 00000000..14c799a3 --- /dev/null +++ b/js/search/utils/randomString.js @@ -0,0 +1,15 @@ +const searchConfig = require('../searchConfig.js'); + +function randomString() { + let text = ""; + + for (let i = 0; i < 7; i++) { + text += searchConfig.RANDOM_STRING.charAt(Math.floor(Math.random() * searchConfig.RANDOM_STRING.length)); + } + let time = new Date().getTime(); + return text + time; +} + +module.exports = { + randomString: randomString +}; \ No newline at end of file diff --git a/package.json b/package.json index 939573a6..13e2ebc3 100644 --- a/package.json +++ b/package.json @@ -117,7 +117,6 @@ "lodash.omit": "^4.5.0", "lodash.pick": "^4.4.0", "parse-domain": "^2.0.0", - "randomstring": "^1.1.5", "ref": "^1.3.4", "shell-path": "^2.1.0", "winreg": "^1.2.3" diff --git a/tests/Search.test.js b/tests/Search.test.js index cb49e735..3ec4988e 100644 --- a/tests/Search.test.js +++ b/tests/Search.test.js @@ -98,10 +98,6 @@ describe('Tests for Search', function() { setTimeout(function () { expect(SearchApi.isInitialized).toBe(true); - expect(SearchApi.indexFolderName).toBe(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${userId}`); - expect(SearchApi.dataFolder).toBe(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH); - expect(SearchApi.realTimeIndex).toBe(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX); - expect(SearchApi.batchIndex).toBe(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER); expect(SearchApi.messageData).toEqual([]); expect(SearchApi.isRealTimeIndexing).toBe(false);