if/SymphonyElectron
1
0
Fork 0
mirror of https://github.com/finos/SymphonyElectron.git synced 2024-12-31 19:27:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #302 from keerthi16/SEARCH-622

Browse Source 
SEARCH-622 (Remove the "randomString" dependency from electron & refactoring)
This commit is contained in:
Vikas Shashidhar 2018-02-16 15:16:55 +05:30 committed by GitHub
commit 182465ef7f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 103 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
demo/search.html
View File

@ -115,11 +115,6 @@
<p>Results:</p>
<p id="results"></p>
<table id="table" class="hidden" style="width:100%">
<tr>
<th>ThreadId</th>
<th>SenderId</th>
<th>Text</th>
</tr>
</table>
<br>
</div>
@ -189,24 +184,38 @@
if (result.messages.length > 0) {
out = result;
var th = document.createElement('tr');
var avatarTh = document.createElement('td');
avatarTh.innerText = "Avatar";
var senderNameTh = document.createElement('td');
senderNameTh.innerText = "Sender Name";
var th1 = document.createElement('td');
th1.innerText = "ThreadId";
var th2 = document.createElement('td');
th2.innerText = 'SenderId';
var th3 = document.createElement('td');
th3.innerText = 'Text';
th.appendChild(avatarTh);
th.appendChild(senderNameTh);
th.appendChild(th1);
th.appendChild(th2);
th.appendChild(th3);
table.appendChild(th);
out.messages.forEach(function (msg) {
var tr = document.createElement('tr');
var avatar = document.createElement('img');
avatar.src = msg.senderAvatar;
avatar.style.maxWidth = '75px';
avatar.style.maxHeight = '75px';
var senderName = document.createElement('td');
senderName.innerText = msg.senderName;
var t1 = document.createElement('td');
t1.innerText = msg.threadId;
var t2 = document.createElement('td');
t2.innerText = msg.senderId;
var t3 = document.createElement('td');
t3.innerText = msg.text;
tr.appendChild(avatar);
tr.appendChild(senderName);
tr.appendChild(t1);
tr.appendChild(t2);
tr.appendChild(t3);
@ -280,4 +289,4 @@
});
</script>
</body>
</html>
</html>

45
js/cryptoLib/index.js
View File

@ -18,13 +18,8 @@ class Crypto {
* @param key
*/
constructor(userId, key) {
this.indexDataFolder = `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${userId}`;
this.permanentIndexName = `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${userId}`;
this.userId = userId;
this.key = key;
this.encryptedIndex = `${DUMP_PATH}/${this.permanentIndexName}.enc`;
this.dataFolder = searchConfig.FOLDERS_CONSTANTS.INDEX_PATH;
this.lz4Temp = `${DUMP_PATH}/${this.permanentIndexName}${searchConfig.TAR_LZ4_EXT}`;
this.decryptedTemp = `${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`;
}
/**
@ -35,14 +30,14 @@ class Crypto {
encryption(key) {
return new Promise((resolve, reject) => {
if (!fs.existsSync(this.indexDataFolder)){
if (!fs.existsSync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`)){
log.send(logLevels.ERROR, 'Crypto: User index folder not found');
reject();
return;
}
lz4.compression(`${searchConfig.FOLDERS_CONSTANTS.INDEX_FOLDER_NAME}/${this.permanentIndexName}`,
`${this.permanentIndexName}`, (error, response) => {
lz4.compression(`${searchConfig.FOLDERS_CONSTANTS.INDEX_FOLDER_NAME}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}`,
`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}`, (error, response) => {
if (error) {
log.send(logLevels.ERROR, 'Crypto: Error while compressing to lz4: ' + error);
reject(error);
@ -52,8 +47,8 @@ class Crypto {
if (response && response.stderr) {
log.send(logLevels.WARN, 'Crypto: Child process stderr while compression, ' + response.stderr);
}
const input = fs.createReadStream(this.lz4Temp);
const outputEncryption = fs.createWriteStream(this.encryptedIndex);
const input = fs.createReadStream(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`);
const outputEncryption = fs.createWriteStream(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}.enc`);
let config = {
key: key
};
@ -62,8 +57,8 @@ class Crypto {
encrypt = crypto.encrypt(config);
} catch (e) {
log.send(logLevels.ERROR, 'Error encrypting : ' + e);
if (fs.existsSync(this.lz4Temp)) {
fs.unlinkSync(this.lz4Temp);
if (fs.existsSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`)) {
fs.unlinkSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`);
}
reject();
return;
@ -74,14 +69,14 @@ class Crypto {
encryptionProcess.on('finish', (err) => {
if (err) {
log.send(logLevels.ERROR, 'Crypto: Error while encrypting the compressed file: ' + err);
if (fs.existsSync(this.lz4Temp)) {
fs.unlinkSync(this.lz4Temp);
if (fs.existsSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`)) {
fs.unlinkSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`);
}
reject(new Error(err));
return;
}
if (fs.existsSync(this.lz4Temp)) {
fs.unlinkSync(this.lz4Temp);
if (fs.existsSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`)) {
fs.unlinkSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}${searchConfig.TAR_LZ4_EXT}`);
}
resolve('Success');
});
@ -97,14 +92,14 @@ class Crypto {
decryption() {
return new Promise((resolve, reject) => {
if (!fs.existsSync(this.encryptedIndex)){
if (!fs.existsSync(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}.enc`)){
log.send(logLevels.ERROR, 'Crypto: Encrypted file not found');
reject();
return;
}
const input = fs.createReadStream(this.encryptedIndex);
const output = fs.createWriteStream(this.decryptedTemp);
const input = fs.createReadStream(`${DUMP_PATH}/${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${this.userId}.enc`);
const output = fs.createWriteStream(`${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`);
let config = {
key: this.key
};
@ -113,8 +108,8 @@ class Crypto {
decrypt = crypto.decrypt(config);
} catch (e) {
log.send(logLevels.ERROR, 'Error decrypting : ' + e);
if (fs.existsSync(this.decryptedTemp)) {
fs.unlinkSync(this.decryptedTemp);
if (fs.existsSync(`${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`)) {
fs.unlinkSync(`${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`);
}
reject();
return;
@ -124,13 +119,13 @@ class Crypto {
decryptionProcess.on('finish', () => {
if (!fs.existsSync(this.decryptedTemp)){
if (!fs.existsSync(`${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`)){
log.send(logLevels.ERROR, 'decrypted.tar.lz4 file not found');
reject();
return;
}
lz4.deCompression(this.decryptedTemp,(error, response) => {
lz4.deCompression(`${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`,(error, response) => {
if (error) {
log.send(logLevels.ERROR, 'Crypto: Error while deCompression, ' + error);
// no return, need to unlink if error
@ -139,7 +134,7 @@ class Crypto {
if (response && response.stderr) {
log.send(logLevels.WARN, 'Crypto: Child process stderr while deCompression, ' + response.stderr);
}
fs.unlink(this.decryptedTemp, () => {
fs.unlink(`${DUMP_PATH}/decrypted${searchConfig.TAR_LZ4_EXT}`, () => {
resolve('success');
});
})

97
js/search/search.js
View File

@ -1,7 +1,7 @@
'use strict';
const fs = require('fs');
const randomString = require('randomstring');
const { randomString } = require('../search/utils/randomString.js');
const childProcess = require('child_process');
const path = require('path');
const isDevEnv = require('../utils/misc.js').isDevEnv;
@ -31,10 +31,6 @@ class Search {
this.isInitialized = false;
this.userId = userId;
this.key = key;
this.indexFolderName = `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`;
this.dataFolder = searchConfig.FOLDERS_CONSTANTS.INDEX_PATH;
this.realTimeIndex = searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX;
this.batchIndex = searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER;
this.messageData = [];
this.isRealTimeIndexing = false;
this.crypto = new Crypto(userId, key);
@ -70,14 +66,14 @@ class Search {
*/
init() {
libSymphonySearch.symSEInit();
libSymphonySearch.symSEEnsureFolderExists(this.dataFolder);
Search.deleteIndexFolders(this.realTimeIndex);
Search.deleteIndexFolders(this.batchIndex);
Search.indexValidator(this.indexFolderName);
Search.indexValidator(this.realTimeIndex);
libSymphonySearch.symSEEnsureFolderExists(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH);
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX);
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER);
Search.indexValidator(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`);
Search.indexValidator(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX);
let indexDateStartFrom = new Date().getTime() - searchConfig.SEARCH_PERIOD_SUBTRACTOR;
// Deleting all the messages except 3 Months from now
libSymphonySearch.symSEDeleteMessages(this.indexFolderName, null,
libSymphonySearch.symSEDeleteMessages(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, null,
searchConfig.MINIMUM_DATE, indexDateStartFrom.toString());
this.isInitialized = true;
}
@ -115,14 +111,14 @@ class Search {
return;
}
if (!fs.existsSync(this.dataFolder)) {
if (!fs.existsSync(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH)) {
log.send(logLevels.ERROR, 'User index folder not found');
reject(new Error('User index folder not found'));
return;
}
const indexId = randomString.generate(searchConfig.BATCH_RANDOM_INDEX_PATH_LENGTH);
libSymphonySearch.symSECreatePartialIndexAsync(this.batchIndex, indexId, messages, (err, res) => {
const indexId = randomString();
libSymphonySearch.symSECreatePartialIndexAsync(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER, indexId, messages, (err, res) => {
if (err) {
log.send(logLevels.ERROR, 'Batch Indexing: error ->' + err);
reject(new Error(err));
@ -140,19 +136,19 @@ class Search {
mergeIndexBatches() {
return new Promise((resolve, reject) => {
if (!fs.existsSync(this.dataFolder)) {
if (!fs.existsSync(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH)) {
log.send(logLevels.ERROR, 'User index folder not found');
reject(new Error('User index folder not found'));
return;
}
libSymphonySearch.symSEMergePartialIndexAsync(this.indexFolderName, this.batchIndex, (err, res) => {
libSymphonySearch.symSEMergePartialIndexAsync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER, (err, res) => {
if (err) {
log.send(logLevels.ERROR, 'Error merging the index ->' + err);
reject(new Error(err));
return;
}
Search.deleteIndexFolders(this.batchIndex);
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER);
resolve(res);
});
});
@ -199,13 +195,13 @@ class Search {
throw new Error('Library not initialized');
}
if (!fs.existsSync(this.dataFolder)) {
if (!fs.existsSync(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH)) {
log.send(logLevels.ERROR, 'User index folder not found');
throw new Error('User index folder not found');
}
this.isRealTimeIndexing = true;
return libSymphonySearch.symSEIndexRealTimeAsync(this.realTimeIndex, message, (err, result) => {
return libSymphonySearch.symSEIndexRealTimeAsync(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX, message, (err, result) => {
this.isRealTimeIndexing = false;
if (err) {
log.send(logLevels.ERROR, 'RealTime Indexing: error -> ' + err);
@ -222,26 +218,7 @@ class Search {
* @returns {Promise}
*/
readJson(batch) {
return new Promise((resolve, reject) => {
let dirPath = path.join(searchConfig.FOLDERS_CONSTANTS.EXEC_PATH, isMac ? '..' : '', 'msgsjson', batch);
let messageFolderPath = isDevEnv ? path.join('./msgsjson', batch) : dirPath;
let files = fs.readdirSync(messageFolderPath);
this.messageData = [];
files.forEach((file) => {
let tempPath = path.join(messageFolderPath, file);
let data = fs.readFileSync(tempPath, "utf8");
if (data) {
try {
this.messageData.push(JSON.parse(data));
} catch (err) {
reject(new Error(err))
}
} else {
reject(new Error('Error reading batch'))
}
});
resolve(this.messageData);
});
return readFile.call(this, batch);
}
/**
@ -280,7 +257,7 @@ class Search {
return;
}
if (!fs.existsSync(this.indexFolderName) || !fs.existsSync(this.realTimeIndex)) {
if (!fs.existsSync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`) || !fs.existsSync(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX)) {
log.send(logLevels.ERROR, 'Index folder does not exist.');
reject(new Error('Index folder does not exist.'));
return;
@ -322,7 +299,7 @@ class Search {
_sortOrder = searchConfig.SORT_BY_SCORE;
}
const returnedResult = libSymphonySearch.symSESearch(this.indexFolderName, this.realTimeIndex, q, startDateTime.toString(), endDateTime.toString(), _offset, _limit, _sortOrder);
const returnedResult = libSymphonySearch.symSESearch(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX, q, startDateTime.toString(), endDateTime.toString(), _offset, _limit, _sortOrder);
try {
let ret = returnedResult.readCString();
resolve(JSON.parse(ret));
@ -345,13 +322,13 @@ class Search {
return;
}
if (!fs.existsSync(this.indexFolderName)) {
if (!fs.existsSync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`)) {
log.send(logLevels.ERROR, 'Index folder does not exist.');
reject(new Error('Index folder does not exist.'));
return;
}
libSymphonySearch.symSEGetLastMessageTimestampAsync(this.indexFolderName, (err, res) => {
libSymphonySearch.symSEGetLastMessageTimestampAsync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, (err, res) => {
if (err) {
log.send(logLevels.ERROR, 'Error getting the index timestamp ->' + err);
reject(new Error(err));
@ -367,9 +344,10 @@ class Search {
});
}
/*eslint class-methods-use-this: ["error", { "exceptMethods": ["deleteRealTimeFolder"] }] */
deleteRealTimeFolder() {
Search.deleteIndexFolders(this.realTimeIndex);
Search.indexValidator(this.realTimeIndex);
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX);
Search.indexValidator(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX);
}
/**
@ -587,6 +565,35 @@ function deleteIndexFolder() {
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH);
}
/**
* Reads the file from the msgjson
* this is only for the demo page
* @param batch
* @returns {Promise<Array>}
*/
function readFile(batch) {
return new Promise((resolve, reject) => {
let dirPath = path.join(searchConfig.FOLDERS_CONSTANTS.EXEC_PATH, isMac ? '..' : '', 'msgsjson', batch);
let messageFolderPath = isDevEnv ? path.join('./msgsjson', batch) : dirPath;
let files = fs.readdirSync(messageFolderPath);
this.messageData = [];
files.forEach((file) => {
let tempPath = path.join(messageFolderPath, file);
let data = fs.readFileSync(tempPath, "utf8");
if (data) {
try {
this.messageData.push(JSON.parse(data));
} catch (err) {
reject(new Error(err))
}
} else {
reject(new Error('Error reading batch'))
}
});
resolve(this.messageData);
});
}
/**
* Exporting the search library
* @type {{Search: Search}}

1
js/search/searchConfig.js
View File

@ -57,6 +57,7 @@ const searchConfig = {
LIBRARY_CONSTANTS: libraryPaths,
FOLDERS_CONSTANTS: folderPaths,
TAR_LZ4_EXT: '.tar.lz4',
RANDOM_STRING: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
MINIMUM_DISK_SPACE: 300000000 // in bytes
};

15
js/search/utils/randomString.js Normal file
View File

@ -0,0 +1,15 @@
const searchConfig = require('../searchConfig.js');
function randomString() {
let text = "";
for (let i = 0; i < 7; i++) {
text += searchConfig.RANDOM_STRING.charAt(Math.floor(Math.random() * searchConfig.RANDOM_STRING.length));
}
let time = new Date().getTime();
return text + time;
}
module.exports = {
randomString: randomString
};

1
package.json
View File

@ -117,7 +117,6 @@
"lodash.omit": "^4.5.0",
"lodash.pick": "^4.4.0",
"parse-domain": "^2.0.0",
"randomstring": "^1.1.5",
"ref": "^1.3.4",
"shell-path": "^2.1.0",
"winreg": "^1.2.3"

4
tests/Search.test.js
View File

@ -98,10 +98,6 @@ describe('Tests for Search', function() {
setTimeout(function () {
expect(SearchApi.isInitialized).toBe(true);
expect(SearchApi.indexFolderName).toBe(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${userId}`);
expect(SearchApi.dataFolder).toBe(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH);
expect(SearchApi.realTimeIndex).toBe(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX);
expect(SearchApi.batchIndex).toBe(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER);
expect(SearchApi.messageData).toEqual([]);
expect(SearchApi.isRealTimeIndexing).toBe(false);