if/SymphonyElectron
1
0
Fork 0
mirror of https://github.com/finos/SymphonyElectron.git synced 2025-01-03 12:47:13 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #248 from keerthi16/SEARCH-440

Browse Source 
SEARCH-440 (Search configuration file for electron)
This commit is contained in:
Vikas Shashidhar 2017-11-28 16:19:26 +05:30 committed by GitHub
commit 41db5a220b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 107 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
js/compressionLib/index.js
View File

@ -1,20 +1,9 @@
const electron = require('electron');
const child = require('child_process');
const app = electron.app;
const path = require('path');
const userData = path.join(app.getPath('userData'));
const isMac = require('../utils/misc.js').isMac;
const isDevEnv = require('../utils/misc.js').isDevEnv;
const DATA_FOLDER_PATH = isDevEnv ? path.join(__dirname, '..', '..') : userData;
const execPath = path.dirname(app.getPath('exe'));
// lz4 library path
const libraryFolderPath = isDevEnv ? path.join(__dirname, '..', '..', 'library') : path.join(execPath, 'library');
const winArchPath = process.arch === 'ia32' ? 'lz4-win-x86.exe' : 'lz4-win-x64.exe';
const productionPath = path.join(libraryFolderPath, winArchPath);
const devPath = path.join(__dirname, '..', '..', 'library', winArchPath);
const macLibraryPath = isDevEnv ? path.join(__dirname, '..', '..', 'library') : path.join(execPath, '..', 'library');
const lz4Path = isDevEnv ? devPath : productionPath;
const searchConfig = require('../search/searchConfig.js');
const ROOT_PATH = isDevEnv ? path.join(__dirname, '..', '..') : searchConfig.FOLDERS_CONSTANTS.USER_DATA_PATH;
/**
* Using the child process to execute the tar and lz4
@ -26,7 +15,7 @@ const lz4Path = isDevEnv ? devPath : productionPath;
*/
function compression(pathToFolder, outputPath, callback) {
if (isMac) {
child.exec(`cd "${DATA_FOLDER_PATH}" && tar cf - "${pathToFolder}" | "${macLibraryPath}/lz4.exec" > "${outputPath}.tar.lz4"`, (error, stdout, stderr) => {
child.exec(`cd "${ROOT_PATH}" && tar cf - "${pathToFolder}" | "${searchConfig.LIBRARY_CONSTANTS.MAC_LIBRARY_FOLDER}/lz4.exec" > "${outputPath}.tar.lz4"`, (error, stdout, stderr) => {
if (error) {
return callback(new Error(error), null);
}
@ -36,7 +25,7 @@ function compression(pathToFolder, outputPath, callback) {
});
})
} else {
child.exec(`cd "${DATA_FOLDER_PATH}" && "${libraryFolderPath}\\tar-win.exe" cf - "${pathToFolder}" | "${lz4Path}" > "${outputPath}.tar.lz4"`, (error, stdout, stderr) => {
child.exec(`cd "${ROOT_PATH}" && "${searchConfig.LIBRARY_CONSTANTS.WIN_LIBRARY_FOLDER}\\tar-win.exe" cf - "${pathToFolder}" | "${searchConfig.LIBRARY_CONSTANTS.LZ4_PATH}" > "${outputPath}.tar.lz4"`, (error, stdout, stderr) => {
if (error) {
return callback(new Error(error), null);
}
@ -57,7 +46,7 @@ function compression(pathToFolder, outputPath, callback) {
*/
function deCompression(pathName, callback) {
if (isMac) {
child.exec(`cd "${DATA_FOLDER_PATH}" && "${macLibraryPath}/lz4.exec" -d "${pathName}" | tar -xf - `, (error, stdout, stderr) => {
child.exec(`cd "${ROOT_PATH}" && "${searchConfig.LIBRARY_CONSTANTS.MAC_LIBRARY_FOLDER}/lz4.exec" -d "${pathName}" | tar -xf - `, (error, stdout, stderr) => {
if (error) {
return callback(new Error(error), null);
}
@ -67,7 +56,7 @@ function deCompression(pathName, callback) {
});
})
} else {
child.exec(`cd "${DATA_FOLDER_PATH}" && "${lz4Path}" -d "${pathName}" | "${libraryFolderPath}\\tar-win.exe" xf - `, (error, stdout, stderr) => {
child.exec(`cd "${ROOT_PATH}" && "${searchConfig.LIBRARY_CONSTANTS.LZ4_PATH}" -d "${pathName}" | "${searchConfig.LIBRARY_CONSTANTS.WIN_LIBRARY_FOLDER}\\tar-win.exe" xf - `, (error, stdout, stderr) => {
if (error) {
return callback(new Error(error), null);
}

1
js/cryptoLib/crypto.js
View File

@ -53,7 +53,6 @@ exports.createSalt = function(length) {
try {
return crypto.randomBytes(length);
} catch (ex) {
console.error('Problem reading random data and generating salt!');
throw ex;
}
};

88
js/cryptoLib/index.js
View File

@ -1,6 +1,4 @@
'use strict';
const electron = require('electron');
const app = electron.app;
const path = require('path');
const fs = require('fs');
const lz4 = require('../compressionLib');
@ -8,22 +6,19 @@ const isDevEnv = require('../utils/misc.js').isDevEnv;
const crypto = require('./crypto');
const log = require('../log.js');
const logLevels = require('../enums/logLevels.js');
const searchConfig = require('../search/searchConfig.js');
const userData = path.join(app.getPath('userData'));
const DATA_FOLDER = isDevEnv ? './data' : path.join(userData, 'data');
const INDEX_DATA_FOLDER = isDevEnv ? './data/search_index' : path.join(userData, 'data/search_index');
const TEMPORARY_PATH = isDevEnv ? path.join(__dirname, '..', '..') : userData;
const DUMP_PATH = isDevEnv ? path.join(__dirname, '..', '..') : searchConfig.FOLDERS_CONSTANTS.USER_DATA_PATH;
class Crypto {
constructor(userId, key) {
let INDEX_VERSION = 'v1';
this.indexDataFolder = INDEX_DATA_FOLDER + '_' + userId + '_' + INDEX_VERSION;
this.permanentIndexFolderName = 'search_index_' + userId + '_' + INDEX_VERSION;
this.dump = TEMPORARY_PATH;
this.indexDataFolder = `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${userId}_${searchConfig.INDEX_VERSION}`;
this.permanentIndexName = `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${userId}_${searchConfig.INDEX_VERSION}`;
this.dump = DUMP_PATH;
this.key = key;
this.encryptedIndex = `${TEMPORARY_PATH}/${this.permanentIndexFolderName}.enc`;
this.dataFolder = DATA_FOLDER;
this.encryptedIndex = `${DUMP_PATH}/${this.permanentIndexName}.enc`;
this.dataFolder = searchConfig.FOLDERS_CONSTANTS.INDEX_PATH;
}
/**
@ -35,40 +30,41 @@ class Crypto {
return new Promise((resolve, reject) => {
if (!fs.existsSync(this.indexDataFolder)){
log.send(logLevels.ERROR, 'user index folder not found');
log.send(logLevels.ERROR, 'Crypto: User index folder not found');
reject();
return;
}
lz4.compression(`data/${this.permanentIndexFolderName}`, `${this.permanentIndexFolderName}`, (error, response) => {
if (error) {
log.send(logLevels.ERROR, 'lz4 compression error: ' + error);
reject(error);
return;
}
if (response && response.stderr) {
log.send(logLevels.WARN, 'compression stderr, ' + response.stderr);
}
const input = fs.createReadStream(`${this.dump}/${this.permanentIndexFolderName}.tar.lz4`);
const outputEncryption = fs.createWriteStream(this.encryptedIndex);
let config = {
key: this.key
};
const encrypt = crypto.encrypt(config);
let encryptionProcess = input.pipe(encrypt).pipe(outputEncryption);
encryptionProcess.on('finish', (err) => {
if (err) {
log.send(logLevels.ERROR, 'encryption error: ' + err);
reject(new Error(err));
lz4.compression(`${searchConfig.FOLDERS_CONSTANTS.INDEX_FOLDER_NAME}/${this.permanentIndexName}`,
`${this.permanentIndexName}`, (error, response) => {
if (error) {
log.send(logLevels.ERROR, 'Crypto: Error while compressing to lz4: ' + error);
reject(error);
return;
}
fs.unlinkSync(`${this.dump}/${this.permanentIndexFolderName}.tar.lz4`);
resolve('Success');
if (response && response.stderr) {
log.send(logLevels.WARN, 'Crypto: Child process stderr while compression, ' + response.stderr);
}
const input = fs.createReadStream(`${this.dump}/${this.permanentIndexName}${searchConfig.TAR_LZ4_EXT}`);
const outputEncryption = fs.createWriteStream(this.encryptedIndex);
let config = {
key: this.key
};
const encrypt = crypto.encrypt(config);
let encryptionProcess = input.pipe(encrypt).pipe(outputEncryption);
encryptionProcess.on('finish', (err) => {
if (err) {
log.send(logLevels.ERROR, 'Crypto: Error while encrypting the compressed file: ' + err);
reject(new Error(err));
return;
}
fs.unlinkSync(`${this.dump}/${this.permanentIndexName}${searchConfig.TAR_LZ4_EXT}`);
resolve('Success');
});
});
});
});
}
@ -81,13 +77,13 @@ class Crypto {
return new Promise((resolve, reject) => {
if (!fs.existsSync(this.encryptedIndex)){
log.send(logLevels.ERROR, 'encrypted file not found');
log.send(logLevels.ERROR, 'Crypto: Encrypted file not found');
reject();
return;
}
const input = fs.createReadStream(this.encryptedIndex);
const output = fs.createWriteStream(`${this.dump}/decrypted.tar.lz4`);
const output = fs.createWriteStream(`${this.dump}/decrypted${searchConfig.TAR_LZ4_EXT}`);
let config = {
key: this.key
};
@ -97,22 +93,22 @@ class Crypto {
decryptionProcess.on('finish', () => {
if (!fs.existsSync(`${this.dump}/decrypted.tar.lz4`)){
if (!fs.existsSync(`${this.dump}/decrypted${searchConfig.TAR_LZ4_EXT}`)){
log.send(logLevels.ERROR, 'decrypted.tar.lz4 file not found');
reject();
return;
}
lz4.deCompression(`${this.dump}/decrypted.tar.lz4`,(error, response) => {
lz4.deCompression(`${this.dump}/decrypted${searchConfig.TAR_LZ4_EXT}`,(error, response) => {
if (error) {
log.send(logLevels.ERROR, 'lz4 deCompression error, ' + error);
log.send(logLevels.ERROR, 'Crypto: Error while deCompression, ' + error);
// no return, need to unlink if error
}
if (response && response.stderr) {
log.send(logLevels.WARN, 'deCompression stderr, ' + response.stderr);
log.send(logLevels.WARN, 'Crypto: Child process stderr while deCompression, ' + response.stderr);
}
fs.unlink(`${this.dump}/decrypted.tar.lz4`, () => {
fs.unlink(`${this.dump}/decrypted${searchConfig.TAR_LZ4_EXT}`, () => {
resolve('success');
});
})

38
js/search/search.js
View File

@ -2,9 +2,7 @@
const fs = require('fs');
const randomString = require('randomstring');
const electron = require('electron');
const childProcess = require('child_process');
const app = electron.app;
const path = require('path');
const isDevEnv = require('../utils/misc.js').isDevEnv;
const isMac = require('../utils/misc.js').isMac;
@ -13,30 +11,10 @@ const searchConfig = require('./searchConfig');
const log = require('../log.js');
const logLevels = require('../enums/logLevels.js');
// Search library
const libSymphonySearch = require('./searchLibrary');
// Crypto Library
const Crypto = require('../cryptoLib');
// Path for the exec file and the user data folder
const userData = path.join(app.getPath('userData'));
const execPath = path.dirname(app.getPath('exe'));
// Constants paths for temp indexing folders
const TEMP_BATCH_INDEX_FOLDER = isDevEnv ? './data/temp_batch_indexes' : path.join(userData, 'data/temp_batch_indexes');
const TEMP_REAL_TIME_INDEX = isDevEnv ? './data/temp_realtime_index' : path.join(userData, 'data/temp_realtime_index');
// Main User Index path
const INDEX_PREFIX = isDevEnv ? './data/search_index' : path.join(userData, 'data/search_index');
// Folder contains real time, batch and user index
const INDEX_DATA_FOLDER = isDevEnv ? './data' : path.join(userData, 'data');
// library path contractor
const winArchPath = process.arch === 'ia32' ? 'library/indexvalidator-x86.exe' : 'library/indexvalidator-x64.exe';
const rootPath = isMac ? 'library/indexvalidator.exec' : winArchPath;
const productionPath = path.join(execPath, isMac ? '..' : '', rootPath);
const devPath = path.join(__dirname, '..', '..', rootPath);
const INDEX_VALIDATOR = isDevEnv ? devPath : productionPath;
const INDEX_VALIDATOR = searchConfig.LIBRARY_CONSTANTS.INDEX_VALIDATOR;
/**
* This search class communicates with the SymphonySearchEngine C library via node-ffi.
@ -50,14 +28,13 @@ class Search {
* @param key
*/
constructor(userId, key) {
console.time('Decrypting');
this.isInitialized = false;
this.userId = userId;
this.key = key;
this.indexFolderName = INDEX_PREFIX + '_' + this.userId + '_' + searchConfig.INDEX_VERSION;
this.dataFolder = INDEX_DATA_FOLDER;
this.realTimeIndex = TEMP_REAL_TIME_INDEX;
this.batchIndex = TEMP_BATCH_INDEX_FOLDER;
this.indexFolderName = `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}_${searchConfig.INDEX_VERSION}`;
this.dataFolder = searchConfig.FOLDERS_CONSTANTS.INDEX_PATH;
this.realTimeIndex = searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX;
this.batchIndex = searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER;
this.messageData = [];
this.isRealTimeIndexing = false;
this.crypto = new Crypto(userId, key);
@ -72,7 +49,6 @@ class Search {
*/
decryptAndInit() {
this.crypto.decryption().then(() => {
console.timeEnd('Decrypting');
this.init();
}).catch(() => {
this.init();
@ -233,7 +209,7 @@ class Search {
*/
readJson(batch) {
return new Promise((resolve, reject) => {
let dirPath = path.join(execPath, isMac ? '..' : '', 'msgsjson', batch);
let dirPath = path.join(searchConfig.FOLDERS_CONSTANTS.EXEC_PATH, isMac ? '..' : '', 'msgsjson', batch);
let messageFolderPath = isDevEnv ? path.join('./msgsjson', batch) : dirPath;
let files = fs.readdirSync(messageFolderPath);
this.messageData = [];
@ -541,7 +517,7 @@ class Search {
* when the app is closed/signed-out/navigates
*/
function deleteIndexFolder() {
Search.deleteFolderRecursive(INDEX_DATA_FOLDER);
Search.deleteFolderRecursive(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH);
}
/**

50
js/search/searchConfig.js
View File

@ -1,3 +1,48 @@
const electron = require('electron');
const app = electron.app;
const path = require('path');
const userData = path.join(app.getPath('userData'));
const execPath = path.dirname(app.getPath('exe'));
const isDevEnv = require('../utils/misc.js').isDevEnv;
const isMac = require('../utils/misc.js').isMac;
const INDEX_FOLDER_NAME = 'data';
const winLibraryPath = isDevEnv ? path.join(__dirname, '..', '..', 'library') : path.join(execPath, 'library');
const macLibraryPath = isDevEnv ? path.join(__dirname, '..', '..', 'library') : path.join(execPath, '..', 'library');
const arch = process.arch === 'ia32';
const winIndexValidatorArch = arch ? 'indexvalidator-x86.exe' : 'indexvalidator-x64.exe';
const indexValidatorPath = isMac ? path.join(macLibraryPath, 'indexvalidator.exec') : path.join(winLibraryPath, winIndexValidatorArch);
const winLZ4ArchPath = arch ? 'lz4-win-x86.exe' : 'lz4-win-x64.exe';
const lz4Path = path.join(winLibraryPath, winLZ4ArchPath);
const indexFolderPath = isDevEnv ? `./${INDEX_FOLDER_NAME}` : path.join(userData, INDEX_FOLDER_NAME);
const winSearchLibArchPath = arch ? 'libsymphonysearch-x86.dll' : 'libsymphonysearch-x64.dll';
const libraryPath = isMac ? path.join(macLibraryPath, 'libsymphonysearch.dylib') : path.join(winLibraryPath, winSearchLibArchPath);
const libraryPaths = {
INDEX_VALIDATOR: indexValidatorPath,
LZ4_PATH: lz4Path,
MAC_LIBRARY_FOLDER: macLibraryPath,
WIN_LIBRARY_FOLDER: winLibraryPath,
SEARCH_LIBRARY_PATH: libraryPath
};
const folderPaths = {
INDEX_PATH: indexFolderPath,
TEMP_BATCH_INDEX_FOLDER: indexFolderPath + '/temp_batch_indexes',
TEMP_REAL_TIME_INDEX: indexFolderPath + '/temp_realtime_index',
PREFIX_NAME: 'search_index',
PREFIX_NAME_PATH: indexFolderPath + '/search_index',
EXEC_PATH: execPath,
USER_DATA_PATH: userData,
INDEX_FOLDER_NAME: INDEX_FOLDER_NAME
};
const searchConfig = {
SEARCH_PERIOD_SUBTRACTOR: 3 * 31 * 24 * 60 * 60 * 1000,
REAL_TIME_INDEXING_TIME: 60000,
@ -5,7 +50,10 @@ const searchConfig = {
MAXIMUM_DATE: '9999999999999',
INDEX_VERSION: 'v1',
SORT_BY_SCORE: 0,
BATCH_RANDOM_INDEX_PATH_LENGTH: 20
BATCH_RANDOM_INDEX_PATH_LENGTH: 20,
LIBRARY_CONSTANTS: libraryPaths,
FOLDERS_CONSTANTS: folderPaths,
TAR_LZ4_EXT: '.tar.lz4'
};
module.exports = searchConfig;

20
js/search/searchLibrary.js
View File

@ -2,31 +2,17 @@
const ffi = require('ffi');
const ref = require('ref');
const electron = require('electron');
const app = electron.app;
const path = require('path');
const isDevEnv = require('../utils/misc.js').isDevEnv;
const isMac = require('../utils/misc.js').isMac;
// ref types
const searchConfig = require('../search/searchConfig.js');
const symLucyIndexer = ref.types.void;
const symLucyIndexerPtr = ref.refType(symLucyIndexer);
// application execution path
const execPath = path.dirname(app.getPath('exe'));
// library path contractor
const winArchPath = process.arch === 'ia32' ? 'library/libsymphonysearch-x86.dll' : 'library/libsymphonysearch-x64.dll';
const rootPath = isMac ? 'library/libsymphonysearch.dylib' : winArchPath;
const productionPath = path.join(execPath, isMac ? '..' : '', rootPath);
const devPath = path.join(__dirname, '..', '..', rootPath);
const libraryPath = isDevEnv ? devPath : productionPath;
/**
* Initializing the C SymphonySearchEngine library
* using the node-ffi
*/
let libSymphonySearch = ffi.Library(libraryPath, {
let libSymphonySearch = ffi.Library(searchConfig.LIBRARY_CONSTANTS.SEARCH_LIBRARY_PATH, {
//init
'symSE_init': ['void', []],
'symSE_remove_folder': ['int', ['string']],