if/SymphonyElectron
1
0
Fork 0
mirror of https://github.com/finos/SymphonyElectron.git synced 2025-02-25 18:55:29 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into SEARCH-605-new

Browse Source 
# Conflicts:
#	js/search/searchConfig.js
This commit is contained in:
Keerthi Niranjan
2018-02-16 15:34:29 +05:30
parent 6449393c7f 182465ef7f
commit 70f37644e9
7 changed files with 103 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
js/search/search.js
View File

@@ -1,7 +1,7 @@
'use strict';
const fs = require('fs');
const randomString = require('randomstring');
const { randomString } = require('../search/utils/randomString.js');
const childProcess = require('child_process');
const path = require('path');
const isDevEnv = require('../utils/misc.js').isDevEnv;
@@ -31,10 +31,6 @@ class Search {
this.isInitialized = false;
this.userId = userId;
this.key = key;
this.indexFolderName = `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`;
this.dataFolder = searchConfig.FOLDERS_CONSTANTS.INDEX_PATH;
this.realTimeIndex = searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX;
this.batchIndex = searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER;
this.messageData = [];
this.isRealTimeIndexing = false;
this.crypto = new Crypto(userId, key);
@@ -70,14 +66,14 @@ class Search {
*/
init() {
libSymphonySearch.symSEInit();
libSymphonySearch.symSEEnsureFolderExists(this.dataFolder);
Search.deleteIndexFolders(this.realTimeIndex);
Search.deleteIndexFolders(this.batchIndex);
Search.indexValidator(this.indexFolderName);
Search.indexValidator(this.realTimeIndex);
libSymphonySearch.symSEEnsureFolderExists(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH);
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX);
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER);
Search.indexValidator(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`);
Search.indexValidator(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX);
let indexDateStartFrom = new Date().getTime() - searchConfig.SEARCH_PERIOD_SUBTRACTOR;
// Deleting all the messages except 3 Months from now
libSymphonySearch.symSEDeleteMessages(this.indexFolderName, null,
libSymphonySearch.symSEDeleteMessages(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, null,
searchConfig.MINIMUM_DATE, indexDateStartFrom.toString());
this.isInitialized = true;
}
@@ -115,14 +111,14 @@ class Search {
return;
}
if (!fs.existsSync(this.dataFolder)) {
if (!fs.existsSync(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH)) {
log.send(logLevels.ERROR, 'User index folder not found');
reject(new Error('User index folder not found'));
return;
}
const indexId = randomString.generate(searchConfig.BATCH_RANDOM_INDEX_PATH_LENGTH);
libSymphonySearch.symSECreatePartialIndexAsync(this.batchIndex, indexId, messages, (err, res) => {
const indexId = randomString();
libSymphonySearch.symSECreatePartialIndexAsync(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER, indexId, messages, (err, res) => {
if (err) {
log.send(logLevels.ERROR, 'Batch Indexing: error ->' + err);
reject(new Error(err));
@@ -140,19 +136,19 @@ class Search {
mergeIndexBatches() {
return new Promise((resolve, reject) => {
if (!fs.existsSync(this.dataFolder)) {
if (!fs.existsSync(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH)) {
log.send(logLevels.ERROR, 'User index folder not found');
reject(new Error('User index folder not found'));
return;
}
libSymphonySearch.symSEMergePartialIndexAsync(this.indexFolderName, this.batchIndex, (err, res) => {
libSymphonySearch.symSEMergePartialIndexAsync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER, (err, res) => {
if (err) {
log.send(logLevels.ERROR, 'Error merging the index ->' + err);
reject(new Error(err));
return;
}
Search.deleteIndexFolders(this.batchIndex);
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER);
resolve(res);
});
});
@@ -199,13 +195,13 @@ class Search {
throw new Error('Library not initialized');
}
if (!fs.existsSync(this.dataFolder)) {
if (!fs.existsSync(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH)) {
log.send(logLevels.ERROR, 'User index folder not found');
throw new Error('User index folder not found');
}
this.isRealTimeIndexing = true;
return libSymphonySearch.symSEIndexRealTimeAsync(this.realTimeIndex, message, (err, result) => {
return libSymphonySearch.symSEIndexRealTimeAsync(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX, message, (err, result) => {
this.isRealTimeIndexing = false;
if (err) {
log.send(logLevels.ERROR, 'RealTime Indexing: error -> ' + err);
@@ -222,26 +218,7 @@ class Search {
* @returns {Promise}
*/
readJson(batch) {
return new Promise((resolve, reject) => {
let dirPath = path.join(searchConfig.FOLDERS_CONSTANTS.EXEC_PATH, isMac ? '..' : '', 'msgsjson', batch);
let messageFolderPath = isDevEnv ? path.join('./msgsjson', batch) : dirPath;
let files = fs.readdirSync(messageFolderPath);
this.messageData = [];
files.forEach((file) => {
let tempPath = path.join(messageFolderPath, file);
let data = fs.readFileSync(tempPath, "utf8");
if (data) {
try {
this.messageData.push(JSON.parse(data));
} catch (err) {
reject(new Error(err))
}
} else {
reject(new Error('Error reading batch'))
}
});
resolve(this.messageData);
});
return readFile.call(this, batch);
}
/**
@@ -280,7 +257,7 @@ class Search {
return;
}
if (!fs.existsSync(this.indexFolderName) || !fs.existsSync(this.realTimeIndex)) {
if (!fs.existsSync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`) || !fs.existsSync(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX)) {
log.send(logLevels.ERROR, 'Index folder does not exist.');
reject(new Error('Index folder does not exist.'));
return;
@@ -322,7 +299,7 @@ class Search {
_sortOrder = searchConfig.SORT_BY_SCORE;
}
const returnedResult = libSymphonySearch.symSESearch(this.indexFolderName, this.realTimeIndex, q, startDateTime.toString(), endDateTime.toString(), _offset, _limit, _sortOrder);
const returnedResult = libSymphonySearch.symSESearch(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX, q, startDateTime.toString(), endDateTime.toString(), _offset, _limit, _sortOrder);
try {
let ret = returnedResult.readCString();
resolve(JSON.parse(ret));
@@ -345,13 +322,13 @@ class Search {
return;
}
if (!fs.existsSync(this.indexFolderName)) {
if (!fs.existsSync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`)) {
log.send(logLevels.ERROR, 'Index folder does not exist.');
reject(new Error('Index folder does not exist.'));
return;
}
libSymphonySearch.symSEGetLastMessageTimestampAsync(this.indexFolderName, (err, res) => {
libSymphonySearch.symSEGetLastMessageTimestampAsync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, (err, res) => {
if (err) {
log.send(logLevels.ERROR, 'Error getting the index timestamp ->' + err);
reject(new Error(err));
@@ -367,9 +344,10 @@ class Search {
});
}
/*eslint class-methods-use-this: ["error", { "exceptMethods": ["deleteRealTimeFolder"] }] */
deleteRealTimeFolder() {
Search.deleteIndexFolders(this.realTimeIndex);
Search.indexValidator(this.realTimeIndex);
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX);
Search.indexValidator(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX);
}
/**
@@ -587,6 +565,35 @@ function deleteIndexFolder() {
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH);
}
/**
* Reads the file from the msgjson
* this is only for the demo page
* @param batch
* @returns {Promise<Array>}
*/
function readFile(batch) {
return new Promise((resolve, reject) => {
let dirPath = path.join(searchConfig.FOLDERS_CONSTANTS.EXEC_PATH, isMac ? '..' : '', 'msgsjson', batch);
let messageFolderPath = isDevEnv ? path.join('./msgsjson', batch) : dirPath;
let files = fs.readdirSync(messageFolderPath);
this.messageData = [];
files.forEach((file) => {
let tempPath = path.join(messageFolderPath, file);
let data = fs.readFileSync(tempPath, "utf8");
if (data) {
try {
this.messageData.push(JSON.parse(data));
} catch (err) {
reject(new Error(err))
}
} else {
reject(new Error('Error reading batch'))
}
});
resolve(this.messageData);
});
}
/**
* Exporting the search library
* @type {{Search: Search}}

1
js/search/searchConfig.js
View File

@@ -57,6 +57,7 @@ const searchConfig = {
LIBRARY_CONSTANTS: libraryPaths,
FOLDERS_CONSTANTS: folderPaths,
TAR_LZ4_EXT: '.tar.lz4',
RANDOM_STRING: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
MINIMUM_DISK_SPACE: 300000000, // in bytes
PERMISSION_ERROR: "The FSUTIL utility requires that you have administrative privileges.",
WIN_PATH_ERROR: "Error: The system cannot find the path specified.",

15
js/search/utils/randomString.js Normal file
View File

@@ -0,0 +1,15 @@
const searchConfig = require('../searchConfig.js');
function randomString() {
let text = "";
for (let i = 0; i < 7; i++) {
text += searchConfig.RANDOM_STRING.charAt(Math.floor(Math.random() * searchConfig.RANDOM_STRING.length));
}
let time = new Date().getTime();
return text + time;
}
module.exports = {
randomString: randomString
};