if/SymphonyElectron
1
0
Fork 0
mirror of https://github.com/finos/SymphonyElectron.git synced 2025-02-25 18:55:29 -06:00
Code Issues Packages Projects Releases Wiki Activity

Swift Search to Master (#405)

Browse Source 
* SEARCH-627 & SEARCH-628

- Add implementation for memory indexing
- Batch indexing and Real-time indexing

* ELECTRON-426 (#345)

- Changes innerHTML to innerText
- Removes 'replaceStrongTag' and 'replaceHTMLTags' function and changing to innerHTML to innerText

* SEARCH-764

- Rework "deleteIndexFolders" in the Electron Preload Script
- Fixes the Security Vulnerability

* SEARCH-764

- Updates spectron test preload api version

* SEARCH-766

- Rework "readFile" in Electron Preload Script
- Removing this method as this is only for the demo app

* SEARCH-766

- Removes merge method

* SEARCH-767

- Rework "path" in Electron Preload Script
- Removes the constructor to use hardcoded path

* SEARCH-767

- Updates unit test

* SEARCH-775

- Adds search api version to match with the client app

* SEARCH-775

- Spectron test fix

* SEARCH-770 (#391)

- constructor is required which was removed as part of SEARCH-767
This commit is contained in:
Keerthi Niranjan 2018-06-27 14:12:16 +05:30 committed by Vishwas Shashidhar
parent 65c304766f
commit 4aaa9ac539
10 changed files with 210 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
demo/index.html
View File

@ -92,12 +92,14 @@
<th>Container Identifier</th> <th>Container Identifier</th>
<th>Container Version</th> <th>Container Version</th>
<th>Build Number</th> <th>Build Number</th>
<th>Search Api Version</th>
</tr> </tr>
<tr> <tr>
<td id="api-version"></td> <td id="api-version"></td>
<td id="container-identifier"></td> <td id="container-identifier"></td>
<td id="container-ver"></td> <td id="container-ver"></td>
<td id="build-number"></td> <td id="build-number"></td>
<td id="search-api-ver"></td>
</tr> </tr>
</table> </table>
</body> </body>
@ -250,11 +252,13 @@
let containerIdentifier = document.getElementById('container-identifier'); let containerIdentifier = document.getElementById('container-identifier');
let version = document.getElementById('container-ver'); let version = document.getElementById('container-ver');
let buildNumber = document.getElementById('build-number'); let buildNumber = document.getElementById('build-number');
let searchApiVer = document.getElementById('search-api-ver');
apiVersionInfo.innerText = verInfo.apiVer; apiVersionInfo.innerText = verInfo.apiVer;
containerIdentifier.innerText = verInfo.containerIdentifier; containerIdentifier.innerText = verInfo.containerIdentifier;
version.innerText = verInfo.containerVer; version.innerText = verInfo.containerVer;
buildNumber.innerText = verInfo.buildNumber; buildNumber.innerText = verInfo.buildNumber;
searchApiVer.innerText = verInfo.searchApiVer;
}); });
}); });

37
demo/search.html
View File

@ -79,13 +79,7 @@
</div> </div>
<br> <br>
<div> <div>
<label for="batchNumber">Batch Number: </label><input placeholder="Ex: batch1, batch2" id="batchNumber"> <label for="getLatestMessageTimestamp">Get Latest Message Timestamp</label>
<button id='index'>Index Messages</button>
<button id='merge'>Merge</button>
</div>
<br>
<div>
<label for="batchNumber">Get Latest Message Timestamp</label>
<button id='getLatestMessageTimestamp'>Click</button> <button id='getLatestMessageTimestamp'>Click</button>
</div> </div>
<br> <br>
@ -125,8 +119,6 @@
var search = new ssf.Search("12345678910112", "jjjehdnctsjyieoalskcjdhsnahsadndfnusdfsdfsd="); var search = new ssf.Search("12345678910112", "jjjehdnctsjyieoalskcjdhsnahsadndfnusdfsdfsd=");
var searchUtils = new ssf.SearchUtils(); var searchUtils = new ssf.SearchUtils();
var buttonEl = document.getElementById('search'); var buttonEl = document.getElementById('search');
var merge = document.getElementById('merge');
var buttonIndex = document.getElementById('index');
var queryEl = document.getElementById('query'); var queryEl = document.getElementById('query');
var offsetEl = document.getElementById('offset'); var offsetEl = document.getElementById('offset');
var limitEl = document.getElementById('limit'); var limitEl = document.getElementById('limit');
@ -139,7 +131,6 @@
var table = document.getElementById('table'); var table = document.getElementById('table');
var sendMessage = document.getElementById('sendMessage'); var sendMessage = document.getElementById('sendMessage');
var realTimeIndexing = document.getElementById('realTimeIndexing'); var realTimeIndexing = document.getElementById('realTimeIndexing');
var batchNumber = document.getElementById('batchNumber');
var timestamp = document.getElementById('getLatestMessageTimestamp'); var timestamp = document.getElementById('getLatestMessageTimestamp');
var has = document.getElementById('has'); var has = document.getElementById('has');
var checkFreeSpace = document.getElementById('checkFreeSpace'); var checkFreeSpace = document.getElementById('checkFreeSpace');
@ -148,18 +139,6 @@
var rotationId = document.getElementById('rotationId'); var rotationId = document.getElementById('rotationId');
var updateUserConfig = document.getElementById('updateUserConfig'); var updateUserConfig = document.getElementById('updateUserConfig');
buttonIndex.addEventListener('click', function () {
let batchIndex = batchNumber.value;
search.readJson(batchIndex).then(function (res) {
search.indexBatch(JSON.stringify(res)).then(function () {
resultsEl.innerHTML = "Index created";
});
}).catch(function (err) {
console.log(err);
});
});
buttonEl.addEventListener('click', function () { buttonEl.addEventListener('click', function () {
if (!search.isLibInit()) { if (!search.isLibInit()) {
search.init(); search.init();
@ -238,20 +217,6 @@
} }
}); });
merge.addEventListener('click', function () {
search.mergeIndexBatches().then(function () {
search.encryptIndex('jjjehdnctsjyieoalskcjdhsnahsadndfnusdfsdfsd=').then(function () {
searchUtils.updateUserConfig(12345678910112, {rotationId:0, version: 1}).then(function (res) {
resultsEl.innerHTML = JSON.stringify(res);
}).catch(function (err) {
resultsEl.innerHTML = JSON.stringify(err);
});
});
}).catch(function (err) {
resultsEl.innerHTML = 'Error: ' + err;
});
});
timestamp.addEventListener('click', function () { timestamp.addEventListener('click', function () {
search.getLatestMessageTimestamp().then(function (res) { search.getLatestMessageTimestamp().then(function (res) {
resultsEl.innerHTML = res; resultsEl.innerHTML = res;

3
js/preload/preloadMain.js
View File

@ -110,7 +110,8 @@ function createAPI() {
containerIdentifier: appName, containerIdentifier: appName,
containerVer: appVer, containerVer: appVer,
buildNumber: buildNumber, buildNumber: buildNumber,
apiVer: '1.0.0' apiVer: '2.0.0',
searchApiVer: '2.0.0'
}; };
resolve(verInfo); resolve(verInfo);
}); });

138
js/search/search.js
View File

@ -1,7 +1,7 @@
'use strict'; 'use strict';
const fs = require('fs'); const fs = require('fs');
const { randomString } = require('../search/utils/randomString.js'); const ref = require('ref');
const childProcess = require('child_process'); const childProcess = require('child_process');
const path = require('path'); const path = require('path');
const isDevEnv = require('../utils/misc.js').isDevEnv; const isDevEnv = require('../utils/misc.js').isDevEnv;
@ -17,6 +17,7 @@ const Crypto = require('../cryptoLib');
const INDEX_VALIDATOR = searchConfig.LIBRARY_CONSTANTS.INDEX_VALIDATOR; const INDEX_VALIDATOR = searchConfig.LIBRARY_CONSTANTS.INDEX_VALIDATOR;
/*eslint class-methods-use-this: ["error", { "exceptMethods": ["deleteRealTimeFolder"] }] */
/** /**
* This search class communicates with the SymphonySearchEngine C library via node-ffi. * This search class communicates with the SymphonySearchEngine C library via node-ffi.
* There should be only 1 instance of this class in the Electron * There should be only 1 instance of this class in the Electron
@ -67,16 +68,18 @@ class Search {
* and creates a folder in the userData * and creates a folder in the userData
*/ */
init() { init() {
libSymphonySearch.symSEDestroy();
libSymphonySearch.symSEInit(); libSymphonySearch.symSEInit();
libSymphonySearch.symSEClearMainRAMIndex();
libSymphonySearch.symSEClearRealtimeRAMIndex();
libSymphonySearch.symSEEnsureFolderExists(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH); libSymphonySearch.symSEEnsureFolderExists(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH);
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX);
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER);
Search.indexValidator(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`); Search.indexValidator(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`);
Search.indexValidator(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX);
let indexDateStartFrom = new Date().getTime() - searchConfig.SEARCH_PERIOD_SUBTRACTOR; let indexDateStartFrom = new Date().getTime() - searchConfig.SEARCH_PERIOD_SUBTRACTOR;
libSymphonySearch.symSEMainFSIndexToRAMIndex(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`);
// Deleting all the messages except 3 Months from now // Deleting all the messages except 3 Months from now
libSymphonySearch.symSEDeleteMessages(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, null, libSymphonySearch.symSEDeleteMessagesFromRAMIndex(null,
searchConfig.MINIMUM_DATE, indexDateStartFrom.toString()); searchConfig.MINIMUM_DATE, indexDateStartFrom.toString());
Search.deleteIndexFolders();
this.isInitialized = true; this.isInitialized = true;
} }
@ -113,17 +116,10 @@ class Search {
return; return;
} }
if (!fs.existsSync(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH)) { libSymphonySearch.symSEIndexMainRAMAsync(messages, function (err, res) {
log.send(logLevels.ERROR, 'User index folder not found');
reject(new Error('User index folder not found'));
return;
}
const indexId = randomString();
libSymphonySearch.symSECreatePartialIndexAsync(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER, indexId, messages, (err, res) => {
if (err) { if (err) {
log.send(logLevels.ERROR, 'Batch Indexing: error ->' + err); log.send(logLevels.ERROR, `IndexBatch: Error indexing messages to memory : ${err}`);
reject(new Error(err)); reject(new Error('IndexBatch: Error indexing messages to memory '));
return; return;
} }
resolve(res); resolve(res);
@ -135,22 +131,24 @@ class Search {
* Merging the temporary * Merging the temporary
* created from indexBatch() * created from indexBatch()
*/ */
mergeIndexBatches() { memoryIndexToFSIndex() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
Search.deleteIndexFolders();
libSymphonySearch.symSEEnsureFolderExists(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH);
if (!fs.existsSync(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH)) { if (!fs.existsSync(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH)) {
log.send(logLevels.ERROR, 'User index folder not found'); log.send(logLevels.ERROR, 'User index folder not found');
reject(new Error('User index folder not found')); reject(new Error('User index folder not found'));
return; return;
} }
libSymphonySearch.symSEMergePartialIndexAsync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER, (err, res) => { libSymphonySearch.symSEMainRAMIndexToFSIndexAsync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, (err, res) => {
if (err) { if (err) {
log.send(logLevels.ERROR, 'Error merging the index ->' + err); log.send(logLevels.ERROR, 'Error merging the index ->' + err);
reject(new Error(err)); reject(new Error(err));
return; return;
} }
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_BATCH_INDEX_FOLDER);
resolve(res); resolve(res);
}); });
}); });
@ -197,13 +195,8 @@ class Search {
throw new Error('Library not initialized'); throw new Error('Library not initialized');
} }
if (!fs.existsSync(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH)) {
log.send(logLevels.ERROR, 'User index folder not found');
throw new Error('User index folder not found');
}
this.isRealTimeIndexing = true; this.isRealTimeIndexing = true;
return libSymphonySearch.symSEIndexRealTimeAsync(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX, message, (err, result) => { return libSymphonySearch.symSEIndexRealtimeRAMAsync(message, (err, result) => {
this.isRealTimeIndexing = false; this.isRealTimeIndexing = false;
if (err) { if (err) {
log.send(logLevels.ERROR, 'RealTime Indexing: error -> ' + err); log.send(logLevels.ERROR, 'RealTime Indexing: error -> ' + err);
@ -213,16 +206,6 @@ class Search {
}); });
} }
/**
* Reading a json file
* for the demo search app only
* @param {String} batch
* @returns {Promise}
*/
readJson(batch) {
return readFile.call(this, batch);
}
/** /**
* Encrypting the index after the merging the index * Encrypting the index after the merging the index
* to the main user index * to the main user index
@ -259,12 +242,6 @@ class Search {
return; return;
} }
if (!fs.existsSync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`) || !fs.existsSync(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX)) {
log.send(logLevels.ERROR, 'Index folder does not exist.');
reject(new Error('Index folder does not exist.'));
return;
}
let q = Search.constructQuery(query, senderIds, threadIds, fileType); let q = Search.constructQuery(query, senderIds, threadIds, fileType);
if (q === undefined) { if (q === undefined) {
@ -301,9 +278,9 @@ class Search {
_sortOrder = searchConfig.SORT_BY_SCORE; _sortOrder = searchConfig.SORT_BY_SCORE;
} }
const returnedResult = libSymphonySearch.symSESearch(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX, q, startDateTime.toString(), endDateTime.toString(), _offset, _limit, _sortOrder); const returnedResult = libSymphonySearch.symSERAMIndexSearch(q, startDateTime.toString(), endDateTime.toString(), _offset, _limit, _sortOrder);
try { try {
let ret = returnedResult.readCString(); let ret = ref.readCString(returnedResult);
resolve(JSON.parse(ret)); resolve(JSON.parse(ret));
} finally { } finally {
libSymphonySearch.symSEFreeResult(returnedResult); libSymphonySearch.symSEFreeResult(returnedResult);
@ -324,20 +301,14 @@ class Search {
return; return;
} }
if (!fs.existsSync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`)) { libSymphonySearch.symSEMainRAMIndexGetLastMessageTimestampAsync((err, res) => {
log.send(logLevels.ERROR, 'Index folder does not exist.');
reject(new Error('Index folder does not exist.'));
return;
}
libSymphonySearch.symSEGetLastMessageTimestampAsync(`${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME_PATH}_${this.userId}`, (err, res) => {
if (err) { if (err) {
log.send(logLevels.ERROR, 'Error getting the index timestamp ->' + err); log.send(logLevels.ERROR, 'Error getting the index timestamp ->' + err);
reject(new Error(err)); reject(new Error(err));
} }
const returnedResult = res; const returnedResult = res;
try { try {
let ret = returnedResult.readCString(); let ret = ref.readCString(returnedResult);
resolve(ret); resolve(ret);
} finally { } finally {
libSymphonySearch.symSEFreeResult(returnedResult); libSymphonySearch.symSEFreeResult(returnedResult);
@ -346,10 +317,8 @@ class Search {
}); });
} }
/*eslint class-methods-use-this: ["error", { "exceptMethods": ["deleteRealTimeFolder"] }] */
deleteRealTimeFolder() { deleteRealTimeFolder() {
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX); libSymphonySearch.symSEClearRealtimeRAMIndex();
Search.indexValidator(searchConfig.FOLDERS_CONSTANTS.TEMP_REAL_TIME_INDEX);
} }
/** /**
@ -541,20 +510,22 @@ class Search {
/** /**
* Removing all the folders and files inside the data folder * Removing all the folders and files inside the data folder
* @param location
*/ */
static deleteIndexFolders(location) { static deleteIndexFolders() {
if (fs.existsSync(location)) { function removeFiles(filePath) {
fs.readdirSync(location).forEach((file) => { if (fs.existsSync(filePath)) {
let curPath = location + "/" + file; fs.readdirSync(filePath).forEach((file) => {
if (fs.lstatSync(curPath).isDirectory()) { let curPath = filePath + "/" + file;
Search.deleteIndexFolders(curPath); if (fs.lstatSync(curPath).isDirectory()) {
} else { removeFiles(curPath);
fs.unlinkSync(curPath); } else {
} fs.unlinkSync(curPath);
}); }
fs.rmdirSync(location); });
fs.rmdirSync(filePath);
}
} }
removeFiles(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH);
} }
} }
@ -562,39 +533,16 @@ class Search {
/** /**
* Deleting the data index folder * Deleting the data index folder
* when the app is closed/signed-out/navigates * when the app is closed/signed-out/navigates
* isEncryption if that is true
* will not clear the memory index
*/ */
function deleteIndexFolder() { function deleteIndexFolder(isEncryption) {
Search.deleteIndexFolders(searchConfig.FOLDERS_CONSTANTS.INDEX_PATH); if (!isEncryption) {
libSymphonySearch.symSEDestroy();
}
Search.deleteIndexFolders();
} }
/**
* Reads the file from the msgjson
* this is only for the demo page
* @param batch
* @returns {Promise<Array>}
*/
function readFile(batch) {
return new Promise((resolve, reject) => {
let dirPath = path.join(searchConfig.FOLDERS_CONSTANTS.EXEC_PATH, isMac ? '..' : '', 'msgsjson', batch);
let messageFolderPath = isDevEnv ? path.join('./msgsjson', batch) : dirPath;
let files = fs.readdirSync(messageFolderPath);
this.messageData = [];
files.forEach((file) => {
let tempPath = path.join(messageFolderPath, file);
let data = fs.readFileSync(tempPath, "utf8");
if (data) {
try {
this.messageData.push(JSON.parse(data));
} catch (err) {
reject(new Error(err))
}
} else {
reject(new Error('Error reading batch'))
}
});
resolve(this.messageData);
});
}
/** /**
* Creating launch agent for handling the deletion of * Creating launch agent for handling the deletion of

1
js/search/searchConfig.js
View File

@ -53,7 +53,6 @@ const libraryPaths = {
const folderPaths = { const folderPaths = {
INDEX_PATH: indexFolderPath, INDEX_PATH: indexFolderPath,
TEMP_BATCH_INDEX_FOLDER: indexFolderPath + '/temp_batch_indexes', TEMP_BATCH_INDEX_FOLDER: indexFolderPath + '/temp_batch_indexes',
TEMP_REAL_TIME_INDEX: indexFolderPath + '/temp_realtime_index',
PREFIX_NAME: 'search_index', PREFIX_NAME: 'search_index',
PREFIX_NAME_PATH: indexFolderPath + '/search_index', PREFIX_NAME_PATH: indexFolderPath + '/search_index',
EXEC_PATH: execPath, EXEC_PATH: execPath,

44
js/search/searchLibrary.js
View File

@ -13,6 +13,22 @@ const symLucyIndexerPtr = ref.refType(symLucyIndexer);
* using the node-ffi * using the node-ffi
*/ */
let libSymphonySearch = ffi.Library(searchConfig.LIBRARY_CONSTANTS.SEARCH_LIBRARY_PATH, { let libSymphonySearch = ffi.Library(searchConfig.LIBRARY_CONSTANTS.SEARCH_LIBRARY_PATH, {
//New Memory Indexing API
'symSE_index_main_RAM': ['int', [ 'string' ] ],
'symSE_index_realtime_RAM': ['int', [ 'string' ] ],
'symSE_main_RAM_index_to_FS_index': ['int', [ 'string' ] ],
'symSE_realtime_RAM_index_to_FS_index': ['int', [ 'string' ] ],
'symSE_main_RAM_index_get_last_message_timestamp': ['char *', [] ],
'symSE_RAM_index_search': ['char *', [ 'string', 'string', 'string', 'int', 'int', 'int' ] ],
'symSE_main_FS_index_to_RAM_index': ['int', [ 'string' ] ],
'symSE_realtime_FS_index_to_RAM_index': ['int', [ 'string' ] ],
'symSE_clear_realtime_RAM_index': ['int', [] ],
'symSE_clear_main_RAM_index': ['int', [] ],
'symSE_delete_messages_from_RAM_index': ['int', [ 'string', 'string', 'string' ] ],
'symSE_destroy': ['int', [] ],
//init //init
'symSE_init': ['void', []], 'symSE_init': ['void', []],
'symSE_remove_folder': ['int', ['string']], 'symSE_remove_folder': ['int', ['string']],
@ -40,6 +56,32 @@ let libSymphonySearch = ffi.Library(searchConfig.LIBRARY_CONSTANTS.SEARCH_LIBRAR
}); });
module.exports = { module.exports = {
// New Memory Indexing API
symSEIndexMainRAM: libSymphonySearch.symSE_index_main_RAM,
symSEIndexRealtimeRAM: libSymphonySearch.symSE_index_realtime_RAM,
symSEMainRAMIndexToFSIndex: libSymphonySearch.symSE_main_RAM_index_to_FS_index,
symSERealtimeRAMIndexToFSIndex: libSymphonySearch.symSE_realtime_RAM_index_to_FS_index,
symSEMainRAMIndexGetLastMessageTimestamp: libSymphonySearch.symSE_main_RAM_index_get_last_message_timestamp,
symSERAMIndexSearch: libSymphonySearch.symSE_RAM_index_search,
symSEMainFSIndexToRAMIndex: libSymphonySearch.symSE_main_FS_index_to_RAM_index,
symSERealtimeFSIndexToRAMIndex: libSymphonySearch.symSE_realtime_FS_index_to_RAM_index,
symSEClearRealtimeRAMIndex: libSymphonySearch.symSE_clear_realtime_RAM_index,
symSEClearMainRAMIndex: libSymphonySearch.symSE_clear_main_RAM_index,
symSEDeleteMessagesFromRAMIndex: libSymphonySearch.symSE_delete_messages_from_RAM_index,
symSEDestroy: libSymphonySearch.symSE_destroy,
symSEIndexMainRAMAsync: libSymphonySearch.symSE_index_main_RAM.async,
symSEIndexRealtimeRAMAsync: libSymphonySearch.symSE_index_realtime_RAM.async,
symSEMainRAMIndexToFSIndexAsync: libSymphonySearch.symSE_main_RAM_index_to_FS_index.async,
symSERealtimeRAMIndexToFSIndexAsync: libSymphonySearch.symSE_realtime_RAM_index_to_FS_index.async,
symSEMainRAMIndexGetLastMessageTimestampAsync: libSymphonySearch.symSE_main_RAM_index_get_last_message_timestamp.async,
symSERAMIndexSearchAsync: libSymphonySearch.symSE_RAM_index_search.async,
symSEMainFSIndexToRAMIndexAsync: libSymphonySearch.symSE_main_FS_index_to_RAM_index.async,
symSERealtimeFSIndexToRAMIndexAsync: libSymphonySearch.symSE_realtime_FS_index_to_RAM_index.async,
symSEClearRealtimeRAMIndexAsync: libSymphonySearch.symSE_clear_realtime_RAM_index.async,
symSEClearMainRAMIndexAsync: libSymphonySearch.symSE_clear_main_RAM_index.async,
symSEDeleteMessagesFromRAMIndexAsync: libSymphonySearch.symSE_delete_messages_from_RAM_index.async,
symSEDestroyAsync: libSymphonySearch.symSE_destroy.async,
symSEInit: libSymphonySearch.symSE_init, symSEInit: libSymphonySearch.symSE_init,
symSERemoveFolder: libSymphonySearch.symSE_remove_folder, symSERemoveFolder: libSymphonySearch.symSE_remove_folder,
symSEEnsureIndexExists: libSymphonySearch.symSE_ensure_index_exists, symSEEnsureIndexExists: libSymphonySearch.symSE_ensure_index_exists,
@ -69,5 +111,5 @@ module.exports = {
symSEDeleteMessagesAsync: libSymphonySearch.symSE_delete_messages.async, symSEDeleteMessagesAsync: libSymphonySearch.symSE_delete_messages.async,
symSECommitIndexAsync: libSymphonySearch.symSE_commit_index.async, symSECommitIndexAsync: libSymphonySearch.symSE_commit_index.async,
symSEFreeResultAsync: libSymphonySearch.symSE_free_results.async, symSEFreeResultAsync: libSymphonySearch.symSE_free_results.async,
symSEGetLastMessageTimestampAsync: libSymphonySearch.symSE_get_last_message_timestamp.async symSEGetLastMessageTimestampAsync: libSymphonySearch.symSE_get_last_message_timestamp.async,
}; };

13
js/search/searchUtils.js
View File

@ -7,12 +7,12 @@ const { isMac } = require('../utils/misc.js');
* Utils to validate users config data and * Utils to validate users config data and
* available disk space to enable electron search * available disk space to enable electron search
*/ */
/*eslint class-methods-use-this: ["error", { "exceptMethods": ["checkFreeSpace"] }] */
class SearchUtils { class SearchUtils {
constructor() { constructor() {
this.path = searchConfig.FOLDERS_CONSTANTS.USER_DATA_PATH; this.indexVersion = searchConfig.INDEX_VERSION;
} }
/** /**
* This function returns true if the available disk space * This function returns true if the available disk space
* is more than the constant MINIMUM_DISK_SPACE * is more than the constant MINIMUM_DISK_SPACE
@ -20,14 +20,15 @@ class SearchUtils {
*/ */
checkFreeSpace() { checkFreeSpace() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let userDataPath = searchConfig.FOLDERS_CONSTANTS.USER_DATA_PATH;
if (!isMac) { if (!isMac) {
try { try {
this.path = this.path.substring(0, 1); userDataPath = userDataPath.substring(0, 1);
} catch (e) { } catch (e) {
reject(new Error('Invalid Path : ' + e)); reject(new Error('Invalid Path : ' + e));
} }
} }
checkDiskSpace(this.path, resolve, reject); checkDiskSpace(userDataPath, resolve, reject);
}); });
} }
@ -120,7 +121,7 @@ function createUserConfigFile(userId, data) {
let createStream = fs.createWriteStream(searchConfig.FOLDERS_CONSTANTS.USER_CONFIG_FILE); let createStream = fs.createWriteStream(searchConfig.FOLDERS_CONSTANTS.USER_CONFIG_FILE);
if (userData) { if (userData) {
if (!userData.indexVersion) { if (!userData.indexVersion) {
userData.indexVersion = searchConfig.INDEX_VERSION; userData.indexVersion = this.indexVersion;
} }
try { try {
userData = JSON.stringify(userData); userData = JSON.stringify(userData);
@ -146,7 +147,7 @@ function updateConfig(userId, data, resolve, reject) {
let userData = data; let userData = data;
if (userData && !userData.indexVersion) { if (userData && !userData.indexVersion) {
userData.indexVersion = searchConfig.INDEX_VERSION; userData.indexVersion = this.indexVersion;
} }
let configPath = searchConfig.FOLDERS_CONSTANTS.USER_CONFIG_FILE; let configPath = searchConfig.FOLDERS_CONSTANTS.USER_CONFIG_FILE;

161
tests/Search.test.js
View File

@ -7,6 +7,7 @@ let userConfigDir = null;
let searchConfig; let searchConfig;
let SearchApi; let SearchApi;
let libSymphonySearch;
jest.mock('electron', function() { jest.mock('electron', function() {
return { return {
@ -37,9 +38,6 @@ describe('Tests for Search', function() {
let userId; let userId;
let key; let key;
let dataFolderPath; let dataFolderPath;
let realTimeIndexPath;
let tempBatchPath;
let launchAgent;
let currentDate = new Date().getTime(); let currentDate = new Date().getTime();
jasmine.DEFAULT_TIMEOUT_INTERVAL = 60000; jasmine.DEFAULT_TIMEOUT_INTERVAL = 60000;
@ -48,28 +46,32 @@ describe('Tests for Search', function() {
userId = 12345678910112; userId = 12345678910112;
key = 'jjjehdnctsjyieoalskcjdhsnahsadndfnusdfsdfsd='; key = 'jjjehdnctsjyieoalskcjdhsnahsadndfnusdfsdfsd=';
executionPath = path.join(__dirname, 'library'); executionPath = path.join(__dirname, 'library');
if (isWindowsOS) { if (isWindowsOS) {
executionPath = path.join(__dirname, '..', 'library'); executionPath = path.join(__dirname, '..', 'library');
} }
userConfigDir = path.join(__dirname, '..'); userConfigDir = path.join(__dirname, '..');
libSymphonySearch = require('../js/search/searchLibrary.js');
searchConfig = require('../js/search/searchConfig.js');
let root = path.join(userConfigDir, `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${userId}.enc`);
if (fs.existsSync(root)) {
fs.unlinkSync(root);
}
const { Search } = require('../js/search/search.js');
SearchApi = new Search(userId, key);
searchConfig = require('../js/search/searchConfig.js'); libSymphonySearch.symSEDestroy();
const { Search } = require('../js/search/search.js'); dataFolderPath = path.join(userConfigDir, 'data');
SearchApi = new Search(userId, key); if (fs.existsSync(dataFolderPath)) {
launchAgent = require('../js/search/utils/search-launchd.js'); deleteIndexFolders(dataFolderPath)
realTimeIndexPath = path.join(userConfigDir, 'data', 'temp_realtime_index'); }
tempBatchPath = path.join(userConfigDir, 'data', 'temp_batch_indexes'); done();
dataFolderPath = path.join(userConfigDir, 'data');
if (fs.existsSync(dataFolderPath)) {
deleteIndexFolders(dataFolderPath);
}
done();
}); });
afterAll(function (done) { afterAll(function (done) {
setTimeout(function () { setTimeout(function () {
libSymphonySearch.symSEDestroy();
deleteIndexFolders(dataFolderPath); deleteIndexFolders(dataFolderPath);
let root = path.join(userConfigDir, `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${userId}.enc`); let root = path.join(userConfigDir, `${searchConfig.FOLDERS_CONSTANTS.PREFIX_NAME}_${userId}.enc`);
if (fs.existsSync(root)) { if (fs.existsSync(root)) {
@ -83,12 +85,15 @@ describe('Tests for Search', function() {
}, 3000); }, 3000);
}); });
function deleteIndexFolders(location) { function deleteIndexFolders(location, isEncryption) {
if (!isEncryption) {
libSymphonySearch.symSEDestroy();
}
if (fs.existsSync(location)) { if (fs.existsSync(location)) {
fs.readdirSync(location).forEach(function(file) { fs.readdirSync(location).forEach(function(file) {
let curPath = path.join(location, file); let curPath = path.join(location, file);
if (fs.lstatSync(curPath).isDirectory()) { if (fs.lstatSync(curPath).isDirectory()) {
deleteIndexFolders(curPath); deleteIndexFolders(curPath, true);
} else { } else {
fs.unlinkSync(curPath); fs.unlinkSync(curPath);
} }
@ -123,13 +128,9 @@ describe('Tests for Search', function() {
}); });
it('should exist index folder', function() { it('should exist index folder', function() {
expect(fs.existsSync(path.join(userConfigDir, 'data', 'search_index_12345678910112'))).toBe(true); expect(fs.existsSync(path.join(userConfigDir, 'data', 'search_index_12345678910112'))).toBe(false);
expect(fs.existsSync(realTimeIndexPath)).toBe(true);
}); });
it('should not exist index folder', function() {
expect(fs.existsSync(tempBatchPath)).toBe(false);
});
}); });
describe('Batch indexing process tests', function () { describe('Batch indexing process tests', function () {
@ -165,7 +166,6 @@ describe('Tests for Search', function() {
}]; }];
const indexBatch = jest.spyOn(SearchApi, 'indexBatch'); const indexBatch = jest.spyOn(SearchApi, 'indexBatch');
SearchApi.indexBatch(JSON.stringify(messages)).then(function () { SearchApi.indexBatch(JSON.stringify(messages)).then(function () {
expect(fs.existsSync(tempBatchPath)).toBe(true);
expect(indexBatch).toHaveBeenCalledWith(JSON.stringify(messages)); expect(indexBatch).toHaveBeenCalledWith(JSON.stringify(messages));
done(); done();
}); });
@ -232,28 +232,22 @@ describe('Tests for Search', function() {
it('should match messages length after batch indexing', function (done) { it('should match messages length after batch indexing', function (done) {
const searchQuery = jest.spyOn(SearchApi, 'searchQuery'); const searchQuery = jest.spyOn(SearchApi, 'searchQuery');
SearchApi.searchQuery('it works', [], [], '', undefined, undefined, 25, 0, 0).then(function (res) { SearchApi.searchQuery('it works', [], [], '', undefined, undefined, 25, 0, 0).then(function (res) {
expect(res.messages.length).toEqual(0); expect(res.messages.length).toEqual(3);
expect(searchQuery).toHaveBeenCalled(); expect(searchQuery).toHaveBeenCalled();
done() done()
}); });
}); });
it('should merge batch index to user index', function (done) { it('should merge batch index to user index', function (done) {
const mergeIndexBatches = jest.spyOn(SearchApi, 'mergeIndexBatches'); const memoryIndexToFSIndex = jest.spyOn(SearchApi, 'memoryIndexToFSIndex');
SearchApi.mergeIndexBatches().then(function () { SearchApi.memoryIndexToFSIndex().then(function () {
expect(fs.existsSync(tempBatchPath)).toBe(false); expect(memoryIndexToFSIndex).toHaveBeenCalled();
expect(mergeIndexBatches).toHaveBeenCalled();
done(); done();
}); });
}); });
it('should match messages length after batch indexing', function (done) { it('should exist data folder after ram index to fs index', function () {
const searchQuery = jest.spyOn(SearchApi, 'searchQuery'); expect(fs.existsSync(dataFolderPath)).toBe(true);
SearchApi.searchQuery('it works', [], [], '', undefined, undefined, 25, 0, 0).then(function (res) {
expect(res.messages.length).toEqual(3);
expect(searchQuery).toHaveBeenCalled();
done();
});
}); });
}); });
@ -280,7 +274,6 @@ describe('Tests for Search', function() {
const searchQuery = jest.spyOn(SearchApi, 'searchQuery'); const searchQuery = jest.spyOn(SearchApi, 'searchQuery');
SearchApi.searchQuery('realtime working', ["71811853189212"], ["Au8O2xKHyX1LtE6zW019GX///rZYegAtdA=="], '', undefined, undefined, 25, 0, 0).then(function (res) { SearchApi.searchQuery('realtime working', ["71811853189212"], ["Au8O2xKHyX1LtE6zW019GX///rZYegAtdA=="], '', undefined, undefined, 25, 0, 0).then(function (res) {
expect(res.messages.length).toEqual(3); expect(res.messages.length).toEqual(3);
expect(fs.existsSync(realTimeIndexPath)).toBe(true);
expect(searchQuery).toHaveBeenCalled(); expect(searchQuery).toHaveBeenCalled();
done(); done();
}) })
@ -309,8 +302,6 @@ describe('Tests for Search', function() {
SearchApi.searchQuery('isRealTimeIndexing', [], [], '', undefined, undefined, 25, 0, 0).then(function (res) { SearchApi.searchQuery('isRealTimeIndexing', [], [], '', undefined, undefined, 25, 0, 0).then(function (res) {
expect(res.messages.length).toEqual(0); expect(res.messages.length).toEqual(0);
expect(fs.existsSync(realTimeIndexPath)).toBe(true);
done(); done();
}); });
}, 6000) }, 6000)
@ -378,7 +369,6 @@ describe('Tests for Search', function() {
it('should delete realtime index', function () { it('should delete realtime index', function () {
const deleteRealTimeFolder = jest.spyOn(SearchApi, 'deleteRealTimeFolder'); const deleteRealTimeFolder = jest.spyOn(SearchApi, 'deleteRealTimeFolder');
SearchApi.deleteRealTimeFolder(); SearchApi.deleteRealTimeFolder();
expect(fs.existsSync(realTimeIndexPath)).toBe(true);
expect(deleteRealTimeFolder).toHaveBeenCalled(); expect(deleteRealTimeFolder).toHaveBeenCalled();
}); });
}); });
@ -425,13 +415,14 @@ describe('Tests for Search', function() {
}); });
}); });
it('should not get the latest timestamp', function (done) { it('should be equal to 0000000000000', function (done) {
const getLatestMessageTimestamp = jest.spyOn(SearchApi, 'getLatestMessageTimestamp'); const getLatestMessageTimestamp = jest.spyOn(SearchApi, 'getLatestMessageTimestamp');
deleteIndexFolders(dataFolderPath); libSymphonySearch.symSEClearMainRAMIndex();
SearchApi.getLatestMessageTimestamp().catch(function (err) { libSymphonySearch.symSEClearRealtimeRAMIndex();
expect(err).toEqual(new Error('Index folder does not exist.')); SearchApi.getLatestMessageTimestamp().then(function (res) {
expect(getLatestMessageTimestamp).toHaveBeenCalled(); expect(getLatestMessageTimestamp).toHaveBeenCalled();
expect(getLatestMessageTimestamp).toHaveBeenCalledTimes(3); expect(getLatestMessageTimestamp).toHaveBeenCalledTimes(3);
expect(res).toEqual('0000000000000');
done(); done();
}); });
}); });
@ -479,26 +470,25 @@ describe('Tests for Search', function() {
const searchQuery = jest.spyOn(SearchApi, 'searchQuery'); const searchQuery = jest.spyOn(SearchApi, 'searchQuery');
SearchApi.searchQuery('works', [], [], '', '', '', 2, 0, 0).then(function (res) { SearchApi.searchQuery('works', [], [], '', '', '', 2, 0, 0).then(function (res) {
expect(res.messages.length).toBe(2); expect(res.messages.length).toBe(2);
expect(searchQuery).toHaveBeenCalledTimes(7); expect(searchQuery).toHaveBeenCalledTimes(6);
expect(searchQuery).toHaveBeenCalled(); expect(searchQuery).toHaveBeenCalled();
done(); done();
}); });
}); });
it('should search fails index folder not fund', function (done) { it('should search fails result cleared', function (done) {
deleteIndexFolders(dataFolderPath); libSymphonySearch.symSEClearMainRAMIndex();
setTimeout(function () { libSymphonySearch.symSEClearRealtimeRAMIndex();
const searchQuery = jest.spyOn(SearchApi, 'searchQuery'); const searchQuery = jest.spyOn(SearchApi, 'searchQuery');
SearchApi.searchQuery('it works', [], [], '', '', '', 25, 0, 0).catch(function (err) { SearchApi.searchQuery('it works', [], [], '', '', '', 25, 0, 0).then(function (res) {
expect(err).toEqual(new Error('Index folder does not exist.')); expect(res.messages.length).toBe(0);
expect(searchQuery).toHaveBeenCalledTimes(8); expect(searchQuery).toHaveBeenCalledTimes(7);
expect(searchQuery).toHaveBeenCalled(); expect(searchQuery).toHaveBeenCalled();
SearchApi = undefined; SearchApi = undefined;
const { Search } = require('../js/search/search.js'); const { Search } = require('../js/search/search.js');
SearchApi = new Search(userId, key); SearchApi = new Search(userId, key);
done(); done();
}) });
}, 3000);
}); });
it('should search fails query is undefined', function (done) { it('should search fails query is undefined', function (done) {
@ -531,5 +521,52 @@ describe('Tests for Search', function() {
done(); done();
}); });
}); });
it('should index for testing quote', function (done) {
let messages = [{
messageId: "Jc+4K8RtPxHJfyuDQU9atX///qN3KHYXdA==",
threadId: "Au8O2xKHyX1LtE6zW019GX///rZYegAtdA==",
ingestionDate: currentDate.toString(),
senderId: "71811853189212",
chatType: "CHATROOM",
isPublic: "false",
sendingApp: "lc",
text: "quote search"
}, {
messageId: "Jc+4K8RtPxHJfyuDQU9atX///qN3KHYXdA==",
threadId: "Au8O2xKHyX1LtE6zW019GX///rZYegAtdA==",
ingestionDate: currentDate.toString(),
senderId: "71811853189212",
chatType: "CHATROOM",
isPublic: "false",
sendingApp: "lc",
text: "search"
}];
const indexBatch = jest.spyOn(SearchApi, 'indexBatch');
SearchApi.indexBatch(JSON.stringify(messages)).then(function () {
expect(indexBatch).toHaveBeenCalledWith(JSON.stringify(messages));
done();
});
});
it('should search without quote', function (done) {
const searchQuery = jest.spyOn(SearchApi, 'searchQuery');
SearchApi.searchQuery('search', [], [], undefined, '', '', 25, 0, 0).then(function (res) {
expect(res.messages.length).toEqual(2);
expect(searchQuery).toHaveBeenCalled();
expect(searchQuery).toHaveBeenCalledTimes(4);
done();
});
});
it('should quote search', function (done) {
const searchQuery = jest.spyOn(SearchApi, 'searchQuery');
SearchApi.searchQuery('\"quote search\"', [], [], undefined, '', '', 25, 0, 0).then(function (res) {
expect(res.messages.length).toEqual(1);
expect(searchQuery).toHaveBeenCalled();
expect(searchQuery).toHaveBeenCalledTimes(5);
done();
});
});
}); });
}); });

6
tests/SearchUtils.test.js
View File

@ -65,14 +65,14 @@ describe('Tests for Search Utils', function() {
it('should return error', function (done) { it('should return error', function (done) {
const checkFreeSpace = jest.spyOn(SearchUtilsAPI, 'checkFreeSpace'); const checkFreeSpace = jest.spyOn(SearchUtilsAPI, 'checkFreeSpace');
if (isMac) { if (isMac) {
SearchUtilsAPI.path = undefined; searchConfig.FOLDERS_CONSTANTS.USER_DATA_PATH = undefined;
SearchUtilsAPI.checkFreeSpace().catch(function (err) { SearchUtilsAPI.checkFreeSpace().catch(function (err) {
expect(err).toEqual(new Error("Please provide path")); expect(err).toEqual(new Error("Please provide path"));
expect(checkFreeSpace).toHaveBeenCalled(); expect(checkFreeSpace).toHaveBeenCalled();
done(); done();
}); });
} else { } else {
SearchUtilsAPI.path = undefined; searchConfig.FOLDERS_CONSTANTS.USER_DATA_PATH = undefined;
SearchUtilsAPI.checkFreeSpace().catch(function (err) { SearchUtilsAPI.checkFreeSpace().catch(function (err) {
expect(err).toBeTruthy(); expect(err).toBeTruthy();
expect(checkFreeSpace).toHaveBeenCalled(); expect(checkFreeSpace).toHaveBeenCalled();
@ -85,7 +85,7 @@ describe('Tests for Search Utils', function() {
const checkFreeSpace = jest.spyOn(SearchUtilsAPI, 'checkFreeSpace'); const checkFreeSpace = jest.spyOn(SearchUtilsAPI, 'checkFreeSpace');
SearchUtilsAPI.path = './tp'; SearchUtilsAPI.path = './tp';
if (isWindowsOS) { if (isWindowsOS) {
SearchUtilsAPI.path = 'A://test'; searchConfig.FOLDERS_CONSTANTS.USER_DATA_PATH = 'A://test';
searchConfig.LIBRARY_CONSTANTS.FREE_DISK_SPACE = path.join(__dirname, '..', searchConfig.LIBRARY_CONSTANTS.FREE_DISK_SPACE = path.join(__dirname, '..',
"node_modules/electron-utils/FreeDiskSpace/bin/Release/FreeDiskSpace.exe"); "node_modules/electron-utils/FreeDiskSpace/bin/Release/FreeDiskSpace.exe");
} }

8
tests/spectron/getVersionInfo.spectron.js
View File

@ -4,6 +4,8 @@ const { buildNumber } = require('../../package');
const electronVersion = require('../../package').devDependencies.electron; const electronVersion = require('../../package').devDependencies.electron;
const bluebird = require('bluebird'); const bluebird = require('bluebird');
const SEARCH_API_VERSION = '2.0.0';
let app = new Application({}); let app = new Application({});
describe('Tests for getVersionInfo API', () => { describe('Tests for getVersionInfo API', () => {
@ -56,14 +58,16 @@ describe('Tests for getVersionInfo API', () => {
'#api-version', '#api-version',
'#container-identifier', '#container-identifier',
'#container-ver', '#container-ver',
'#build-number' '#build-number',
'#search-api-ver'
]).mapSeries((string) => { ]).mapSeries((string) => {
return app.client.getText(string) return app.client.getText(string)
}).then((values) => { }).then((values) => {
expect(values[ 0 ]).toBe('1.0.0'); expect(values[ 0 ]).toBe('2.0.0');
expect(values[ 1 ]).toBe('Electron'); expect(values[ 1 ]).toBe('Electron');
expect(values[ 2 ]).toBe(electronVersion); expect(values[ 2 ]).toBe(electronVersion);
expect(values[ 3 ]).toBe(buildNumber); expect(values[ 3 ]).toBe(buildNumber);
expect(values[ 4 ]).toBe(SEARCH_API_VERSION);
done(); done();
}); });
}); });