SymphonyElectron/js/search/search.js

'use strict';

const fs = require('fs');
const randomString = require('randomstring');
const electron = require('electron');
const childProcess = require('child_process');
const app = electron.app;
const path = require('path');
const isDevEnv = require('../utils/misc.js').isDevEnv;
const isMac = require('../utils/misc.js').isMac;

// Search library
const libSymphonySearch = require('./searchLibrary');

// Crypto Library
const Crypto = require('../cryptoLib');

// Path for the exec file and the user data folder
const userData = path.join(app.getPath('userData'));
const execPath = path.dirname(app.getPath('exe'));

// Constants paths for temp indexing folders
const TEMP_BATCH_INDEX_FOLDER = isDevEnv ? './data/temp_batch_indexes' : path.join(userData, 'data/temp_batch_indexes');
const TEMP_REAL_TIME_INDEX = isDevEnv ? './data/temp_realtime_index' : path.join(userData, 'data/temp_realtime_index');
// Main User Index path
const INDEX_PREFIX = isDevEnv ? './data/search_index' : path.join(userData, 'data/search_index');
// Folder contains real time, batch and user index
const INDEX_DATA_FOLDER = isDevEnv ? './data' : path.join(userData, 'data');
//3 Months
const SEARCH_PERIOD_SUBTRACTOR = 3 * 31 * 24 * 60 * 60 * 1000;
const MINIMUM_DATE = '0000000000000';
const MAXIMUM_DATE = '9999999999999';
const INDEX_VERSION = 'v1';

const SORT_BY_SCORE = 0;
const BATCH_RANDOM_INDEX_PATH_LENGTH = 20;

// library path contractor
const winArchPath = process.arch === 'ia32' ? 'library/indexvalidator-x86.exe' : 'library/indexvalidator-x64.exe';
const rootPath = isMac ? 'library/indexvalidator.exec' : winArchPath;
const productionPath = path.join(execPath, isMac ? '..' : '', rootPath);
const devPath = path.join(__dirname, '..', '..', rootPath);
const INDEX_VALIDATOR = isDevEnv ? devPath : productionPath;

/**
 * This search class communicates with the SymphonySearchEngine C library via node-ffi.
 * There should be only 1 instance of this class in the Electron
 */
class Search {

    /**
     * Constructor for the SymphonySearchEngine library
     * @param userId (for the index folder name)
     * @param key
     */
    constructor(userId, key) {
        this.isInitialized = false;
        this.userId = userId;
        this.key = key;
        this.indexFolderName = INDEX_PREFIX + '_' + this.userId + '_' + INDEX_VERSION;
        this.dataFolder = INDEX_DATA_FOLDER;
        this.realTimeIndex = TEMP_REAL_TIME_INDEX;
        this.batchIndex = TEMP_BATCH_INDEX_FOLDER;
        this.messageData = [];
        this.crypto = new Crypto(userId, key);
        this.decryptAndInit();
    }

    decryptAndInit() {
        this.crypto.decryption().then(() => {
            this.init();
        }).catch(() => {
            this.init();
        });
    }

    /**
     * returns isInitialized boolean
     * @returns {boolean}
     */
    isLibInit() {
        return this.isInitialized;
    }

    /**
     * This init function
     * initialise the SymphonySearchEngine library
     * and creates a folder in the userData
     */
    init() {
        libSymphonySearch.symSEInit();
        libSymphonySearch.symSEEnsureFolderExists(this.dataFolder);
        libSymphonySearch.symSERemoveFolder(this.realTimeIndex);
        libSymphonySearch.symSERemoveFolder(this.batchIndex);
        Search.indexValidator(this.indexFolderName);
        Search.indexValidator(this.realTimeIndex);
        let indexDateStartFrom = new Date().getTime() - SEARCH_PERIOD_SUBTRACTOR;
        // Deleting all the messages except 3 Months from now
        libSymphonySearch.symSEDeleteMessages(this.indexFolderName, null,
            MINIMUM_DATE, indexDateStartFrom.toString());
        this.isInitialized = true;
    }

    /**
     * An array of messages is passed for indexing
     * it will be indexed in a temporary index folder
     * @param {Array} messages
     * @returns {Promise}
     */
    indexBatch(messages) {
        return new Promise((resolve, reject) => {
            if (!messages) {
                reject(new Error('Messages is required'));
                return;
            }

            if (!(JSON.parse(messages) instanceof Array)) {
                reject(new Error('Messages must be an array'));
                return;
            }

            if (!this.isInitialized) {
                reject(new Error('Library not initialized'));
                return;
            }

            const indexId = randomString.generate(BATCH_RANDOM_INDEX_PATH_LENGTH);
            libSymphonySearch.symSECreatePartialIndexAsync(this.batchIndex, indexId, messages, (err, res) => {
                if (err) {
                    reject(new Error(err));
                }
                resolve(res);
            });
        });
    }

    /**
     * Merging the temporary
     * created from indexBatch()
     */
    mergeIndexBatches() {
        return new Promise((resolve, reject) => {
            libSymphonySearch.symSEMergePartialIndexAsync(this.indexFolderName, this.batchIndex, (err, res) => {
                if (err) {
                    reject(new Error(err));
                }
                libSymphonySearch.symSERemoveFolder(this.batchIndex);
                resolve(res);
            });
        });
    }

    /**
     * An array of messages to be indexed
     * in real time
     * @param message
     */
    realTimeIndexing(message) {
        if (!message) {
            return new Error('Message is required');
        }

        if (!(JSON.parse(message) instanceof Array)){
            return new Error('Message must be an array');
        }

        if (!this.isInitialized) {
            return new Error('Library not initialized');
        }

        let result = libSymphonySearch.symSEIndexRealTime(this.realTimeIndex, message);
        return result === 0 ? "Successful" : result
    }

    /**
     * Reading a json file
     * for the demo search app only
     * @param {String} batch
     * @returns {Promise}
     */
    readJson(batch) {
        return new Promise((resolve, reject) => {
            let dirPath = path.join(execPath, isMac ? '..' : '', 'msgsjson', batch);
            let messageFolderPath = isDevEnv ? path.join('./msgsjson', batch) : dirPath;
            let files = fs.readdirSync(messageFolderPath);
            this.messageData = [];
            files.forEach((file) => {
                let tempPath = path.join(messageFolderPath, file);
                let data = fs.readFileSync(tempPath, "utf8");
                if (data) {
                    try {
                        this.messageData.push(JSON.parse(data));
                    } catch (err) {
                        reject(new Error(err))
                    }
                } else {
                    reject(new Error('Error reading batch'))
                }
            });
            resolve(this.messageData);
        });
    }

    /**
     * Encrypting the index after the merging the index
     * to the main user index
     */
    encryptIndex() {
        this.crypto.encryption().then(() => {
            return 'Success'
        }).catch((e) => {
            throw new Error(e)
        });
    }

    /**
     * This returns the search results
     * which returns a char *
     * @param {String} query
     * @param {Array} senderIds
     * @param {Array} threadIds
     * @param {String} fileType
     * @param {String} startDate
     * @param {String} endDate
     * @param {Number} limit
     * @param {Number} offset
     * @param {Number} sortOrder
     * @returns {Promise}
     */
    searchQuery(query, senderIds, threadIds, fileType, startDate,
                endDate, limit, offset, sortOrder) {

        let _limit = limit;
        let _offset = offset;
        let _sortOrder = sortOrder;

        return new Promise((resolve, reject) => {
            if (!this.isInitialized) {
                reject(new Error('Library not initialized'));
                return;
            }

            if (!fs.existsSync(this.indexFolderName) || !fs.existsSync(this.realTimeIndex)) {
                reject('Index folder does not exist.');
                return;
            }

            let q = Search.constructQuery(query, senderIds, threadIds, fileType);

            if (q === undefined) {
                reject(new Error('Search query error'));
                return;
            }

            let sd = new Date().getTime() - SEARCH_PERIOD_SUBTRACTOR;
            let sd_time = MINIMUM_DATE;
            if (startDate && startDate !== "" && typeof startDate === 'object') {
                sd_time = new Date(startDate).getTime();
                if (sd_time >= sd) {
                    sd_time = sd;
                }
            }

            let ed_time = MAXIMUM_DATE;
            if (endDate && endDate !== "" && typeof endDate === 'object') {
                ed_time = new Date(endDate).getTime();
            }

            if (!_limit && _limit === "" && typeof _limit !== 'number' && Math.round(_limit) !== _limit) {
                _limit = 25;
            }

            if (!_offset && _offset === "" && typeof _offset !== 'number' && Math.round(_offset) !== _offset) {
                _offset = 0
            }

            if (!_sortOrder && _sortOrder === "" && typeof _sortOrder !== 'number' && Math.round(_sortOrder) !== _sortOrder) {
                _sortOrder = SORT_BY_SCORE;
            }

            const returnedResult = libSymphonySearch.symSESearch(this.indexFolderName, this.realTimeIndex, q, sd_time.toString(), ed_time.toString(), _offset, _limit, _sortOrder);
            try {
                let ret = returnedResult.readCString();
                resolve(JSON.parse(ret));
            } finally {
                libSymphonySearch.symSEFreeResult(returnedResult);
            }
        });
    }

    /**
     * returns the latest message timestamp
     * from the indexed data
     * @returns {Promise}
     */
    getLatestMessageTimestamp() {
        return new Promise((resolve, reject) => {
            if (!this.isInitialized) {
                reject('Not initialized');
                return;
            }

            if (!fs.existsSync(this.indexFolderName)) {
                reject('Index folder does not exist.');
                return;
            }

            libSymphonySearch.symSEGetLastMessageTimestampAsync(this.indexFolderName, (err, res) => {
                if (err) {
                    reject(new Error(err));
                }
                const returnedResult = res;
                try {
                    let ret = returnedResult.readCString();
                    resolve(ret);
                } finally {
                    libSymphonySearch.symSEFreeResult(returnedResult);
                }
            });
        });
    }

    /**
     * This the query constructor
     * for the search function
     * @param {String} searchQuery
     * @param {Array} senderId
     * @param {Array} threadId
     * @param {String} fileType
     * @returns {string}
     */
    static constructQuery(searchQuery, senderId, threadId, fileType) {

        let searchText = "";
        if(searchQuery !== undefined) {
            searchText = searchQuery.trim().toLowerCase(); //to prevent injection of AND and ORs
        }
        let q = "";
        let hashTags = Search.getHashTags(searchText);
        let hashCashTagQuery = "";

        if(hashTags.length > 0) {
            hashCashTagQuery = " OR tags:(";
            hashTags.forEach((item) => {
                hashCashTagQuery = hashCashTagQuery + "\"" + item + "\" "
            });
            hashCashTagQuery += ")";
        }

        let hasAttachments = false;
        let additionalAttachmentQuery = "";
        if(fileType) {
            hasAttachments = true;
            if(fileType === "attachment") {
                additionalAttachmentQuery = "(hasfiles:true)";
            } else {
                additionalAttachmentQuery = "(filetype:(" + fileType +"))";
            }
        }


        if (searchText.length > 0 ) {
            q = "((text:(" + searchText + "))" + hashCashTagQuery ;
            if(hasAttachments) {
                q += " OR (filename:(" + searchText + "))" ;
            }
            q = q + ")";
        }

        q = Search.appendFilterQuery(q, "senderId", senderId);
        q = Search.appendFilterQuery(q, "threadId", threadId);

        if(q === "") {
            if(hasAttachments) {
                q = additionalAttachmentQuery;
            } else {
                q = undefined; //will be handled in the search function
            }
        } else {
            if(hasAttachments){
                q = q + " AND " + additionalAttachmentQuery
            }
        }
        return q;
    }

    /**
     * appending the senderId and threadId for the query
     * @param {String} searchText
     * @param {String} fieldName
     * @param {Array} valueArray
     * @returns {string}
     */
    static appendFilterQuery(searchText, fieldName, valueArray) {
        let q = "";
        if (valueArray && valueArray.length > 0 ) {

            q += "(" + fieldName +":(";
            valueArray.forEach((item)=>{
                q+= "\"" + item + "\" ";
            });
            q += "))";
            if(searchText.length > 0 ) {
                q = searchText + " AND " + q;
            }

        } else {
            q = searchText;
        }

        return q;
    }

    // hashtags can have any characters(before the latest release it was
    // not like this). So the only regex is splitting the search query based on
    // whitespaces
    /**
     * return the hash cash
     * tags from the query
     * @param {String} searchText
     * @returns {Array}
     */
    static getHashTags(searchText) {
        let hashTags = [];
        let tokens = searchText.toLowerCase()
            .trim()
            .replace(/\s\s+/g, ' ')
            .split(' ').filter((el) => {return el.length !== 0});
        tokens.forEach((item) => {
            if (item.startsWith('#') || item.startsWith('$')) {
                hashTags.push(item);
            }
        });
        return hashTags;
    }

    /**
     * Validate the index folder exist or not
     * @param {String} file
     * @returns {*}
     */
    static indexValidator(file) {
        let data;
        let result = childProcess.execFileSync(INDEX_VALIDATOR, [file]).toString();
        try {
            data = JSON.parse(result);
            if (data.status === 'OK') {
                return data;
            }
            return new Error('Unable validate index folder')
        } catch (err) {
            throw new Error(err);
        }
    }
}

/**
 * Exporting the search library
 * @type {{Search: Search}}
 */
module.exports = {
    Search: Search
};