Spartan/wiki
1
0
Fork 0
mirror of https://github.com/requarks/wiki.git synced 2026-08-26 13:17:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: rebuild search index + isSearchable flag

Browse Source
This commit is contained in:
NGPixel
2023-07-07 04:56:44 +00:00
parent 17034040ac
commit c87f4ce770
16 changed files with 199 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/db/migrations/3.0.0.mjs
+17 -2
View File
@@ -12,6 +12,7 @@ export async function up (knex) {
// =====================================
await knex.raw('CREATE EXTENSION IF NOT EXISTS ltree;')
await knex.raw('CREATE EXTENSION IF NOT EXISTS pgcrypto;')
await knex.raw('CREATE EXTENSION IF NOT EXISTS pg_trgm;')
await knex.schema
// =====================================
@@ -235,6 +236,8 @@ export async function up (knex) {
table.string('editor').notNullable()
table.string('contentType').notNullable()
table.boolean('isBrowsable').notNullable().defaultTo(true)
table.boolean('isSearchable').notNullable().defaultTo(true)
table.specificType('isSearchableComputed', `boolean GENERATED ALWAYS AS ("publishState" != 'draft' AND "isSearchable") STORED`).index()
table.string('password')
table.integer('ratingScore').notNullable().defaultTo(0)
table.integer('ratingCount').notNullable().defaultTo(0)
@@ -393,6 +396,13 @@ export async function up (knex) {
.table('userKeys', table => {
table.uuid('userId').notNullable().references('id').inTable('users')
})
// =====================================
// TS WORD SUGGESTION TABLE
// =====================================
.createTable('autocomplete', table => {
table.text('word')
})
.raw(`CREATE INDEX "autocomplete_idx" ON "autocomplete" USING GIN (word gin_trgm_ops)`)
// =====================================
// DEFAULT DATA
@@ -518,8 +528,8 @@ export async function up (knex) {
key: 'update',
value: {
lastCheckedAt: null,
version: null,
versionDate: null
version: WIKI.version,
versionDate: WIKI.releaseDate
}
},
{
@@ -771,6 +781,11 @@ export async function up (knex) {
cron: '5 0 * * *',
type: 'system'
},
{
task: 'refreshAutocomplete',
cron: '0 */3 * * *',
type: 'system'
},
{
task: 'updateLocales',
cron: '0 0 * * *',
server/graph/resolvers/page.mjs
+1
View File
@@ -77,6 +77,7 @@ export default {
.select(searchCols)
.fromRaw('pages, websearch_to_tsquery(?, ?) query', [dictName, args.query])
.where('siteId', args.siteId)
.where('isSearchableComputed', true)
.where(builder => {
if (args.path) {
builder.where('path', 'ILIKE', `${args.path}%`)
server/graph/resolvers/search.mjs
-16
View File
@@ -1,16 +0,0 @@
import { generateError, generateSuccess } from '../../helpers/graph.mjs'
export default {
Mutation: {
async rebuildSearchIndex (obj, args, context) {
try {
await WIKI.data.searchEngine.rebuild()
return {
responseResult: generateSuccess('Index rebuilt successfully')
}
} catch (err) {
return generateError(err)
}
}
}
}
server/graph/resolvers/system.mjs
+13
View File
@@ -141,6 +141,19 @@ export default {
return generateError(err)
}
},
async rebuildSearchIndex (obj, args, context) {
try {
await WIKI.scheduler.addJob({
task: 'rebuildSearchIndex',
maxRetries: 0
})
return {
operation: generateSuccess('Search index rebuild has been scheduled and will start shortly.')
}
} catch (err) {
return generateError(err)
}
},
async retryJob (obj, args, context) {
WIKI.logger.info(`Admin requested rescheduling of job ${args.id}...`)
try {
server/graph/schemas/page.graphql
+8
View File
@@ -65,6 +65,11 @@ extend type Query {
limit: Int
): PageSearchResponse
searchPagesAutocomplete(
siteId: UUID!
query: String!
): [String]
pages(
limit: Int
orderBy: PageOrderBy
@@ -130,6 +135,7 @@ extend type Mutation {
editor: String!
icon: String
isBrowsable: Boolean
isSearchable: Boolean
locale: String!
path: String!
publishState: PagePublishState!
@@ -231,6 +237,7 @@ type Page {
icon: String
id: UUID
isBrowsable: Boolean
isSearchable: Boolean
locale: String
password: String
path: String
@@ -393,6 +400,7 @@ input PageUpdateInput {
description: String
icon: String
isBrowsable: Boolean
isSearchable: Boolean
locale: String
password: String
path: String
server/graph/schemas/search.graphql
-11
View File
@@ -1,11 +0,0 @@
# ===============================================
# SEARCH
# ===============================================
extend type Mutation {
rebuildSearchIndex: DefaultResponse
}
# -----------------------------------------------
# TYPES
# -----------------------------------------------
server/graph/schemas/system.graphql
+2
View File
@@ -29,6 +29,8 @@ extend type Mutation {
key: String!
): DefaultResponse
rebuildSearchIndex: DefaultResponse
retryJob(
id: UUID!
): DefaultResponse
server/locales/en.json
+4 -1
View File
@@ -525,7 +525,7 @@
"admin.scheduler.waitUntil": "Start",
"admin.search.configSaveSuccess": "Search engine configuration saved successfully.",
"admin.search.dictOverrides": "PostgreSQL Dictionary Mapping Overrides",
"admin.search.dictOverridesHint": "JSON object of 2 letters locale codes and their PostgreSQL dictionary association. e.g. { \"en\": \"english\" }",
"admin.search.dictOverridesHint": "JSON object of 2 letters locale codes and their PostgreSQL dictionary association. e.g. {0}",
"admin.search.engineConfig": "Engine Configuration",
"admin.search.engineNoConfig": "This engine has no configuration options you can modify.",
"admin.search.highlighting": "Enable Term Highlighting",
@@ -533,10 +533,12 @@
"admin.search.indexRebuildSuccess": "Index rebuilt successfully.",
"admin.search.listRefreshSuccess": "List of search engines has been refreshed.",
"admin.search.rebuildIndex": "Rebuild Index",
"admin.search.rebuildInitSuccess": "A search index rebuild has been initiated and will start shortly.",
"admin.search.saveSuccess": "Search engine configuration saved successfully",
"admin.search.searchEngine": "Search Engine",
"admin.search.subtitle": "Configure the search capabilities of your wiki",
"admin.search.title": "Search Engine",
"admin.searchRebuildIndex": "Rebuild Index",
"admin.security.cors": "CORS (Cross-Origin Resource Sharing)",
"admin.security.corsHostnames": "Hostnames Whitelist",
"admin.security.corsHostnamesHint": "Enter one hostname per line",
@@ -1535,6 +1537,7 @@
"editor.props.draftHint": "Visible to users with write access only.",
"editor.props.icon": "Icon",
"editor.props.info": "Info",
"editor.props.isSearchable": "Include in Search Results",
"editor.props.jsLoad": "Javascript - On Load",
"editor.props.jsLoadHint": "Execute javascript once the page is loaded",
"editor.props.jsUnload": "Javascript - On Unload",
server/models/pages.mjs
+38 -13
View File
@@ -344,6 +344,7 @@ export class Page extends Model {
hash: generateHash({ path: opts.path, locale: opts.locale }),
icon: opts.icon,
isBrowsable: opts.isBrowsable ?? true,
isSearchable: opts.isSearchable ?? true,
localeCode: opts.locale,
ownerId: opts.user.id,
path: opts.path,
@@ -388,7 +389,7 @@ export class Page extends Model {
})
// -> Update search vector
WIKI.db.pages.updatePageSearchVector(page.id)
WIKI.db.pages.updatePageSearchVector({ id: page.id })
// // -> Add to Storage
// if (!opts.skipStorage) {
@@ -507,14 +508,17 @@ export class Page extends Model {
historyData.affectedFields.push('publishEndDate')
}
// -> Page Config
// -> Browsable / Searchable Flags
if ('isBrowsable' in opts.patch) {
patch.config = {
...patch.config ?? ogPage.config ?? {},
isBrowsable: opts.patch.isBrowsable
}
patch.isBrowsable = opts.patch.isBrowsable
historyData.affectedFields.push('isBrowsable')
}
if ('isSearchable' in opts.patch) {
patch.isSearchable = opts.patch.isSearchable
historyData.affectedFields.push('isSearchable')
}
// -> Page Config
if ('allowComments' in opts.patch) {
patch.config = {
...patch.config ?? ogPage.config ?? {},
@@ -644,7 +648,7 @@ export class Page extends Model {
// -> Save Tags
if (opts.patch.tags) {
await WIKI.db.tags.associateTags({ tags: opts.patch.tags, page })
// await WIKI.db.tags.associateTags({ tags: opts.patch.tags, page })
}
// -> Render page to HTML
@@ -672,7 +676,7 @@ export class Page extends Model {
// -> Update search vector
if (shouldUpdateSearch) {
WIKI.db.pages.updatePageSearchVector(page.id)
WIKI.db.pages.updatePageSearchVector({ id: page.id })
}
// -> Update on Storage
@@ -710,13 +714,21 @@ export class Page extends Model {
/**
* Update a page text search vector value
*
* @param {String} id Page UUID
* @param {Object} opts - Options
* @param {string} [opts.id] - Page ID to update (fetch from DB)
* @param {Object} [opts.page] - Page object to update (use directly)
*/
static async updatePageSearchVector (id) {
const page = await WIKI.db.pages.query().findById(id).select('localeCode', 'render')
const safeContent = WIKI.db.pages.cleanHTML(page.render)
static async updatePageSearchVector ({ id, page }) {
if (!page) {
if (!id) {
throw new Error('Must provide either the page ID or the page object.')
}
page = await WIKI.db.pages.query().findById(id).select('id', 'localeCode', 'render', 'password')
}
// -> Exclude password-protected content from being indexed
const safeContent = page.password ? '' : WIKI.db.pages.cleanHTML(page.render)
const dictName = getDictNameFromLocale(page.localeCode)
return WIKI.db.knex('pages').where('id', id).update({
return WIKI.db.knex('pages').where('id', page.id).update({
searchContent: safeContent,
ts: WIKI.db.knex.raw(`
setweight(to_tsvector('${dictName}', coalesce(title,'')), 'A') ||
@@ -725,6 +737,19 @@ export class Page extends Model {
})
}
/**
* Refresh Autocomplete Index
*/
static async refreshAutocompleteIndex () {
await WIKI.db.knex('autocomplete').truncate()
await WIKI.db.knex.raw(`
INSERT INTO "autocomplete" (word)
SELECT word FROM ts_stat(
'SELECT to_tsvector(''simple'', "title") || to_tsvector(''simple'', "description") || to_tsvector(''simple'', "searchContent") FROM "pages" WHERE isSearchableComputed IS TRUE'
)
`)
}
/**
* Convert an Existing Page
*
server/tasks/simple/refresh-autocomplete.mjs
+13
View File
@@ -0,0 +1,13 @@
export async function task (payload) {
WIKI.logger.info('Refreshing autocomplete word index...')
try {
await WIKI.db.pages.refreshAutocompleteIndex()
WIKI.logger.info('Refreshed autocomplete word index: [ COMPLETED ]')
} catch (err) {
WIKI.logger.error('Refreshing autocomplete word index: [ FAILED ]')
WIKI.logger.error(err.message)
throw err
}
}
server/tasks/workers/purge-uploads.mjs
+1
View File
@@ -22,5 +22,6 @@ export async function task ({ payload }) {
} catch (err) {
WIKI.logger.error('Purging orphaned upload files: [ FAILED ]')
WIKI.logger.error(err.message)
throw err
}
}
server/tasks/workers/rebuild-search-index.mjs
+35
View File
@@ -0,0 +1,35 @@
import { pipeline } from 'node:stream/promises'
import { Transform } from 'node:stream'
export async function task ({ payload }) {
WIKI.logger.info('Rebuilding search index...')
try {
await WIKI.ensureDb()
let idx = 0
await pipeline(
WIKI.db.knex.select('id', 'title', 'description', 'localeCode', 'render', 'password').from('pages').stream(),
new Transform({
objectMode: true,
transform: async (page, enc, cb) => {
idx++
await WIKI.db.pages.updatePageSearchVector({ page })
if (idx % 50 === 0) {
WIKI.logger.info(`Rebuilding search index... (${idx} processed)`)
}
cb()
}
})
)
WIKI.logger.info('Refreshing autocomplete index...')
await WIKI.db.pages.refreshAutocompleteIndex()
WIKI.logger.info('Rebuilt search index: [ COMPLETED ]')
} catch (err) {
WIKI.logger.error('Rebuilding search index: [ FAILED ]')
WIKI.logger.error(err.message)
throw err
}
}