From 8c4bbd946d2247c2e239cbbf8773d2d31c1a57aa Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 10 Jan 2023 11:09:30 +0100 Subject: Refactor model utils --- server/models/shared/index.ts | 4 + server/models/shared/model-cache.ts | 90 +++++++++++++++++ server/models/shared/query.ts | 75 +++++++++++++- server/models/shared/sequelize-helpers.ts | 39 ++++++++ server/models/shared/sort.ts | 160 ++++++++++++++++++++++++++++++ server/models/shared/sql.ts | 68 +++++++++++++ server/models/shared/update.ts | 14 ++- 7 files changed, 441 insertions(+), 9 deletions(-) create mode 100644 server/models/shared/model-cache.ts create mode 100644 server/models/shared/sequelize-helpers.ts create mode 100644 server/models/shared/sort.ts create mode 100644 server/models/shared/sql.ts (limited to 'server/models/shared') diff --git a/server/models/shared/index.ts b/server/models/shared/index.ts index 04528929c..5a7621e4d 100644 --- a/server/models/shared/index.ts +++ b/server/models/shared/index.ts @@ -1,4 +1,8 @@ export * from './abstract-run-query' export * from './model-builder' +export * from './model-cache' export * from './query' +export * from './sequelize-helpers' +export * from './sort' +export * from './sql' export * from './update' diff --git a/server/models/shared/model-cache.ts b/server/models/shared/model-cache.ts new file mode 100644 index 000000000..3651267e7 --- /dev/null +++ b/server/models/shared/model-cache.ts @@ -0,0 +1,90 @@ +import { Model } from 'sequelize-typescript' +import { logger } from '@server/helpers/logger' + +type ModelCacheType = + 'local-account-name' + | 'local-actor-name' + | 'local-actor-url' + | 'load-video-immutable-id' + | 'load-video-immutable-url' + +type DeleteKey = + 'video' + +class ModelCache { + + private static instance: ModelCache + + private readonly localCache: { [id in ModelCacheType]: Map } = { + 'local-account-name': new Map(), + 'local-actor-name': new Map(), + 'local-actor-url': new Map(), + 'load-video-immutable-id': new Map(), + 'load-video-immutable-url': new Map() + } + + private readonly deleteIds: { + [deleteKey in DeleteKey]: Map + } = { + video: new Map() + } + + private constructor () { + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + doCache (options: { + cacheType: ModelCacheType + key: string + fun: () => Promise + whitelist?: () => boolean + deleteKey?: DeleteKey + }) { + const { cacheType, key, fun, whitelist, deleteKey } = options + + if (whitelist && whitelist() !== true) return fun() + + const cache = this.localCache[cacheType] + + if (cache.has(key)) { + logger.debug('Model cache hit for %s -> %s.', cacheType, key) + return Promise.resolve(cache.get(key)) + } + + return fun().then(m => { + if (!m) return m + + if (!whitelist || whitelist()) cache.set(key, m) + + if (deleteKey) { + const map = this.deleteIds[deleteKey] + if (!map.has(m.id)) map.set(m.id, []) + + const a = map.get(m.id) + a.push({ cacheType, key }) + } + + return m + }) + } + + invalidateCache (deleteKey: DeleteKey, modelId: number) { + const map = this.deleteIds[deleteKey] + + if (!map.has(modelId)) return + + for (const toDelete of map.get(modelId)) { + logger.debug('Removing %s -> %d of model cache %s -> %s.', deleteKey, modelId, toDelete.cacheType, toDelete.key) + this.localCache[toDelete.cacheType].delete(toDelete.key) + } + + map.delete(modelId) + } +} + +export { + ModelCache +} diff --git a/server/models/shared/query.ts b/server/models/shared/query.ts index 036cc13c6..934acc21f 100644 --- a/server/models/shared/query.ts +++ b/server/models/shared/query.ts @@ -1,17 +1,82 @@ -import { BindOrReplacements, QueryTypes } from 'sequelize' -import { sequelizeTypescript } from '@server/initializers/database' +import { BindOrReplacements, Op, QueryTypes, Sequelize } from 'sequelize' +import validator from 'validator' +import { forceNumber } from '@shared/core-utils' -function doesExist (query: string, bind?: BindOrReplacements) { +function doesExist (sequelize: Sequelize, query: string, bind?: BindOrReplacements) { const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind, raw: true } - return sequelizeTypescript.query(query, options) + return sequelize.query(query, options) .then(results => results.length === 1) } +function createSimilarityAttribute (col: string, value: string) { + return Sequelize.fn( + 'similarity', + + searchTrigramNormalizeCol(col), + + searchTrigramNormalizeValue(value) + ) +} + +function buildWhereIdOrUUID (id: number | string) { + return validator.isInt('' + id) ? { id } : { uuid: id } +} + +function parseAggregateResult (result: any) { + if (!result) return 0 + + const total = forceNumber(result) + if (isNaN(total)) return 0 + + return total +} + +function parseRowCountResult (result: any) { + if (result.length !== 0) return result[0].total + + return 0 +} + +function createSafeIn (sequelize: Sequelize, toEscape: (string | number)[], additionalUnescaped: string[] = []) { + return toEscape.map(t => { + return t === null + ? null + : sequelize.escape('' + t) + }).concat(additionalUnescaped).join(', ') +} + +function searchAttribute (sourceField?: string, targetField?: string) { + if (!sourceField) return {} + + return { + [targetField]: { + // FIXME: ts error + [Op.iLike as any]: `%${sourceField}%` + } + } +} + export { - doesExist + doesExist, + createSimilarityAttribute, + buildWhereIdOrUUID, + parseAggregateResult, + parseRowCountResult, + createSafeIn, + searchAttribute +} + +// --------------------------------------------------------------------------- + +function searchTrigramNormalizeValue (value: string) { + return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', value)) +} + +function searchTrigramNormalizeCol (col: string) { + return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', Sequelize.col(col))) } diff --git a/server/models/shared/sequelize-helpers.ts b/server/models/shared/sequelize-helpers.ts new file mode 100644 index 000000000..7af8471dc --- /dev/null +++ b/server/models/shared/sequelize-helpers.ts @@ -0,0 +1,39 @@ +import { Sequelize } from 'sequelize' + +function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) { + if (!model.createdAt || !model.updatedAt) { + throw new Error('Miss createdAt & updatedAt attributes to model') + } + + const now = Date.now() + const createdAtTime = model.createdAt.getTime() + const updatedAtTime = model.updatedAt.getTime() + + return (now - createdAtTime) > refreshInterval && (now - updatedAtTime) > refreshInterval +} + +function throwIfNotValid (value: any, validator: (value: any) => boolean, fieldName = 'value', nullable = false) { + if (nullable && (value === null || value === undefined)) return + + if (validator(value) === false) { + throw new Error(`"${value}" is not a valid ${fieldName}.`) + } +} + +function buildTrigramSearchIndex (indexName: string, attribute: string) { + return { + name: indexName, + // FIXME: gin_trgm_ops is not taken into account in Sequelize 6, so adding it ourselves in the literal function + fields: [ Sequelize.literal('lower(immutable_unaccent(' + attribute + ')) gin_trgm_ops') as any ], + using: 'gin', + operator: 'gin_trgm_ops' + } +} + +// --------------------------------------------------------------------------- + +export { + throwIfNotValid, + buildTrigramSearchIndex, + isOutdated +} diff --git a/server/models/shared/sort.ts b/server/models/shared/sort.ts new file mode 100644 index 000000000..77e84dcf4 --- /dev/null +++ b/server/models/shared/sort.ts @@ -0,0 +1,160 @@ +import { literal, OrderItem, Sequelize } from 'sequelize' + +// Translate for example "-name" to [ [ 'name', 'DESC' ], [ 'id', 'ASC' ] ] +function getSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] { + const { direction, field } = buildSortDirectionAndField(value) + + let finalField: string | ReturnType + + if (field.toLowerCase() === 'match') { // Search + finalField = Sequelize.col('similarity') + } else { + finalField = field + } + + return [ [ finalField, direction ], lastSort ] +} + +function getAdminUsersSort (value: string): OrderItem[] { + const { direction, field } = buildSortDirectionAndField(value) + + let finalField: string | ReturnType + + if (field === 'videoQuotaUsed') { // Users list + finalField = Sequelize.col('videoQuotaUsed') + } else { + finalField = field + } + + const nullPolicy = direction === 'ASC' + ? 'NULLS FIRST' + : 'NULLS LAST' + + // FIXME: typings + return [ [ finalField as any, direction, nullPolicy ], [ 'id', 'ASC' ] ] +} + +function getPlaylistSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] { + const { direction, field } = buildSortDirectionAndField(value) + + if (field.toLowerCase() === 'name') { + return [ [ 'displayName', direction ], lastSort ] + } + + return getSort(value, lastSort) +} + +function getCommentSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] { + const { direction, field } = buildSortDirectionAndField(value) + + if (field === 'totalReplies') { + return [ + [ Sequelize.literal('"totalReplies"'), direction ], + lastSort + ] + } + + return getSort(value, lastSort) +} + +function getVideoSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] { + const { direction, field } = buildSortDirectionAndField(value) + + if (field.toLowerCase() === 'trending') { // Sort by aggregation + return [ + [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoViews.views')), '0'), direction ], + + [ Sequelize.col('VideoModel.views'), direction ], + + lastSort + ] + } else if (field === 'publishedAt') { + return [ + [ 'ScheduleVideoUpdate', 'updateAt', direction + ' NULLS LAST' ], + + [ Sequelize.col('VideoModel.publishedAt'), direction ], + + lastSort + ] + } + + let finalField: string | ReturnType + + // Alias + if (field.toLowerCase() === 'match') { // Search + finalField = Sequelize.col('similarity') + } else { + finalField = field + } + + const firstSort: OrderItem = typeof finalField === 'string' + ? finalField.split('.').concat([ direction ]) as OrderItem + : [ finalField, direction ] + + return [ firstSort, lastSort ] +} + +function getBlacklistSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] { + const { direction, field } = buildSortDirectionAndField(value) + + const videoFields = new Set([ 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid' ]) + + if (videoFields.has(field)) { + return [ + [ literal(`"Video.${field}" ${direction}`) ], + lastSort + ] as OrderItem[] + } + + return getSort(value, lastSort) +} + +function getInstanceFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] { + const { direction, field } = buildSortDirectionAndField(value) + + if (field === 'redundancyAllowed') { + return [ + [ 'ActorFollowing.Server.redundancyAllowed', direction ], + lastSort + ] + } + + return getSort(value, lastSort) +} + +function getChannelSyncSort (value: string): OrderItem[] { + const { direction, field } = buildSortDirectionAndField(value) + if (field.toLowerCase() === 'videochannel') { + return [ + [ literal('"VideoChannel.name"'), direction ] + ] + } + return [ [ field, direction ] ] +} + +function buildSortDirectionAndField (value: string) { + let field: string + let direction: 'ASC' | 'DESC' + + if (value.substring(0, 1) === '-') { + direction = 'DESC' + field = value.substring(1) + } else { + direction = 'ASC' + field = value + } + + return { direction, field } +} + +export { + buildSortDirectionAndField, + getPlaylistSort, + getSort, + getCommentSort, + getAdminUsersSort, + getVideoSort, + getBlacklistSort, + getChannelSyncSort, + getInstanceFollowsSort +} diff --git a/server/models/shared/sql.ts b/server/models/shared/sql.ts new file mode 100644 index 000000000..5aaeb49f0 --- /dev/null +++ b/server/models/shared/sql.ts @@ -0,0 +1,68 @@ +import { literal, Model, ModelStatic } from 'sequelize' +import { forceNumber } from '@shared/core-utils' +import { AttributesOnly } from '@shared/typescript-utils' + +function buildLocalAccountIdsIn () { + return literal( + '(SELECT "account"."id" FROM "account" INNER JOIN "actor" ON "actor"."id" = "account"."actorId" AND "actor"."serverId" IS NULL)' + ) +} + +function buildLocalActorIdsIn () { + return literal( + '(SELECT "actor"."id" FROM "actor" WHERE "actor"."serverId" IS NULL)' + ) +} + +function buildBlockedAccountSQL (blockerIds: number[]) { + const blockerIdsString = blockerIds.join(', ') + + return 'SELECT "targetAccountId" AS "id" FROM "accountBlocklist" WHERE "accountId" IN (' + blockerIdsString + ')' + + ' UNION ' + + 'SELECT "account"."id" AS "id" FROM account INNER JOIN "actor" ON account."actorId" = actor.id ' + + 'INNER JOIN "serverBlocklist" ON "actor"."serverId" = "serverBlocklist"."targetServerId" ' + + 'WHERE "serverBlocklist"."accountId" IN (' + blockerIdsString + ')' +} + +function buildServerIdsFollowedBy (actorId: any) { + const actorIdNumber = forceNumber(actorId) + + return '(' + + 'SELECT "actor"."serverId" FROM "actorFollow" ' + + 'INNER JOIN "actor" ON actor.id = "actorFollow"."targetActorId" ' + + 'WHERE "actorFollow"."actorId" = ' + actorIdNumber + + ')' +} + +function buildSQLAttributes (options: { + model: ModelStatic + tableName: string + + excludeAttributes?: Exclude, symbol>[] + aliasPrefix?: string +}) { + const { model, tableName, aliasPrefix, excludeAttributes } = options + + const attributes = Object.keys(model.getAttributes()) as Exclude, symbol>[] + + return attributes + .filter(a => { + if (!excludeAttributes) return true + if (excludeAttributes.includes(a)) return false + + return true + }) + .map(a => { + return `"${tableName}"."${a}" AS "${aliasPrefix || ''}${a}"` + }) +} + +// --------------------------------------------------------------------------- + +export { + buildSQLAttributes, + buildBlockedAccountSQL, + buildServerIdsFollowedBy, + buildLocalAccountIdsIn, + buildLocalActorIdsIn +} diff --git a/server/models/shared/update.ts b/server/models/shared/update.ts index d338211e3..d02c4535d 100644 --- a/server/models/shared/update.ts +++ b/server/models/shared/update.ts @@ -1,9 +1,15 @@ -import { QueryTypes, Transaction } from 'sequelize' -import { sequelizeTypescript } from '@server/initializers/database' +import { QueryTypes, Sequelize, Transaction } from 'sequelize' // Sequelize always skip the update if we only update updatedAt field -function setAsUpdated (table: string, id: number, transaction?: Transaction) { - return sequelizeTypescript.query( +function setAsUpdated (options: { + sequelize: Sequelize + table: string + id: number + transaction?: Transaction +}) { + const { sequelize, table, id, transaction } = options + + return sequelize.query( `UPDATE "${table}" SET "updatedAt" = :updatedAt WHERE id = :id`, { replacements: { table, id, updatedAt: new Date() }, -- cgit v1.2.3