Refactor model utils

7 files changed, 441 insertions, 9 deletions

diff --git a/server/models/shared/index.ts b/server/models/shared/index.ts
index 04528929c..5a7621e4d 100644
--- a/server/models/shared/index.ts
+++ b/server/models/shared/index.ts
@@ -1,4 +1,8 @@
 export * from './abstract-run-query'
 export * from './model-builder'
+export * from './model-cache'
 export * from './query'
+export * from './sequelize-helpers'
+export * from './sort'
+export * from './sql'
 export * from './update'
diff --git a/server/models/shared/model-cache.ts b/server/models/shared/model-cache.ts
new file mode 100644
index 000000000..3651267e7
--- /dev/null
+++ b/server/models/shared/model-cache.ts
@@ -0,0 +1,90 @@
+import { Model } from 'sequelize-typescript'
+import { logger } from '@server/helpers/logger'
+
+type ModelCacheType =
+  'local-account-name'
+  | 'local-actor-name'
+  | 'local-actor-url'
+  | 'load-video-immutable-id'
+  | 'load-video-immutable-url'
+
+type DeleteKey =
+  'video'
+
+class ModelCache {
+
+  private static instance: ModelCache
+
+  private readonly localCache: { [id in ModelCacheType]: Map<string, any> } = {
+    'local-account-name': new Map(),
+    'local-actor-name': new Map(),
+    'local-actor-url': new Map(),
+    'load-video-immutable-id': new Map(),
+    'load-video-immutable-url': new Map()
+  }
+
+  private readonly deleteIds: {
+    [deleteKey in DeleteKey]: Map<number, { cacheType: ModelCacheType, key: string }[]>
+  } = {
+    video: new Map()
+  }
+
+  private constructor () {
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  doCache<T extends Model> (options: {
+    cacheType: ModelCacheType
+    key: string
+    fun: () => Promise<T>
+    whitelist?: () => boolean
+    deleteKey?: DeleteKey
+  }) {
+    const { cacheType, key, fun, whitelist, deleteKey } = options
+
+    if (whitelist && whitelist() !== true) return fun()
+
+    const cache = this.localCache[cacheType]
+
+    if (cache.has(key)) {
+      logger.debug('Model cache hit for %s -> %s.', cacheType, key)
+      return Promise.resolve<T>(cache.get(key))
+    }
+
+    return fun().then(m => {
+      if (!m) return m
+
+      if (!whitelist || whitelist()) cache.set(key, m)
+
+      if (deleteKey) {
+        const map = this.deleteIds[deleteKey]
+        if (!map.has(m.id)) map.set(m.id, [])
+
+        const a = map.get(m.id)
+        a.push({ cacheType, key })
+      }
+
+      return m
+    })
+  }
+
+  invalidateCache (deleteKey: DeleteKey, modelId: number) {
+    const map = this.deleteIds[deleteKey]
+
+    if (!map.has(modelId)) return
+
+    for (const toDelete of map.get(modelId)) {
+      logger.debug('Removing %s -> %d of model cache %s -> %s.', deleteKey, modelId, toDelete.cacheType, toDelete.key)
+      this.localCache[toDelete.cacheType].delete(toDelete.key)
+    }
+
+    map.delete(modelId)
+  }
+}
+
+export {
+  ModelCache
+}
diff --git a/server/models/shared/query.ts b/server/models/shared/query.ts
index 036cc13c6..934acc21f 100644
--- a/server/models/shared/query.ts
+++ b/server/models/shared/query.ts
@@ -1,17 +1,82 @@
-import { BindOrReplacements, QueryTypes } from 'sequelize'
-import { sequelizeTypescript } from '@server/initializers/database'
+import { BindOrReplacements, Op, QueryTypes, Sequelize } from 'sequelize'
+import validator from 'validator'
+import { forceNumber } from '@shared/core-utils'
 
-function doesExist (query: string, bind?: BindOrReplacements) {
+function doesExist (sequelize: Sequelize, query: string, bind?: BindOrReplacements) {
   const options = {
     type: QueryTypes.SELECT as QueryTypes.SELECT,
     bind,
     raw: true
   }
 
-  return sequelizeTypescript.query(query, options)
+  return sequelize.query(query, options)
             .then(results => results.length === 1)
 }
 
+function createSimilarityAttribute (col: string, value: string) {
+  return Sequelize.fn(
+    'similarity',
+
+    searchTrigramNormalizeCol(col),
+
+    searchTrigramNormalizeValue(value)
+  )
+}
+
+function buildWhereIdOrUUID (id: number | string) {
+  return validator.isInt('' + id) ? { id } : { uuid: id }
+}
+
+function parseAggregateResult (result: any) {
+  if (!result) return 0
+
+  const total = forceNumber(result)
+  if (isNaN(total)) return 0
+
+  return total
+}
+
+function parseRowCountResult (result: any) {
+  if (result.length !== 0) return result[0].total
+
+  return 0
+}
+
+function createSafeIn (sequelize: Sequelize, toEscape: (string | number)[], additionalUnescaped: string[] = []) {
+  return toEscape.map(t => {
+    return t === null
+      ? null
+      : sequelize.escape('' + t)
+  }).concat(additionalUnescaped).join(', ')
+}
+
+function searchAttribute (sourceField?: string, targetField?: string) {
+  if (!sourceField) return {}
+
+  return {
+    [targetField]: {
+      // FIXME: ts error
+      [Op.iLike as any]: `%${sourceField}%`
+    }
+  }
+}
+
 export {
-  doesExist
+  doesExist,
+  createSimilarityAttribute,
+  buildWhereIdOrUUID,
+  parseAggregateResult,
+  parseRowCountResult,
+  createSafeIn,
+  searchAttribute
+}
+
+// ---------------------------------------------------------------------------
+
+function searchTrigramNormalizeValue (value: string) {
+  return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', value))
+}
+
+function searchTrigramNormalizeCol (col: string) {
+  return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', Sequelize.col(col)))
 }
diff --git a/server/models/shared/sequelize-helpers.ts b/server/models/shared/sequelize-helpers.ts
new file mode 100644
index 000000000..7af8471dc
--- /dev/null
+++ b/server/models/shared/sequelize-helpers.ts
@@ -0,0 +1,39 @@
+import { Sequelize } from 'sequelize'
+
+function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
+  if (!model.createdAt || !model.updatedAt) {
+    throw new Error('Miss createdAt & updatedAt attributes to model')
+  }
+
+  const now = Date.now()
+  const createdAtTime = model.createdAt.getTime()
+  const updatedAtTime = model.updatedAt.getTime()
+
+  return (now - createdAtTime) > refreshInterval && (now - updatedAtTime) > refreshInterval
+}
+
+function throwIfNotValid (value: any, validator: (value: any) => boolean, fieldName = 'value', nullable = false) {
+  if (nullable && (value === null || value === undefined)) return
+
+  if (validator(value) === false) {
+    throw new Error(`"${value}" is not a valid ${fieldName}.`)
+  }
+}
+
+function buildTrigramSearchIndex (indexName: string, attribute: string) {
+  return {
+    name: indexName,
+    // FIXME: gin_trgm_ops is not taken into account in Sequelize 6, so adding it ourselves in the literal function
+    fields: [ Sequelize.literal('lower(immutable_unaccent(' + attribute + ')) gin_trgm_ops') as any ],
+    using: 'gin',
+    operator: 'gin_trgm_ops'
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  throwIfNotValid,
+  buildTrigramSearchIndex,
+  isOutdated
+}
diff --git a/server/models/shared/sort.ts b/server/models/shared/sort.ts
new file mode 100644
index 000000000..77e84dcf4
--- /dev/null
+++ b/server/models/shared/sort.ts
@@ -0,0 +1,160 @@
+import { literal, OrderItem, Sequelize } from 'sequelize'
+
+// Translate for example "-name" to [ [ 'name', 'DESC' ], [ 'id', 'ASC' ] ]
+function getSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  let finalField: string | ReturnType<typeof Sequelize.col>
+
+  if (field.toLowerCase() === 'match') { // Search
+    finalField = Sequelize.col('similarity')
+  } else {
+    finalField = field
+  }
+
+  return [ [ finalField, direction ], lastSort ]
+}
+
+function getAdminUsersSort (value: string): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  let finalField: string | ReturnType<typeof Sequelize.col>
+
+  if (field === 'videoQuotaUsed') { // Users list
+    finalField = Sequelize.col('videoQuotaUsed')
+  } else {
+    finalField = field
+  }
+
+  const nullPolicy = direction === 'ASC'
+    ? 'NULLS FIRST'
+    : 'NULLS LAST'
+
+  // FIXME: typings
+  return [ [ finalField as any, direction, nullPolicy ], [ 'id', 'ASC' ] ]
+}
+
+function getPlaylistSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  if (field.toLowerCase() === 'name') {
+    return [ [ 'displayName', direction ], lastSort ]
+  }
+
+  return getSort(value, lastSort)
+}
+
+function getCommentSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  if (field === 'totalReplies') {
+    return [
+      [ Sequelize.literal('"totalReplies"'), direction ],
+      lastSort
+    ]
+  }
+
+  return getSort(value, lastSort)
+}
+
+function getVideoSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  if (field.toLowerCase() === 'trending') { // Sort by aggregation
+    return [
+      [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoViews.views')), '0'), direction ],
+
+      [ Sequelize.col('VideoModel.views'), direction ],
+
+      lastSort
+    ]
+  } else if (field === 'publishedAt') {
+    return [
+      [ 'ScheduleVideoUpdate', 'updateAt', direction + ' NULLS LAST' ],
+
+      [ Sequelize.col('VideoModel.publishedAt'), direction ],
+
+      lastSort
+    ]
+  }
+
+  let finalField: string | ReturnType<typeof Sequelize.col>
+
+  // Alias
+  if (field.toLowerCase() === 'match') { // Search
+    finalField = Sequelize.col('similarity')
+  } else {
+    finalField = field
+  }
+
+  const firstSort: OrderItem = typeof finalField === 'string'
+    ? finalField.split('.').concat([ direction ]) as OrderItem
+    : [ finalField, direction ]
+
+  return [ firstSort, lastSort ]
+}
+
+function getBlacklistSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  const videoFields = new Set([ 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid' ])
+
+  if (videoFields.has(field)) {
+    return [
+      [ literal(`"Video.${field}" ${direction}`) ],
+      lastSort
+    ] as OrderItem[]
+  }
+
+  return getSort(value, lastSort)
+}
+
+function getInstanceFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+
+  if (field === 'redundancyAllowed') {
+    return [
+      [ 'ActorFollowing.Server.redundancyAllowed', direction ],
+      lastSort
+    ]
+  }
+
+  return getSort(value, lastSort)
+}
+
+function getChannelSyncSort (value: string): OrderItem[] {
+  const { direction, field } = buildSortDirectionAndField(value)
+  if (field.toLowerCase() === 'videochannel') {
+    return [
+      [ literal('"VideoChannel.name"'), direction ]
+    ]
+  }
+  return [ [ field, direction ] ]
+}
+
+function buildSortDirectionAndField (value: string) {
+  let field: string
+  let direction: 'ASC' | 'DESC'
+
+  if (value.substring(0, 1) === '-') {
+    direction = 'DESC'
+    field = value.substring(1)
+  } else {
+    direction = 'ASC'
+    field = value
+  }
+
+  return { direction, field }
+}
+
+export {
+  buildSortDirectionAndField,
+  getPlaylistSort,
+  getSort,
+  getCommentSort,
+  getAdminUsersSort,
+  getVideoSort,
+  getBlacklistSort,
+  getChannelSyncSort,
+  getInstanceFollowsSort
+}
diff --git a/server/models/shared/sql.ts b/server/models/shared/sql.ts
new file mode 100644
index 000000000..5aaeb49f0
--- /dev/null
+++ b/server/models/shared/sql.ts
@@ -0,0 +1,68 @@
+import { literal, Model, ModelStatic } from 'sequelize'
+import { forceNumber } from '@shared/core-utils'
+import { AttributesOnly } from '@shared/typescript-utils'
+
+function buildLocalAccountIdsIn () {
+  return literal(
+    '(SELECT "account"."id" FROM "account" INNER JOIN "actor" ON "actor"."id" = "account"."actorId" AND "actor"."serverId" IS NULL)'
+  )
+}
+
+function buildLocalActorIdsIn () {
+  return literal(
+    '(SELECT "actor"."id" FROM "actor" WHERE "actor"."serverId" IS NULL)'
+  )
+}
+
+function buildBlockedAccountSQL (blockerIds: number[]) {
+  const blockerIdsString = blockerIds.join(', ')
+
+  return 'SELECT "targetAccountId" AS "id" FROM "accountBlocklist" WHERE "accountId" IN (' + blockerIdsString + ')' +
+    ' UNION ' +
+    'SELECT "account"."id" AS "id" FROM account INNER JOIN "actor" ON account."actorId" = actor.id ' +
+    'INNER JOIN "serverBlocklist" ON "actor"."serverId" = "serverBlocklist"."targetServerId" ' +
+    'WHERE "serverBlocklist"."accountId" IN (' + blockerIdsString + ')'
+}
+
+function buildServerIdsFollowedBy (actorId: any) {
+  const actorIdNumber = forceNumber(actorId)
+
+  return '(' +
+    'SELECT "actor"."serverId" FROM "actorFollow" ' +
+    'INNER JOIN "actor" ON actor.id = "actorFollow"."targetActorId" ' +
+    'WHERE "actorFollow"."actorId" = ' + actorIdNumber +
+    ')'
+}
+
+function buildSQLAttributes<M extends Model> (options: {
+  model: ModelStatic<M>
+  tableName: string
+
+  excludeAttributes?: Exclude<keyof AttributesOnly<M>, symbol>[]
+  aliasPrefix?: string
+}) {
+  const { model, tableName, aliasPrefix, excludeAttributes } = options
+
+  const attributes = Object.keys(model.getAttributes()) as Exclude<keyof AttributesOnly<M>, symbol>[]
+
+  return attributes
+    .filter(a => {
+      if (!excludeAttributes) return true
+      if (excludeAttributes.includes(a)) return false
+
+      return true
+    })
+    .map(a => {
+      return `"${tableName}"."${a}" AS "${aliasPrefix || ''}${a}"`
+    })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  buildSQLAttributes,
+  buildBlockedAccountSQL,
+  buildServerIdsFollowedBy,
+  buildLocalAccountIdsIn,
+  buildLocalActorIdsIn
+}
diff --git a/server/models/shared/update.ts b/server/models/shared/update.ts
index d338211e3..d02c4535d 100644
--- a/server/models/shared/update.ts
+++ b/server/models/shared/update.ts
@@ -1,9 +1,15 @@
-import { QueryTypes, Transaction } from 'sequelize'
-import { sequelizeTypescript } from '@server/initializers/database'
+import { QueryTypes, Sequelize, Transaction } from 'sequelize'
 
 // Sequelize always skip the update if we only update updatedAt field
-function setAsUpdated (table: string, id: number, transaction?: Transaction) {
-  return sequelizeTypescript.query(
+function setAsUpdated (options: {
+  sequelize: Sequelize
+  table: string
+  id: number
+  transaction?: Transaction
+}) {
+  const { sequelize, table, id, transaction } = options
+
+  return sequelize.query(
     `UPDATE "${table}" SET "updatedAt" = :updatedAt WHERE id = :id`,
     {
       replacements: { table, id, updatedAt: new Date() },