aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/models/shared
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2023-07-31 14:34:36 +0200
committerChocobozzz <me@florianbigard.com>2023-08-11 15:02:33 +0200
commit3a4992633ee62d5edfbb484d9c6bcb3cf158489d (patch)
treee4510b39bdac9c318fdb4b47018d08f15368b8f0 /server/models/shared
parent04d1da5621d25d59bd5fa1543b725c497bf5d9a8 (diff)
downloadPeerTube-3a4992633ee62d5edfbb484d9c6bcb3cf158489d.tar.gz
PeerTube-3a4992633ee62d5edfbb484d9c6bcb3cf158489d.tar.zst
PeerTube-3a4992633ee62d5edfbb484d9c6bcb3cf158489d.zip
Migrate server to ESM
Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports)
Diffstat (limited to 'server/models/shared')
-rw-r--r--server/models/shared/abstract-run-query.ts32
-rw-r--r--server/models/shared/index.ts8
-rw-r--r--server/models/shared/model-builder.ts118
-rw-r--r--server/models/shared/model-cache.ts90
-rw-r--r--server/models/shared/query.ts82
-rw-r--r--server/models/shared/sequelize-helpers.ts39
-rw-r--r--server/models/shared/sort.ts146
-rw-r--r--server/models/shared/sql.ts68
-rw-r--r--server/models/shared/update.ts34
9 files changed, 0 insertions, 617 deletions
diff --git a/server/models/shared/abstract-run-query.ts b/server/models/shared/abstract-run-query.ts
deleted file mode 100644
index 7f27a0c4b..000000000
--- a/server/models/shared/abstract-run-query.ts
+++ /dev/null
@@ -1,32 +0,0 @@
1import { QueryTypes, Sequelize, Transaction } from 'sequelize'
2
3/**
4 *
5 * Abstract builder to run video SQL queries
6 *
7 */
8
9export class AbstractRunQuery {
10 protected query: string
11 protected replacements: any = {}
12
13 constructor (protected readonly sequelize: Sequelize) {
14
15 }
16
17 protected runQuery (options: { nest?: boolean, transaction?: Transaction, logging?: boolean } = {}) {
18 const queryOptions = {
19 transaction: options.transaction,
20 logging: options.logging,
21 replacements: this.replacements,
22 type: QueryTypes.SELECT as QueryTypes.SELECT,
23 nest: options.nest ?? false
24 }
25
26 return this.sequelize.query<any>(this.query, queryOptions)
27 }
28
29 protected buildSelect (entities: string[]) {
30 return `SELECT ${entities.join(', ')} `
31 }
32}
diff --git a/server/models/shared/index.ts b/server/models/shared/index.ts
deleted file mode 100644
index 5a7621e4d..000000000
--- a/server/models/shared/index.ts
+++ /dev/null
@@ -1,8 +0,0 @@
1export * from './abstract-run-query'
2export * from './model-builder'
3export * from './model-cache'
4export * from './query'
5export * from './sequelize-helpers'
6export * from './sort'
7export * from './sql'
8export * from './update'
diff --git a/server/models/shared/model-builder.ts b/server/models/shared/model-builder.ts
deleted file mode 100644
index 07f7c4038..000000000
--- a/server/models/shared/model-builder.ts
+++ /dev/null
@@ -1,118 +0,0 @@
1import { isPlainObject } from 'lodash'
2import { Model as SequelizeModel, ModelStatic, Sequelize } from 'sequelize'
3import { logger } from '@server/helpers/logger'
4
5/**
6 *
7 * Build Sequelize models from sequelize raw query (that must use { nest: true } options)
8 *
9 * In order to sequelize to correctly build the JSON this class will ingest,
10 * the columns selected in the raw query should be in the following form:
11 * * All tables must be Pascal Cased (for example "VideoChannel")
12 * * Root table must end with `Model` (for example "VideoCommentModel")
13 * * Joined tables must contain the origin table name + '->JoinedTable'. For example:
14 * * "Actor" is joined to "Account": "Actor" table must be renamed "Account->Actor"
15 * * "Account->Actor" is joined to "Server": "Server" table must be renamed to "Account->Actor->Server"
16 * * Selected columns must be renamed to contain the JSON path:
17 * * "videoComment"."id": "VideoCommentModel"."id"
18 * * "Account"."Actor"."Server"."id": "Account.Actor.Server.id"
19 * * All tables must contain the row id
20 */
21
22export class ModelBuilder <T extends SequelizeModel> {
23 private readonly modelRegistry = new Map<string, T>()
24
25 constructor (private readonly sequelize: Sequelize) {
26
27 }
28
29 createModels (jsonArray: any[], baseModelName: string): T[] {
30 const result: T[] = []
31
32 for (const json of jsonArray) {
33 const { created, model } = this.createModel(json, baseModelName, json.id + '.' + baseModelName)
34
35 if (created) result.push(model)
36 }
37
38 return result
39 }
40
41 private createModel (json: any, modelName: string, keyPath: string) {
42 if (!json.id) return { created: false, model: null }
43
44 const { created, model } = this.createOrFindModel(json, modelName, keyPath)
45
46 for (const key of Object.keys(json)) {
47 const value = json[key]
48 if (!value) continue
49
50 // Child model
51 if (isPlainObject(value)) {
52 const { created, model: subModel } = this.createModel(value, key, keyPath + '.' + json.id + '.' + key)
53 if (!created || !subModel) continue
54
55 const Model = this.findModelBuilder(modelName)
56 const association = Model.associations[key]
57
58 if (!association) {
59 logger.error('Cannot find association %s of model %s', key, modelName, { associations: Object.keys(Model.associations) })
60 continue
61 }
62
63 if (association.isMultiAssociation) {
64 if (!Array.isArray(model[key])) model[key] = []
65
66 model[key].push(subModel)
67 } else {
68 model[key] = subModel
69 }
70 }
71 }
72
73 return { created, model }
74 }
75
76 private createOrFindModel (json: any, modelName: string, keyPath: string) {
77 const registryKey = this.getModelRegistryKey(json, keyPath)
78 if (this.modelRegistry.has(registryKey)) {
79 return {
80 created: false,
81 model: this.modelRegistry.get(registryKey)
82 }
83 }
84
85 const Model = this.findModelBuilder(modelName)
86
87 if (!Model) {
88 logger.error(
89 'Cannot build model %s that does not exist', this.buildSequelizeModelName(modelName),
90 { existing: this.sequelize.modelManager.all.map(m => m.name) }
91 )
92 return { created: false, model: null }
93 }
94
95 const model = Model.build(json, { raw: true, isNewRecord: false })
96
97 this.modelRegistry.set(registryKey, model)
98
99 return { created: true, model }
100 }
101
102 private findModelBuilder (modelName: string) {
103 return this.sequelize.modelManager.getModel(this.buildSequelizeModelName(modelName)) as ModelStatic<T>
104 }
105
106 private buildSequelizeModelName (modelName: string) {
107 if (modelName === 'Avatars') return 'ActorImageModel'
108 if (modelName === 'ActorFollowing') return 'ActorModel'
109 if (modelName === 'ActorFollower') return 'ActorModel'
110 if (modelName === 'FlaggedAccount') return 'AccountModel'
111
112 return modelName + 'Model'
113 }
114
115 private getModelRegistryKey (json: any, keyPath: string) {
116 return keyPath + json.id
117 }
118}
diff --git a/server/models/shared/model-cache.ts b/server/models/shared/model-cache.ts
deleted file mode 100644
index 3651267e7..000000000
--- a/server/models/shared/model-cache.ts
+++ /dev/null
@@ -1,90 +0,0 @@
1import { Model } from 'sequelize-typescript'
2import { logger } from '@server/helpers/logger'
3
4type ModelCacheType =
5 'local-account-name'
6 | 'local-actor-name'
7 | 'local-actor-url'
8 | 'load-video-immutable-id'
9 | 'load-video-immutable-url'
10
11type DeleteKey =
12 'video'
13
14class ModelCache {
15
16 private static instance: ModelCache
17
18 private readonly localCache: { [id in ModelCacheType]: Map<string, any> } = {
19 'local-account-name': new Map(),
20 'local-actor-name': new Map(),
21 'local-actor-url': new Map(),
22 'load-video-immutable-id': new Map(),
23 'load-video-immutable-url': new Map()
24 }
25
26 private readonly deleteIds: {
27 [deleteKey in DeleteKey]: Map<number, { cacheType: ModelCacheType, key: string }[]>
28 } = {
29 video: new Map()
30 }
31
32 private constructor () {
33 }
34
35 static get Instance () {
36 return this.instance || (this.instance = new this())
37 }
38
39 doCache<T extends Model> (options: {
40 cacheType: ModelCacheType
41 key: string
42 fun: () => Promise<T>
43 whitelist?: () => boolean
44 deleteKey?: DeleteKey
45 }) {
46 const { cacheType, key, fun, whitelist, deleteKey } = options
47
48 if (whitelist && whitelist() !== true) return fun()
49
50 const cache = this.localCache[cacheType]
51
52 if (cache.has(key)) {
53 logger.debug('Model cache hit for %s -> %s.', cacheType, key)
54 return Promise.resolve<T>(cache.get(key))
55 }
56
57 return fun().then(m => {
58 if (!m) return m
59
60 if (!whitelist || whitelist()) cache.set(key, m)
61
62 if (deleteKey) {
63 const map = this.deleteIds[deleteKey]
64 if (!map.has(m.id)) map.set(m.id, [])
65
66 const a = map.get(m.id)
67 a.push({ cacheType, key })
68 }
69
70 return m
71 })
72 }
73
74 invalidateCache (deleteKey: DeleteKey, modelId: number) {
75 const map = this.deleteIds[deleteKey]
76
77 if (!map.has(modelId)) return
78
79 for (const toDelete of map.get(modelId)) {
80 logger.debug('Removing %s -> %d of model cache %s -> %s.', deleteKey, modelId, toDelete.cacheType, toDelete.key)
81 this.localCache[toDelete.cacheType].delete(toDelete.key)
82 }
83
84 map.delete(modelId)
85 }
86}
87
88export {
89 ModelCache
90}
diff --git a/server/models/shared/query.ts b/server/models/shared/query.ts
deleted file mode 100644
index 934acc21f..000000000
--- a/server/models/shared/query.ts
+++ /dev/null
@@ -1,82 +0,0 @@
1import { BindOrReplacements, Op, QueryTypes, Sequelize } from 'sequelize'
2import validator from 'validator'
3import { forceNumber } from '@shared/core-utils'
4
5function doesExist (sequelize: Sequelize, query: string, bind?: BindOrReplacements) {
6 const options = {
7 type: QueryTypes.SELECT as QueryTypes.SELECT,
8 bind,
9 raw: true
10 }
11
12 return sequelize.query(query, options)
13 .then(results => results.length === 1)
14}
15
16function createSimilarityAttribute (col: string, value: string) {
17 return Sequelize.fn(
18 'similarity',
19
20 searchTrigramNormalizeCol(col),
21
22 searchTrigramNormalizeValue(value)
23 )
24}
25
26function buildWhereIdOrUUID (id: number | string) {
27 return validator.isInt('' + id) ? { id } : { uuid: id }
28}
29
30function parseAggregateResult (result: any) {
31 if (!result) return 0
32
33 const total = forceNumber(result)
34 if (isNaN(total)) return 0
35
36 return total
37}
38
39function parseRowCountResult (result: any) {
40 if (result.length !== 0) return result[0].total
41
42 return 0
43}
44
45function createSafeIn (sequelize: Sequelize, toEscape: (string | number)[], additionalUnescaped: string[] = []) {
46 return toEscape.map(t => {
47 return t === null
48 ? null
49 : sequelize.escape('' + t)
50 }).concat(additionalUnescaped).join(', ')
51}
52
53function searchAttribute (sourceField?: string, targetField?: string) {
54 if (!sourceField) return {}
55
56 return {
57 [targetField]: {
58 // FIXME: ts error
59 [Op.iLike as any]: `%${sourceField}%`
60 }
61 }
62}
63
64export {
65 doesExist,
66 createSimilarityAttribute,
67 buildWhereIdOrUUID,
68 parseAggregateResult,
69 parseRowCountResult,
70 createSafeIn,
71 searchAttribute
72}
73
74// ---------------------------------------------------------------------------
75
76function searchTrigramNormalizeValue (value: string) {
77 return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', value))
78}
79
80function searchTrigramNormalizeCol (col: string) {
81 return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', Sequelize.col(col)))
82}
diff --git a/server/models/shared/sequelize-helpers.ts b/server/models/shared/sequelize-helpers.ts
deleted file mode 100644
index 7af8471dc..000000000
--- a/server/models/shared/sequelize-helpers.ts
+++ /dev/null
@@ -1,39 +0,0 @@
1import { Sequelize } from 'sequelize'
2
3function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
4 if (!model.createdAt || !model.updatedAt) {
5 throw new Error('Miss createdAt & updatedAt attributes to model')
6 }
7
8 const now = Date.now()
9 const createdAtTime = model.createdAt.getTime()
10 const updatedAtTime = model.updatedAt.getTime()
11
12 return (now - createdAtTime) > refreshInterval && (now - updatedAtTime) > refreshInterval
13}
14
15function throwIfNotValid (value: any, validator: (value: any) => boolean, fieldName = 'value', nullable = false) {
16 if (nullable && (value === null || value === undefined)) return
17
18 if (validator(value) === false) {
19 throw new Error(`"${value}" is not a valid ${fieldName}.`)
20 }
21}
22
23function buildTrigramSearchIndex (indexName: string, attribute: string) {
24 return {
25 name: indexName,
26 // FIXME: gin_trgm_ops is not taken into account in Sequelize 6, so adding it ourselves in the literal function
27 fields: [ Sequelize.literal('lower(immutable_unaccent(' + attribute + ')) gin_trgm_ops') as any ],
28 using: 'gin',
29 operator: 'gin_trgm_ops'
30 }
31}
32
33// ---------------------------------------------------------------------------
34
35export {
36 throwIfNotValid,
37 buildTrigramSearchIndex,
38 isOutdated
39}
diff --git a/server/models/shared/sort.ts b/server/models/shared/sort.ts
deleted file mode 100644
index d923072f2..000000000
--- a/server/models/shared/sort.ts
+++ /dev/null
@@ -1,146 +0,0 @@
1import { literal, OrderItem, Sequelize } from 'sequelize'
2
3// Translate for example "-name" to [ [ 'name', 'DESC' ], [ 'id', 'ASC' ] ]
4function getSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
5 const { direction, field } = buildSortDirectionAndField(value)
6
7 let finalField: string | ReturnType<typeof Sequelize.col>
8
9 if (field.toLowerCase() === 'match') { // Search
10 finalField = Sequelize.col('similarity')
11 } else {
12 finalField = field
13 }
14
15 return [ [ finalField, direction ], lastSort ]
16}
17
18function getAdminUsersSort (value: string): OrderItem[] {
19 const { direction, field } = buildSortDirectionAndField(value)
20
21 let finalField: string | ReturnType<typeof Sequelize.col>
22
23 if (field === 'videoQuotaUsed') { // Users list
24 finalField = Sequelize.col('videoQuotaUsed')
25 } else {
26 finalField = field
27 }
28
29 const nullPolicy = direction === 'ASC'
30 ? 'NULLS FIRST'
31 : 'NULLS LAST'
32
33 // FIXME: typings
34 return [ [ finalField as any, direction, nullPolicy ], [ 'id', 'ASC' ] ]
35}
36
37function getPlaylistSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
38 const { direction, field } = buildSortDirectionAndField(value)
39
40 if (field.toLowerCase() === 'name') {
41 return [ [ 'displayName', direction ], lastSort ]
42 }
43
44 return getSort(value, lastSort)
45}
46
47function getVideoSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
48 const { direction, field } = buildSortDirectionAndField(value)
49
50 if (field.toLowerCase() === 'trending') { // Sort by aggregation
51 return [
52 [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoViews.views')), '0'), direction ],
53
54 [ Sequelize.col('VideoModel.views'), direction ],
55
56 lastSort
57 ]
58 } else if (field === 'publishedAt') {
59 return [
60 [ 'ScheduleVideoUpdate', 'updateAt', direction + ' NULLS LAST' ],
61
62 [ Sequelize.col('VideoModel.publishedAt'), direction ],
63
64 lastSort
65 ]
66 }
67
68 let finalField: string | ReturnType<typeof Sequelize.col>
69
70 // Alias
71 if (field.toLowerCase() === 'match') { // Search
72 finalField = Sequelize.col('similarity')
73 } else {
74 finalField = field
75 }
76
77 const firstSort: OrderItem = typeof finalField === 'string'
78 ? finalField.split('.').concat([ direction ]) as OrderItem
79 : [ finalField, direction ]
80
81 return [ firstSort, lastSort ]
82}
83
84function getBlacklistSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
85 const { direction, field } = buildSortDirectionAndField(value)
86
87 const videoFields = new Set([ 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid' ])
88
89 if (videoFields.has(field)) {
90 return [
91 [ literal(`"Video.${field}" ${direction}`) ],
92 lastSort
93 ] as OrderItem[]
94 }
95
96 return getSort(value, lastSort)
97}
98
99function getInstanceFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] {
100 const { direction, field } = buildSortDirectionAndField(value)
101
102 if (field === 'redundancyAllowed') {
103 return [
104 [ 'ActorFollowing.Server.redundancyAllowed', direction ],
105 lastSort
106 ]
107 }
108
109 return getSort(value, lastSort)
110}
111
112function getChannelSyncSort (value: string): OrderItem[] {
113 const { direction, field } = buildSortDirectionAndField(value)
114 if (field.toLowerCase() === 'videochannel') {
115 return [
116 [ literal('"VideoChannel.name"'), direction ]
117 ]
118 }
119 return [ [ field, direction ] ]
120}
121
122function buildSortDirectionAndField (value: string) {
123 let field: string
124 let direction: 'ASC' | 'DESC'
125
126 if (value.substring(0, 1) === '-') {
127 direction = 'DESC'
128 field = value.substring(1)
129 } else {
130 direction = 'ASC'
131 field = value
132 }
133
134 return { direction, field }
135}
136
137export {
138 buildSortDirectionAndField,
139 getPlaylistSort,
140 getSort,
141 getAdminUsersSort,
142 getVideoSort,
143 getBlacklistSort,
144 getChannelSyncSort,
145 getInstanceFollowsSort
146}
diff --git a/server/models/shared/sql.ts b/server/models/shared/sql.ts
deleted file mode 100644
index 5aaeb49f0..000000000
--- a/server/models/shared/sql.ts
+++ /dev/null
@@ -1,68 +0,0 @@
1import { literal, Model, ModelStatic } from 'sequelize'
2import { forceNumber } from '@shared/core-utils'
3import { AttributesOnly } from '@shared/typescript-utils'
4
5function buildLocalAccountIdsIn () {
6 return literal(
7 '(SELECT "account"."id" FROM "account" INNER JOIN "actor" ON "actor"."id" = "account"."actorId" AND "actor"."serverId" IS NULL)'
8 )
9}
10
11function buildLocalActorIdsIn () {
12 return literal(
13 '(SELECT "actor"."id" FROM "actor" WHERE "actor"."serverId" IS NULL)'
14 )
15}
16
17function buildBlockedAccountSQL (blockerIds: number[]) {
18 const blockerIdsString = blockerIds.join(', ')
19
20 return 'SELECT "targetAccountId" AS "id" FROM "accountBlocklist" WHERE "accountId" IN (' + blockerIdsString + ')' +
21 ' UNION ' +
22 'SELECT "account"."id" AS "id" FROM account INNER JOIN "actor" ON account."actorId" = actor.id ' +
23 'INNER JOIN "serverBlocklist" ON "actor"."serverId" = "serverBlocklist"."targetServerId" ' +
24 'WHERE "serverBlocklist"."accountId" IN (' + blockerIdsString + ')'
25}
26
27function buildServerIdsFollowedBy (actorId: any) {
28 const actorIdNumber = forceNumber(actorId)
29
30 return '(' +
31 'SELECT "actor"."serverId" FROM "actorFollow" ' +
32 'INNER JOIN "actor" ON actor.id = "actorFollow"."targetActorId" ' +
33 'WHERE "actorFollow"."actorId" = ' + actorIdNumber +
34 ')'
35}
36
37function buildSQLAttributes<M extends Model> (options: {
38 model: ModelStatic<M>
39 tableName: string
40
41 excludeAttributes?: Exclude<keyof AttributesOnly<M>, symbol>[]
42 aliasPrefix?: string
43}) {
44 const { model, tableName, aliasPrefix, excludeAttributes } = options
45
46 const attributes = Object.keys(model.getAttributes()) as Exclude<keyof AttributesOnly<M>, symbol>[]
47
48 return attributes
49 .filter(a => {
50 if (!excludeAttributes) return true
51 if (excludeAttributes.includes(a)) return false
52
53 return true
54 })
55 .map(a => {
56 return `"${tableName}"."${a}" AS "${aliasPrefix || ''}${a}"`
57 })
58}
59
60// ---------------------------------------------------------------------------
61
62export {
63 buildSQLAttributes,
64 buildBlockedAccountSQL,
65 buildServerIdsFollowedBy,
66 buildLocalAccountIdsIn,
67 buildLocalActorIdsIn
68}
diff --git a/server/models/shared/update.ts b/server/models/shared/update.ts
deleted file mode 100644
index 96db43730..000000000
--- a/server/models/shared/update.ts
+++ /dev/null
@@ -1,34 +0,0 @@
1import { QueryTypes, Sequelize, Transaction } from 'sequelize'
2
3const updating = new Set<string>()
4
5// Sequelize always skip the update if we only update updatedAt field
6async function setAsUpdated (options: {
7 sequelize: Sequelize
8 table: string
9 id: number
10 transaction?: Transaction
11}) {
12 const { sequelize, table, id, transaction } = options
13 const key = table + '-' + id
14
15 if (updating.has(key)) return
16 updating.add(key)
17
18 try {
19 await sequelize.query(
20 `UPDATE "${table}" SET "updatedAt" = :updatedAt WHERE id = :id`,
21 {
22 replacements: { table, id, updatedAt: new Date() },
23 type: QueryTypes.UPDATE,
24 transaction
25 }
26 )
27 } finally {
28 updating.delete(key)
29 }
30}
31
32export {
33 setAsUpdated
34}
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-28 17:16:28 +0000