]> git.immae.eu Git - github/Chocobozzz/PeerTube.git/blob - server/models/video/sql/videos-id-list-query-builder.ts
Add ability to search by uuids/actor names
[github/Chocobozzz/PeerTube.git] / server / models / video / sql / videos-id-list-query-builder.ts
1 import { Sequelize } from 'sequelize'
2 import validator from 'validator'
3 import { exists } from '@server/helpers/custom-validators/misc'
4 import { WEBSERVER } from '@server/initializers/constants'
5 import { buildDirectionAndField, createSafeIn } from '@server/models/utils'
6 import { MUserAccountId, MUserId } from '@server/types/models'
7 import { VideoFilter, VideoPrivacy, VideoState } from '@shared/models'
8 import { AbstractVideosQueryBuilder } from './shared/abstract-videos-query-builder'
9
10 /**
11 *
12 * Build videos list SQL query to fetch rows
13 *
14 */
15
16 export type BuildVideosListQueryOptions = {
17 attributes?: string[]
18
19 serverAccountId: number
20 followerActorId: number
21 includeLocalVideos: boolean
22
23 count: number
24 start: number
25 sort: string
26
27 nsfw?: boolean
28 filter?: VideoFilter
29 host?: string
30 isLive?: boolean
31
32 categoryOneOf?: number[]
33 licenceOneOf?: number[]
34 languageOneOf?: string[]
35 tagsOneOf?: string[]
36 tagsAllOf?: string[]
37
38 uuids?: string[]
39
40 withFiles?: boolean
41
42 accountId?: number
43 videoChannelId?: number
44
45 videoPlaylistId?: number
46
47 trendingAlgorithm?: string // best, hot, or any other algorithm implemented
48 trendingDays?: number
49
50 user?: MUserAccountId
51 historyOfUser?: MUserId
52
53 startDate?: string // ISO 8601
54 endDate?: string // ISO 8601
55 originallyPublishedStartDate?: string
56 originallyPublishedEndDate?: string
57
58 durationMin?: number // seconds
59 durationMax?: number // seconds
60
61 search?: string
62
63 isCount?: boolean
64
65 group?: string
66 having?: string
67 }
68
69 export class VideosIdListQueryBuilder extends AbstractVideosQueryBuilder {
70 protected replacements: any = {}
71
72 private attributes: string[]
73 private joins: string[] = []
74
75 private readonly and: string[] = []
76
77 private readonly cte: string[] = []
78
79 private group = ''
80 private having = ''
81
82 private sort = ''
83 private limit = ''
84 private offset = ''
85
86 constructor (protected readonly sequelize: Sequelize) {
87 super()
88 }
89
90 queryVideoIds (options: BuildVideosListQueryOptions) {
91 this.buildIdsListQuery(options)
92
93 return this.runQuery()
94 }
95
96 countVideoIds (countOptions: BuildVideosListQueryOptions): Promise<number> {
97 this.buildIdsListQuery(countOptions)
98
99 return this.runQuery().then(rows => rows.length !== 0 ? rows[0].total : 0)
100 }
101
102 getIdsListQueryAndSort (options: BuildVideosListQueryOptions) {
103 this.buildIdsListQuery(options)
104 return { query: this.query, sort: this.sort, replacements: this.replacements }
105 }
106
107 private buildIdsListQuery (options: BuildVideosListQueryOptions) {
108 this.attributes = options.attributes || [ '"video"."id"' ]
109
110 if (options.group) this.group = options.group
111 if (options.having) this.having = options.having
112
113 this.joins = this.joins.concat([
114 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId"',
115 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId"',
116 'INNER JOIN "actor" "accountActor" ON "account"."actorId" = "accountActor"."id"'
117 ])
118
119 this.whereNotBlacklisted()
120
121 if (options.serverAccountId) {
122 this.whereNotBlocked(options.serverAccountId, options.user)
123 }
124
125 // Only list public/published videos
126 if (!options.filter || (options.filter !== 'all-local' && options.filter !== 'all')) {
127 this.whereStateAndPrivacyAvailable(options.user)
128 }
129
130 if (options.videoPlaylistId) {
131 this.joinPlaylist(options.videoPlaylistId)
132 }
133
134 if (options.filter && (options.filter === 'local' || options.filter === 'all-local')) {
135 this.whereOnlyLocal()
136 }
137
138 if (options.host) {
139 this.whereHost(options.host)
140 }
141
142 if (options.accountId) {
143 this.whereAccountId(options.accountId)
144 }
145
146 if (options.videoChannelId) {
147 this.whereChannelId(options.videoChannelId)
148 }
149
150 if (options.followerActorId) {
151 this.whereFollowerActorId(options.followerActorId, options.includeLocalVideos)
152 }
153
154 if (options.withFiles === true) {
155 this.whereFileExists()
156 }
157
158 if (options.tagsOneOf) {
159 this.whereTagsOneOf(options.tagsOneOf)
160 }
161
162 if (options.tagsAllOf) {
163 this.whereTagsAllOf(options.tagsAllOf)
164 }
165
166 if (options.uuids) {
167 this.whereUUIDs(options.uuids)
168 }
169
170 if (options.nsfw === true) {
171 this.whereNSFW()
172 } else if (options.nsfw === false) {
173 this.whereSFW()
174 }
175
176 if (options.isLive === true) {
177 this.whereLive()
178 } else if (options.isLive === false) {
179 this.whereVOD()
180 }
181
182 if (options.categoryOneOf) {
183 this.whereCategoryOneOf(options.categoryOneOf)
184 }
185
186 if (options.licenceOneOf) {
187 this.whereLicenceOneOf(options.licenceOneOf)
188 }
189
190 if (options.languageOneOf) {
191 this.whereLanguageOneOf(options.languageOneOf)
192 }
193
194 // We don't exclude results in this so if we do a count we don't need to add this complex clause
195 if (options.isCount !== true) {
196 if (options.trendingDays) {
197 this.groupForTrending(options.trendingDays)
198 } else if ([ 'best', 'hot' ].includes(options.trendingAlgorithm)) {
199 this.groupForHotOrBest(options.trendingAlgorithm, options.user)
200 }
201 }
202
203 if (options.historyOfUser) {
204 this.joinHistory(options.historyOfUser.id)
205 }
206
207 if (options.startDate) {
208 this.whereStartDate(options.startDate)
209 }
210
211 if (options.endDate) {
212 this.whereEndDate(options.endDate)
213 }
214
215 if (options.originallyPublishedStartDate) {
216 this.whereOriginallyPublishedStartDate(options.originallyPublishedStartDate)
217 }
218
219 if (options.originallyPublishedEndDate) {
220 this.whereOriginallyPublishedEndDate(options.originallyPublishedEndDate)
221 }
222
223 if (options.durationMin) {
224 this.whereDurationMin(options.durationMin)
225 }
226
227 if (options.durationMax) {
228 this.whereDurationMax(options.durationMax)
229 }
230
231 this.whereSearch(options.search)
232
233 if (options.isCount === true) {
234 this.setCountAttribute()
235 } else {
236 if (exists(options.sort)) {
237 this.setSort(options.sort)
238 }
239
240 if (exists(options.count)) {
241 this.setLimit(options.count)
242 }
243
244 if (exists(options.start)) {
245 this.setOffset(options.start)
246 }
247 }
248
249 const cteString = this.cte.length !== 0
250 ? `WITH ${this.cte.join(', ')} `
251 : ''
252
253 this.query = cteString +
254 'SELECT ' + this.attributes.join(', ') + ' ' +
255 'FROM "video" ' + this.joins.join(' ') + ' ' +
256 'WHERE ' + this.and.join(' AND ') + ' ' +
257 this.group + ' ' +
258 this.having + ' ' +
259 this.sort + ' ' +
260 this.limit + ' ' +
261 this.offset
262 }
263
264 private setCountAttribute () {
265 this.attributes = [ 'COUNT(*) as "total"' ]
266 }
267
268 private joinHistory (userId: number) {
269 this.joins.push('INNER JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId"')
270
271 this.and.push('"userVideoHistory"."userId" = :historyOfUser')
272
273 this.replacements.historyOfUser = userId
274 }
275
276 private joinPlaylist (playlistId: number) {
277 this.joins.push(
278 'INNER JOIN "videoPlaylistElement" "video"."id" = "videoPlaylistElement"."videoId" ' +
279 'AND "videoPlaylistElement"."videoPlaylistId" = :videoPlaylistId'
280 )
281
282 this.replacements.videoPlaylistId = playlistId
283 }
284
285 private whereStateAndPrivacyAvailable (user?: MUserAccountId) {
286 this.and.push(
287 `("video"."state" = ${VideoState.PUBLISHED} OR ` +
288 `("video"."state" = ${VideoState.TO_TRANSCODE} AND "video"."waitTranscoding" IS false))`
289 )
290
291 if (user) {
292 this.and.push(
293 `("video"."privacy" = ${VideoPrivacy.PUBLIC} OR "video"."privacy" = ${VideoPrivacy.INTERNAL})`
294 )
295 } else { // Or only public videos
296 this.and.push(
297 `"video"."privacy" = ${VideoPrivacy.PUBLIC}`
298 )
299 }
300 }
301
302 private whereOnlyLocal () {
303 this.and.push('"video"."remote" IS FALSE')
304 }
305
306 private whereHost (host: string) {
307 // Local instance
308 if (host === WEBSERVER.HOST) {
309 this.and.push('"accountActor"."serverId" IS NULL')
310 return
311 }
312
313 this.joins.push('INNER JOIN "server" ON "server"."id" = "accountActor"."serverId"')
314
315 this.and.push('"server"."host" = :host')
316 this.replacements.host = host
317 }
318
319 private whereAccountId (accountId: number) {
320 this.and.push('"account"."id" = :accountId')
321 this.replacements.accountId = accountId
322 }
323
324 private whereChannelId (channelId: number) {
325 this.and.push('"videoChannel"."id" = :videoChannelId')
326 this.replacements.videoChannelId = channelId
327 }
328
329 private whereFollowerActorId (followerActorId: number, includeLocalVideos: boolean) {
330 let query =
331 '(' +
332 ' EXISTS (' + // Videos shared by actors we follow
333 ' SELECT 1 FROM "videoShare" ' +
334 ' INNER JOIN "actorFollow" "actorFollowShare" ON "actorFollowShare"."targetActorId" = "videoShare"."actorId" ' +
335 ' AND "actorFollowShare"."actorId" = :followerActorId AND "actorFollowShare"."state" = \'accepted\' ' +
336 ' WHERE "videoShare"."videoId" = "video"."id"' +
337 ' )' +
338 ' OR' +
339 ' EXISTS (' + // Videos published by accounts we follow
340 ' SELECT 1 from "actorFollow" ' +
341 ' WHERE "actorFollow"."targetActorId" = "account"."actorId" AND "actorFollow"."actorId" = :followerActorId ' +
342 ' AND "actorFollow"."state" = \'accepted\'' +
343 ' )'
344
345 if (includeLocalVideos) {
346 query += ' OR "video"."remote" IS FALSE'
347 }
348
349 query += ')'
350
351 this.and.push(query)
352 this.replacements.followerActorId = followerActorId
353 }
354
355 private whereFileExists () {
356 this.and.push(
357 '(' +
358 ' EXISTS (SELECT 1 FROM "videoFile" WHERE "videoFile"."videoId" = "video"."id") ' +
359 ' OR EXISTS (' +
360 ' SELECT 1 FROM "videoStreamingPlaylist" ' +
361 ' INNER JOIN "videoFile" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ' +
362 ' WHERE "videoStreamingPlaylist"."videoId" = "video"."id"' +
363 ' )' +
364 ')'
365 )
366 }
367
368 private whereTagsOneOf (tagsOneOf: string[]) {
369 const tagsOneOfLower = tagsOneOf.map(t => t.toLowerCase())
370
371 this.and.push(
372 'EXISTS (' +
373 ' SELECT 1 FROM "videoTag" ' +
374 ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' +
375 ' WHERE lower("tag"."name") IN (' + createSafeIn(this.sequelize, tagsOneOfLower) + ') ' +
376 ' AND "video"."id" = "videoTag"."videoId"' +
377 ')'
378 )
379 }
380
381 private whereTagsAllOf (tagsAllOf: string[]) {
382 const tagsAllOfLower = tagsAllOf.map(t => t.toLowerCase())
383
384 this.and.push(
385 'EXISTS (' +
386 ' SELECT 1 FROM "videoTag" ' +
387 ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' +
388 ' WHERE lower("tag"."name") IN (' + createSafeIn(this.sequelize, tagsAllOfLower) + ') ' +
389 ' AND "video"."id" = "videoTag"."videoId" ' +
390 ' GROUP BY "videoTag"."videoId" HAVING COUNT(*) = ' + tagsAllOfLower.length +
391 ')'
392 )
393 }
394
395 private whereUUIDs (uuids: string[]) {
396 this.and.push('"video"."uuid" IN (' + createSafeIn(this.sequelize, uuids) + ')')
397 }
398
399 private whereCategoryOneOf (categoryOneOf: number[]) {
400 this.and.push('"video"."category" IN (:categoryOneOf)')
401 this.replacements.categoryOneOf = categoryOneOf
402 }
403
404 private whereLicenceOneOf (licenceOneOf: number[]) {
405 this.and.push('"video"."licence" IN (:licenceOneOf)')
406 this.replacements.licenceOneOf = licenceOneOf
407 }
408
409 private whereLanguageOneOf (languageOneOf: string[]) {
410 const languages = languageOneOf.filter(l => l && l !== '_unknown')
411 const languagesQueryParts: string[] = []
412
413 if (languages.length !== 0) {
414 languagesQueryParts.push('"video"."language" IN (:languageOneOf)')
415 this.replacements.languageOneOf = languages
416
417 languagesQueryParts.push(
418 'EXISTS (' +
419 ' SELECT 1 FROM "videoCaption" WHERE "videoCaption"."language" ' +
420 ' IN (' + createSafeIn(this.sequelize, languages) + ') AND ' +
421 ' "videoCaption"."videoId" = "video"."id"' +
422 ')'
423 )
424 }
425
426 if (languageOneOf.includes('_unknown')) {
427 languagesQueryParts.push('"video"."language" IS NULL')
428 }
429
430 if (languagesQueryParts.length !== 0) {
431 this.and.push('(' + languagesQueryParts.join(' OR ') + ')')
432 }
433 }
434
435 private whereNSFW () {
436 this.and.push('"video"."nsfw" IS TRUE')
437 }
438
439 private whereSFW () {
440 this.and.push('"video"."nsfw" IS FALSE')
441 }
442
443 private whereLive () {
444 this.and.push('"video"."isLive" IS TRUE')
445 }
446
447 private whereVOD () {
448 this.and.push('"video"."isLive" IS FALSE')
449 }
450
451 private whereNotBlocked (serverAccountId: number, user?: MUserAccountId) {
452 const blockerIds = [ serverAccountId ]
453 if (user) blockerIds.push(user.Account.id)
454
455 const inClause = createSafeIn(this.sequelize, blockerIds)
456
457 this.and.push(
458 'NOT EXISTS (' +
459 ' SELECT 1 FROM "accountBlocklist" ' +
460 ' WHERE "accountBlocklist"."accountId" IN (' + inClause + ') ' +
461 ' AND "accountBlocklist"."targetAccountId" = "account"."id" ' +
462 ')' +
463 'AND NOT EXISTS (' +
464 ' SELECT 1 FROM "serverBlocklist" WHERE "serverBlocklist"."accountId" IN (' + inClause + ') ' +
465 ' AND "serverBlocklist"."targetServerId" = "accountActor"."serverId"' +
466 ')'
467 )
468 }
469
470 private whereSearch (search?: string) {
471 if (!search) {
472 this.attributes.push('0 as similarity')
473 return
474 }
475
476 const escapedSearch = this.sequelize.escape(search)
477 const escapedLikeSearch = this.sequelize.escape('%' + search + '%')
478
479 this.cte.push(
480 '"trigramSearch" AS (' +
481 ' SELECT "video"."id", ' +
482 ` similarity(lower(immutable_unaccent("video"."name")), lower(immutable_unaccent(${escapedSearch}))) as similarity ` +
483 ' FROM "video" ' +
484 ' WHERE lower(immutable_unaccent("video"."name")) % lower(immutable_unaccent(' + escapedSearch + ')) OR ' +
485 ' lower(immutable_unaccent("video"."name")) LIKE lower(immutable_unaccent(' + escapedLikeSearch + '))' +
486 ')'
487 )
488
489 this.joins.push('LEFT JOIN "trigramSearch" ON "video"."id" = "trigramSearch"."id"')
490
491 let base = '(' +
492 ' "trigramSearch"."id" IS NOT NULL OR ' +
493 ' EXISTS (' +
494 ' SELECT 1 FROM "videoTag" ' +
495 ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' +
496 ` WHERE lower("tag"."name") = ${escapedSearch} ` +
497 ' AND "video"."id" = "videoTag"."videoId"' +
498 ' )'
499
500 if (validator.isUUID(search)) {
501 base += ` OR "video"."uuid" = ${escapedSearch}`
502 }
503
504 base += ')'
505
506 this.and.push(base)
507 this.attributes.push(`COALESCE("trigramSearch"."similarity", 0) as similarity`)
508 }
509
510 private whereNotBlacklisted () {
511 this.and.push('"video"."id" NOT IN (SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")')
512 }
513
514 private whereStartDate (startDate: string) {
515 this.and.push('"video"."publishedAt" >= :startDate')
516 this.replacements.startDate = startDate
517 }
518
519 private whereEndDate (endDate: string) {
520 this.and.push('"video"."publishedAt" <= :endDate')
521 this.replacements.endDate = endDate
522 }
523
524 private whereOriginallyPublishedStartDate (startDate: string) {
525 this.and.push('"video"."originallyPublishedAt" >= :originallyPublishedStartDate')
526 this.replacements.originallyPublishedStartDate = startDate
527 }
528
529 private whereOriginallyPublishedEndDate (endDate: string) {
530 this.and.push('"video"."originallyPublishedAt" <= :originallyPublishedEndDate')
531 this.replacements.originallyPublishedEndDate = endDate
532 }
533
534 private whereDurationMin (durationMin: number) {
535 this.and.push('"video"."duration" >= :durationMin')
536 this.replacements.durationMin = durationMin
537 }
538
539 private whereDurationMax (durationMax: number) {
540 this.and.push('"video"."duration" <= :durationMax')
541 this.replacements.durationMax = durationMax
542 }
543
544 private groupForTrending (trendingDays: number) {
545 const viewsGteDate = new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays)
546
547 this.joins.push('LEFT JOIN "videoView" ON "video"."id" = "videoView"."videoId" AND "videoView"."startDate" >= :viewsGteDate')
548 this.replacements.viewsGteDate = viewsGteDate
549
550 this.attributes.push('COALESCE(SUM("videoView"."views"), 0) AS "score"')
551
552 this.group = 'GROUP BY "video"."id"'
553 }
554
555 private groupForHotOrBest (trendingAlgorithm: string, user?: MUserAccountId) {
556 /**
557 * "Hotness" is a measure based on absolute view/comment/like/dislike numbers,
558 * with fixed weights only applied to their log values.
559 *
560 * This algorithm gives little chance for an old video to have a good score,
561 * for which recent spikes in interactions could be a sign of "hotness" and
562 * justify a better score. However there are multiple ways to achieve that
563 * goal, which is left for later. Yes, this is a TODO :)
564 *
565 * notes:
566 * - weights and base score are in number of half-days.
567 * - all comments are counted, regardless of being written by the video author or not
568 * see https://github.com/reddit-archive/reddit/blob/master/r2/r2/lib/db/_sorts.pyx#L47-L58
569 * - we have less interactions than on reddit, so multiply weights by an arbitrary factor
570 */
571 const weights = {
572 like: 3 * 50,
573 dislike: -3 * 50,
574 view: Math.floor((1 / 3) * 50),
575 comment: 2 * 50, // a comment takes more time than a like to do, but can be done multiple times
576 history: -2 * 50
577 }
578
579 this.joins.push('LEFT JOIN "videoComment" ON "video"."id" = "videoComment"."videoId"')
580
581 let attribute =
582 `LOG(GREATEST(1, "video"."likes" - 1)) * ${weights.like} ` + // likes (+)
583 `+ LOG(GREATEST(1, "video"."dislikes" - 1)) * ${weights.dislike} ` + // dislikes (-)
584 `+ LOG("video"."views" + 1) * ${weights.view} ` + // views (+)
585 `+ LOG(GREATEST(1, COUNT(DISTINCT "videoComment"."id"))) * ${weights.comment} ` + // comments (+)
586 '+ (SELECT (EXTRACT(epoch FROM "video"."publishedAt") - 1446156582) / 47000) ' // base score (in number of half-days)
587
588 if (trendingAlgorithm === 'best' && user) {
589 this.joins.push(
590 'LEFT JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId" AND "userVideoHistory"."userId" = :bestUser'
591 )
592 this.replacements.bestUser = user.id
593
594 attribute += `+ POWER(COUNT(DISTINCT "userVideoHistory"."id"), 2.0) * ${weights.history} `
595 }
596
597 attribute += 'AS "score"'
598 this.attributes.push(attribute)
599
600 this.group = 'GROUP BY "video"."id"'
601 }
602
603 private setSort (sort: string) {
604 if (sort === '-originallyPublishedAt' || sort === 'originallyPublishedAt') {
605 this.attributes.push('COALESCE("video"."originallyPublishedAt", "video"."publishedAt") AS "publishedAtForOrder"')
606 }
607
608 this.sort = this.buildOrder(sort)
609 }
610
611 private buildOrder (value: string) {
612 const { direction, field } = buildDirectionAndField(value)
613 if (field.match(/^[a-zA-Z."]+$/) === null) throw new Error('Invalid sort column ' + field)
614
615 if (field.toLowerCase() === 'random') return 'ORDER BY RANDOM()'
616
617 if ([ 'trending', 'hot', 'best' ].includes(field.toLowerCase())) { // Sort by aggregation
618 return `ORDER BY "score" ${direction}, "video"."views" ${direction}`
619 }
620
621 let firstSort: string
622
623 if (field.toLowerCase() === 'match') { // Search
624 firstSort = '"similarity"'
625 } else if (field === 'originallyPublishedAt') {
626 firstSort = '"publishedAtForOrder"'
627 } else if (field.includes('.')) {
628 firstSort = field
629 } else {
630 firstSort = `"video"."${field}"`
631 }
632
633 return `ORDER BY ${firstSort} ${direction}, "video"."id" ASC`
634 }
635
636 private setLimit (countArg: number) {
637 const count = parseInt(countArg + '', 10)
638 this.limit = `LIMIT ${count}`
639 }
640
641 private setOffset (startArg: number) {
642 const start = parseInt(startArg + '', 10)
643 this.offset = `OFFSET ${start}`
644 }
645 }