1 import { VideoFilter, VideoPrivacy, VideoState } from '@shared/models'
2 import { buildDirectionAndField, createSafeIn } from '@server/models/utils'
3 import { Model } from 'sequelize-typescript'
4 import { MUserAccountId, MUserId } from '@server/types/models'
5 import validator from 'validator'
6 import { exists } from '@server/helpers/custom-validators/misc'
8 export type BuildVideosQueryOptions = {
11 serverAccountId: number
12 followerActorId: number
13 includeLocalVideos: boolean
20 categoryOneOf?: number[]
22 licenceOneOf?: number[]
23 languageOneOf?: string[]
30 videoChannelId?: number
32 videoPlaylistId?: number
38 historyOfUser?: MUserId
40 startDate?: string // ISO 8601
41 endDate?: string // ISO 8601
42 originallyPublishedStartDate?: string
43 originallyPublishedEndDate?: string
45 durationMin?: number // seconds
46 durationMax?: number // seconds
56 function buildListQuery (model: typeof Model, options: BuildVideosQueryOptions) {
57 const and: string[] = []
58 const joins: string[] = []
59 const replacements: any = {}
60 const cte: string[] = []
62 let attributes: string[] = options.attributes || [ '"video"."id"' ]
63 let group = options.group || ''
64 const having = options.having || ''
67 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId"' +
68 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId"' +
69 'INNER JOIN "actor" "accountActor" ON "account"."actorId" = "accountActor"."id"'
72 and.push('"video"."id" NOT IN (SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")')
74 if (options.serverAccountId) {
75 const blockerIds = [ options.serverAccountId ]
76 if (options.user) blockerIds.push(options.user.Account.id)
78 const inClause = createSafeIn(model, blockerIds)
82 ' SELECT 1 FROM "accountBlocklist" ' +
83 ' WHERE "accountBlocklist"."accountId" IN (' + inClause + ') ' +
84 ' AND "accountBlocklist"."targetAccountId" = "account"."id" ' +
87 ' SELECT 1 FROM "serverBlocklist" WHERE "serverBlocklist"."accountId" IN (' + inClause + ') ' +
88 ' AND "serverBlocklist"."targetServerId" = "accountActor"."serverId"' +
93 // Only list public/published videos
94 if (!options.filter || (options.filter !== 'all-local' && options.filter !== 'all')) {
96 `("video"."state" = ${VideoState.PUBLISHED} OR ` +
97 `("video"."state" = ${VideoState.TO_TRANSCODE} AND "video"."waitTranscoding" IS false))`
102 `("video"."privacy" = ${VideoPrivacy.PUBLIC} OR "video"."privacy" = ${VideoPrivacy.INTERNAL})`
104 } else { // Or only public videos
106 `"video"."privacy" = ${VideoPrivacy.PUBLIC}`
111 if (options.videoPlaylistId) {
113 'INNER JOIN "videoPlaylistElement" "video"."id" = "videoPlaylistElement"."videoId" ' +
114 'AND "videoPlaylistElement"."videoPlaylistId" = :videoPlaylistId'
117 replacements.videoPlaylistId = options.videoPlaylistId
120 if (options.filter && (options.filter === 'local' || options.filter === 'all-local')) {
121 and.push('"video"."remote" IS FALSE')
124 if (options.accountId) {
125 and.push('"account"."id" = :accountId')
126 replacements.accountId = options.accountId
129 if (options.videoChannelId) {
130 and.push('"videoChannel"."id" = :videoChannelId')
131 replacements.videoChannelId = options.videoChannelId
134 if (options.followerActorId) {
138 ' SELECT 1 FROM "videoShare" ' +
139 ' INNER JOIN "actorFollow" "actorFollowShare" ON "actorFollowShare"."targetActorId" = "videoShare"."actorId" ' +
140 ' AND "actorFollowShare"."actorId" = :followerActorId AND "actorFollowShare"."state" = \'accepted\' ' +
141 ' WHERE "videoShare"."videoId" = "video"."id"' +
145 ' SELECT 1 from "actorFollow" ' +
146 ' WHERE "actorFollow"."targetActorId" = "videoChannel"."actorId" AND "actorFollow"."actorId" = :followerActorId ' +
147 ' AND "actorFollow"."state" = \'accepted\'' +
150 if (options.includeLocalVideos) {
151 query += ' OR "video"."remote" IS FALSE'
157 replacements.followerActorId = options.followerActorId
160 if (options.withFiles === true) {
163 ' EXISTS (SELECT 1 FROM "videoFile" WHERE "videoFile"."videoId" = "video"."id") ' +
165 ' SELECT 1 FROM "videoStreamingPlaylist" ' +
166 ' INNER JOIN "videoFile" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ' +
167 ' WHERE "videoStreamingPlaylist"."videoId" = "video"."id"' +
173 if (options.tagsOneOf) {
174 const tagsOneOfLower = options.tagsOneOf.map(t => t.toLowerCase())
178 ' SELECT 1 FROM "videoTag" ' +
179 ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' +
180 ' WHERE lower("tag"."name") IN (' + createSafeIn(model, tagsOneOfLower) + ') ' +
181 ' AND "video"."id" = "videoTag"."videoId"' +
186 if (options.tagsAllOf) {
187 const tagsAllOfLower = options.tagsAllOf.map(t => t.toLowerCase())
191 ' SELECT 1 FROM "videoTag" ' +
192 ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' +
193 ' WHERE lower("tag"."name") IN (' + createSafeIn(model, tagsAllOfLower) + ') ' +
194 ' AND "video"."id" = "videoTag"."videoId" ' +
195 ' GROUP BY "videoTag"."videoId" HAVING COUNT(*) = ' + tagsAllOfLower.length +
200 if (options.nsfw === true) {
201 and.push('"video"."nsfw" IS TRUE')
204 if (options.nsfw === false) {
205 and.push('"video"."nsfw" IS FALSE')
208 if (options.categoryOneOf) {
209 and.push('"video"."category" IN (:categoryOneOf)')
210 replacements.categoryOneOf = options.categoryOneOf
213 if (options.licenceOneOf) {
214 and.push('"video"."licence" IN (:licenceOneOf)')
215 replacements.licenceOneOf = options.licenceOneOf
218 if (options.languageOneOf) {
219 const languages = options.languageOneOf.filter(l => l && l !== '_unknown')
220 const languagesQueryParts: string[] = []
222 if (languages.length !== 0) {
223 languagesQueryParts.push('"video"."language" IN (:languageOneOf)')
224 replacements.languageOneOf = languages
226 languagesQueryParts.push(
228 ' SELECT 1 FROM "videoCaption" WHERE "videoCaption"."language" ' +
229 ' IN (' + createSafeIn(model, languages) + ') AND ' +
230 ' "videoCaption"."videoId" = "video"."id"' +
235 if (options.languageOneOf.includes('_unknown')) {
236 languagesQueryParts.push('"video"."language" IS NULL')
239 if (languagesQueryParts.length !== 0) {
240 and.push('(' + languagesQueryParts.join(' OR ') + ')')
244 // We don't exclude results in this so if we do a count we don't need to add this complex clause
245 if (options.isCount !== true) {
246 if (options.trendingDays) {
247 const viewsGteDate = new Date(new Date().getTime() - (24 * 3600 * 1000) * options.trendingDays)
249 joins.push('LEFT JOIN "videoView" ON "video"."id" = "videoView"."videoId" AND "videoView"."startDate" >= :viewsGteDate')
250 replacements.viewsGteDate = viewsGteDate
252 attributes.push('COALESCE(SUM("videoView"."views"), 0) AS "score"')
254 group = 'GROUP BY "video"."id"'
255 } else if (options.hot) {
257 * "Hotness" is a measure based on absolute view/comment/like/dislike numbers,
258 * with fixed weights only applied to their log values.
260 * This algorithm gives little chance for an old video to have a good score,
261 * for which recent spikes in interactions could be a sign of "hotness" and
262 * justify a better score. However there are multiple ways to achieve that
263 * goal, which is left for later. Yes, this is a TODO :)
266 * - weights and base score are in number of half-days.
267 * - all comments are counted, regardless of being written by the video author or not
268 * see https://github.com/reddit-archive/reddit/blob/master/r2/r2/lib/db/_sorts.pyx#L47-L58
274 comment: 2 // a comment takes more time than a like to do, but can be done multiple times
277 joins.push('LEFT JOIN "videoComment" ON "video"."id" = "videoComment"."videoId"')
280 `LOG(GREATEST(1, "video"."likes" - 1)) * ${weights.like} ` + // likes (+)
281 `- LOG(GREATEST(1, "video"."dislikes" - 1)) * ${weights.dislike} ` + // dislikes (-)
282 `+ LOG("video"."views" + 1) * ${weights.view} ` + // views (+)
283 `+ LOG(GREATEST(1, COUNT(DISTINCT "videoComment"."id"))) * ${weights.comment} ` + // comments (+)
284 '+ (SELECT EXTRACT(epoch FROM "video"."publishedAt") / 47000) ' + // base score (in number of half-days)
288 group = 'GROUP BY "video"."id"'
292 if (options.historyOfUser) {
293 joins.push('INNER JOIN "userVideoHistory" on "video"."id" = "userVideoHistory"."videoId"')
295 and.push('"userVideoHistory"."userId" = :historyOfUser')
296 replacements.historyOfUser = options.historyOfUser.id
299 if (options.startDate) {
300 and.push('"video"."publishedAt" >= :startDate')
301 replacements.startDate = options.startDate
304 if (options.endDate) {
305 and.push('"video"."publishedAt" <= :endDate')
306 replacements.endDate = options.endDate
309 if (options.originallyPublishedStartDate) {
310 and.push('"video"."originallyPublishedAt" >= :originallyPublishedStartDate')
311 replacements.originallyPublishedStartDate = options.originallyPublishedStartDate
314 if (options.originallyPublishedEndDate) {
315 and.push('"video"."originallyPublishedAt" <= :originallyPublishedEndDate')
316 replacements.originallyPublishedEndDate = options.originallyPublishedEndDate
319 if (options.durationMin) {
320 and.push('"video"."duration" >= :durationMin')
321 replacements.durationMin = options.durationMin
324 if (options.durationMax) {
325 and.push('"video"."duration" <= :durationMax')
326 replacements.durationMax = options.durationMax
329 if (options.search) {
330 const escapedSearch = model.sequelize.escape(options.search)
331 const escapedLikeSearch = model.sequelize.escape('%' + options.search + '%')
334 '"trigramSearch" AS (' +
335 ' SELECT "video"."id", ' +
336 ` similarity(lower(immutable_unaccent("video"."name")), lower(immutable_unaccent(${escapedSearch}))) as similarity ` +
338 ' WHERE lower(immutable_unaccent("video"."name")) % lower(immutable_unaccent(' + escapedSearch + ')) OR ' +
339 ' lower(immutable_unaccent("video"."name")) LIKE lower(immutable_unaccent(' + escapedLikeSearch + '))' +
343 joins.push('LEFT JOIN "trigramSearch" ON "video"."id" = "trigramSearch"."id"')
346 ' "trigramSearch"."id" IS NOT NULL OR ' +
348 ' SELECT 1 FROM "videoTag" ' +
349 ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' +
350 ` WHERE lower("tag"."name") = ${escapedSearch} ` +
351 ' AND "video"."id" = "videoTag"."videoId"' +
354 if (validator.isUUID(options.search)) {
355 base += ` OR "video"."uuid" = ${escapedSearch}`
361 attributes.push(`COALESCE("trigramSearch"."similarity", 0) as similarity`)
363 attributes.push('0 as similarity')
366 if (options.isCount === true) attributes = [ 'COUNT(*) as "total"' ]
370 if (options.isCount !== true) {
372 if (exists(options.sort)) {
373 if (options.sort === '-originallyPublishedAt' || options.sort === 'originallyPublishedAt') {
374 attributes.push('COALESCE("video"."originallyPublishedAt", "video"."publishedAt") AS "publishedAtForOrder"')
377 order = buildOrder(options.sort)
378 suffix += `${order} `
381 if (exists(options.count)) {
382 const count = parseInt(options.count + '', 10)
383 suffix += `LIMIT ${count} `
386 if (exists(options.start)) {
387 const start = parseInt(options.start + '', 10)
388 suffix += `OFFSET ${start} `
392 const cteString = cte.length !== 0
393 ? `WITH ${cte.join(', ')} `
396 const query = cteString +
397 'SELECT ' + attributes.join(', ') + ' ' +
398 'FROM "video" ' + joins.join(' ') + ' ' +
399 'WHERE ' + and.join(' AND ') + ' ' +
404 return { query, replacements, order }
407 function buildOrder (value: string) {
408 const { direction, field } = buildDirectionAndField(value)
409 if (field.match(/^[a-zA-Z."]+$/) === null) throw new Error('Invalid sort column ' + field)
411 if (field.toLowerCase() === 'random') return 'ORDER BY RANDOM()'
413 if ([ 'trending', 'hot' ].includes(field.toLowerCase())) { // Sort by aggregation
414 return `ORDER BY "score" ${direction}, "video"."views" ${direction}`
417 let firstSort: string
419 if (field.toLowerCase() === 'match') { // Search
420 firstSort = '"similarity"'
421 } else if (field === 'originallyPublishedAt') {
422 firstSort = '"publishedAtForOrder"'
423 } else if (field.includes('.')) {
426 firstSort = `"video"."${field}"`
429 return `ORDER BY ${firstSort} ${direction}, "video"."id" ASC`
432 function wrapForAPIResults (baseQuery: string, replacements: any, options: BuildVideosQueryOptions, order: string) {
435 '"VideoChannel"."id"': '"VideoChannel.id"',
436 '"VideoChannel"."name"': '"VideoChannel.name"',
437 '"VideoChannel"."description"': '"VideoChannel.description"',
438 '"VideoChannel"."actorId"': '"VideoChannel.actorId"',
439 '"VideoChannel->Actor"."id"': '"VideoChannel.Actor.id"',
440 '"VideoChannel->Actor"."preferredUsername"': '"VideoChannel.Actor.preferredUsername"',
441 '"VideoChannel->Actor"."url"': '"VideoChannel.Actor.url"',
442 '"VideoChannel->Actor"."serverId"': '"VideoChannel.Actor.serverId"',
443 '"VideoChannel->Actor"."avatarId"': '"VideoChannel.Actor.avatarId"',
444 '"VideoChannel->Account"."id"': '"VideoChannel.Account.id"',
445 '"VideoChannel->Account"."name"': '"VideoChannel.Account.name"',
446 '"VideoChannel->Account->Actor"."id"': '"VideoChannel.Account.Actor.id"',
447 '"VideoChannel->Account->Actor"."preferredUsername"': '"VideoChannel.Account.Actor.preferredUsername"',
448 '"VideoChannel->Account->Actor"."url"': '"VideoChannel.Account.Actor.url"',
449 '"VideoChannel->Account->Actor"."serverId"': '"VideoChannel.Account.Actor.serverId"',
450 '"VideoChannel->Account->Actor"."avatarId"': '"VideoChannel.Account.Actor.avatarId"',
451 '"VideoChannel->Actor->Server"."id"': '"VideoChannel.Actor.Server.id"',
452 '"VideoChannel->Actor->Server"."host"': '"VideoChannel.Actor.Server.host"',
453 '"VideoChannel->Actor->Avatar"."id"': '"VideoChannel.Actor.Avatar.id"',
454 '"VideoChannel->Actor->Avatar"."filename"': '"VideoChannel.Actor.Avatar.filename"',
455 '"VideoChannel->Actor->Avatar"."fileUrl"': '"VideoChannel.Actor.Avatar.fileUrl"',
456 '"VideoChannel->Actor->Avatar"."onDisk"': '"VideoChannel.Actor.Avatar.onDisk"',
457 '"VideoChannel->Actor->Avatar"."createdAt"': '"VideoChannel.Actor.Avatar.createdAt"',
458 '"VideoChannel->Actor->Avatar"."updatedAt"': '"VideoChannel.Actor.Avatar.updatedAt"',
459 '"VideoChannel->Account->Actor->Server"."id"': '"VideoChannel.Account.Actor.Server.id"',
460 '"VideoChannel->Account->Actor->Server"."host"': '"VideoChannel.Account.Actor.Server.host"',
461 '"VideoChannel->Account->Actor->Avatar"."id"': '"VideoChannel.Account.Actor.Avatar.id"',
462 '"VideoChannel->Account->Actor->Avatar"."filename"': '"VideoChannel.Account.Actor.Avatar.filename"',
463 '"VideoChannel->Account->Actor->Avatar"."fileUrl"': '"VideoChannel.Account.Actor.Avatar.fileUrl"',
464 '"VideoChannel->Account->Actor->Avatar"."onDisk"': '"VideoChannel.Account.Actor.Avatar.onDisk"',
465 '"VideoChannel->Account->Actor->Avatar"."createdAt"': '"VideoChannel.Account.Actor.Avatar.createdAt"',
466 '"VideoChannel->Account->Actor->Avatar"."updatedAt"': '"VideoChannel.Account.Actor.Avatar.updatedAt"',
467 '"Thumbnails"."id"': '"Thumbnails.id"',
468 '"Thumbnails"."type"': '"Thumbnails.type"',
469 '"Thumbnails"."filename"': '"Thumbnails.filename"'
473 'INNER JOIN "video" ON "tmp"."id" = "video"."id"',
475 'INNER JOIN "videoChannel" AS "VideoChannel" ON "video"."channelId" = "VideoChannel"."id"',
476 'INNER JOIN "actor" AS "VideoChannel->Actor" ON "VideoChannel"."actorId" = "VideoChannel->Actor"."id"',
477 'INNER JOIN "account" AS "VideoChannel->Account" ON "VideoChannel"."accountId" = "VideoChannel->Account"."id"',
478 'INNER JOIN "actor" AS "VideoChannel->Account->Actor" ON "VideoChannel->Account"."actorId" = "VideoChannel->Account->Actor"."id"',
480 'LEFT OUTER JOIN "server" AS "VideoChannel->Actor->Server" ON "VideoChannel->Actor"."serverId" = "VideoChannel->Actor->Server"."id"',
481 'LEFT OUTER JOIN "avatar" AS "VideoChannel->Actor->Avatar" ON "VideoChannel->Actor"."avatarId" = "VideoChannel->Actor->Avatar"."id"',
483 'LEFT OUTER JOIN "server" AS "VideoChannel->Account->Actor->Server" ' +
484 'ON "VideoChannel->Account->Actor"."serverId" = "VideoChannel->Account->Actor->Server"."id"',
486 'LEFT OUTER JOIN "avatar" AS "VideoChannel->Account->Actor->Avatar" ' +
487 'ON "VideoChannel->Account->Actor"."avatarId" = "VideoChannel->Account->Actor->Avatar"."id"',
489 'LEFT OUTER JOIN "thumbnail" AS "Thumbnails" ON "video"."id" = "Thumbnails"."videoId"'
492 if (options.withFiles) {
493 joins.push('LEFT JOIN "videoFile" AS "VideoFiles" ON "VideoFiles"."videoId" = "video"."id"')
495 joins.push('LEFT JOIN "videoStreamingPlaylist" AS "VideoStreamingPlaylists" ON "VideoStreamingPlaylists"."videoId" = "video"."id"')
497 'LEFT JOIN "videoFile" AS "VideoStreamingPlaylists->VideoFiles" ' +
498 'ON "VideoStreamingPlaylists->VideoFiles"."videoStreamingPlaylistId" = "VideoStreamingPlaylists"."id"'
501 Object.assign(attributes, {
502 '"VideoFiles"."id"': '"VideoFiles.id"',
503 '"VideoFiles"."createdAt"': '"VideoFiles.createdAt"',
504 '"VideoFiles"."updatedAt"': '"VideoFiles.updatedAt"',
505 '"VideoFiles"."resolution"': '"VideoFiles.resolution"',
506 '"VideoFiles"."size"': '"VideoFiles.size"',
507 '"VideoFiles"."extname"': '"VideoFiles.extname"',
508 '"VideoFiles"."infoHash"': '"VideoFiles.infoHash"',
509 '"VideoFiles"."fps"': '"VideoFiles.fps"',
510 '"VideoFiles"."videoId"': '"VideoFiles.videoId"',
512 '"VideoStreamingPlaylists"."id"': '"VideoStreamingPlaylists.id"',
513 '"VideoStreamingPlaylists"."playlistUrl"': '"VideoStreamingPlaylists.playlistUrl"',
514 '"VideoStreamingPlaylists"."type"': '"VideoStreamingPlaylists.type"',
515 '"VideoStreamingPlaylists->VideoFiles"."id"': '"VideoStreamingPlaylists.VideoFiles.id"',
516 '"VideoStreamingPlaylists->VideoFiles"."createdAt"': '"VideoStreamingPlaylists.VideoFiles.createdAt"',
517 '"VideoStreamingPlaylists->VideoFiles"."updatedAt"': '"VideoStreamingPlaylists.VideoFiles.updatedAt"',
518 '"VideoStreamingPlaylists->VideoFiles"."resolution"': '"VideoStreamingPlaylists.VideoFiles.resolution"',
519 '"VideoStreamingPlaylists->VideoFiles"."size"': '"VideoStreamingPlaylists.VideoFiles.size"',
520 '"VideoStreamingPlaylists->VideoFiles"."extname"': '"VideoStreamingPlaylists.VideoFiles.extname"',
521 '"VideoStreamingPlaylists->VideoFiles"."infoHash"': '"VideoStreamingPlaylists.VideoFiles.infoHash"',
522 '"VideoStreamingPlaylists->VideoFiles"."fps"': '"VideoStreamingPlaylists.VideoFiles.fps"',
523 '"VideoStreamingPlaylists->VideoFiles"."videoStreamingPlaylistId"': '"VideoStreamingPlaylists.VideoFiles.videoStreamingPlaylistId"',
524 '"VideoStreamingPlaylists->VideoFiles"."videoId"': '"VideoStreamingPlaylists.VideoFiles.videoId"'
530 'LEFT OUTER JOIN "userVideoHistory" ' +
531 'ON "video"."id" = "userVideoHistory"."videoId" AND "userVideoHistory"."userId" = :userVideoHistoryId'
533 replacements.userVideoHistoryId = options.user.id
535 Object.assign(attributes, {
536 '"userVideoHistory"."id"': '"userVideoHistory.id"',
537 '"userVideoHistory"."currentTime"': '"userVideoHistory.currentTime"'
541 if (options.videoPlaylistId) {
543 'INNER JOIN "videoPlaylistElement" as "VideoPlaylistElement" ON "videoPlaylistElement"."videoId" = "video"."id" ' +
544 'AND "VideoPlaylistElement"."videoPlaylistId" = :videoPlaylistId'
546 replacements.videoPlaylistId = options.videoPlaylistId
548 Object.assign(attributes, {
549 '"VideoPlaylistElement"."createdAt"': '"VideoPlaylistElement.createdAt"',
550 '"VideoPlaylistElement"."updatedAt"': '"VideoPlaylistElement.updatedAt"',
551 '"VideoPlaylistElement"."url"': '"VideoPlaylistElement.url"',
552 '"VideoPlaylistElement"."position"': '"VideoPlaylistElement.position"',
553 '"VideoPlaylistElement"."startTimestamp"': '"VideoPlaylistElement.startTimestamp"',
554 '"VideoPlaylistElement"."stopTimestamp"': '"VideoPlaylistElement.stopTimestamp"',
555 '"VideoPlaylistElement"."videoPlaylistId"': '"VideoPlaylistElement.videoPlaylistId"'
559 const select = 'SELECT ' + Object.keys(attributes).map(key => {
560 const value = attributes[key]
561 if (value) return `${key} AS ${value}`
566 return `${select} FROM (${baseQuery}) AS "tmp" ${joins.join(' ')} ${order}`