import { VideoFilter, VideoPrivacy, VideoState } from '@shared/models'
import { buildDirectionAndField, createSafeIn } from '@server/models/utils'
import { Model } from 'sequelize-typescript'
-import { MUserAccountId, MUserId } from '@server/typings/models'
+import { MUserAccountId, MUserId } from '@server/types/models'
import validator from 'validator'
+import { exists } from '@server/helpers/custom-validators/misc'
export type BuildVideosQueryOptions = {
attributes?: string[]
videoPlaylistId?: number
+ trendingAlgorithm?: string // best, hot, or any other algorithm implemented
trendingDays?: number
+
user?: MUserAccountId
historyOfUser?: MUserId
}
// Only list public/published videos
- if (!options.filter || options.filter !== 'all-local') {
+ if (!options.filter || (options.filter !== 'all-local' && options.filter !== 'all')) {
and.push(
`("video"."state" = ${VideoState.PUBLISHED} OR ` +
`("video"."state" = ${VideoState.TO_TRANSCODE} AND "video"."waitTranscoding" IS false))`
' EXISTS (' +
' SELECT 1 FROM "videoShare" ' +
' INNER JOIN "actorFollow" "actorFollowShare" ON "actorFollowShare"."targetActorId" = "videoShare"."actorId" ' +
- ' AND "actorFollowShare"."actorId" = :followerActorId WHERE "videoShare"."videoId" = "video"."id"' +
+ ' AND "actorFollowShare"."actorId" = :followerActorId AND "actorFollowShare"."state" = \'accepted\' ' +
+ ' WHERE "videoShare"."videoId" = "video"."id"' +
' )' +
' OR' +
' EXISTS (' +
' SELECT 1 from "actorFollow" ' +
- ' WHERE "actorFollow"."targetActorId" = "videoChannel"."actorId" AND "actorFollow"."actorId" = :followerActorId' +
+ ' WHERE "actorFollow"."targetActorId" = "videoChannel"."actorId" AND "actorFollow"."actorId" = :followerActorId ' +
+ ' AND "actorFollow"."state" = \'accepted\'' +
' )'
if (options.includeLocalVideos) {
}
if (options.withFiles === true) {
- and.push('EXISTS (SELECT 1 FROM "videoFile" WHERE "videoFile"."videoId" = "video"."id")')
+ and.push(
+ '(' +
+ ' EXISTS (SELECT 1 FROM "videoFile" WHERE "videoFile"."videoId" = "video"."id") ' +
+ ' OR EXISTS (' +
+ ' SELECT 1 FROM "videoStreamingPlaylist" ' +
+ ' INNER JOIN "videoFile" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ' +
+ ' WHERE "videoStreamingPlaylist"."videoId" = "video"."id"' +
+ ' )' +
+ ')'
+ )
}
if (options.tagsOneOf) {
}
if (options.languageOneOf) {
- replacements.languageOneOf = options.languageOneOf.filter(l => l && l !== '_unknown')
-
- let languagesQuery = '("video"."language" IN (:languageOneOf) OR '
+ const languages = options.languageOneOf.filter(l => l && l !== '_unknown')
+ const languagesQueryParts: string[] = []
+
+ if (languages.length !== 0) {
+ languagesQueryParts.push('"video"."language" IN (:languageOneOf)')
+ replacements.languageOneOf = languages
+
+ languagesQueryParts.push(
+ 'EXISTS (' +
+ ' SELECT 1 FROM "videoCaption" WHERE "videoCaption"."language" ' +
+ ' IN (' + createSafeIn(model, languages) + ') AND ' +
+ ' "videoCaption"."videoId" = "video"."id"' +
+ ')'
+ )
+ }
if (options.languageOneOf.includes('_unknown')) {
- languagesQuery += '"video"."language" IS NULL OR '
+ languagesQueryParts.push('"video"."language" IS NULL')
}
- and.push(
- languagesQuery +
- ' EXISTS (' +
- ' SELECT 1 FROM "videoCaption" WHERE "videoCaption"."language" ' +
- ' IN (' + createSafeIn(model, options.languageOneOf) + ') AND ' +
- ' "videoCaption"."videoId" = "video"."id"' +
- ' )' +
- ')'
- )
+ if (languagesQueryParts.length !== 0) {
+ and.push('(' + languagesQueryParts.join(' OR ') + ')')
+ }
}
- // We don't exclude results in this if so if we do a count we don't need to add this complex clauses
- if (options.trendingDays && options.isCount !== true) {
- const viewsGteDate = new Date(new Date().getTime() - (24 * 3600 * 1000) * options.trendingDays)
-
- joins.push('LEFT JOIN "videoView" ON "video"."id" = "videoView"."videoId" AND "videoView"."startDate" >= :viewsGteDate')
- replacements.viewsGteDate = viewsGteDate
-
- attributes.push('COALESCE(SUM("videoView"."views"), 0) AS "videoViewsSum"')
-
- group = 'GROUP BY "video"."id"'
+ // We don't exclude results in this so if we do a count we don't need to add this complex clause
+ if (options.isCount !== true) {
+ if (options.trendingDays) {
+ const viewsGteDate = new Date(new Date().getTime() - (24 * 3600 * 1000) * options.trendingDays)
+
+ joins.push('LEFT JOIN "videoView" ON "video"."id" = "videoView"."videoId" AND "videoView"."startDate" >= :viewsGteDate')
+ replacements.viewsGteDate = viewsGteDate
+
+ attributes.push('COALESCE(SUM("videoView"."views"), 0) AS "score"')
+
+ group = 'GROUP BY "video"."id"'
+ } else if ([ 'best', 'hot' ].includes(options.trendingAlgorithm)) {
+ /**
+ * "Hotness" is a measure based on absolute view/comment/like/dislike numbers,
+ * with fixed weights only applied to their log values.
+ *
+ * This algorithm gives little chance for an old video to have a good score,
+ * for which recent spikes in interactions could be a sign of "hotness" and
+ * justify a better score. However there are multiple ways to achieve that
+ * goal, which is left for later. Yes, this is a TODO :)
+ *
+ * notes:
+ * - weights and base score are in number of half-days.
+ * - all comments are counted, regardless of being written by the video author or not
+ * see https://github.com/reddit-archive/reddit/blob/master/r2/r2/lib/db/_sorts.pyx#L47-L58
+ */
+ const weights = {
+ like: 3,
+ dislike: -3,
+ view: 1 / 12,
+ comment: 2, // a comment takes more time than a like to do, but can be done multiple times
+ history: -2
+ }
+
+ joins.push('LEFT JOIN "videoComment" ON "video"."id" = "videoComment"."videoId"')
+
+ let attribute =
+ `LOG(GREATEST(1, "video"."likes" - 1)) * ${weights.like} ` + // likes (+)
+ `+ LOG(GREATEST(1, "video"."dislikes" - 1)) * ${weights.dislike} ` + // dislikes (-)
+ `+ LOG("video"."views" + 1) * ${weights.view} ` + // views (+)
+ `+ LOG(GREATEST(1, COUNT(DISTINCT "videoComment"."id"))) * ${weights.comment} ` + // comments (+)
+ '+ (SELECT EXTRACT(epoch FROM "video"."publishedAt") / 47000) ' // base score (in number of half-days)
+
+ if (options.trendingAlgorithm === 'best' && options.user) {
+ joins.push(
+ 'LEFT JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId" AND "userVideoHistory"."userId" = :bestUser'
+ )
+ replacements.bestUser = options.user.id
+
+ attribute += `+ POWER(COUNT(DISTINCT "userVideoHistory"."id"), 2.0) * ${weights.history} `
+ }
+
+ attribute += 'AS "score"'
+ attributes.push(attribute)
+
+ group = 'GROUP BY "video"."id"'
+ }
}
if (options.historyOfUser) {
- joins.push('INNER JOIN "userVideoHistory" on "video"."id" = "userVideoHistory"."videoId"')
+ joins.push('INNER JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId"')
and.push('"userVideoHistory"."userId" = :historyOfUser')
replacements.historyOfUser = options.historyOfUser.id
let suffix = ''
let order = ''
if (options.isCount !== true) {
- const count = parseInt(options.count + '', 10)
- const start = parseInt(options.start + '', 10)
- order = buildOrder(model, options.sort)
+ if (exists(options.sort)) {
+ if (options.sort === '-originallyPublishedAt' || options.sort === 'originallyPublishedAt') {
+ attributes.push('COALESCE("video"."originallyPublishedAt", "video"."publishedAt") AS "publishedAtForOrder"')
+ }
+
+ order = buildOrder(options.sort)
+ suffix += `${order} `
+ }
+
+ if (exists(options.count)) {
+ const count = parseInt(options.count + '', 10)
+ suffix += `LIMIT ${count} `
+ }
- suffix = order + ' ' +
- 'LIMIT ' + count + ' ' +
- 'OFFSET ' + start
+ if (exists(options.start)) {
+ const start = parseInt(options.start + '', 10)
+ suffix += `OFFSET ${start} `
+ }
}
const cteString = cte.length !== 0
return { query, replacements, order }
}
-function buildOrder (model: typeof Model, value: string) {
+function buildOrder (value: string) {
const { direction, field } = buildDirectionAndField(value)
if (field.match(/^[a-zA-Z."]+$/) === null) throw new Error('Invalid sort column ' + field)
if (field.toLowerCase() === 'random') return 'ORDER BY RANDOM()'
- if (field.toLowerCase() === 'trending') { // Sort by aggregation
- return `ORDER BY "videoViewsSum" ${direction}, "video"."views" ${direction}`
+ if ([ 'trending', 'hot', 'best' ].includes(field.toLowerCase())) { // Sort by aggregation
+ return `ORDER BY "score" ${direction}, "video"."views" ${direction}`
}
let firstSort: string
if (field.toLowerCase() === 'match') { // Search
firstSort = '"similarity"'
+ } else if (field === 'originallyPublishedAt') {
+ firstSort = '"publishedAtForOrder"'
} else if (field.includes('.')) {
firstSort = field
} else {
]
if (options.withFiles) {
- joins.push('INNER JOIN "videoFile" AS "VideoFiles" ON "VideoFiles"."videoId" = "video"."id"')
+ joins.push('LEFT JOIN "videoFile" AS "VideoFiles" ON "VideoFiles"."videoId" = "video"."id"')
+
+ joins.push('LEFT JOIN "videoStreamingPlaylist" AS "VideoStreamingPlaylists" ON "VideoStreamingPlaylists"."videoId" = "video"."id"')
+ joins.push(
+ 'LEFT JOIN "videoFile" AS "VideoStreamingPlaylists->VideoFiles" ' +
+ 'ON "VideoStreamingPlaylists->VideoFiles"."videoStreamingPlaylistId" = "VideoStreamingPlaylists"."id"'
+ )
Object.assign(attributes, {
'"VideoFiles"."id"': '"VideoFiles.id"',
'"VideoFiles"."resolution"': '"VideoFiles.resolution"',
'"VideoFiles"."size"': '"VideoFiles.size"',
'"VideoFiles"."extname"': '"VideoFiles.extname"',
+ '"VideoFiles"."filename"': '"VideoFiles.filename"',
+ '"VideoFiles"."fileUrl"': '"VideoFiles.fileUrl"',
+ '"VideoFiles"."torrentFilename"': '"VideoFiles.torrentFilename"',
+ '"VideoFiles"."torrentUrl"': '"VideoFiles.torrentUrl"',
'"VideoFiles"."infoHash"': '"VideoFiles.infoHash"',
'"VideoFiles"."fps"': '"VideoFiles.fps"',
- '"VideoFiles"."videoId"': '"VideoFiles.videoId"'
+ '"VideoFiles"."videoId"': '"VideoFiles.videoId"',
+
+ '"VideoStreamingPlaylists"."id"': '"VideoStreamingPlaylists.id"',
+ '"VideoStreamingPlaylists"."playlistUrl"': '"VideoStreamingPlaylists.playlistUrl"',
+ '"VideoStreamingPlaylists"."type"': '"VideoStreamingPlaylists.type"',
+ '"VideoStreamingPlaylists->VideoFiles"."id"': '"VideoStreamingPlaylists.VideoFiles.id"',
+ '"VideoStreamingPlaylists->VideoFiles"."createdAt"': '"VideoStreamingPlaylists.VideoFiles.createdAt"',
+ '"VideoStreamingPlaylists->VideoFiles"."updatedAt"': '"VideoStreamingPlaylists.VideoFiles.updatedAt"',
+ '"VideoStreamingPlaylists->VideoFiles"."resolution"': '"VideoStreamingPlaylists.VideoFiles.resolution"',
+ '"VideoStreamingPlaylists->VideoFiles"."size"': '"VideoStreamingPlaylists.VideoFiles.size"',
+ '"VideoStreamingPlaylists->VideoFiles"."extname"': '"VideoStreamingPlaylists.VideoFiles.extname"',
+ '"VideoStreamingPlaylists->VideoFiles"."filename"': '"VideoStreamingPlaylists.VideoFiles.filename"',
+ '"VideoStreamingPlaylists->VideoFiles"."fileUrl"': '"VideoStreamingPlaylists.VideoFiles.fileUrl"',
+ '"VideoStreamingPlaylists->VideoFiles"."torrentFilename"': '"VideoStreamingPlaylists.VideoFiles.torrentFilename"',
+ '"VideoStreamingPlaylists->VideoFiles"."torrentUrl"': '"VideoStreamingPlaylists.VideoFiles.torrentUrl"',
+ '"VideoStreamingPlaylists->VideoFiles"."infoHash"': '"VideoStreamingPlaylists.VideoFiles.infoHash"',
+ '"VideoStreamingPlaylists->VideoFiles"."fps"': '"VideoStreamingPlaylists.VideoFiles.fps"',
+ '"VideoStreamingPlaylists->VideoFiles"."videoStreamingPlaylistId"': '"VideoStreamingPlaylists.VideoFiles.videoStreamingPlaylistId"',
+ '"VideoStreamingPlaylists->VideoFiles"."videoId"': '"VideoStreamingPlaylists.VideoFiles.videoId"'
})
}