+import { sample } from 'lodash'
+import { literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
import {
AllowNull,
BeforeDestroy,
Table,
UpdatedAt
} from 'sequelize-typescript'
-import { ActorModel } from '../activitypub/actor'
-import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
+import { getServerActor } from '@server/models/application/application'
+import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
+import {
+ CacheFileObject,
+ FileRedundancyInformation,
+ StreamingPlaylistRedundancyInformation,
+ VideoPrivacy,
+ VideoRedundanciesTarget,
+ VideoRedundancy,
+ VideoRedundancyStrategy,
+ VideoRedundancyStrategyWithManual
+} from '@shared/models'
+import { AttributesOnly } from '@shared/typescript-utils'
+import { isTestInstance } from '../../helpers/core-utils'
import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
+import { logger } from '../../helpers/logger'
+import { CONFIG } from '../../initializers/config'
import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
-import { VideoFileModel } from '../video/video-file'
-import { getServerActor } from '../../helpers/utils'
+import { ActorModel } from '../actor/actor'
+import { ServerModel } from '../server/server'
+import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../shared'
+import { ScheduleVideoUpdateModel } from '../video/schedule-video-update'
import { VideoModel } from '../video/video'
-import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
-import { logger } from '../../helpers/logger'
-import { CacheFileObject, VideoPrivacy } from '../../../shared'
import { VideoChannelModel } from '../video/video-channel'
-import { ServerModel } from '../server/server'
-import { sample } from 'lodash'
-import { isTestInstance } from '../../helpers/core-utils'
-import * as Bluebird from 'bluebird'
-import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize'
+import { VideoFileModel } from '../video/video-file'
import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
-import { CONFIG } from '../../initializers/config'
-import { MVideoRedundancy, MVideoRedundancyVideo } from '@server/typings/models'
export enum ScopeNames {
WITH_VIDEO = 'WITH_VIDEO'
}
@Scopes(() => ({
- [ ScopeNames.WITH_VIDEO ]: {
+ [ScopeNames.WITH_VIDEO]: {
include: [
{
model: VideoFileModel,
{
fields: [ 'actorId' ]
},
+ {
+ fields: [ 'expiresOn' ]
+ },
{
fields: [ 'url' ],
unique: true
}
]
})
-export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
+export class VideoRedundancyModel extends Model<Partial<AttributesOnly<VideoRedundancyModel>>> {
@CreatedAt
createdAt: Date
@UpdatedAt
updatedAt: Date
- @AllowNull(false)
+ @AllowNull(true)
@Column
expiresOn: Date
const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
logger.info('Removing duplicated video file %s.', logIdentifier)
- videoFile.Video.removeFile(videoFile, true)
- .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
+ videoFile.Video.removeWebTorrentFile(videoFile, true)
+ .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
}
if (instance.videoStreamingPlaylistId) {
const videoUUID = videoStreamingPlaylist.Video.uuid
logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
- videoStreamingPlaylist.Video.removeStreamingPlaylist(true)
- .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
+ videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
+ .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
}
return undefined
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
}
+ static async listLocalByVideoId (videoId: number): Promise<MVideoRedundancyVideo[]> {
+ const actor = await getServerActor()
+
+ const queryStreamingPlaylist = {
+ where: {
+ actorId: actor.id
+ },
+ include: [
+ {
+ model: VideoStreamingPlaylistModel.unscoped(),
+ required: true,
+ include: [
+ {
+ model: VideoModel.unscoped(),
+ required: true,
+ where: {
+ id: videoId
+ }
+ }
+ ]
+ }
+ ]
+ }
+
+ const queryFiles = {
+ where: {
+ actorId: actor.id
+ },
+ include: [
+ {
+ model: VideoFileModel,
+ required: true,
+ include: [
+ {
+ model: VideoModel,
+ required: true,
+ where: {
+ id: videoId
+ }
+ }
+ ]
+ }
+ ]
+ }
+
+ return Promise.all([
+ VideoRedundancyModel.findAll(queryStreamingPlaylist),
+ VideoRedundancyModel.findAll(queryFiles)
+ ]).then(([ r1, r2 ]) => r1.concat(r2))
+ }
+
static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
const actor = await getServerActor()
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
}
- static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> {
+ static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MVideoRedundancyVideo> {
+ const query = {
+ where: { id },
+ transaction
+ }
+
+ return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
+ }
+
+ static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoRedundancy> {
const query = {
where: {
url
},
include: [
{
- attributes: [ ],
+ attributes: [],
model: VideoFileModel,
required: true,
include: [
{
- attributes: [ ],
+ attributes: [],
model: VideoModel,
required: true,
where: {
}
return VideoRedundancyModel.findOne(query)
- .then(r => !!r)
+ .then(r => !!r)
}
- static async getVideoSample (p: Bluebird<VideoModel[]>) {
+ static async getVideoSample (p: Promise<VideoModel[]>) {
const rows = await p
if (rows.length === 0) return undefined
}
static async findMostViewToDuplicate (randomizedFactor: number) {
+ const peertubeActor = await getServerActor()
+
// On VideoModel!
const query = {
attributes: [ 'id', 'views' ],
limit: randomizedFactor,
order: getVideoSort('-views'),
where: {
- privacy: VideoPrivacy.PUBLIC
+ privacy: VideoPrivacy.PUBLIC,
+ isLive: false,
+ ...this.buildVideoIdsForDuplication(peertubeActor)
},
include: [
- await VideoRedundancyModel.buildVideoFileForDuplication(),
VideoRedundancyModel.buildServerRedundancyInclude()
]
}
}
static async findTrendingToDuplicate (randomizedFactor: number) {
+ const peertubeActor = await getServerActor()
+
// On VideoModel!
const query = {
attributes: [ 'id', 'views' ],
limit: randomizedFactor,
order: getVideoSort('-trending'),
where: {
- privacy: VideoPrivacy.PUBLIC
+ privacy: VideoPrivacy.PUBLIC,
+ isLive: false,
+ ...this.buildVideoIdsForDuplication(peertubeActor)
},
include: [
- await VideoRedundancyModel.buildVideoFileForDuplication(),
VideoRedundancyModel.buildServerRedundancyInclude(),
VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
}
static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
+ const peertubeActor = await getServerActor()
+
// On VideoModel!
const query = {
attributes: [ 'id', 'publishedAt' ],
order: getVideoSort('-publishedAt'),
where: {
privacy: VideoPrivacy.PUBLIC,
+ isLive: false,
views: {
- [ Op.gte ]: minViews
- }
+ [Op.gte]: minViews
+ },
+ ...this.buildVideoIdsForDuplication(peertubeActor)
},
include: [
- await VideoRedundancyModel.buildVideoFileForDuplication(),
- VideoRedundancyModel.buildServerRedundancyInclude()
+ VideoRedundancyModel.buildServerRedundancyInclude(),
+
+ // Required by publishedAt sort
+ {
+ model: ScheduleVideoUpdateModel.unscoped(),
+ required: false
+ }
]
}
actorId: actor.id,
strategy,
createdAt: {
- [ Op.lt ]: expiredDate
+ [Op.lt]: expiredDate
}
}
}
return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
}
- static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
- const actor = await getServerActor()
- const redundancyInclude = {
- attributes: [],
- model: VideoRedundancyModel,
- required: true,
- where: {
- actorId: actor.id,
- strategy
- }
- }
-
- const queryFiles: FindOptions = {
- include: [ redundancyInclude ]
- }
-
- const queryStreamingPlaylists: FindOptions = {
- include: [
- {
- attributes: [],
- model: VideoModel.unscoped(),
- required: true,
- include: [
- {
- required: true,
- attributes: [],
- model: VideoStreamingPlaylistModel.unscoped(),
- include: [
- redundancyInclude
- ]
- }
- ]
- }
- ]
- }
-
- return Promise.all([
- VideoFileModel.aggregate('size', 'SUM', queryFiles),
- VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
- ]).then(([ r1, r2 ]) => {
- return parseAggregateResult(r1) + parseAggregateResult(r2)
- })
- }
-
- static async listLocalExpired () {
+ static async listLocalExpired (): Promise<MVideoRedundancyVideo[]> {
const actor = await getServerActor()
const query = {
where: {
actorId: actor.id,
expiresOn: {
- [ Op.lt ]: new Date()
+ [Op.lt]: new Date()
}
}
}
[Op.ne]: actor.id
},
expiresOn: {
- [ Op.lt ]: new Date()
+ [Op.lt]: new Date(),
+ [Op.ne]: null
}
}
}
const query = {
where: {
- actorId: actor.id
+ [Op.and]: [
+ {
+ actorId: actor.id
+ },
+ {
+ [Op.or]: [
+ {
+ '$VideoStreamingPlaylist.id$': {
+ [Op.ne]: null
+ }
+ },
+ {
+ '$VideoFile.id$': {
+ [Op.ne]: null
+ }
+ }
+ ]
+ }
+ ]
},
include: [
{
- model: VideoFileModel,
+ model: VideoFileModel.unscoped(),
required: false,
include: [ buildVideoInclude() ]
},
{
- model: VideoStreamingPlaylistModel,
+ model: VideoStreamingPlaylistModel.unscoped(),
required: false,
include: [ buildVideoInclude() ]
}
return VideoRedundancyModel.findAll(query)
}
- static async getStats (strategy: VideoRedundancyStrategy) {
- const actor = await getServerActor()
+ static listForApi (options: {
+ start: number
+ count: number
+ sort: string
+ target: VideoRedundanciesTarget
+ strategy?: string
+ }) {
+ const { start, count, sort, target, strategy } = options
+ const redundancyWhere: WhereOptions = {}
+ const videosWhere: WhereOptions = {}
+ let redundancySqlSuffix = ''
+
+ if (target === 'my-videos') {
+ Object.assign(videosWhere, { remote: false })
+ } else if (target === 'remote-videos') {
+ Object.assign(videosWhere, { remote: true })
+ Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
+ redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
+ }
- const query: FindOptions = {
- raw: true,
- attributes: [
- [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
- [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
- [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
- ],
- where: {
- strategy,
- actorId: actor.id
- },
+ if (strategy) {
+ Object.assign(redundancyWhere, { strategy })
+ }
+
+ const videoFilterWhere = {
+ [Op.and]: [
+ {
+ [Op.or]: [
+ {
+ id: {
+ [Op.in]: literal(
+ '(' +
+ 'SELECT "videoId" FROM "videoFile" ' +
+ 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
+ redundancySqlSuffix +
+ ')'
+ )
+ }
+ },
+ {
+ id: {
+ [Op.in]: literal(
+ '(' +
+ 'select "videoId" FROM "videoStreamingPlaylist" ' +
+ 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
+ redundancySqlSuffix +
+ ')'
+ )
+ }
+ }
+ ]
+ },
+
+ videosWhere
+ ]
+ }
+
+ // /!\ On video model /!\
+ const findOptions = {
+ offset: start,
+ limit: count,
+ order: getSort(sort),
include: [
{
- attributes: [],
+ required: false,
model: VideoFileModel,
- required: true
+ include: [
+ {
+ model: VideoRedundancyModel.unscoped(),
+ required: false,
+ where: redundancyWhere
+ }
+ ]
+ },
+ {
+ required: false,
+ model: VideoStreamingPlaylistModel.unscoped(),
+ include: [
+ {
+ model: VideoRedundancyModel.unscoped(),
+ required: false,
+ where: redundancyWhere
+ },
+ {
+ model: VideoFileModel,
+ required: false
+ }
+ ]
}
- ]
+ ],
+ where: videoFilterWhere
}
- return VideoRedundancyModel.findOne(query)
- .then((r: any) => ({
- totalUsed: parseAggregateResult(r.totalUsed),
- totalVideos: r.totalVideos,
- totalVideoFiles: r.totalVideoFiles
- }))
+ // /!\ On video model /!\
+ const countOptions = {
+ where: videoFilterWhere
+ }
+
+ return Promise.all([
+ VideoModel.findAll(findOptions),
+
+ VideoModel.count(countOptions)
+ ]).then(([ data, total ]) => ({ total, data }))
+ }
+
+ static async getStats (strategy: VideoRedundancyStrategyWithManual) {
+ const actor = await getServerActor()
+
+ const sql = `WITH "tmp" AS ` +
+ `(` +
+ `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
+ `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
+ `FROM "videoRedundancy" AS "videoRedundancy" ` +
+ `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
+ `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
+ `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
+ `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
+ `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
+ `), ` +
+ `"videoIds" AS (` +
+ `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
+ `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
+ `) ` +
+ `SELECT ` +
+ `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
+ `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
+ `COUNT(*) AS "totalVideoFiles" ` +
+ `FROM "tmp"`
+
+ return VideoRedundancyModel.sequelize.query<any>(sql, {
+ replacements: { strategy, actorId: actor.id },
+ type: QueryTypes.SELECT
+ }).then(([ row ]) => ({
+ totalUsed: parseAggregateResult(row.totalUsed),
+ totalVideos: row.totalVideos,
+ totalVideoFiles: row.totalVideoFiles
+ }))
+ }
+
+ static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
+ const filesRedundancies: FileRedundancyInformation[] = []
+ const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
+
+ for (const file of video.VideoFiles) {
+ for (const redundancy of file.RedundancyVideos) {
+ filesRedundancies.push({
+ id: redundancy.id,
+ fileUrl: redundancy.fileUrl,
+ strategy: redundancy.strategy,
+ createdAt: redundancy.createdAt,
+ updatedAt: redundancy.updatedAt,
+ expiresOn: redundancy.expiresOn,
+ size: file.size
+ })
+ }
+ }
+
+ for (const playlist of video.VideoStreamingPlaylists) {
+ const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
+
+ for (const redundancy of playlist.RedundancyVideos) {
+ streamingPlaylistsRedundancies.push({
+ id: redundancy.id,
+ fileUrl: redundancy.fileUrl,
+ strategy: redundancy.strategy,
+ createdAt: redundancy.createdAt,
+ updatedAt: redundancy.updatedAt,
+ expiresOn: redundancy.expiresOn,
+ size
+ })
+ }
+ }
+
+ return {
+ id: video.id,
+ name: video.name,
+ url: video.url,
+ uuid: video.uuid,
+
+ redundancies: {
+ files: filesRedundancies,
+ streamingPlaylists: streamingPlaylistsRedundancies
+ }
+ }
}
getVideo () {
- if (this.VideoFile) return this.VideoFile.Video
+ if (this.VideoFile?.Video) return this.VideoFile.Video
- return this.VideoStreamingPlaylist.Video
+ if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video
+
+ return undefined
+ }
+
+ getVideoUUID () {
+ const video = this.getVideo()
+ if (!video) return undefined
+
+ return video.uuid
}
isOwned () {
return !!this.strategy
}
- toActivityPubObject (): CacheFileObject {
+ toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
if (this.VideoStreamingPlaylist) {
return {
id: this.url,
type: 'CacheFile' as 'CacheFile',
object: this.VideoStreamingPlaylist.Video.url,
- expires: this.expiresOn.toISOString(),
+ expires: this.expiresOn ? this.expiresOn.toISOString() : null,
url: {
type: 'Link',
- mimeType: 'application/x-mpegURL',
mediaType: 'application/x-mpegURL',
href: this.fileUrl
}
id: this.url,
type: 'CacheFile' as 'CacheFile',
object: this.VideoFile.Video.url,
- expires: this.expiresOn.toISOString(),
+ expires: this.expiresOn ? this.expiresOn.toISOString() : null,
url: {
type: 'Link',
- mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
- mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
+ mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
href: this.fileUrl,
height: this.VideoFile.resolution,
size: this.VideoFile.size,
}
// Don't include video files we already duplicated
- private static async buildVideoFileForDuplication () {
- const actor = await getServerActor()
-
+ private static buildVideoIdsForDuplication (peertubeActor: MActor) {
const notIn = literal(
'(' +
- `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
+ `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
+ `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
+ `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
+ `UNION ` +
+ `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
+ `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
+ `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
')'
)
return {
- attributes: [],
- model: VideoFileModel.unscoped(),
- required: true,
- where: {
- id: {
- [ Op.notIn ]: notIn
- }
+ id: {
+ [Op.notIn]: notIn
}
}
}
projects / github / Chocobozzz / PeerTube.git / blobdiff