+import { sample } from 'lodash'
+import { literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
import {
AllowNull,
BeforeDestroy,
Table,
UpdatedAt
} from 'sequelize-typescript'
-import { ActorModel } from '../activitypub/actor'
-import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
+import { getServerActor } from '@server/models/application/application'
+import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
+import {
+ CacheFileObject,
+ FileRedundancyInformation,
+ StreamingPlaylistRedundancyInformation,
+ VideoPrivacy,
+ VideoRedundanciesTarget,
+ VideoRedundancy,
+ VideoRedundancyStrategy,
+ VideoRedundancyStrategyWithManual
+} from '@shared/models'
+import { AttributesOnly } from '@shared/typescript-utils'
+import { isTestInstance } from '../../helpers/core-utils'
import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
+import { logger } from '../../helpers/logger'
+import { CONFIG } from '../../initializers/config'
import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
-import { VideoFileModel } from '../video/video-file'
+import { ActorModel } from '../actor/actor'
+import { ServerModel } from '../server/server'
+import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../shared'
+import { ScheduleVideoUpdateModel } from '../video/schedule-video-update'
import { VideoModel } from '../video/video'
-import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
-import { logger } from '../../helpers/logger'
-import { CacheFileObject, VideoPrivacy } from '../../../shared'
import { VideoChannelModel } from '../video/video-channel'
-import { ServerModel } from '../server/server'
-import { sample } from 'lodash'
-import { isTestInstance } from '../../helpers/core-utils'
-import * as Bluebird from 'bluebird'
-import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
+import { VideoFileModel } from '../video/video-file'
import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
-import { CONFIG } from '../../initializers/config'
-import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
-import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
-import {
- FileRedundancyInformation,
- StreamingPlaylistRedundancyInformation,
- VideoRedundancy
-} from '@shared/models/redundancy/video-redundancy.model'
-import { getServerActor } from '@server/models/application/application'
export enum ScopeNames {
WITH_VIDEO = 'WITH_VIDEO'
{
fields: [ 'actorId' ]
},
+ {
+ fields: [ 'expiresOn' ]
+ },
{
fields: [ 'url' ],
unique: true
}
]
})
-export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
+export class VideoRedundancyModel extends Model<Partial<AttributesOnly<VideoRedundancyModel>>> {
@CreatedAt
createdAt: Date
const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
logger.info('Removing duplicated video file %s.', logIdentifier)
- videoFile.Video.removeFile(videoFile, true)
- .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
+ videoFile.Video.removeWebTorrentFile(videoFile, true)
+ .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
}
if (instance.videoStreamingPlaylistId) {
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
}
+ static async listLocalByVideoId (videoId: number): Promise<MVideoRedundancyVideo[]> {
+ const actor = await getServerActor()
+
+ const queryStreamingPlaylist = {
+ where: {
+ actorId: actor.id
+ },
+ include: [
+ {
+ model: VideoStreamingPlaylistModel.unscoped(),
+ required: true,
+ include: [
+ {
+ model: VideoModel.unscoped(),
+ required: true,
+ where: {
+ id: videoId
+ }
+ }
+ ]
+ }
+ ]
+ }
+
+ const queryFiles = {
+ where: {
+ actorId: actor.id
+ },
+ include: [
+ {
+ model: VideoFileModel,
+ required: true,
+ include: [
+ {
+ model: VideoModel,
+ required: true,
+ where: {
+ id: videoId
+ }
+ }
+ ]
+ }
+ ]
+ }
+
+ return Promise.all([
+ VideoRedundancyModel.findAll(queryStreamingPlaylist),
+ VideoRedundancyModel.findAll(queryFiles)
+ ]).then(([ r1, r2 ]) => r1.concat(r2))
+ }
+
static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
const actor = await getServerActor()
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
}
- static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird<MVideoRedundancyVideo> {
+ static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MVideoRedundancyVideo> {
const query = {
where: { id },
transaction
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
}
- static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> {
+ static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoRedundancy> {
const query = {
where: {
url
.then(r => !!r)
}
- static async getVideoSample (p: Bluebird<VideoModel[]>) {
+ static async getVideoSample (p: Promise<VideoModel[]>) {
const rows = await p
if (rows.length === 0) return undefined
}
static async findMostViewToDuplicate (randomizedFactor: number) {
+ const peertubeActor = await getServerActor()
+
// On VideoModel!
const query = {
attributes: [ 'id', 'views' ],
limit: randomizedFactor,
order: getVideoSort('-views'),
where: {
- privacy: VideoPrivacy.PUBLIC
+ privacy: VideoPrivacy.PUBLIC,
+ isLive: false,
+ ...this.buildVideoIdsForDuplication(peertubeActor)
},
include: [
- await VideoRedundancyModel.buildVideoFileForDuplication(),
VideoRedundancyModel.buildServerRedundancyInclude()
]
}
}
static async findTrendingToDuplicate (randomizedFactor: number) {
+ const peertubeActor = await getServerActor()
+
// On VideoModel!
const query = {
attributes: [ 'id', 'views' ],
limit: randomizedFactor,
order: getVideoSort('-trending'),
where: {
- privacy: VideoPrivacy.PUBLIC
+ privacy: VideoPrivacy.PUBLIC,
+ isLive: false,
+ ...this.buildVideoIdsForDuplication(peertubeActor)
},
include: [
- await VideoRedundancyModel.buildVideoFileForDuplication(),
VideoRedundancyModel.buildServerRedundancyInclude(),
VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
}
static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
+ const peertubeActor = await getServerActor()
+
// On VideoModel!
const query = {
attributes: [ 'id', 'publishedAt' ],
order: getVideoSort('-publishedAt'),
where: {
privacy: VideoPrivacy.PUBLIC,
+ isLive: false,
views: {
[Op.gte]: minViews
- }
+ },
+ ...this.buildVideoIdsForDuplication(peertubeActor)
},
include: [
- await VideoRedundancyModel.buildVideoFileForDuplication(),
- VideoRedundancyModel.buildServerRedundancyInclude()
+ VideoRedundancyModel.buildServerRedundancyInclude(),
+
+ // Required by publishedAt sort
+ {
+ model: ScheduleVideoUpdateModel.unscoped(),
+ required: false
+ }
]
}
return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
}
- static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
- const actor = await getServerActor()
- const redundancyInclude = {
- attributes: [],
- model: VideoRedundancyModel,
- required: true,
- where: {
- actorId: actor.id,
- strategy
- }
- }
-
- const queryFiles: FindOptions = {
- include: [ redundancyInclude ]
- }
-
- const queryStreamingPlaylists: FindOptions = {
- include: [
- {
- attributes: [],
- model: VideoModel.unscoped(),
- required: true,
- include: [
- {
- required: true,
- attributes: [],
- model: VideoStreamingPlaylistModel.unscoped(),
- include: [
- redundancyInclude
- ]
- }
- ]
- }
- ]
- }
-
- return Promise.all([
- VideoFileModel.aggregate('size', 'SUM', queryFiles),
- VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
- ]).then(([ r1, r2 ]) => {
- return parseAggregateResult(r1) + parseAggregateResult(r2)
- })
- }
-
- static async listLocalExpired () {
+ static async listLocalExpired (): Promise<MVideoRedundancyVideo[]> {
const actor = await getServerActor()
const query = {
const query = {
where: {
- actorId: actor.id
+ [Op.and]: [
+ {
+ actorId: actor.id
+ },
+ {
+ [Op.or]: [
+ {
+ '$VideoStreamingPlaylist.id$': {
+ [Op.ne]: null
+ }
+ },
+ {
+ '$VideoFile.id$': {
+ [Op.ne]: null
+ }
+ }
+ ]
+ }
+ ]
},
include: [
{
- model: VideoFileModel,
+ model: VideoFileModel.unscoped(),
required: false,
include: [ buildVideoInclude() ]
},
{
- model: VideoStreamingPlaylistModel,
+ model: VideoStreamingPlaylistModel.unscoped(),
required: false,
include: [ buildVideoInclude() ]
}
}
if (strategy) {
- Object.assign(redundancyWhere, { strategy: strategy })
+ Object.assign(redundancyWhere, { strategy })
}
const videoFilterWhere = {
static async getStats (strategy: VideoRedundancyStrategyWithManual) {
const actor = await getServerActor()
- const query: FindOptions = {
- raw: true,
- attributes: [
- [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
- [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
- [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
- ],
- where: {
- strategy,
- actorId: actor.id
- },
- include: [
- {
- attributes: [],
- model: VideoFileModel,
- required: true
- }
- ]
- }
-
- return VideoRedundancyModel.findOne(query)
- .then((r: any) => ({
- totalUsed: parseAggregateResult(r.totalUsed),
- totalVideos: r.totalVideos,
- totalVideoFiles: r.totalVideoFiles
- }))
+ const sql = `WITH "tmp" AS ` +
+ `(` +
+ `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
+ `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
+ `FROM "videoRedundancy" AS "videoRedundancy" ` +
+ `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
+ `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
+ `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
+ `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
+ `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
+ `), ` +
+ `"videoIds" AS (` +
+ `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
+ `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
+ `) ` +
+ `SELECT ` +
+ `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
+ `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
+ `COUNT(*) AS "totalVideoFiles" ` +
+ `FROM "tmp"`
+
+ return VideoRedundancyModel.sequelize.query<any>(sql, {
+ replacements: { strategy, actorId: actor.id },
+ type: QueryTypes.SELECT
+ }).then(([ row ]) => ({
+ totalUsed: parseAggregateResult(row.totalUsed),
+ totalVideos: row.totalVideos,
+ totalVideoFiles: row.totalVideoFiles
+ }))
}
static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
}
getVideo () {
- if (this.VideoFile) return this.VideoFile.Video
+ if (this.VideoFile?.Video) return this.VideoFile.Video
+
+ if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video
- return this.VideoStreamingPlaylist.Video
+ return undefined
+ }
+
+ getVideoUUID () {
+ const video = this.getVideo()
+ if (!video) return undefined
+
+ return video.uuid
}
isOwned () {
}
// Don't include video files we already duplicated
- private static async buildVideoFileForDuplication () {
- const actor = await getServerActor()
-
+ private static buildVideoIdsForDuplication (peertubeActor: MActor) {
const notIn = literal(
'(' +
- `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
+ `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
+ `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
+ `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
+ `UNION ` +
+ `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
+ `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
+ `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
')'
)
return {
- attributes: [],
- model: VideoFileModel,
- required: true,
- where: {
- id: {
- [Op.notIn]: notIn
- }
+ id: {
+ [Op.notIn]: notIn
}
}
}
projects / github / Chocobozzz / PeerTube.git / blobdiff