From 764b1a14fc494f2cfd7ea590d2f07b01df65c7ad Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Fri, 23 Jul 2021 11:20:00 +0200
Subject: Use random names for VOD HLS playlists

---
 server/models/actor/actor-follow.ts                | 10 +--
 server/models/redundancy/video-redundancy.ts       |  4 +-
 .../models/video/formatter/video-format-utils.ts   |  8 +-
 server/models/video/sql/shared/video-tables.ts     |  3 +-
 server/models/video/video-file.ts                  | 45 +++++++++---
 server/models/video/video-streaming-playlist.ts    | 85 ++++++++++++++--------
 server/models/video/video.ts                       | 12 +--
 7 files changed, 105 insertions(+), 62 deletions(-)

(limited to 'server/models')

diff --git a/server/models/actor/actor-follow.ts b/server/models/actor/actor-follow.ts
index 83c00a22d..3080e02a6 100644
--- a/server/models/actor/actor-follow.ts
+++ b/server/models/actor/actor-follow.ts
@@ -19,8 +19,8 @@ import {
   UpdatedAt
 } from 'sequelize-typescript'
 import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
+import { doesExist } from '@server/helpers/database-utils'
 import { getServerActor } from '@server/models/application/application'
-import { VideoModel } from '@server/models/video/video'
 import {
   MActorFollowActorsDefault,
   MActorFollowActorsDefaultSubscription,
@@ -166,14 +166,8 @@ export class ActorFollowModel extends Model<Partial<AttributesOnly<ActorFollowMo
 
   static isFollowedBy (actorId: number, followerActorId: number) {
     const query = 'SELECT 1 FROM "actorFollow" WHERE "actorId" = $followerActorId AND "targetActorId" = $actorId LIMIT 1'
-    const options = {
-      type: QueryTypes.SELECT as QueryTypes.SELECT,
-      bind: { actorId, followerActorId },
-      raw: true
-    }
 
-    return VideoModel.sequelize.query(query, options)
-                     .then(results => results.length === 1)
+    return doesExist(query, { actorId, followerActorId })
   }
 
   static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Promise<MActorFollowActorsDefault> {
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index ccda023e0..d645be248 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -160,8 +160,8 @@ export class VideoRedundancyModel extends Model<Partial<AttributesOnly<VideoRedu
       const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
       logger.info('Removing duplicated video file %s.', logIdentifier)
 
-      videoFile.Video.removeFile(videoFile, true)
-               .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
+      videoFile.Video.removeFileAndTorrent(videoFile, true)
+        .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
     }
 
     if (instance.videoStreamingPlaylistId) {
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts
index 6b1e59063..3310b3b46 100644
--- a/server/models/video/formatter/video-format-utils.ts
+++ b/server/models/video/formatter/video-format-utils.ts
@@ -182,8 +182,8 @@ function streamingPlaylistsModelToFormattedJSON (
       return {
         id: playlist.id,
         type: playlist.type,
-        playlistUrl: playlist.playlistUrl,
-        segmentsSha256Url: playlist.segmentsSha256Url,
+        playlistUrl: playlist.getMasterPlaylistUrl(video),
+        segmentsSha256Url: playlist.getSha256SegmentsUrl(video),
         redundancies,
         files
       }
@@ -331,7 +331,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
       type: 'Link',
       name: 'sha256',
       mediaType: 'application/json' as 'application/json',
-      href: playlist.segmentsSha256Url
+      href: playlist.getSha256SegmentsUrl(video)
     })
 
     addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || [])
@@ -339,7 +339,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
     url.push({
       type: 'Link',
       mediaType: 'application/x-mpegURL' as 'application/x-mpegURL',
-      href: playlist.playlistUrl,
+      href: playlist.getMasterPlaylistUrl(video),
       tag
     })
   }
diff --git a/server/models/video/sql/shared/video-tables.ts b/server/models/video/sql/shared/video-tables.ts
index abdd22188..742d19099 100644
--- a/server/models/video/sql/shared/video-tables.ts
+++ b/server/models/video/sql/shared/video-tables.ts
@@ -92,12 +92,13 @@ export class VideoTables {
   }
 
   getStreamingPlaylistAttributes () {
-    let playlistKeys = [ 'id', 'playlistUrl', 'type' ]
+    let playlistKeys = [ 'id', 'playlistUrl', 'playlistFilename', 'type' ]
 
     if (this.mode === 'get') {
       playlistKeys = playlistKeys.concat([
         'p2pMediaLoaderInfohashes',
         'p2pMediaLoaderPeerVersion',
+        'segmentsSha256Filename',
         'segmentsSha256Url',
         'videoId',
         'createdAt',
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 22cf63804..797a85a4e 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -1,7 +1,7 @@
 import { remove } from 'fs-extra'
 import * as memoizee from 'memoizee'
 import { join } from 'path'
-import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize'
+import { FindOptions, Op, Transaction } from 'sequelize'
 import {
   AllowNull,
   BelongsTo,
@@ -21,6 +21,7 @@ import {
 import { Where } from 'sequelize/types/lib/utils'
 import validator from 'validator'
 import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
+import { doesExist } from '@server/helpers/database-utils'
 import { logger } from '@server/helpers/logger'
 import { extractVideo } from '@server/helpers/video'
 import { getTorrentFilePath } from '@server/lib/video-paths'
@@ -250,14 +251,8 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
 
   static doesInfohashExist (infoHash: string) {
     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
-    const options = {
-      type: QueryTypes.SELECT as QueryTypes.SELECT,
-      bind: { infoHash },
-      raw: true
-    }
 
-    return VideoModel.sequelize.query(query, options)
-              .then(results => results.length === 1)
+    return doesExist(query, { infoHash })
   }
 
   static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
@@ -266,6 +261,33 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
     return !!videoFile
   }
 
+  static async doesOwnedTorrentFileExist (filename: string) {
+    const query = 'SELECT 1 FROM "videoFile" ' +
+                  'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
+                  'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
+                  'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
+                  'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
+
+    return doesExist(query, { filename })
+  }
+
+  static async doesOwnedWebTorrentVideoFileExist (filename: string) {
+    const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
+                  'WHERE "filename" = $filename LIMIT 1'
+
+    return doesExist(query, { filename })
+  }
+
+  static loadByFilename (filename: string) {
+    const query = {
+      where: {
+        filename
+      }
+    }
+
+    return VideoFileModel.findOne(query)
+  }
+
   static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
     const query = {
       where: {
@@ -443,10 +465,9 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
   }
 
   getFileDownloadUrl (video: MVideoWithHost) {
-    const basePath = this.isHLS()
-      ? STATIC_DOWNLOAD_PATHS.HLS_VIDEOS
-      : STATIC_DOWNLOAD_PATHS.VIDEOS
-    const path = join(basePath, this.filename)
+    const path = this.isHLS()
+      ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
+      : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
 
     if (video.isOwned()) return WEBSERVER.URL + path
 
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index d627e8c9d..b15d20cf9 100644
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -1,19 +1,27 @@
 import * as memoizee from 'memoizee'
 import { join } from 'path'
-import { Op, QueryTypes } from 'sequelize'
+import { Op } from 'sequelize'
 import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { doesExist } from '@server/helpers/database-utils'
 import { VideoFileModel } from '@server/models/video/video-file'
-import { MStreamingPlaylist } from '@server/types/models'
+import { MStreamingPlaylist, MVideo } from '@server/types/models'
+import { AttributesOnly } from '@shared/core-utils'
 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
 import { sha1 } from '../../helpers/core-utils'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
 import { isArrayOf } from '../../helpers/custom-validators/misc'
 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
-import { CONSTRAINTS_FIELDS, MEMOIZE_LENGTH, MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, STATIC_PATHS } from '../../initializers/constants'
+import {
+  CONSTRAINTS_FIELDS,
+  MEMOIZE_LENGTH,
+  MEMOIZE_TTL,
+  P2P_MEDIA_LOADER_PEER_VERSION,
+  STATIC_PATHS,
+  WEBSERVER
+} from '../../initializers/constants'
 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
 import { throwIfNotValid } from '../utils'
 import { VideoModel } from './video'
-import { AttributesOnly } from '@shared/core-utils'
 
 @Table({
   tableName: 'videoStreamingPlaylist',
@@ -43,7 +51,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
   type: VideoStreamingPlaylistType
 
   @AllowNull(false)
-  @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url'))
+  @Column
+  playlistFilename: string
+
+  @AllowNull(true)
+  @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url', true))
   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max))
   playlistUrl: string
 
@@ -57,7 +69,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
   p2pMediaLoaderPeerVersion: number
 
   @AllowNull(false)
-  @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url'))
+  @Column
+  segmentsSha256Filename: string
+
+  @AllowNull(true)
+  @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url', true))
   @Column
   segmentsSha256Url: string
 
@@ -98,14 +114,8 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
 
   static doesInfohashExist (infoHash: string) {
     const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1'
-    const options = {
-      type: QueryTypes.SELECT as QueryTypes.SELECT,
-      bind: { infoHash },
-      raw: true
-    }
 
-    return VideoModel.sequelize.query<object>(query, options)
-              .then(results => results.length === 1)
+    return doesExist(query, { infoHash })
   }
 
   static buildP2PMediaLoaderInfoHashes (playlistUrl: string, files: unknown[]) {
@@ -125,7 +135,13 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
         p2pMediaLoaderPeerVersion: {
           [Op.ne]: P2P_MEDIA_LOADER_PEER_VERSION
         }
-      }
+      },
+      include: [
+        {
+          model: VideoModel.unscoped(),
+          required: true
+        }
+      ]
     }
 
     return VideoStreamingPlaylistModel.findAll(query)
@@ -144,7 +160,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
     return VideoStreamingPlaylistModel.findByPk(id, options)
   }
 
-  static loadHLSPlaylistByVideo (videoId: number) {
+  static loadHLSPlaylistByVideo (videoId: number): Promise<MStreamingPlaylist> {
     const options = {
       where: {
         type: VideoStreamingPlaylistType.HLS,
@@ -155,30 +171,29 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
     return VideoStreamingPlaylistModel.findOne(options)
   }
 
-  static getHlsPlaylistFilename (resolution: number) {
-    return resolution + '.m3u8'
-  }
+  static async loadOrGenerate (video: MVideo) {
+    let playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
+    if (!playlist) playlist = new VideoStreamingPlaylistModel()
 
-  static getMasterHlsPlaylistFilename () {
-    return 'master.m3u8'
+    return Object.assign(playlist, { videoId: video.id, Video: video })
   }
 
-  static getHlsSha256SegmentsFilename () {
-    return 'segments-sha256.json'
-  }
+  assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) {
+    const masterPlaylistUrl = this.getMasterPlaylistUrl(video)
 
-  static getHlsMasterPlaylistStaticPath (videoUUID: string) {
-    return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
+    this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files)
   }
 
-  static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) {
-    return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
+  getMasterPlaylistUrl (video: MVideo) {
+    if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid)
+
+    return this.playlistUrl
   }
 
-  static getHlsSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) {
-    if (isLive) return join('/live', 'segments-sha256', videoUUID)
+  getSha256SegmentsUrl (video: MVideo) {
+    if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive)
 
-    return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
+    return this.segmentsSha256Url
   }
 
   getStringType () {
@@ -195,4 +210,14 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
     return this.type === other.type &&
       this.videoId === other.videoId
   }
+
+  private getMasterPlaylistStaticPath (videoUUID: string) {
+    return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename)
+  }
+
+  private getSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) {
+    if (isLive) return join('/live', 'segments-sha256', videoUUID)
+
+    return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename)
+  }
 }
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 1e5648a36..0f0f894e4 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -762,8 +762,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
 
       // Remove physical files and torrents
       instance.VideoFiles.forEach(file => {
-        tasks.push(instance.removeFile(file))
-        tasks.push(file.removeTorrent())
+        tasks.push(instance.removeFileAndTorrent(file))
       })
 
       // Remove playlists file
@@ -1670,10 +1669,13 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
                                        .concat(toAdd)
   }
 
-  removeFile (videoFile: MVideoFile, isRedundancy = false) {
+  removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) {
     const filePath = getVideoFilePath(this, videoFile, isRedundancy)
-    return remove(filePath)
-      .catch(err => logger.warn('Cannot delete file %s.', filePath, { err }))
+
+    const promises: Promise<any>[] = [ remove(filePath) ]
+    if (!isRedundancy) promises.push(videoFile.removeTorrent())
+
+    return Promise.all(promises)
   }
 
   async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) {
-- 
cgit v1.2.3