Add ability to disable webtorrent

In favour of HLS

8 files changed, 296 insertions, 182 deletions

diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index 61d9a5612..77f83d8aa 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -497,7 +497,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
         expires: this.expiresOn.toISOString(),
         url: {
           type: 'Link',
-          mimeType: 'application/x-mpegURL',
           mediaType: 'application/x-mpegURL',
           href: this.fileUrl
         }
@@ -511,7 +510,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
       expires: this.expiresOn.toISOString(),
       url: {
         type: 'Link',
-        mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
         mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
         href: this.fileUrl,
         height: this.VideoFile.resolution,
diff --git a/server/models/utils.ts b/server/models/utils.ts
index e7e6ddde1..ccdbcd1cf 100644
--- a/server/models/utils.ts
+++ b/server/models/utils.ts
@@ -1,7 +1,7 @@
 import { Model, Sequelize } from 'sequelize-typescript'
 import * as validator from 'validator'
 import { Col } from 'sequelize/types/lib/utils'
-import { col, literal, OrderItem } from 'sequelize'
+import { literal, OrderItem } from 'sequelize'
 
 type SortType = { sortModel: string, sortValue: string }
 
diff --git a/server/models/video/schedule-video-update.ts b/server/models/video/schedule-video-update.ts
index fc2a424aa..eefc10f14 100644
--- a/server/models/video/schedule-video-update.ts
+++ b/server/models/video/schedule-video-update.ts
@@ -2,7 +2,7 @@ import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Model, Ta
 import { ScopeNames as VideoScopeNames, VideoModel } from './video'
 import { VideoPrivacy } from '../../../shared/models/videos'
 import { Op, Transaction } from 'sequelize'
-import { MScheduleVideoUpdateFormattable } from '@server/typings/models'
+import { MScheduleVideoUpdateFormattable, MScheduleVideoUpdateVideoAll } from '@server/typings/models'
 
 @Table({
   tableName: 'scheduleVideoUpdate',
@@ -72,10 +72,12 @@ export class ScheduleVideoUpdateModel extends Model<ScheduleVideoUpdateModel> {
         {
           model: VideoModel.scope(
             [
-              VideoScopeNames.WITH_FILES,
+              VideoScopeNames.WITH_WEBTORRENT_FILES,
+              VideoScopeNames.WITH_STREAMING_PLAYLISTS,
               VideoScopeNames.WITH_ACCOUNT_DETAILS,
               VideoScopeNames.WITH_BLACKLISTED,
-              VideoScopeNames.WITH_THUMBNAILS
+              VideoScopeNames.WITH_THUMBNAILS,
+              VideoScopeNames.WITH_TAGS
             ]
           )
         }
@@ -83,7 +85,7 @@ export class ScheduleVideoUpdateModel extends Model<ScheduleVideoUpdateModel> {
       transaction: t
     }
 
-    return ScheduleVideoUpdateModel.findAll(query)
+    return ScheduleVideoUpdateModel.findAll<MScheduleVideoUpdateVideoAll>(query)
   }
 
   static deleteByVideoId (videoId: number, t: Transaction) {
diff --git a/server/models/video/video-change-ownership.ts b/server/models/video/video-change-ownership.ts
index f7a351329..3259b6c02 100644
--- a/server/models/video/video-change-ownership.ts
+++ b/server/models/video/video-change-ownership.ts
@@ -43,7 +43,11 @@ enum ScopeNames {
   [ScopeNames.WITH_VIDEO]: {
     include: [
       {
-        model: VideoModel.scope([ VideoScopeNames.WITH_THUMBNAILS, VideoScopeNames.WITH_FILES ]),
+        model: VideoModel.scope([
+          VideoScopeNames.WITH_THUMBNAILS,
+          VideoScopeNames.WITH_WEBTORRENT_FILES,
+          VideoScopeNames.WITH_STREAMING_PLAYLISTS
+        ]),
         required: true
       }
     ]
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 68e2d562a..cacef0106 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -23,22 +23,52 @@ import { parseAggregateResult, throwIfNotValid } from '../utils'
 import { VideoModel } from './video'
 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
-import { FindOptions, QueryTypes, Transaction } from 'sequelize'
+import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize'
 import { MIMETYPES } from '../../initializers/constants'
-import { MVideoFile } from '@server/typings/models'
+import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../typings/models/video/video-file'
+import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/typings/models'
 
 @Table({
   tableName: 'videoFile',
   indexes: [
     {
-      fields: [ 'videoId' ]
+      fields: [ 'videoId' ],
+      where: {
+        videoId: {
+          [Op.ne]: null
+        }
+      }
+    },
+    {
+      fields: [ 'videoStreamingPlaylistId' ],
+      where: {
+        videoStreamingPlaylistId: {
+          [Op.ne]: null
+        }
+      }
     },
+
     {
       fields: [ 'infoHash' ]
     },
+
     {
       fields: [ 'videoId', 'resolution', 'fps' ],
-      unique: true
+      unique: true,
+      where: {
+        videoId: {
+          [Op.ne]: null
+        }
+      }
+    },
+    {
+      fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
+      unique: true,
+      where: {
+        videoStreamingPlaylistId: {
+          [Op.ne]: null
+        }
+      }
     }
   ]
 })
@@ -81,12 +111,24 @@ export class VideoFileModel extends Model<VideoFileModel> {
 
   @BelongsTo(() => VideoModel, {
     foreignKey: {
-      allowNull: false
+      allowNull: true
     },
     onDelete: 'CASCADE'
   })
   Video: VideoModel
 
+  @ForeignKey(() => VideoStreamingPlaylistModel)
+  @Column
+  videoStreamingPlaylistId: number
+
+  @BelongsTo(() => VideoStreamingPlaylistModel, {
+    foreignKey: {
+      allowNull: true
+    },
+    onDelete: 'CASCADE'
+  })
+  VideoStreamingPlaylist: VideoStreamingPlaylistModel
+
   @HasMany(() => VideoRedundancyModel, {
     foreignKey: {
       allowNull: true
@@ -163,6 +205,36 @@ export class VideoFileModel extends Model<VideoFileModel> {
       }))
   }
 
+  // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
+  static async customUpsert (
+    videoFile: MVideoFile,
+    mode: 'streaming-playlist' | 'video',
+    transaction: Transaction
+  ) {
+    const baseWhere = {
+      fps: videoFile.fps,
+      resolution: videoFile.resolution
+    }
+
+    if (mode === 'streaming-playlist') Object.assign(baseWhere, { videoStreamingPlaylistId: videoFile.videoStreamingPlaylistId })
+    else Object.assign(baseWhere, { videoId: videoFile.videoId })
+
+    const element = await VideoFileModel.findOne({ where: baseWhere, transaction })
+    if (!element) return videoFile.save({ transaction })
+
+    for (const k of Object.keys(videoFile.toJSON())) {
+      element[k] = videoFile[k]
+    }
+
+    return element.save({ transaction })
+  }
+
+  getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
+    if (this.videoId) return (this as MVideoFileVideo).Video
+
+    return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
+  }
+
   isAudio () {
     return !!MIMETYPES.AUDIO.EXT_MIMETYPE[this.extname]
   }
@@ -170,6 +242,9 @@ export class VideoFileModel extends Model<VideoFileModel> {
   hasSameUniqueKeysThan (other: MVideoFile) {
     return this.fps === other.fps &&
       this.resolution === other.resolution &&
-      this.videoId === other.videoId
+      (
+        (this.videoId !== null && this.videoId === other.videoId) ||
+        (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
+      )
   }
 }
diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts
index 2987aa780..9fed2d49d 100644
--- a/server/models/video/video-format-utils.ts
+++ b/server/models/video/video-format-utils.ts
@@ -1,11 +1,6 @@
-import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
+import { Video, VideoDetails } from '../../../shared/models/videos'
 import { VideoModel } from './video'
-import {
-  ActivityPlaylistInfohashesObject,
-  ActivityPlaylistSegmentHashesObject,
-  ActivityUrlObject,
-  VideoTorrentObject
-} from '../../../shared/models/activitypub/objects'
+import { ActivityTagObject, ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects'
 import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
 import { VideoCaptionModel } from './video-caption'
 import {
@@ -16,9 +11,18 @@ import {
 } from '../../lib/activitypub'
 import { isArray } from '../../helpers/custom-validators/misc'
 import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
-import { MStreamingPlaylistRedundanciesOpt, MVideo, MVideoAP, MVideoFormattable, MVideoFormattableDetails } from '../../typings/models'
-import { MStreamingPlaylistRedundancies } from '../../typings/models/video/video-streaming-playlist'
+import {
+  MStreamingPlaylistRedundanciesOpt,
+  MStreamingPlaylistVideo,
+  MVideo,
+  MVideoAP,
+  MVideoFile,
+  MVideoFormattable,
+  MVideoFormattableDetails
+} from '../../typings/models'
 import { MVideoFileRedundanciesOpt } from '../../typings/models/video/video-file'
+import { VideoFile } from '@shared/models/videos/video-file.model'
+import { generateMagnetUri } from '@server/helpers/webtorrent'
 
 export type VideoFormattingJSONOptions = {
   completeDescription?: boolean
@@ -115,7 +119,7 @@ function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): Vid
 
   const tags = video.Tags ? video.Tags.map(t => t.name) : []
 
-  const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video.VideoStreamingPlaylists)
+  const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists)
 
   const detailsJson = {
     support: video.support,
@@ -138,33 +142,43 @@ function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): Vid
   }
 
   // Format and sort video files
-  detailsJson.files = videoFilesModelToFormattedJSON(video, video.VideoFiles)
+  detailsJson.files = videoFilesModelToFormattedJSON(video, baseUrlHttp, baseUrlWs, video.VideoFiles)
 
   return Object.assign(formattedJson, detailsJson)
 }
 
-function streamingPlaylistsModelToFormattedJSON (playlists: MStreamingPlaylistRedundanciesOpt[]): VideoStreamingPlaylist[] {
+function streamingPlaylistsModelToFormattedJSON (video: MVideo, playlists: MStreamingPlaylistRedundanciesOpt[]): VideoStreamingPlaylist[] {
   if (isArray(playlists) === false) return []
 
+  const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
+
   return playlists
     .map(playlist => {
+      const playlistWithVideo = Object.assign(playlist, { Video: video })
+
       const redundancies = isArray(playlist.RedundancyVideos)
         ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl }))
         : []
 
+      const files = videoFilesModelToFormattedJSON(playlistWithVideo, baseUrlHttp, baseUrlWs, playlist.VideoFiles)
+
       return {
         id: playlist.id,
         type: playlist.type,
         playlistUrl: playlist.playlistUrl,
         segmentsSha256Url: playlist.segmentsSha256Url,
-        redundancies
-      } as VideoStreamingPlaylist
+        redundancies,
+        files
+      }
     })
 }
 
-function videoFilesModelToFormattedJSON (video: MVideo, videoFiles: MVideoFileRedundanciesOpt[]): VideoFile[] {
-  const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
-
+function videoFilesModelToFormattedJSON (
+  model: MVideo | MStreamingPlaylistVideo,
+  baseUrlHttp: string,
+  baseUrlWs: string,
+  videoFiles: MVideoFileRedundanciesOpt[]
+): VideoFile[] {
   return videoFiles
     .map(videoFile => {
       let resolutionLabel = videoFile.resolution + 'p'
@@ -174,13 +188,13 @@ function videoFilesModelToFormattedJSON (video: MVideo, videoFiles: MVideoFileRe
           id: videoFile.resolution,
           label: resolutionLabel
         },
-        magnetUri: video.generateMagnetUri(videoFile, baseUrlHttp, baseUrlWs),
+        magnetUri: generateMagnetUri(model, videoFile, baseUrlHttp, baseUrlWs),
         size: videoFile.size,
         fps: videoFile.fps,
-        torrentUrl: video.getTorrentUrl(videoFile, baseUrlHttp),
-        torrentDownloadUrl: video.getTorrentDownloadUrl(videoFile, baseUrlHttp),
-        fileUrl: video.getVideoFileUrl(videoFile, baseUrlHttp),
-        fileDownloadUrl: video.getVideoFileDownloadUrl(videoFile, baseUrlHttp)
+        torrentUrl: model.getTorrentUrl(videoFile, baseUrlHttp),
+        torrentDownloadUrl: model.getTorrentDownloadUrl(videoFile, baseUrlHttp),
+        fileUrl: model.getVideoFileUrl(videoFile, baseUrlHttp),
+        fileDownloadUrl: model.getVideoFileDownloadUrl(videoFile, baseUrlHttp)
       } as VideoFile
     })
     .sort((a, b) => {
@@ -190,6 +204,39 @@ function videoFilesModelToFormattedJSON (video: MVideo, videoFiles: MVideoFileRe
     })
 }
 
+function addVideoFilesInAPAcc (
+  acc: ActivityUrlObject[] | ActivityTagObject[],
+  model: MVideoAP | MStreamingPlaylistVideo,
+  baseUrlHttp: string,
+  baseUrlWs: string,
+  files: MVideoFile[]
+) {
+  for (const file of files) {
+    acc.push({
+      type: 'Link',
+      mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ file.extname ] as any,
+      href: model.getVideoFileUrl(file, baseUrlHttp),
+      height: file.resolution,
+      size: file.size,
+      fps: file.fps
+    })
+
+    acc.push({
+      type: 'Link',
+      mediaType: 'application/x-bittorrent' as 'application/x-bittorrent',
+      href: model.getTorrentUrl(file, baseUrlHttp),
+      height: file.resolution
+    })
+
+    acc.push({
+      type: 'Link',
+      mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet',
+      href: generateMagnetUri(model, file, baseUrlHttp, baseUrlWs),
+      height: file.resolution
+    })
+  }
+}
+
 function videoModelToActivityPubObject (video: MVideoAP): VideoTorrentObject {
   const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
   if (!video.Tags) video.Tags = []
@@ -224,50 +271,25 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoTorrentObject {
   }
 
   const url: ActivityUrlObject[] = []
-  for (const file of video.VideoFiles) {
-    url.push({
-      type: 'Link',
-      mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ file.extname ] as any,
-      mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ file.extname ] as any,
-      href: video.getVideoFileUrl(file, baseUrlHttp),
-      height: file.resolution,
-      size: file.size,
-      fps: file.fps
-    })
-
-    url.push({
-      type: 'Link',
-      mimeType: 'application/x-bittorrent' as 'application/x-bittorrent',
-      mediaType: 'application/x-bittorrent' as 'application/x-bittorrent',
-      href: video.getTorrentUrl(file, baseUrlHttp),
-      height: file.resolution
-    })
-
-    url.push({
-      type: 'Link',
-      mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet',
-      mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet',
-      href: video.generateMagnetUri(file, baseUrlHttp, baseUrlWs),
-      height: file.resolution
-    })
-  }
+  addVideoFilesInAPAcc(url, video, baseUrlHttp, baseUrlWs, video.VideoFiles || [])
 
   for (const playlist of (video.VideoStreamingPlaylists || [])) {
-    let tag: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[]
+    let tag: ActivityTagObject[]
 
     tag = playlist.p2pMediaLoaderInfohashes
                   .map(i => ({ type: 'Infohash' as 'Infohash', name: i }))
     tag.push({
       type: 'Link',
       name: 'sha256',
-      mimeType: 'application/json' as 'application/json',
       mediaType: 'application/json' as 'application/json',
       href: playlist.segmentsSha256Url
     })
 
+    const playlistWithVideo = Object.assign(playlist, { Video: video })
+    addVideoFilesInAPAcc(tag, playlistWithVideo, baseUrlHttp, baseUrlWs, playlist.VideoFiles || [])
+
     url.push({
       type: 'Link',
-      mimeType: 'application/x-mpegURL' as 'application/x-mpegURL',
       mediaType: 'application/x-mpegURL' as 'application/x-mpegURL',
       href: playlist.playlistUrl,
       tag
@@ -277,7 +299,6 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoTorrentObject {
   // Add video url too
   url.push({
     type: 'Link',
-    mimeType: 'text/html',
     mediaType: 'text/html',
     href: WEBSERVER.URL + '/videos/watch/' + video.uuid
   })
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index 0ea90d28c..faad4cc2d 100644
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -5,12 +5,14 @@ import { VideoModel } from './video'
 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
-import { CONSTRAINTS_FIELDS, P2P_MEDIA_LOADER_PEER_VERSION, STATIC_PATHS } from '../../initializers/constants'
+import { CONSTRAINTS_FIELDS, P2P_MEDIA_LOADER_PEER_VERSION, STATIC_DOWNLOAD_PATHS, STATIC_PATHS } from '../../initializers/constants'
 import { join } from 'path'
 import { sha1 } from '../../helpers/core-utils'
 import { isArrayOf } from '../../helpers/custom-validators/misc'
 import { Op, QueryTypes } from 'sequelize'
 import { MStreamingPlaylist, MVideoFile } from '@server/typings/models'
+import { VideoFileModel } from '@server/models/video/video-file'
+import { getTorrentFileName, getVideoFilename } from '@server/lib/video-paths'
 
 @Table({
   tableName: 'videoStreamingPlaylist',
@@ -70,6 +72,14 @@ export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistMod
   })
   Video: VideoModel
 
+  @HasMany(() => VideoFileModel, {
+    foreignKey: {
+      allowNull: true
+    },
+    onDelete: 'CASCADE'
+  })
+  VideoFiles: VideoFileModel[]
+
   @HasMany(() => VideoRedundancyModel, {
     foreignKey: {
       allowNull: false
@@ -91,11 +101,11 @@ export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistMod
               .then(results => results.length === 1)
   }
 
-  static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: MVideoFile[]) {
+  static buildP2PMediaLoaderInfoHashes (playlistUrl: string, files: unknown[]) {
     const hashes: string[] = []
 
     // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L115
-    for (let i = 0; i < videoFiles.length; i++) {
+    for (let i = 0; i < files.length; i++) {
       hashes.push(sha1(`${P2P_MEDIA_LOADER_PEER_VERSION}${playlistUrl}+V${i}`))
     }
 
@@ -139,10 +149,6 @@ export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistMod
     return 'segments-sha256.json'
   }
 
-  static getHlsVideoName (uuid: string, resolution: number) {
-    return `${uuid}-${resolution}-fragmented.mp4`
-  }
-
   static getHlsMasterPlaylistStaticPath (videoUUID: string) {
     return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
   }
@@ -165,6 +171,26 @@ export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistMod
     return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getStringType() + '/' + this.Video.uuid
   }
 
+  getTorrentDownloadUrl (videoFile: MVideoFile, baseUrlHttp: string) {
+    return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + getTorrentFileName(this, videoFile)
+  }
+
+  getVideoFileDownloadUrl (videoFile: MVideoFile, baseUrlHttp: string) {
+    return baseUrlHttp + STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + getVideoFilename(this, videoFile)
+  }
+
+  getVideoFileUrl (videoFile: MVideoFile, baseUrlHttp: string) {
+    return baseUrlHttp + join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, this.Video.uuid, getVideoFilename(this, videoFile))
+  }
+
+  getTorrentUrl (videoFile: MVideoFile, baseUrlHttp: string) {
+    return baseUrlHttp + join(STATIC_PATHS.TORRENTS, getTorrentFileName(this, videoFile))
+  }
+
+  getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) {
+    return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
+  }
+
   hasSameUniqueKeysThan (other: MStreamingPlaylist) {
     return this.type === other.type &&
       this.videoId === other.videoId
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 0d1dbf106..f84a90992 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -1,7 +1,5 @@
 import * as Bluebird from 'bluebird'
 import { maxBy } from 'lodash'
-import * as magnetUtil from 'magnet-uri'
-import * as parseTorrent from 'parse-torrent'
 import { join } from 'path'
 import {
   CountOptions,
@@ -38,11 +36,11 @@ import {
 } from 'sequelize-typescript'
 import { UserRight, VideoPrivacy, VideoState } from '../../../shared'
 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
-import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
+import { Video, VideoDetails } from '../../../shared/models/videos'
 import { VideoFilter } from '../../../shared/models/videos/video-query.type'
 import { peertubeTruncate } from '../../helpers/core-utils'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
-import { isArray, isBooleanValid } from '../../helpers/custom-validators/misc'
+import { isBooleanValid } from '../../helpers/custom-validators/misc'
 import {
   isVideoCategoryValid,
   isVideoDescriptionValid,
@@ -100,7 +98,7 @@ import { VideoTagModel } from './video-tag'
 import { ScheduleVideoUpdateModel } from './schedule-video-update'
 import { VideoCaptionModel } from './video-caption'
 import { VideoBlacklistModel } from './video-blacklist'
-import { remove, writeFile } from 'fs-extra'
+import { remove } from 'fs-extra'
 import { VideoViewModel } from './video-views'
 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
 import {
@@ -117,18 +115,20 @@ import { VideoPlaylistElementModel } from './video-playlist-element'
 import { CONFIG } from '../../initializers/config'
 import { ThumbnailModel } from './thumbnail'
 import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
-import { createTorrentPromise } from '../../helpers/webtorrent'
 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
 import {
   MChannel,
   MChannelAccountDefault,
   MChannelId,
+  MStreamingPlaylist,
+  MStreamingPlaylistFilesVideo,
   MUserAccountId,
   MUserId,
   MVideoAccountLight,
   MVideoAccountLightBlacklistAllFiles,
   MVideoAP,
   MVideoDetails,
+  MVideoFileVideo,
   MVideoFormattable,
   MVideoFormattableDetails,
   MVideoForUser,
@@ -140,8 +140,10 @@ import {
   MVideoWithFile,
   MVideoWithRights
 } from '../../typings/models'
-import { MVideoFile, MVideoFileRedundanciesOpt } from '../../typings/models/video/video-file'
+import { MVideoFile, MVideoFileStreamingPlaylistVideo } from '../../typings/models/video/video-file'
 import { MThumbnail } from '../../typings/models/video/thumbnail'
+import { VideoFile } from '@shared/models/videos/video-file.model'
+import { getTorrentFileName, getTorrentFilePath, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
 
 // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
 const indexes: (ModelIndexesOptions & { where?: WhereOptions })[] = [
@@ -211,7 +213,7 @@ export enum ScopeNames {
   FOR_API = 'FOR_API',
   WITH_ACCOUNT_DETAILS = 'WITH_ACCOUNT_DETAILS',
   WITH_TAGS = 'WITH_TAGS',
-  WITH_FILES = 'WITH_FILES',
+  WITH_WEBTORRENT_FILES = 'WITH_WEBTORRENT_FILES',
   WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE',
   WITH_BLACKLISTED = 'WITH_BLACKLISTED',
   WITH_BLOCKLIST = 'WITH_BLOCKLIST',
@@ -666,7 +668,7 @@ export type AvailableForListIDsOptions = {
       }
     ]
   },
-  [ ScopeNames.WITH_FILES ]: (withRedundancies = false) => {
+  [ ScopeNames.WITH_WEBTORRENT_FILES ]: (withRedundancies = false) => {
     let subInclude: any[] = []
 
     if (withRedundancies === true) {
@@ -691,16 +693,19 @@ export type AvailableForListIDsOptions = {
     }
   },
   [ ScopeNames.WITH_STREAMING_PLAYLISTS ]: (withRedundancies = false) => {
-    let subInclude: any[] = []
+    const subInclude: IncludeOptions[] = [
+      {
+        model: VideoFileModel.unscoped(),
+        required: false
+      }
+    ]
 
     if (withRedundancies === true) {
-      subInclude = [
-        {
-          attributes: [ 'fileUrl' ],
-          model: VideoRedundancyModel.unscoped(),
-          required: false
-        }
-      ]
+      subInclude.push({
+        attributes: [ 'fileUrl' ],
+        model: VideoRedundancyModel.unscoped(),
+        required: false
+      })
     }
 
     return {
@@ -913,7 +918,7 @@ export class VideoModel extends Model<VideoModel> {
   @HasMany(() => VideoFileModel, {
     foreignKey: {
       name: 'videoId',
-      allowNull: false
+      allowNull: true
     },
     hooks: true,
     onDelete: 'cascade'
@@ -1071,7 +1076,7 @@ export class VideoModel extends Model<VideoModel> {
     }
 
     return VideoModel.scope([
-      ScopeNames.WITH_FILES,
+      ScopeNames.WITH_WEBTORRENT_FILES,
       ScopeNames.WITH_STREAMING_PLAYLISTS,
       ScopeNames.WITH_THUMBNAILS
     ]).findAll(query)
@@ -1463,7 +1468,7 @@ export class VideoModel extends Model<VideoModel> {
     }
 
     return VideoModel.scope([
-      ScopeNames.WITH_FILES,
+      ScopeNames.WITH_WEBTORRENT_FILES,
       ScopeNames.WITH_STREAMING_PLAYLISTS,
       ScopeNames.WITH_THUMBNAILS
     ]).findOne(query)
@@ -1500,7 +1505,7 @@ export class VideoModel extends Model<VideoModel> {
 
     return VideoModel.scope([
       ScopeNames.WITH_ACCOUNT_DETAILS,
-      ScopeNames.WITH_FILES,
+      ScopeNames.WITH_WEBTORRENT_FILES,
       ScopeNames.WITH_STREAMING_PLAYLISTS,
       ScopeNames.WITH_THUMBNAILS,
       ScopeNames.WITH_BLACKLISTED
@@ -1521,7 +1526,7 @@ export class VideoModel extends Model<VideoModel> {
       ScopeNames.WITH_BLACKLISTED,
       ScopeNames.WITH_ACCOUNT_DETAILS,
       ScopeNames.WITH_SCHEDULED_UPDATE,
-      ScopeNames.WITH_FILES,
+      ScopeNames.WITH_WEBTORRENT_FILES,
       ScopeNames.WITH_STREAMING_PLAYLISTS,
       ScopeNames.WITH_THUMBNAILS
     ]
@@ -1555,7 +1560,7 @@ export class VideoModel extends Model<VideoModel> {
       ScopeNames.WITH_ACCOUNT_DETAILS,
       ScopeNames.WITH_SCHEDULED_UPDATE,
       ScopeNames.WITH_THUMBNAILS,
-      { method: [ ScopeNames.WITH_FILES, true ] },
+      { method: [ ScopeNames.WITH_WEBTORRENT_FILES, true ] },
       { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] }
     ]
 
@@ -1787,17 +1792,31 @@ export class VideoModel extends Model<VideoModel> {
       this.VideoChannel.Account.isBlocked()
   }
 
-  getOriginalFile <T extends MVideoWithFile> (this: T) {
-    if (Array.isArray(this.VideoFiles) === false) return undefined
+  getMaxQualityFile <T extends MVideoWithFile> (this: T): MVideoFileVideo | MVideoFileStreamingPlaylistVideo {
+    if (Array.isArray(this.VideoFiles) && this.VideoFiles.length !== 0) {
+      const file = maxBy(this.VideoFiles, file => file.resolution)
+
+      return Object.assign(file, { Video: this })
+    }
+
+    // No webtorrent files, try with streaming playlist files
+    if (Array.isArray(this.VideoStreamingPlaylists) && this.VideoStreamingPlaylists.length !== 0) {
+      const streamingPlaylistWithVideo = Object.assign(this.VideoStreamingPlaylists[0], { Video: this })
+
+      const file = maxBy(streamingPlaylistWithVideo.VideoFiles, file => file.resolution)
+      return Object.assign(file, { VideoStreamingPlaylist: streamingPlaylistWithVideo })
+    }
 
-    // The original file is the file that have the higher resolution
-    return maxBy(this.VideoFiles, file => file.resolution)
+    return undefined
   }
 
-  getFile <T extends MVideoWithFile> (this: T, resolution: number) {
+  getWebTorrentFile <T extends MVideoWithFile> (this: T, resolution: number): MVideoFileVideo {
     if (Array.isArray(this.VideoFiles) === false) return undefined
 
-    return this.VideoFiles.find(f => f.resolution === resolution)
+    const file = this.VideoFiles.find(f => f.resolution === resolution)
+    if (!file) return undefined
+
+    return Object.assign(file, { Video: this })
   }
 
   async addAndSaveThumbnail (thumbnail: MThumbnail, transaction: Transaction) {
@@ -1813,10 +1832,6 @@ export class VideoModel extends Model<VideoModel> {
     this.Thumbnails.push(savedThumbnail)
   }
 
-  getVideoFilename (videoFile: MVideoFile) {
-    return this.uuid + '-' + videoFile.resolution + videoFile.extname
-  }
-
   generateThumbnailName () {
     return this.uuid + '.jpg'
   }
@@ -1837,46 +1852,10 @@ export class VideoModel extends Model<VideoModel> {
     return this.Thumbnails.find(t => t.type === ThumbnailType.PREVIEW)
   }
 
-  getTorrentFileName (videoFile: MVideoFile) {
-    const extension = '.torrent'
-    return this.uuid + '-' + videoFile.resolution + extension
-  }
-
   isOwned () {
     return this.remote === false
   }
 
-  getTorrentFilePath (videoFile: MVideoFile) {
-    return join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile))
-  }
-
-  getVideoFilePath (videoFile: MVideoFile) {
-    return join(CONFIG.STORAGE.VIDEOS_DIR, this.getVideoFilename(videoFile))
-  }
-
-  async createTorrentAndSetInfoHash (videoFile: MVideoFile) {
-    const options = {
-      // Keep the extname, it's used by the client to stream the file inside a web browser
-      name: `${this.name} ${videoFile.resolution}p${videoFile.extname}`,
-      createdBy: 'PeerTube',
-      announceList: [
-        [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ],
-        [ WEBSERVER.URL + '/tracker/announce' ]
-      ],
-      urlList: [ WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) ]
-    }
-
-    const torrent = await createTorrentPromise(this.getVideoFilePath(videoFile), options)
-
-    const filePath = join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile))
-    logger.info('Creating torrent %s.', filePath)
-
-    await writeFile(filePath, torrent)
-
-    const parsedTorrent = parseTorrent(torrent)
-    videoFile.infoHash = parsedTorrent.infoHash
-  }
-
   getWatchStaticPath () {
     return '/videos/watch/' + this.uuid
   }
@@ -1909,7 +1888,8 @@ export class VideoModel extends Model<VideoModel> {
   }
 
   getFormattedVideoFilesJSON (): VideoFile[] {
-    return videoFilesModelToFormattedJSON(this, this.VideoFiles)
+    const { baseUrlHttp, baseUrlWs } = this.getBaseUrls()
+    return videoFilesModelToFormattedJSON(this, baseUrlHttp, baseUrlWs, this.VideoFiles)
   }
 
   toActivityPubObject (this: MVideoAP): VideoTorrentObject {
@@ -1923,8 +1903,10 @@ export class VideoModel extends Model<VideoModel> {
     return peertubeTruncate(this.description, { length: maxLength })
   }
 
-  getOriginalFileResolution () {
-    const originalFilePath = this.getVideoFilePath(this.getOriginalFile())
+  getMaxQualityResolution () {
+    const file = this.getMaxQualityFile()
+    const videoOrPlaylist = file.getVideoOrStreamingPlaylist()
+    const originalFilePath = getVideoFilePath(videoOrPlaylist, file)
 
     return getVideoFileResolution(originalFilePath)
   }
@@ -1933,22 +1915,36 @@ export class VideoModel extends Model<VideoModel> {
     return `/api/${API_VERSION}/videos/${this.uuid}/description`
   }
 
-  getHLSPlaylist () {
+  getHLSPlaylist (): MStreamingPlaylistFilesVideo {
     if (!this.VideoStreamingPlaylists) return undefined
 
-    return this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
+    const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
+    playlist.Video = this
+
+    return playlist
   }
 
-  removeFile (videoFile: MVideoFile, isRedundancy = false) {
-    const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR
+  setHLSPlaylist (playlist: MStreamingPlaylist) {
+    const toAdd = [ playlist ] as [ VideoStreamingPlaylistModel ]
 
-    const filePath = join(baseDir, this.getVideoFilename(videoFile))
+    if (Array.isArray(this.VideoStreamingPlaylists) === false || this.VideoStreamingPlaylists.length === 0) {
+      this.VideoStreamingPlaylists = toAdd
+      return
+    }
+
+    this.VideoStreamingPlaylists = this.VideoStreamingPlaylists
+      .filter(s => s.type !== VideoStreamingPlaylistType.HLS)
+      .concat(toAdd)
+  }
+
+  removeFile (videoFile: MVideoFile, isRedundancy = false) {
+    const filePath = getVideoFilePath(this, videoFile, isRedundancy)
     return remove(filePath)
       .catch(err => logger.warn('Cannot delete file %s.', filePath, { err }))
   }
 
   removeTorrent (videoFile: MVideoFile) {
-    const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile))
+    const torrentPath = getTorrentFilePath(this, videoFile)
     return remove(torrentPath)
       .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
   }
@@ -1973,38 +1969,30 @@ export class VideoModel extends Model<VideoModel> {
     return this.save()
   }
 
-  getBaseUrls () {
-    let baseUrlHttp
-    let baseUrlWs
+  async publishIfNeededAndSave (t: Transaction) {
+    if (this.state !== VideoState.PUBLISHED) {
+      this.state = VideoState.PUBLISHED
+      this.publishedAt = new Date()
+      await this.save({ transaction: t })
 
-    if (this.isOwned()) {
-      baseUrlHttp = WEBSERVER.URL
-      baseUrlWs = WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT
-    } else {
-      baseUrlHttp = REMOTE_SCHEME.HTTP + '://' + this.VideoChannel.Account.Actor.Server.host
-      baseUrlWs = REMOTE_SCHEME.WS + '://' + this.VideoChannel.Account.Actor.Server.host
+      return true
     }
 
-    return { baseUrlHttp, baseUrlWs }
+    return false
   }
 
-  generateMagnetUri (videoFile: MVideoFileRedundanciesOpt, baseUrlHttp: string, baseUrlWs: string) {
-    const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
-    const announce = this.getTrackerUrls(baseUrlHttp, baseUrlWs)
-    let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
-
-    const redundancies = videoFile.RedundancyVideos
-    if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl))
-
-    const magnetHash = {
-      xs,
-      announce,
-      urlList,
-      infoHash: videoFile.infoHash,
-      name: this.name
+  getBaseUrls () {
+    if (this.isOwned()) {
+      return {
+        baseUrlHttp: WEBSERVER.URL,
+        baseUrlWs: WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT
+      }
     }
 
-    return magnetUtil.encode(magnetHash)
+    return {
+      baseUrlHttp: REMOTE_SCHEME.HTTP + '://' + this.VideoChannel.Account.Actor.Server.host,
+      baseUrlWs: REMOTE_SCHEME.WS + '://' + this.VideoChannel.Account.Actor.Server.host
+    }
   }
 
   getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) {
@@ -2012,23 +2000,23 @@ export class VideoModel extends Model<VideoModel> {
   }
 
   getTorrentUrl (videoFile: MVideoFile, baseUrlHttp: string) {
-    return baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
+    return baseUrlHttp + STATIC_PATHS.TORRENTS + getTorrentFileName(this, videoFile)
   }
 
   getTorrentDownloadUrl (videoFile: MVideoFile, baseUrlHttp: string) {
-    return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
+    return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + getTorrentFileName(this, videoFile)
   }
 
   getVideoFileUrl (videoFile: MVideoFile, baseUrlHttp: string) {
-    return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
+    return baseUrlHttp + STATIC_PATHS.WEBSEED + getVideoFilename(this, videoFile)
   }
 
   getVideoRedundancyUrl (videoFile: MVideoFile, baseUrlHttp: string) {
-    return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getVideoFilename(videoFile)
+    return baseUrlHttp + STATIC_PATHS.REDUNDANCY + getVideoFilename(this, videoFile)
   }
 
   getVideoFileDownloadUrl (videoFile: MVideoFile, baseUrlHttp: string) {
-    return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
+    return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + getVideoFilename(this, videoFile)
   }
 
   getBandwidthBits (videoFile: MVideoFile) {