Add hls support on server

4 files changed, 364 insertions, 36 deletions

diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 1f1b76c1e..7d1e371b9 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -62,7 +62,7 @@ export class VideoFileModel extends Model<VideoFileModel> {
   extname: string
 
   @AllowNull(false)
-  @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash'))
+  @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash'))
   @Column
   infoHash: string
 
@@ -86,14 +86,14 @@ export class VideoFileModel extends Model<VideoFileModel> {
 
   @HasMany(() => VideoRedundancyModel, {
     foreignKey: {
-      allowNull: false
+      allowNull: true
     },
     onDelete: 'CASCADE',
     hooks: true
   })
   RedundancyVideos: VideoRedundancyModel[]
 
-  static isInfohashExists (infoHash: string) {
+  static doesInfohashExist (infoHash: string) {
     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
     const options = {
       type: Sequelize.QueryTypes.SELECT,
diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts
index de0747f22..e49dbee30 100644
--- a/server/models/video/video-format-utils.ts
+++ b/server/models/video/video-format-utils.ts
@@ -1,7 +1,12 @@
 import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
 import { VideoModel } from './video'
 import { VideoFileModel } from './video-file'
-import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects'
+import {
+  ActivityPlaylistInfohashesObject,
+  ActivityPlaylistSegmentHashesObject,
+  ActivityUrlObject,
+  VideoTorrentObject
+} from '../../../shared/models/activitypub/objects'
 import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers'
 import { VideoCaptionModel } from './video-caption'
 import {
@@ -11,6 +16,8 @@ import {
   getVideoSharesActivityPubUrl
 } from '../../lib/activitypub'
 import { isArray } from '../../helpers/custom-validators/misc'
+import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
+import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
 
 export type VideoFormattingJSONOptions = {
   completeDescription?: boolean
@@ -120,7 +127,12 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
     }
   })
 
+  const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
+
   const tags = video.Tags ? video.Tags.map(t => t.name) : []
+
+  const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists)
+
   const detailsJson = {
     support: video.support,
     descriptionPath: video.getDescriptionAPIPath(),
@@ -133,7 +145,11 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
       id: video.state,
       label: VideoModel.getStateLabel(video.state)
     },
-    files: []
+
+    trackerUrls: video.getTrackerUrls(baseUrlHttp, baseUrlWs),
+
+    files: [],
+    streamingPlaylists
   }
 
   // Format and sort video files
@@ -142,6 +158,25 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
   return Object.assign(formattedJson, detailsJson)
 }
 
+function streamingPlaylistsModelToFormattedJSON (video: VideoModel, playlists: VideoStreamingPlaylistModel[]): VideoStreamingPlaylist[] {
+  if (isArray(playlists) === false) return []
+
+  return playlists
+    .map(playlist => {
+      const redundancies = isArray(playlist.RedundancyVideos)
+        ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl }))
+        : []
+
+      return {
+        id: playlist.id,
+        type: playlist.type,
+        playlistUrl: playlist.playlistUrl,
+        segmentsSha256Url: playlist.segmentsSha256Url,
+        redundancies
+      } as VideoStreamingPlaylist
+    })
+}
+
 function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] {
   const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
 
@@ -232,6 +267,28 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject {
     })
   }
 
+  for (const playlist of (video.VideoStreamingPlaylists || [])) {
+    let tag: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[]
+
+    tag = playlist.p2pMediaLoaderInfohashes
+                  .map(i => ({ type: 'Infohash' as 'Infohash', name: i }))
+    tag.push({
+      type: 'Link',
+      name: 'sha256',
+      mimeType: 'application/json' as 'application/json',
+      mediaType: 'application/json' as 'application/json',
+      href: playlist.segmentsSha256Url
+    })
+
+    url.push({
+      type: 'Link',
+      mimeType: 'application/x-mpegURL' as 'application/x-mpegURL',
+      mediaType: 'application/x-mpegURL' as 'application/x-mpegURL',
+      href: playlist.playlistUrl,
+      tag
+    })
+  }
+
   // Add video url too
   url.push({
     type: 'Link',
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
new file mode 100644
index 000000000..bce537781
--- /dev/null
+++ b/server/models/video/video-streaming-playlist.ts
@@ -0,0 +1,154 @@
+import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
+import { throwIfNotValid } from '../utils'
+import { VideoModel } from './video'
+import * as Sequelize from 'sequelize'
+import { VideoRedundancyModel } from '../redundancy/video-redundancy'
+import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
+import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
+import { CONSTRAINTS_FIELDS, STATIC_PATHS } from '../../initializers'
+import { VideoFileModel } from './video-file'
+import { join } from 'path'
+import { sha1 } from '../../helpers/core-utils'
+import { isArrayOf } from '../../helpers/custom-validators/misc'
+
+@Table({
+  tableName: 'videoStreamingPlaylist',
+  indexes: [
+    {
+      fields: [ 'videoId' ]
+    },
+    {
+      fields: [ 'videoId', 'type' ],
+      unique: true
+    },
+    {
+      fields: [ 'p2pMediaLoaderInfohashes' ],
+      using: 'gin'
+    }
+  ]
+})
+export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistModel> {
+  @CreatedAt
+  createdAt: Date
+
+  @UpdatedAt
+  updatedAt: Date
+
+  @AllowNull(false)
+  @Column
+  type: VideoStreamingPlaylistType
+
+  @AllowNull(false)
+  @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url'))
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max))
+  playlistUrl: string
+
+  @AllowNull(false)
+  @Is('VideoStreamingPlaylistInfoHashes', value => throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes'))
+  @Column(DataType.ARRAY(DataType.STRING))
+  p2pMediaLoaderInfohashes: string[]
+
+  @AllowNull(false)
+  @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url'))
+  @Column
+  segmentsSha256Url: string
+
+  @ForeignKey(() => VideoModel)
+  @Column
+  videoId: number
+
+  @BelongsTo(() => VideoModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'CASCADE'
+  })
+  Video: VideoModel
+
+  @HasMany(() => VideoRedundancyModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'CASCADE',
+    hooks: true
+  })
+  RedundancyVideos: VideoRedundancyModel[]
+
+  static doesInfohashExist (infoHash: string) {
+    const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1'
+    const options = {
+      type: Sequelize.QueryTypes.SELECT,
+      bind: { infoHash },
+      raw: true
+    }
+
+    return VideoModel.sequelize.query(query, options)
+              .then(results => {
+                return results.length === 1
+              })
+  }
+
+  static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: VideoFileModel[]) {
+    const hashes: string[] = []
+
+    // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L97
+    for (let i = 0; i < videoFiles.length; i++) {
+      hashes.push(sha1(`1${playlistUrl}+V${i}`))
+    }
+
+    return hashes
+  }
+
+  static loadWithVideo (id: number) {
+    const options = {
+      include: [
+        {
+          model: VideoModel.unscoped(),
+          required: true
+        }
+      ]
+    }
+
+    return VideoStreamingPlaylistModel.findById(id, options)
+  }
+
+  static getHlsPlaylistFilename (resolution: number) {
+    return resolution + '.m3u8'
+  }
+
+  static getMasterHlsPlaylistFilename () {
+    return 'master.m3u8'
+  }
+
+  static getHlsSha256SegmentsFilename () {
+    return 'segments-sha256.json'
+  }
+
+  static getHlsMasterPlaylistStaticPath (videoUUID: string) {
+    return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
+  }
+
+  static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) {
+    return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
+  }
+
+  static getHlsSha256SegmentsStaticPath (videoUUID: string) {
+    return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
+  }
+
+  getStringType () {
+    if (this.type === VideoStreamingPlaylistType.HLS) return 'hls'
+
+    return 'unknown'
+  }
+
+  getVideoRedundancyUrl (baseUrlHttp: string) {
+    return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getStringType() + '/' + this.Video.uuid
+  }
+
+  hasSameUniqueKeysThan (other: VideoStreamingPlaylistModel) {
+    return this.type === other.type &&
+      this.videoId === other.videoId
+  }
+}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 80a6c7832..702260772 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -52,7 +52,7 @@ import {
   ACTIVITY_PUB,
   API_VERSION,
   CONFIG,
-  CONSTRAINTS_FIELDS,
+  CONSTRAINTS_FIELDS, HLS_PLAYLIST_DIRECTORY, HLS_REDUNDANCY_DIRECTORY,
   PREVIEWS_SIZE,
   REMOTE_SCHEME,
   STATIC_DOWNLOAD_PATHS,
@@ -95,6 +95,7 @@ import * as validator from 'validator'
 import { UserVideoHistoryModel } from '../account/user-video-history'
 import { UserModel } from '../account/user'
 import { VideoImportModel } from './video-import'
+import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
 
 // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
 const indexes: Sequelize.DefineIndexesOptions[] = [
@@ -159,7 +160,9 @@ export enum ScopeNames {
   WITH_FILES = 'WITH_FILES',
   WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE',
   WITH_BLACKLISTED = 'WITH_BLACKLISTED',
-  WITH_USER_HISTORY = 'WITH_USER_HISTORY'
+  WITH_USER_HISTORY = 'WITH_USER_HISTORY',
+  WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS',
+  WITH_USER_ID = 'WITH_USER_ID'
 }
 
 type ForAPIOptions = {
@@ -463,6 +466,22 @@ type AvailableForListIDsOptions = {
 
     return query
   },
+  [ ScopeNames.WITH_USER_ID ]: {
+    include: [
+      {
+        attributes: [ 'accountId' ],
+        model: () => VideoChannelModel.unscoped(),
+        required: true,
+        include: [
+          {
+            attributes: [ 'userId' ],
+            model: () => AccountModel.unscoped(),
+            required: true
+          }
+        ]
+      }
+    ]
+  },
   [ ScopeNames.WITH_ACCOUNT_DETAILS ]: {
     include: [
       {
@@ -527,22 +546,55 @@ type AvailableForListIDsOptions = {
       }
     ]
   },
-  [ ScopeNames.WITH_FILES ]: {
-    include: [
-      {
-        model: () => VideoFileModel.unscoped(),
-        // FIXME: typings
-        [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join
-        required: false,
-        include: [
-          {
-            attributes: [ 'fileUrl' ],
-            model: () => VideoRedundancyModel.unscoped(),
-            required: false
-          }
-        ]
-      }
-    ]
+  [ ScopeNames.WITH_FILES ]: (withRedundancies = false) => {
+    let subInclude: any[] = []
+
+    if (withRedundancies === true) {
+      subInclude = [
+        {
+          attributes: [ 'fileUrl' ],
+          model: VideoRedundancyModel.unscoped(),
+          required: false
+        }
+      ]
+    }
+
+    return {
+      include: [
+        {
+          model: VideoFileModel.unscoped(),
+          // FIXME: typings
+          [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join
+          required: false,
+          include: subInclude
+        }
+      ]
+    }
+  },
+  [ ScopeNames.WITH_STREAMING_PLAYLISTS ]: (withRedundancies = false) => {
+    let subInclude: any[] = []
+
+    if (withRedundancies === true) {
+      subInclude = [
+        {
+          attributes: [ 'fileUrl' ],
+          model: VideoRedundancyModel.unscoped(),
+          required: false
+        }
+      ]
+    }
+
+    return {
+      include: [
+        {
+          model: VideoStreamingPlaylistModel.unscoped(),
+          // FIXME: typings
+          [ 'separate' as any ]: true, // We may have multiple streaming playlists, having multiple redundancies so let's separate this join
+          required: false,
+          include: subInclude
+        }
+      ]
+    }
   },
   [ ScopeNames.WITH_SCHEDULED_UPDATE ]: {
     include: [
@@ -722,6 +774,16 @@ export class VideoModel extends Model<VideoModel> {
   })
   VideoFiles: VideoFileModel[]
 
+  @HasMany(() => VideoStreamingPlaylistModel, {
+    foreignKey: {
+      name: 'videoId',
+      allowNull: false
+    },
+    hooks: true,
+    onDelete: 'cascade'
+  })
+  VideoStreamingPlaylists: VideoStreamingPlaylistModel[]
+
   @HasMany(() => VideoShareModel, {
     foreignKey: {
       name: 'videoId',
@@ -847,6 +909,9 @@ export class VideoModel extends Model<VideoModel> {
         tasks.push(instance.removeFile(file))
         tasks.push(instance.removeTorrent(file))
       })
+
+      // Remove playlists file
+      tasks.push(instance.removeStreamingPlaylist())
     }
 
     // Do not wait video deletion because we could be in a transaction
@@ -858,10 +923,6 @@ export class VideoModel extends Model<VideoModel> {
     return undefined
   }
 
-  static list () {
-    return VideoModel.scope(ScopeNames.WITH_FILES).findAll()
-  }
-
   static listLocal () {
     const query = {
       where: {
@@ -869,7 +930,7 @@ export class VideoModel extends Model<VideoModel> {
       }
     }
 
-    return VideoModel.scope(ScopeNames.WITH_FILES).findAll(query)
+    return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]).findAll(query)
   }
 
   static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) {
@@ -1200,6 +1261,16 @@ export class VideoModel extends Model<VideoModel> {
     return VideoModel.findOne(options)
   }
 
+  static loadWithRights (id: number | string, t?: Sequelize.Transaction) {
+    const where = VideoModel.buildWhereIdOrUUID(id)
+    const options = {
+      where,
+      transaction: t
+    }
+
+    return VideoModel.scope([ ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_USER_ID ]).findOne(options)
+  }
+
   static loadOnlyId (id: number | string, t?: Sequelize.Transaction) {
     const where = VideoModel.buildWhereIdOrUUID(id)
 
@@ -1212,8 +1283,8 @@ export class VideoModel extends Model<VideoModel> {
     return VideoModel.findOne(options)
   }
 
-  static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) {
-    return VideoModel.scope(ScopeNames.WITH_FILES)
+  static loadWithFiles (id: number, t?: Sequelize.Transaction, logging?: boolean) {
+    return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ])
                      .findById(id, { transaction: t, logging })
   }
 
@@ -1224,9 +1295,7 @@ export class VideoModel extends Model<VideoModel> {
       }
     }
 
-    return VideoModel
-      .scope([ ScopeNames.WITH_FILES ])
-      .findOne(options)
+    return VideoModel.findOne(options)
   }
 
   static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
@@ -1248,7 +1317,11 @@ export class VideoModel extends Model<VideoModel> {
       transaction
     }
 
-    return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
+    return VideoModel.scope([
+      ScopeNames.WITH_ACCOUNT_DETAILS,
+      ScopeNames.WITH_FILES,
+      ScopeNames.WITH_STREAMING_PLAYLISTS
+    ]).findOne(query)
   }
 
   static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) {
@@ -1263,9 +1336,37 @@ export class VideoModel extends Model<VideoModel> {
     const scopes = [
       ScopeNames.WITH_TAGS,
       ScopeNames.WITH_BLACKLISTED,
+      ScopeNames.WITH_ACCOUNT_DETAILS,
+      ScopeNames.WITH_SCHEDULED_UPDATE,
       ScopeNames.WITH_FILES,
+      ScopeNames.WITH_STREAMING_PLAYLISTS
+    ]
+
+    if (userId) {
+      scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings
+    }
+
+    return VideoModel
+      .scope(scopes)
+      .findOne(options)
+  }
+
+  static loadForGetAPI (id: number | string, t?: Sequelize.Transaction, userId?: number) {
+    const where = VideoModel.buildWhereIdOrUUID(id)
+
+    const options = {
+      order: [ [ 'Tags', 'name', 'ASC' ] ],
+      where,
+      transaction: t
+    }
+
+    const scopes = [
+      ScopeNames.WITH_TAGS,
+      ScopeNames.WITH_BLACKLISTED,
       ScopeNames.WITH_ACCOUNT_DETAILS,
-      ScopeNames.WITH_SCHEDULED_UPDATE
+      ScopeNames.WITH_SCHEDULED_UPDATE,
+      { method: [ ScopeNames.WITH_FILES, true ] } as any, // FIXME: typings
+      { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } as any // FIXME: typings
     ]
 
     if (userId) {
@@ -1612,6 +1713,14 @@ export class VideoModel extends Model<VideoModel> {
       .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
   }
 
+  removeStreamingPlaylist (isRedundancy = false) {
+    const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_PLAYLIST_DIRECTORY
+
+    const filePath = join(baseDir, this.uuid)
+    return remove(filePath)
+      .catch(err => logger.warn('Cannot delete playlist directory %s.', filePath, { err }))
+  }
+
   isOutdated () {
     if (this.isOwned()) return false
 
@@ -1646,7 +1755,7 @@ export class VideoModel extends Model<VideoModel> {
 
   generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
     const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
-    const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
+    const announce = this.getTrackerUrls(baseUrlHttp, baseUrlWs)
     let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
 
     const redundancies = videoFile.RedundancyVideos
@@ -1663,6 +1772,10 @@ export class VideoModel extends Model<VideoModel> {
     return magnetUtil.encode(magnetHash)
   }
 
+  getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) {
+    return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
+  }
+
   getThumbnailUrl (baseUrlHttp: string) {
     return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName()
   }
@@ -1686,4 +1799,8 @@ export class VideoModel extends Model<VideoModel> {
   getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
     return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
   }
+
+  getBandwidthBits (videoFile: VideoFileModel) {
+    return Math.ceil((videoFile.size * 8) / this.duration)
+  }
 }