6 files changed, 196 insertions, 22 deletions

diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts
index 8a54de3b0..b3c4f390d 100644
--- a/server/models/video/formatter/video-format-utils.ts
+++ b/server/models/video/formatter/video-format-utils.ts
@@ -1,6 +1,6 @@
 import { uuidToShort } from '@server/helpers/uuid'
 import { generateMagnetUri } from '@server/helpers/webtorrent'
-import { getLocalVideoFileMetadataUrl } from '@server/lib/video-paths'
+import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
 import { VideoFile } from '@shared/models/videos/video-file.model'
 import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects'
 import { Video, VideoDetails } from '../../../../shared/models/videos'
diff --git a/server/models/video/sql/shared/video-tables.ts b/server/models/video/sql/shared/video-tables.ts
index 742d19099..75823864d 100644
--- a/server/models/video/sql/shared/video-tables.ts
+++ b/server/models/video/sql/shared/video-tables.ts
@@ -87,7 +87,8 @@ export class VideoTables {
       'fps',
       'metadataUrl',
       'videoStreamingPlaylistId',
-      'videoId'
+      'videoId',
+      'storage'
     ]
   }
 
@@ -102,7 +103,8 @@ export class VideoTables {
         'segmentsSha256Url',
         'videoId',
         'createdAt',
-        'updatedAt'
+        'updatedAt',
+        'storage'
       ])
     }
 
@@ -258,7 +260,8 @@ export class VideoTables {
       'originallyPublishedAt',
       'channelId',
       'createdAt',
-      'updatedAt'
+      'updatedAt',
+      'moveJobsRunning'
     ]
   }
 }
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 09fc5288b..627c95763 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -23,9 +23,11 @@ import validator from 'validator'
 import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
 import { logger } from '@server/helpers/logger'
 import { extractVideo } from '@server/helpers/video'
-import { getTorrentFilePath } from '@server/lib/video-paths'
+import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
+import { getFSTorrentFilePath } from '@server/lib/paths'
 import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
 import { AttributesOnly } from '@shared/core-utils'
+import { VideoStorage } from '@shared/models'
 import {
   isVideoFileExtnameValid,
   isVideoFileInfoHashValid,
@@ -214,6 +216,11 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
   @Column
   videoId: number
 
+  @AllowNull(false)
+  @Default(VideoStorage.FILE_SYSTEM)
+  @Column
+  storage: VideoStorage
+
   @BelongsTo(() => VideoModel, {
     foreignKey: {
       allowNull: true
@@ -273,7 +280,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
 
   static async doesOwnedWebTorrentVideoFileExist (filename: string) {
     const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
-                  'WHERE "filename" = $filename LIMIT 1'
+                  `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1`
 
     return doesExist(query, { filename })
   }
@@ -450,9 +457,20 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
     return !!this.videoStreamingPlaylistId
   }
 
+  getObjectStorageUrl () {
+    if (this.isHLS()) {
+      return getHLSPublicFileUrl(this.fileUrl)
+    }
+
+    return getWebTorrentPublicFileUrl(this.fileUrl)
+  }
+
   getFileUrl (video: MVideo) {
-    if (!this.Video) this.Video = video as VideoModel
+    if (this.storage === VideoStorage.OBJECT_STORAGE) {
+      return this.getObjectStorageUrl()
+    }
 
+    if (!this.Video) this.Video = video as VideoModel
     if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
 
     return this.fileUrl
@@ -503,7 +521,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
   removeTorrent () {
     if (!this.torrentFilename) return null
 
-    const torrentPath = getTorrentFilePath(this)
+    const torrentPath = getFSTorrentFilePath(this)
     return remove(torrentPath)
       .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
   }
diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts
new file mode 100644
index 000000000..7c1fe6734
--- /dev/null
+++ b/server/models/video/video-job-info.ts
@@ -0,0 +1,100 @@
+import { Op, QueryTypes, Transaction } from 'sequelize'
+import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Model, Table, Unique, UpdatedAt } from 'sequelize-typescript'
+import { AttributesOnly } from '@shared/core-utils'
+import { VideoModel } from './video'
+
+@Table({
+  tableName: 'videoJobInfo',
+  indexes: [
+    {
+      fields: [ 'videoId' ],
+      where: {
+        videoId: {
+          [Op.ne]: null
+        }
+      }
+    }
+  ]
+})
+
+export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfoModel>>> {
+  @CreatedAt
+  createdAt: Date
+
+  @UpdatedAt
+  updatedAt: Date
+
+  @AllowNull(false)
+  @Default(0)
+  @IsInt
+  @Column
+  pendingMove: number
+
+  @AllowNull(false)
+  @Default(0)
+  @IsInt
+  @Column
+  pendingTranscode: number
+
+  @ForeignKey(() => VideoModel)
+  @Unique
+  @Column
+  videoId: number
+
+  @BelongsTo(() => VideoModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+  Video: VideoModel
+
+  static load (videoId: number, transaction: Transaction) {
+    const where = {
+      videoId
+    }
+
+    return VideoJobInfoModel.findOne({ where, transaction })
+  }
+
+  static async increaseOrCreate (videoUUID: string, column: 'pendingMove' | 'pendingTranscode'): Promise<number> {
+    const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } }
+
+    const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(`
+    INSERT INTO "videoJobInfo" ("videoId", "${column}", "createdAt", "updatedAt")
+    SELECT
+      "video"."id" AS "videoId", 1, NOW(), NOW()
+    FROM
+      "video"
+    WHERE
+      "video"."uuid" = $videoUUID
+    ON CONFLICT ("videoId") DO UPDATE
+    SET
+      "${column}" = "videoJobInfo"."${column}" + 1,
+      "updatedAt" = NOW()
+    RETURNING
+      "${column}"
+    `, options)
+
+    return pendingMove
+  }
+
+  static async decrease (videoUUID: string, column: 'pendingMove' | 'pendingTranscode'): Promise<number> {
+    const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } }
+
+    const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(`
+    UPDATE
+      "videoJobInfo"
+    SET
+      "${column}" = "videoJobInfo"."${column}" - 1,
+      "updatedAt" = NOW()
+    FROM "video"
+    WHERE
+      "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID
+    RETURNING
+      "${column}";
+    `, options)
+
+    return pendingMove
+  }
+}
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index d591a3134..3e9fd97c7 100644
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -1,10 +1,25 @@
 import * as memoizee from 'memoizee'
 import { join } from 'path'
 import { Op } from 'sequelize'
-import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import {
+  AllowNull,
+  BelongsTo,
+  Column,
+  CreatedAt,
+  DataType,
+  Default,
+  ForeignKey,
+  HasMany,
+  Is,
+  Model,
+  Table,
+  UpdatedAt
+} from 'sequelize-typescript'
+import { getHLSPublicFileUrl } from '@server/lib/object-storage'
 import { VideoFileModel } from '@server/models/video/video-file'
 import { MStreamingPlaylist, MVideo } from '@server/types/models'
 import { AttributesOnly } from '@shared/core-utils'
+import { VideoStorage } from '@shared/models'
 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
 import { sha1 } from '../../helpers/core-utils'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
@@ -81,6 +96,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
   @Column
   videoId: number
 
+  @AllowNull(false)
+  @Default(VideoStorage.FILE_SYSTEM)
+  @Column
+  storage: VideoStorage
+
   @BelongsTo(() => VideoModel, {
     foreignKey: {
       allowNull: false
@@ -185,12 +205,20 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
   }
 
   getMasterPlaylistUrl (video: MVideo) {
+    if (this.storage === VideoStorage.OBJECT_STORAGE) {
+      return getHLSPublicFileUrl(this.playlistUrl)
+    }
+
     if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid)
 
     return this.playlistUrl
   }
 
   getSha256SegmentsUrl (video: MVideo) {
+    if (this.storage === VideoStorage.OBJECT_STORAGE) {
+      return getHLSPublicFileUrl(this.segmentsSha256Url)
+    }
+
     if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive)
 
     return this.segmentsSha256Url
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 56a5b0e18..874ad168a 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -28,14 +28,16 @@ import { buildNSFWFilter } from '@server/helpers/express-utils'
 import { uuidToShort } from '@server/helpers/uuid'
 import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video'
 import { LiveManager } from '@server/lib/live/live-manager'
-import { getHLSDirectory, getVideoFilePath } from '@server/lib/video-paths'
+import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
+import { getHLSDirectory, getHLSRedundancyDirectory } from '@server/lib/paths'
+import { VideoPathManager } from '@server/lib/video-path-manager'
 import { getServerActor } from '@server/models/application/application'
 import { ModelCache } from '@server/models/model-cache'
 import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils'
 import { VideoFile } from '@shared/models/videos/video-file.model'
 import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared'
 import { VideoObject } from '../../../shared/models/activitypub/objects'
-import { Video, VideoDetails, VideoRateType } from '../../../shared/models/videos'
+import { Video, VideoDetails, VideoRateType, VideoStorage } from '../../../shared/models/videos'
 import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
 import { VideoFilter } from '../../../shared/models/videos/video-query.type'
 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
@@ -114,6 +116,7 @@ import { ScopeNames as VideoChannelScopeNames, SummaryOptions, VideoChannelModel
 import { VideoCommentModel } from './video-comment'
 import { VideoFileModel } from './video-file'
 import { VideoImportModel } from './video-import'
+import { VideoJobInfoModel } from './video-job-info'
 import { VideoLiveModel } from './video-live'
 import { VideoPlaylistElementModel } from './video-playlist-element'
 import { VideoShareModel } from './video-share'
@@ -732,6 +735,15 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
   })
   VideoCaptions: VideoCaptionModel[]
 
+  @HasOne(() => VideoJobInfoModel, {
+    foreignKey: {
+      name: 'videoId',
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+  VideoJobInfo: VideoJobInfoModel
+
   @BeforeDestroy
   static async sendDelete (instance: MVideoAccountLight, options) {
     if (!instance.isOwned()) return undefined
@@ -1641,9 +1653,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
   getMaxQualityResolution () {
     const file = this.getMaxQualityFile()
     const videoOrPlaylist = file.getVideoOrStreamingPlaylist()
-    const originalFilePath = getVideoFilePath(videoOrPlaylist, file)
 
-    return getVideoFileResolution(originalFilePath)
+    return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, file, originalFilePath => {
+      return getVideoFileResolution(originalFilePath)
+    })
   }
 
   getDescriptionAPIPath () {
@@ -1673,16 +1686,24 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
   }
 
   removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) {
-    const filePath = getVideoFilePath(this, videoFile, isRedundancy)
+    const filePath = isRedundancy
+      ? VideoPathManager.Instance.getFSRedundancyVideoFilePath(this, videoFile)
+      : VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile)
 
     const promises: Promise<any>[] = [ remove(filePath) ]
     if (!isRedundancy) promises.push(videoFile.removeTorrent())
 
+    if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
+      promises.push(removeWebTorrentObjectStorage(videoFile))
+    }
+
     return Promise.all(promises)
   }
 
   async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) {
-    const directoryPath = getHLSDirectory(this, isRedundancy)
+    const directoryPath = isRedundancy
+      ? getHLSRedundancyDirectory(this)
+      : getHLSDirectory(this)
 
     await remove(directoryPath)
 
@@ -1698,6 +1719,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
       await Promise.all(
         streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent())
       )
+
+      if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
+        await removeHLSObjectStorage(streamingPlaylist, this)
+      }
     }
   }
 
@@ -1741,16 +1766,16 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
       this.privacy === VideoPrivacy.INTERNAL
   }
 
-  async publishIfNeededAndSave (t: Transaction) {
-    if (this.state !== VideoState.PUBLISHED) {
-      this.state = VideoState.PUBLISHED
-      this.publishedAt = new Date()
-      await this.save({ transaction: t })
+  async setNewState (newState: VideoState, transaction: Transaction) {
+    if (this.state === newState) throw new Error('Cannot use same state ' + newState)
+
+    this.state = newState
 
-      return true
+    if (this.state === VideoState.PUBLISHED) {
+      this.publishedAt = new Date()
     }
 
-    return false
+    await this.save({ transaction })
   }
 
   getBandwidthBits (videoFile: MVideoFile) {