6 files changed, 114 insertions, 33 deletions

diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
index 911734fa0..09f587274 100644
--- a/server/initializers/checker-after-init.ts
+++ b/server/initializers/checker-after-init.ts
@@ -153,6 +153,29 @@ function checkConfig () {
     }
   }
 
+  // Object storage
+  if (CONFIG.OBJECT_STORAGE.ENABLED === true) {
+
+    if (!CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME) {
+      return 'videos_bucket should be set when object storage support is enabled.'
+    }
+
+    if (!CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME) {
+      return 'streaming_playlists_bucket should be set when object storage support is enabled.'
+    }
+
+    if (
+      CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME &&
+      CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX
+    ) {
+      if (CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === '') {
+        return 'Object storage bucket prefixes should be set when the same bucket is used for both types of video.'
+      } else {
+        return 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.'
+      }
+    }
+  }
+
   return null
 }
 
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 30a9823b9..0e684eef8 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -73,6 +73,26 @@ const CONFIG = {
     PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')),
     CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides'))
   },
+  OBJECT_STORAGE: {
+    ENABLED: config.get<boolean>('object_storage.enabled'),
+    MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')),
+    ENDPOINT: config.get<string>('object_storage.endpoint'),
+    REGION: config.get<string>('object_storage.region'),
+    CREDENTIALS: {
+      ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'),
+      SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key')
+    },
+    VIDEOS: {
+      BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'),
+      PREFIX: config.get<string>('object_storage.videos.prefix'),
+      BASE_URL: config.get<string>('object_storage.videos.base_url')
+    },
+    STREAMING_PLAYLISTS: {
+      BUCKET_NAME: config.get<string>('object_storage.streaming_playlists.bucket_name'),
+      PREFIX: config.get<string>('object_storage.streaming_playlists.prefix'),
+      BASE_URL: config.get<string>('object_storage.streaming_playlists.base_url')
+    }
+  },
   WEBSERVER: {
     SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http',
     WS: config.get<boolean>('webserver.https') === true ? 'wss' : 'ws',
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 5f121d9a4..8a1526ae8 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 655
+const LAST_MIGRATION_VERSION = 660
 
 // ---------------------------------------------------------------------------
 
@@ -147,7 +147,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
   'videos-views': 1,
   'activitypub-refresher': 1,
   'video-redundancy': 1,
-  'video-live-ending': 1
+  'video-live-ending': 1,
+  'move-to-object-storage': 3
 }
 // Excluded keys are jobs that can be configured by admins
 const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = {
@@ -162,7 +163,8 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
   'videos-views': 1,
   'activitypub-refresher': 1,
   'video-redundancy': 1,
-  'video-live-ending': 10
+  'video-live-ending': 10,
+  'move-to-object-storage': 1
 }
 const JOB_TTL: { [id in JobType]: number } = {
   'activitypub-http-broadcast': 60000 * 10, // 10 minutes
@@ -178,7 +180,8 @@ const JOB_TTL: { [id in JobType]: number } = {
   'videos-views': undefined, // Unlimited
   'activitypub-refresher': 60000 * 10, // 10 minutes
   'video-redundancy': 1000 * 3600 * 3, // 3 hours
-  'video-live-ending': 1000 * 60 * 10 // 10 minutes
+  'video-live-ending': 1000 * 60 * 10, // 10 minutes
+  'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours
 }
 const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = {
   'videos-views': {
@@ -412,7 +415,8 @@ const VIDEO_STATES: { [ id in VideoState ]: string } = {
   [VideoState.TO_TRANSCODE]: 'To transcode',
   [VideoState.TO_IMPORT]: 'To import',
   [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream',
-  [VideoState.LIVE_ENDED]: 'Livestream ended'
+  [VideoState.LIVE_ENDED]: 'Livestream ended',
+  [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage'
 }
 
 const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 38e7a76d0..0e690f6ae 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -45,6 +45,7 @@ import { VideoTagModel } from '../models/video/video-tag'
 import { VideoViewModel } from '../models/video/video-view'
 import { CONFIG } from './config'
 import { ActorCustomPageModel } from '@server/models/account/actor-custom-page'
+import { VideoJobInfoModel } from '@server/models/video/video-job-info'
 
 require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
 
@@ -143,7 +144,8 @@ async function initDatabaseModels (silent: boolean) {
     TrackerModel,
     VideoTrackerModel,
     PluginModel,
-    ActorCustomPageModel
+    ActorCustomPageModel,
+    VideoJobInfoModel
   ])
 
   // Check extensions exist in the database
diff --git a/server/initializers/migrations/0065-video-file-size.ts b/server/initializers/migrations/0065-video-file-size.ts
index 1aeb27f2d..ac952a98c 100644
--- a/server/initializers/migrations/0065-video-file-size.ts
+++ b/server/initializers/migrations/0065-video-file-size.ts
@@ -1,7 +1,4 @@
 import * as Sequelize from 'sequelize'
-import { stat } from 'fs-extra'
-import { VideoModel } from '../../models/video/video'
-import { getVideoFilePath } from '@server/lib/video-paths'
 
 function up (utils: {
   transaction: Sequelize.Transaction
@@ -9,30 +6,7 @@ function up (utils: {
   sequelize: Sequelize.Sequelize
   db: any
 }): Promise<void> {
-  return utils.db.Video.listOwnedAndPopulateAuthorAndTags()
-    .then((videos: VideoModel[]) => {
-      const tasks: Promise<any>[] = []
-
-      videos.forEach(video => {
-        video.VideoFiles.forEach(videoFile => {
-          const p = new Promise((res, rej) => {
-            stat(getVideoFilePath(video, videoFile), (err, stats) => {
-              if (err) return rej(err)
-
-              videoFile.size = stats.size
-              videoFile.save().then(res).catch(rej)
-            })
-          })
-
-          tasks.push(p)
-        })
-      })
-
-      return tasks
-    })
-    .then((tasks: Promise<any>[]) => {
-      return Promise.all(tasks)
-    })
+  throw new Error('Removed, please upgrade from a previous version first.')
 }
 
 function down (options) {
diff --git a/server/initializers/migrations/0660-object-storage.ts b/server/initializers/migrations/0660-object-storage.ts
new file mode 100644
index 000000000..c815c71c6
--- /dev/null
+++ b/server/initializers/migrations/0660-object-storage.ts
@@ -0,0 +1,58 @@
+import * as Sequelize from 'sequelize'
+import { VideoStorage } from '@shared/models'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+  db: any
+}): Promise<void> {
+  {
+    const query = `
+    CREATE TABLE IF NOT EXISTS "videoJobInfo" (
+      "id" serial,
+      "pendingMove" INTEGER NOT NULL,
+      "pendingTranscode" INTEGER NOT NULL,
+      "videoId" serial UNIQUE NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
+      "createdAt" timestamp WITH time zone NOT NULL,
+      "updatedAt" timestamp WITH time zone NOT NULL,
+      PRIMARY KEY ("id")
+    );
+    `
+
+    await utils.sequelize.query(query)
+  }
+
+  {
+    await utils.queryInterface.addColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: true })
+  }
+  {
+    await utils.sequelize.query(
+      `UPDATE "videoFile" SET "storage" = ${VideoStorage.FILE_SYSTEM}`
+    )
+  }
+  {
+    await utils.queryInterface.changeColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: false })
+  }
+
+  {
+    await utils.queryInterface.addColumn('videoStreamingPlaylist', 'storage', { type: Sequelize.INTEGER, allowNull: true })
+  }
+  {
+    await utils.sequelize.query(
+      `UPDATE "videoStreamingPlaylist" SET "storage" = ${VideoStorage.FILE_SYSTEM}`
+    )
+  }
+  {
+    await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'storage', { type: Sequelize.INTEGER, allowNull: false })
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}