Add migrate-to-object-storage script (#4481)

* add migrate-to-object-storage-script

closes #4467

* add migrate-to-unique-playlist-filenames script

* fix(migrate-to-unique-playlist-filenames): update master/segments256

run updateMasterHLSPlaylist and updateSha256VODSegments after
file rename.

* Improve move to object storage scripts

* PR remarks

Co-authored-by: Chocobozzz <me@florianbigard.com>

2 files changed, 115 insertions, 0 deletions

diff --git a/server/tests/cli/create-move-video-storage-job.ts b/server/tests/cli/create-move-video-storage-job.ts
new file mode 100644
index 000000000..b598c8359
--- /dev/null
+++ b/server/tests/cli/create-move-video-storage-job.ts
@@ -0,0 +1,114 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import 'mocha'
+
+import {
+  areObjectStorageTestsDisabled,
+  cleanupTests,
+  createMultipleServers,
+  doubleFollow,
+  expectStartWith,
+  makeRawRequest,
+  ObjectStorageCommand,
+  PeerTubeServer,
+  setAccessTokensToServers,
+  waitJobs
+} from '@shared/extra-utils'
+import { HttpStatusCode, VideoDetails } from '@shared/models'
+
+async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObjectStorage: boolean) {
+  for (const file of video.files) {
+    const start = inObjectStorage
+      ? ObjectStorageCommand.getWebTorrentBaseUrl()
+      : origin.url
+
+    expectStartWith(file.fileUrl, start)
+
+    await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
+  }
+
+  const start = inObjectStorage
+    ? ObjectStorageCommand.getPlaylistBaseUrl()
+    : origin.url
+
+  const hls = video.streamingPlaylists[0]
+  expectStartWith(hls.playlistUrl, start)
+  expectStartWith(hls.segmentsSha256Url, start)
+
+  for (const file of hls.files) {
+    expectStartWith(file.fileUrl, start)
+
+    await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
+  }
+}
+
+describe('Test create move video storage job', function () {
+  if (areObjectStorageTestsDisabled()) return
+
+  let servers: PeerTubeServer[] = []
+  const uuids: string[] = []
+
+  before(async function () {
+    this.timeout(360000)
+
+    // Run server 2 to have transcoding enabled
+    servers = await createMultipleServers(2)
+    await setAccessTokensToServers(servers)
+
+    await doubleFollow(servers[0], servers[1])
+
+    await ObjectStorageCommand.prepareDefaultBuckets()
+
+    await servers[0].config.enableTranscoding()
+
+    for (let i = 0; i < 3; i++) {
+      const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } })
+      uuids.push(uuid)
+    }
+
+    await waitJobs(servers)
+
+    await servers[0].kill()
+    await servers[0].run(ObjectStorageCommand.getDefaultConfig())
+  })
+
+  it('Should move only one file', async function () {
+    this.timeout(120000)
+
+    const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}`
+    await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig())
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      const video = await server.videos.get({ id: uuids[1] })
+
+      await checkFiles(servers[0], video, true)
+
+      for (const id of [ uuids[0], uuids[2] ]) {
+        const video = await server.videos.get({ id })
+
+        await checkFiles(servers[0], video, false)
+      }
+    }
+  })
+
+  it('Should move all files', async function () {
+    this.timeout(120000)
+
+    const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos`
+    await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig())
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      for (const id of [ uuids[0], uuids[2] ]) {
+        const video = await server.videos.get({ id })
+
+        await checkFiles(servers[0], video, true)
+      }
+    }
+  })
+
+  after(async function () {
+    await cleanupTests(servers)
+  })
+})
diff --git a/server/tests/cli/index.ts b/server/tests/cli/index.ts
index c6dd0581a..6e0cbe58b 100644
--- a/server/tests/cli/index.ts
+++ b/server/tests/cli/index.ts
@@ -1,6 +1,7 @@
 // Order of the tests we want to execute
 import './create-import-video-file-job'
 import './create-transcoding-job'
+import './create-move-video-storage-job'
 import './peertube'
 import './plugins'
 import './print-transcode-command'