Put private videos under a specific subdirectory

4 files changed, 181 insertions, 144 deletions

diff --git a/server/tests/api/transcoding/create-transcoding.ts b/server/tests/api/transcoding/create-transcoding.ts
index a50bf7654..372f5332a 100644
--- a/server/tests/api/transcoding/create-transcoding.ts
+++ b/server/tests/api/transcoding/create-transcoding.ts
@@ -20,7 +20,7 @@ import {
 async function checkFilesInObjectStorage (video: VideoDetails) {
   for (const file of video.files) {
     expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
-    await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
+    await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
   }
 
   if (video.streamingPlaylists.length === 0) return
@@ -28,14 +28,14 @@ async function checkFilesInObjectStorage (video: VideoDetails) {
   const hlsPlaylist = video.streamingPlaylists[0]
   for (const file of hlsPlaylist.files) {
     expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
-    await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
+    await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
   }
 
   expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getPlaylistBaseUrl())
-  await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
+  await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
 
   expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getPlaylistBaseUrl())
-  await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
+  await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
 }
 
 function runTests (objectStorage: boolean) {
@@ -234,7 +234,7 @@ function runTests (objectStorage: boolean) {
 
   it('Should have correctly deleted previous files', async function () {
     for (const fileUrl of shouldBeDeleted) {
-      await makeRawRequest(fileUrl, HttpStatusCode.NOT_FOUND_404)
+      await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
     }
   })
 
diff --git a/server/tests/api/transcoding/hls.ts b/server/tests/api/transcoding/hls.ts
index 252422e5d..7b5492cd4 100644
--- a/server/tests/api/transcoding/hls.ts
+++ b/server/tests/api/transcoding/hls.ts
@@ -1,168 +1,48 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
-import { expect } from 'chai'
-import { basename, join } from 'path'
-import {
-  checkDirectoryIsEmpty,
-  checkResolutionsInMasterPlaylist,
-  checkSegmentHash,
-  checkTmpIsEmpty,
-  expectStartWith,
-  hlsInfohashExist
-} from '@server/tests/shared'
-import { areObjectStorageTestsDisabled, removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
-import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models'
+import { join } from 'path'
+import { checkDirectoryIsEmpty, checkTmpIsEmpty, completeCheckHlsPlaylist } from '@server/tests/shared'
+import { areObjectStorageTestsDisabled } from '@shared/core-utils'
+import { HttpStatusCode } from '@shared/models'
 import {
   cleanupTests,
   createMultipleServers,
   doubleFollow,
-  makeRawRequest,
   ObjectStorageCommand,
   PeerTubeServer,
   setAccessTokensToServers,
-  waitJobs,
-  webtorrentAdd
+  waitJobs
 } from '@shared/server-commands'
 import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants'
 
-async function checkHlsPlaylist (options: {
-  servers: PeerTubeServer[]
-  videoUUID: string
-  hlsOnly: boolean
-
-  resolutions?: number[]
-  objectStorageBaseUrl: string
-}) {
-  const { videoUUID, hlsOnly, objectStorageBaseUrl } = options
-
-  const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ]
-
-  for (const server of options.servers) {
-    const videoDetails = await server.videos.get({ id: videoUUID })
-    const baseUrl = `http://${videoDetails.account.host}`
-
-    expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
-
-    const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
-    expect(hlsPlaylist).to.not.be.undefined
-
-    const hlsFiles = hlsPlaylist.files
-    expect(hlsFiles).to.have.lengthOf(resolutions.length)
-
-    if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0)
-    else expect(videoDetails.files).to.have.lengthOf(resolutions.length)
-
-    // Check JSON files
-    for (const resolution of resolutions) {
-      const file = hlsFiles.find(f => f.resolution.id === resolution)
-      expect(file).to.not.be.undefined
-
-      expect(file.magnetUri).to.have.lengthOf.above(2)
-      expect(file.torrentUrl).to.match(
-        new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`)
-      )
-
-      if (objectStorageBaseUrl) {
-        expectStartWith(file.fileUrl, objectStorageBaseUrl)
-      } else {
-        expect(file.fileUrl).to.match(
-          new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`)
-        )
-      }
-
-      expect(file.resolution.label).to.equal(resolution + 'p')
-
-      await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
-      await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
-
-      const torrent = await webtorrentAdd(file.magnetUri, true)
-      expect(torrent.files).to.be.an('array')
-      expect(torrent.files.length).to.equal(1)
-      expect(torrent.files[0].path).to.exist.and.to.not.equal('')
-    }
-
-    // Check master playlist
-    {
-      await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
-
-      const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl })
-
-      let i = 0
-      for (const resolution of resolutions) {
-        expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
-        expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
-
-        const url = 'http://' + videoDetails.account.host
-        await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i)
-
-        i++
-      }
-    }
-
-    // Check resolution playlists
-    {
-      for (const resolution of resolutions) {
-        const file = hlsFiles.find(f => f.resolution.id === resolution)
-        const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
-
-        const url = objectStorageBaseUrl
-          ? `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}`
-          : `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
-
-        const subPlaylist = await server.streamingPlaylists.get({ url })
-
-        expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
-        expect(subPlaylist).to.contain(basename(file.fileUrl))
-      }
-    }
-
-    {
-      const baseUrlAndPath = objectStorageBaseUrl
-        ? objectStorageBaseUrl + 'hls/' + videoUUID
-        : baseUrl + '/static/streaming-playlists/hls/' + videoUUID
-
-      for (const resolution of resolutions) {
-        await checkSegmentHash({
-          server,
-          baseUrlPlaylist: baseUrlAndPath,
-          baseUrlSegment: baseUrlAndPath,
-          resolution,
-          hlsPlaylist
-        })
-      }
-    }
-  }
-}
-
 describe('Test HLS videos', function () {
   let servers: PeerTubeServer[] = []
-  let videoUUID = ''
-  let videoAudioUUID = ''
 
   function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) {
+    const videoUUIDs: string[] = []
 
     it('Should upload a video and transcode it to HLS', async function () {
       this.timeout(120000)
 
       const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 1', fixture: 'video_short.webm' } })
-      videoUUID = uuid
+      videoUUIDs.push(uuid)
 
       await waitJobs(servers)
 
-      await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl })
+      await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
     })
 
     it('Should upload an audio file and transcode it to HLS', async function () {
       this.timeout(120000)
 
       const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video audio', fixture: 'sample.ogg' } })
-      videoAudioUUID = uuid
+      videoUUIDs.push(uuid)
 
       await waitJobs(servers)
 
-      await checkHlsPlaylist({
+      await completeCheckHlsPlaylist({
         servers,
-        videoUUID: videoAudioUUID,
+        videoUUID: uuid,
         hlsOnly,
         resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ],
         objectStorageBaseUrl
@@ -172,31 +52,36 @@ describe('Test HLS videos', function () {
     it('Should update the video', async function () {
       this.timeout(30000)
 
-      await servers[0].videos.update({ id: videoUUID, attributes: { name: 'video 1 updated' } })
+      await servers[0].videos.update({ id: videoUUIDs[0], attributes: { name: 'video 1 updated' } })
 
       await waitJobs(servers)
 
-      await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl })
+      await completeCheckHlsPlaylist({ servers, videoUUID: videoUUIDs[0], hlsOnly, objectStorageBaseUrl })
     })
 
     it('Should delete videos', async function () {
       this.timeout(10000)
 
-      await servers[0].videos.remove({ id: videoUUID })
-      await servers[0].videos.remove({ id: videoAudioUUID })
+      for (const uuid of videoUUIDs) {
+        await servers[0].videos.remove({ id: uuid })
+      }
 
       await waitJobs(servers)
 
       for (const server of servers) {
-        await server.videos.get({ id: videoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
-        await server.videos.get({ id: videoAudioUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
+        for (const uuid of videoUUIDs) {
+          await server.videos.get({ id: uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
+        }
       }
     })
 
     it('Should have the playlists/segment deleted from the disk', async function () {
       for (const server of servers) {
-        await checkDirectoryIsEmpty(server, 'videos')
-        await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls'))
+        await checkDirectoryIsEmpty(server, 'videos', [ 'private' ])
+        await checkDirectoryIsEmpty(server, join('videos', 'private'))
+
+        await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls'), [ 'private' ])
+        await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls', 'private'))
       }
     })
 
diff --git a/server/tests/api/transcoding/index.ts b/server/tests/api/transcoding/index.ts
index 0cc28b4a4..9866418d6 100644
--- a/server/tests/api/transcoding/index.ts
+++ b/server/tests/api/transcoding/index.ts
@@ -2,4 +2,5 @@ export * from './audio-only'
 export * from './create-transcoding'
 export * from './hls'
 export * from './transcoder'
+export * from './update-while-transcoding'
 export * from './video-studio'
diff --git a/server/tests/api/transcoding/update-while-transcoding.ts b/server/tests/api/transcoding/update-while-transcoding.ts
new file mode 100644
index 000000000..5ca923392
--- /dev/null
+++ b/server/tests/api/transcoding/update-while-transcoding.ts
@@ -0,0 +1,151 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { completeCheckHlsPlaylist } from '@server/tests/shared'
+import { areObjectStorageTestsDisabled, wait } from '@shared/core-utils'
+import { VideoPrivacy } from '@shared/models'
+import {
+  cleanupTests,
+  createMultipleServers,
+  doubleFollow,
+  ObjectStorageCommand,
+  PeerTubeServer,
+  setAccessTokensToServers,
+  waitJobs
+} from '@shared/server-commands'
+
+describe('Test update video privacy while transcoding', function () {
+  let servers: PeerTubeServer[] = []
+
+  const videoUUIDs: string[] = []
+
+  function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) {
+
+    it('Should not have an error while quickly updating a private video to public after upload #1', async function () {
+      this.timeout(360_000)
+
+      const attributes = {
+        name: 'quick update',
+        privacy: VideoPrivacy.PRIVATE
+      }
+
+      const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: false })
+      await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
+      videoUUIDs.push(uuid)
+
+      await waitJobs(servers)
+
+      await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
+    })
+
+    it('Should not have an error while quickly updating a private video to public after upload #2', async function () {
+
+      {
+        const attributes = {
+          name: 'quick update 2',
+          privacy: VideoPrivacy.PRIVATE
+        }
+
+        const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true })
+        await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
+        videoUUIDs.push(uuid)
+
+        await waitJobs(servers)
+
+        await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
+      }
+    })
+
+    it('Should not have an error while quickly updating a private video to public after upload #3', async function () {
+      const attributes = {
+        name: 'quick update 3',
+        privacy: VideoPrivacy.PRIVATE
+      }
+
+      const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true })
+      await wait(1000)
+      await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
+      videoUUIDs.push(uuid)
+
+      await waitJobs(servers)
+
+      await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
+    })
+  }
+
+  before(async function () {
+    this.timeout(120000)
+
+    const configOverride = {
+      transcoding: {
+        enabled: true,
+        allow_audio_files: true,
+        hls: {
+          enabled: true
+        }
+      }
+    }
+    servers = await createMultipleServers(2, configOverride)
+
+    // Get the access tokens
+    await setAccessTokensToServers(servers)
+
+    // Server 1 and server 2 follow each other
+    await doubleFollow(servers[0], servers[1])
+  })
+
+  describe('With WebTorrent & HLS enabled', function () {
+    runTestSuite(false)
+  })
+
+  describe('With only HLS enabled', function () {
+
+    before(async function () {
+      await servers[0].config.updateCustomSubConfig({
+        newConfig: {
+          transcoding: {
+            enabled: true,
+            allowAudioFiles: true,
+            resolutions: {
+              '144p': false,
+              '240p': true,
+              '360p': true,
+              '480p': true,
+              '720p': true,
+              '1080p': true,
+              '1440p': true,
+              '2160p': true
+            },
+            hls: {
+              enabled: true
+            },
+            webtorrent: {
+              enabled: false
+            }
+          }
+        }
+      })
+    })
+
+    runTestSuite(true)
+  })
+
+  describe('With object storage enabled', function () {
+    if (areObjectStorageTestsDisabled()) return
+
+    before(async function () {
+      this.timeout(120000)
+
+      const configOverride = ObjectStorageCommand.getDefaultConfig()
+      await ObjectStorageCommand.prepareDefaultBuckets()
+
+      await servers[0].kill()
+      await servers[0].run(configOverride)
+    })
+
+    runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl())
+  })
+
+  after(async function () {
+    await cleanupTests(servers)
+  })
+})