7 files changed, 73 insertions, 90 deletions

diff --git a/scripts/benchmark.ts b/scripts/benchmark.ts
index 0cadb36d9..5dcf9b01b 100644
--- a/scripts/benchmark.ts
+++ b/scripts/benchmark.ts
@@ -2,21 +2,11 @@ import { registerTSPaths } from '../server/helpers/register-ts-paths'
 registerTSPaths()
 
 import * as autocannon from 'autocannon'
-import {
-  addVideoCommentReply,
-  addVideoCommentThread,
-  createVideoCaption,
-  flushAndRunServer,
-  getVideosList,
-  killallServers,
-  ServerInfo,
-  setAccessTokensToServers,
-  uploadVideo
-} from '@shared/extra-utils'
-import { Video, VideoPrivacy } from '@shared/models'
 import { writeJson } from 'fs-extra'
+import { createSingleServer, killallServers, PeerTubeServer, setAccessTokensToServers } from '@shared/extra-utils'
+import { Video, VideoPrivacy } from '@shared/models'
 
-let server: ServerInfo
+let server: PeerTubeServer
 let video: Video
 let threadId: number
 
@@ -25,7 +15,7 @@ const outfile = process.argv[2]
 run()
   .catch(err => console.error(err))
   .finally(() => {
-    if (server) killallServers([ server ])
+    if (server) return killallServers([ server ])
   })
 
 function buildAuthorizationHeader () {
@@ -198,7 +188,7 @@ function runBenchmark (options: {
 }
 
 async function prepare () {
-  server = await flushAndRunServer(1, {
+  server = await createSingleServer(1, {
     rates_limit: {
       api: {
         max: 5_000_000
@@ -207,7 +197,7 @@ async function prepare () {
   })
   await setAccessTokensToServers([ server ])
 
-  const videoAttributes = {
+  const attributes = {
     name: 'my super video',
     category: 2,
     nsfw: true,
@@ -220,33 +210,29 @@ async function prepare () {
   }
 
   for (let i = 0; i < 10; i++) {
-    Object.assign(videoAttributes, { name: 'my super video ' + i })
-    await uploadVideo(server.url, server.accessToken, videoAttributes)
+    await server.videos.upload({ attributes: { ...attributes, name: 'my super video ' + i } })
   }
 
-  const resVideos = await getVideosList(server.url)
-  video = resVideos.body.data.find(v => v.name === 'my super video 1')
+  const { data } = await server.videos.list()
+  video = data.find(v => v.name === 'my super video 1')
 
   for (let i = 0; i < 10; i++) {
     const text = 'my super first comment'
-    const res = await addVideoCommentThread(server.url, server.accessToken, video.id, text)
-    threadId = res.body.comment.id
+    const created = await server.comments.createThread({ videoId: video.id, text })
+    threadId = created.id
 
     const text1 = 'my super answer to thread 1'
-    const childCommentRes = await addVideoCommentReply(server.url, server.accessToken, video.id, threadId, text1)
-    const childCommentId = childCommentRes.body.comment.id
+    const child = await server.comments.addReply({ videoId: video.id, toCommentId: threadId, text: text1 })
 
     const text2 = 'my super answer to answer of thread 1'
-    await addVideoCommentReply(server.url, server.accessToken, video.id, childCommentId, text2)
+    await server.comments.addReply({ videoId: video.id, toCommentId: child.id, text: text2 })
 
     const text3 = 'my second answer to thread 1'
-    await addVideoCommentReply(server.url, server.accessToken, video.id, threadId, text3)
+    await server.comments.addReply({ videoId: video.id, toCommentId: threadId, text: text3 })
   }
 
   for (const caption of [ 'ar', 'fr', 'en', 'zh' ]) {
-    await createVideoCaption({
-      url: server.url,
-      accessToken: server.accessToken,
+    await server.captions.add({
       language: caption,
       videoId: video.id,
       fixture: 'subtitle-good2.vtt'
diff --git a/scripts/ci.sh b/scripts/ci.sh
index 07e37e0ee..71b1be53b 100755
--- a/scripts/ci.sh
+++ b/scripts/ci.sh
@@ -47,11 +47,12 @@ if [ "$1" = "client" ]; then
 
     feedsFiles=$(findTestFiles ./dist/server/tests/feeds)
     helperFiles=$(findTestFiles ./dist/server/tests/helpers)
+    libFiles=$(findTestFiles ./dist/server/tests/lib)
     miscFiles="./dist/server/tests/client.js ./dist/server/tests/misc-endpoints.js"
     # Not in plugin task, it needs an index.html
     pluginFiles="./dist/server/tests/plugins/html-injection.js"
 
-    MOCHA_PARALLEL=true runTest "$1" 2 $feedsFiles $helperFiles $miscFiles $pluginFiles
+    MOCHA_PARALLEL=true runTest "$1" 2 $feedsFiles $helperFiles $miscFiles $pluginFiles $libFiles
 elif [ "$1" = "cli-plugin" ]; then
     npm run build:server
     npm run setup:cli
@@ -76,7 +77,7 @@ elif [ "$1" = "api-2" ]; then
     serverFiles=$(findTestFiles ./dist/server/tests/api/server)
     usersFiles=$(findTestFiles ./dist/server/tests/api/users)
 
-    MOCHA_PARALLEL=true runTest "$1" 3 $serverFiles $usersFiles $liveFiles
+    MOCHA_PARALLEL=true runTest "$1" 3 $liveFiles $serverFiles $usersFiles
 elif [ "$1" = "api-3" ]; then
     npm run build:server
 
diff --git a/scripts/generate-code-contributors.ts b/scripts/generate-code-contributors.ts
index db5af3f91..935ed3c5a 100755
--- a/scripts/generate-code-contributors.ts
+++ b/scripts/generate-code-contributors.ts
@@ -1,7 +1,7 @@
 import { registerTSPaths } from '../server/helpers/register-ts-paths'
 registerTSPaths()
 
-import { execCLI } from '@shared/extra-utils'
+import { CLICommand } from '@shared/extra-utils'
 
 run()
   .then(() => process.exit(0))
@@ -59,7 +59,7 @@ async function run () {
 }
 
 async function getGitContributors () {
-  const output = await execCLI(`git --no-pager shortlog -sn < /dev/tty | sed 's/^\\s\\+[0-9]\\+\\s\\+//g'`)
+  const output = await CLICommand.exec(`git --no-pager shortlog -sn < /dev/tty | sed 's/^\\s\\+[0-9]\\+\\s\\+//g'`)
 
   return output.split('\n')
                .filter(l => !!l)
diff --git a/scripts/optimize-old-videos.ts b/scripts/optimize-old-videos.ts
index 9692d76ba..bde9d1e01 100644
--- a/scripts/optimize-old-videos.ts
+++ b/scripts/optimize-old-videos.ts
@@ -19,13 +19,13 @@ run()
     process.exit(-1)
   })
 
-let currentVideoId = null
-let currentFile = null
+let currentVideoId: string
+let currentFilePath: string
 
 process.on('SIGINT', async function () {
   console.log('Cleaning up temp files')
-  await remove(`${currentFile}_backup`)
-  await remove(`${dirname(currentFile)}/${currentVideoId}-transcoded.mp4`)
+  await remove(`${currentFilePath}_backup`)
+  await remove(`${dirname(currentFilePath)}/${currentVideoId}-transcoded.mp4`)
   process.exit(0)
 })
 
@@ -40,12 +40,12 @@ async function run () {
     currentVideoId = video.id
 
     for (const file of video.VideoFiles) {
-      currentFile = getVideoFilePath(video, file)
+      currentFilePath = getVideoFilePath(video, file)
 
       const [ videoBitrate, fps, resolution ] = await Promise.all([
-        getVideoFileBitrate(currentFile),
-        getVideoFileFPS(currentFile),
-        getVideoFileResolution(currentFile)
+        getVideoFileBitrate(currentFilePath),
+        getVideoFileFPS(currentFilePath),
+        getVideoFileResolution(currentFilePath)
       ])
 
       const maxBitrate = getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)
@@ -53,25 +53,27 @@ async function run () {
       if (isMaxBitrateExceeded) {
         console.log(
           'Optimizing video file %s with bitrate %s kbps (max: %s kbps)',
-          basename(currentFile), videoBitrate / 1000, maxBitrate / 1000
+          basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000
         )
 
-        const backupFile = `${currentFile}_backup`
-        await copy(currentFile, backupFile)
+        const backupFile = `${currentFilePath}_backup`
+        await copy(currentFilePath, backupFile)
 
         await optimizeOriginalVideofile(video, file)
+        // Update file path, the video filename changed
+        currentFilePath = getVideoFilePath(video, file)
 
         const originalDuration = await getDurationFromVideoFile(backupFile)
-        const newDuration = await getDurationFromVideoFile(currentFile)
+        const newDuration = await getDurationFromVideoFile(currentFilePath)
 
         if (originalDuration === newDuration) {
-          console.log('Finished optimizing %s', basename(currentFile))
+          console.log('Finished optimizing %s', basename(currentFilePath))
           await remove(backupFile)
           continue
         }
 
-        console.log('Failed to optimize %s, restoring original', basename(currentFile))
-        await move(backupFile, currentFile, { overwrite: true })
+        console.log('Failed to optimize %s, restoring original', basename(currentFilePath))
+        await move(backupFile, currentFilePath, { overwrite: true })
         await createTorrentAndSetInfoHash(video, file)
         await file.save()
       }
diff --git a/scripts/parse-log.ts b/scripts/parse-log.ts
index c16503589..6cd3a1860 100755
--- a/scripts/parse-log.ts
+++ b/scripts/parse-log.ts
@@ -6,9 +6,8 @@ import { createReadStream, readdir } from 'fs-extra'
 import { join } from 'path'
 import { createInterface } from 'readline'
 import * as winston from 'winston'
-import { labelFormatter } from '../server/helpers/logger'
+import { labelFormatter, mtimeSortFilesDesc } from '../server/helpers/logger'
 import { CONFIG } from '../server/initializers/config'
-import { mtimeSortFilesDesc } from '../shared/core-utils/logs/logs'
 import { inspect } from 'util'
 import { format as sqlFormat } from 'sql-formatter'
 
diff --git a/scripts/prune-storage.ts b/scripts/prune-storage.ts
index 58d24816e..5b029d215 100755
--- a/scripts/prune-storage.ts
+++ b/scripts/prune-storage.ts
@@ -2,11 +2,11 @@ import { registerTSPaths } from '../server/helpers/register-ts-paths'
 registerTSPaths()
 
 import * as prompt from 'prompt'
-import { join } from 'path'
+import { join, basename } from 'path'
 import { CONFIG } from '../server/initializers/config'
 import { VideoModel } from '../server/models/video/video'
 import { initDatabaseModels } from '../server/initializers/database'
-import { readdir, remove } from 'fs-extra'
+import { readdir, remove, stat } from 'fs-extra'
 import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
 import * as Bluebird from 'bluebird'
 import { getUUIDFromFilename } from '../server/helpers/utils'
@@ -14,6 +14,7 @@ import { ThumbnailModel } from '../server/models/video/thumbnail'
 import { ActorImageModel } from '../server/models/actor/actor-image'
 import { uniq, values } from 'lodash'
 import { ThumbnailType } from '@shared/models'
+import { VideoFileModel } from '@server/models/video/video-file'
 
 run()
   .then(() => process.exit(0))
@@ -37,8 +38,8 @@ async function run () {
   console.log('Detecting files to remove, it could take a while...')
 
   toDelete = toDelete.concat(
-    await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
-    await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
+    await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesWebTorrentFileExist()),
+    await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
 
     await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
 
@@ -78,26 +79,27 @@ async function pruneDirectory (directory: string, existFun: ExistFun) {
 
   const toDelete: string[] = []
   await Bluebird.map(files, async file => {
-    if (await existFun(file) !== true) {
-      toDelete.push(join(directory, file))
+    const filePath = join(directory, file)
+
+    if (await existFun(filePath) !== true) {
+      toDelete.push(filePath)
     }
   }, { concurrency: 20 })
 
   return toDelete
 }
 
-function doesVideoExist (keepOnlyOwned: boolean) {
-  return async (file: string) => {
-    const uuid = getUUIDFromFilename(file)
-    const video = await VideoModel.load(uuid)
+function doesWebTorrentFileExist () {
+  return (filePath: string) => VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
+}
 
-    return video && (keepOnlyOwned === false || video.isOwned())
-  }
+function doesTorrentFileExist () {
+  return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
 }
 
 function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
-  return async (file: string) => {
-    const thumbnail = await ThumbnailModel.loadByFilename(file, type)
+  return async (filePath: string) => {
+    const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
     if (!thumbnail) return false
 
     if (keepOnlyOwned) {
@@ -109,21 +111,20 @@ function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
   }
 }
 
-async function doesActorImageExist (file: string) {
-  const image = await ActorImageModel.loadByName(file)
+async function doesActorImageExist (filePath: string) {
+  const image = await ActorImageModel.loadByName(basename(filePath))
 
   return !!image
 }
 
-async function doesRedundancyExist (file: string) {
-  const uuid = getUUIDFromFilename(file)
-  const video = await VideoModel.loadWithFiles(uuid)
-
-  if (!video) return false
-
-  const isPlaylist = file.includes('.') === false
+async function doesRedundancyExist (filePath: string) {
+  const isPlaylist = (await stat(filePath)).isDirectory()
 
   if (isPlaylist) {
+    const uuid = getUUIDFromFilename(filePath)
+    const video = await VideoModel.loadWithFiles(uuid)
+    if (!video) return false
+
     const p = video.getHLSPlaylist()
     if (!p) return false
 
@@ -131,19 +132,10 @@ async function doesRedundancyExist (file: string) {
     return !!redundancy
   }
 
-  const resolution = parseInt(file.split('-')[5], 10)
-  if (isNaN(resolution)) {
-    console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
-    return true
-  }
-
-  const videoFile = video.getWebTorrentFile(resolution)
-  if (!videoFile) {
-    console.error('Cannot find webtorrent file of video %s - %d', video.url, resolution)
-    return true
-  }
+  const file = await VideoFileModel.loadByFilename(basename(filePath))
+  if (!file) return false
 
-  const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
+  const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
   return !!redundancy
 }
 
diff --git a/scripts/update-host.ts b/scripts/update-host.ts
index 592684225..9e8dd41ca 100755
--- a/scripts/update-host.ts
+++ b/scripts/update-host.ts
@@ -16,7 +16,6 @@ import { VideoShareModel } from '../server/models/video/video-share'
 import { VideoCommentModel } from '../server/models/video/video-comment'
 import { AccountModel } from '../server/models/account/account'
 import { VideoChannelModel } from '../server/models/video/video-channel'
-import { VideoStreamingPlaylistModel } from '../server/models/video/video-streaming-playlist'
 import { initDatabaseModels } from '../server/initializers/database'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { getServerActor } from '@server/models/application/application'
@@ -128,13 +127,17 @@ async function run () {
     for (const file of video.VideoFiles) {
       console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid)
       await createTorrentAndSetInfoHash(video, file)
+
+      await file.save()
     }
 
-    for (const playlist of video.VideoStreamingPlaylists) {
-      playlist.playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
-      playlist.segmentsSha256Url = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive)
+    const playlist = video.getHLSPlaylist()
+    for (const file of (playlist?.VideoFiles || [])) {
+      console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid)
+
+      await createTorrentAndSetInfoHash(video, file)
 
-      await playlist.save()
+      await file.save()
     }
   }
 }