server/tests/cli/prune-storage.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { createFile, readdir } from 'fs-extra'
   5 import { join } from 'path'
   6 import { wait } from '@shared/core-utils'
   7 import { buildUUID } from '@shared/extra-utils'
   8 import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models'
   9 import {
  10   cleanupTests,
  11   CLICommand,
  12   createMultipleServers,
  13   doubleFollow,
  14   killallServers,
  15   makeGetRequest,
  16   PeerTubeServer,
  17   setAccessTokensToServers,
  18   setDefaultVideoChannel,
  19   waitJobs
  20 } from '@shared/server-commands'
  21
  22 async function countFiles (server: PeerTubeServer, directory: string) {
  23   const files = await readdir(server.servers.buildDirectory(directory))
  24
  25   return files.length
  26 }
  27
  28 async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
  29   const files = await readdir(server.servers.buildDirectory(directory))
  30
  31   for (const f of files) {
  32     expect(f).to.not.contain(substring)
  33   }
  34 }
  35
  36 async function assertCountAreOkay (servers: PeerTubeServer[]) {
  37   for (const server of servers) {
  38     const videosCount = await countFiles(server, 'videos')
  39     expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory
  40
  41     const privateVideosCount = await countFiles(server, 'videos/private')
  42     expect(privateVideosCount).to.equal(4)
  43
  44     const torrentsCount = await countFiles(server, 'torrents')
  45     expect(torrentsCount).to.equal(24)
  46
  47     const previewsCount = await countFiles(server, 'previews')
  48     expect(previewsCount).to.equal(3)
  49
  50     const thumbnailsCount = await countFiles(server, 'thumbnails')
  51     expect(thumbnailsCount).to.equal(7) // 3 local videos, 1 local playlist, 2 remotes videos and 1 remote playlist
  52
  53     const avatarsCount = await countFiles(server, 'avatars')
  54     expect(avatarsCount).to.equal(4)
  55
  56     const hlsRootCount = await countFiles(server, join('streaming-playlists', 'hls'))
  57     expect(hlsRootCount).to.equal(3) // 2 videos + private directory
  58
  59     const hlsPrivateRootCount = await countFiles(server, join('streaming-playlists', 'hls', 'private'))
  60     expect(hlsPrivateRootCount).to.equal(1)
  61   }
  62 }
  63
  64 describe('Test prune storage scripts', function () {
  65   let servers: PeerTubeServer[]
  66   const badNames: { [directory: string]: string[] } = {}
  67
  68   before(async function () {
  69     this.timeout(120000)
  70
  71     servers = await createMultipleServers(2, { transcoding: { enabled: true } })
  72     await setAccessTokensToServers(servers)
  73     await setDefaultVideoChannel(servers)
  74
  75     for (const server of servers) {
  76       await server.videos.upload({ attributes: { name: 'video 1', privacy: VideoPrivacy.PUBLIC } })
  77       await server.videos.upload({ attributes: { name: 'video 2', privacy: VideoPrivacy.PUBLIC } })
  78
  79       await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } })
  80
  81       await server.users.updateMyAvatar({ fixture: 'avatar.png' })
  82
  83       await server.playlists.create({
  84         attributes: {
  85           displayName: 'playlist',
  86           privacy: VideoPlaylistPrivacy.PUBLIC,
  87           videoChannelId: server.store.channel.id,
  88           thumbnailfile: 'thumbnail.jpg'
  89         }
  90       })
  91     }
  92
  93     await doubleFollow(servers[0], servers[1])
  94
  95     // Lazy load the remote avatars
  96     {
  97       const account = await servers[0].accounts.get({ accountName: 'root@' + servers[1].host })
  98
  99       for (const avatar of account.avatars) {
 100         await makeGetRequest({
 101           url: servers[0].url,
 102           path: avatar.path,
 103           expectedStatus: HttpStatusCode.OK_200
 104         })
 105       }
 106     }
 107
 108     {
 109       const account = await servers[1].accounts.get({ accountName: 'root@' + servers[0].host })
 110       for (const avatar of account.avatars) {
 111         await makeGetRequest({
 112           url: servers[1].url,
 113           path: avatar.path,
 114           expectedStatus: HttpStatusCode.OK_200
 115         })
 116       }
 117     }
 118
 119     await wait(1000)
 120
 121     await waitJobs(servers)
 122     await killallServers(servers)
 123
 124     await wait(1000)
 125   })
 126
 127   it('Should have the files on the disk', async function () {
 128     await assertCountAreOkay(servers)
 129   })
 130
 131   it('Should create some dirty files', async function () {
 132     for (let i = 0; i < 2; i++) {
 133       {
 134         const basePublic = servers[0].servers.buildDirectory('videos')
 135         const basePrivate = servers[0].servers.buildDirectory(join('videos', 'private'))
 136
 137         const n1 = buildUUID() + '.mp4'
 138         const n2 = buildUUID() + '.webm'
 139
 140         await createFile(join(basePublic, n1))
 141         await createFile(join(basePublic, n2))
 142         await createFile(join(basePrivate, n1))
 143         await createFile(join(basePrivate, n2))
 144
 145         badNames['videos'] = [ n1, n2 ]
 146       }
 147
 148       {
 149         const base = servers[0].servers.buildDirectory('torrents')
 150
 151         const n1 = buildUUID() + '-240.torrent'
 152         const n2 = buildUUID() + '-480.torrent'
 153
 154         await createFile(join(base, n1))
 155         await createFile(join(base, n2))
 156
 157         badNames['torrents'] = [ n1, n2 ]
 158       }
 159
 160       {
 161         const base = servers[0].servers.buildDirectory('thumbnails')
 162
 163         const n1 = buildUUID() + '.jpg'
 164         const n2 = buildUUID() + '.jpg'
 165
 166         await createFile(join(base, n1))
 167         await createFile(join(base, n2))
 168
 169         badNames['thumbnails'] = [ n1, n2 ]
 170       }
 171
 172       {
 173         const base = servers[0].servers.buildDirectory('previews')
 174
 175         const n1 = buildUUID() + '.jpg'
 176         const n2 = buildUUID() + '.jpg'
 177
 178         await createFile(join(base, n1))
 179         await createFile(join(base, n2))
 180
 181         badNames['previews'] = [ n1, n2 ]
 182       }
 183
 184       {
 185         const base = servers[0].servers.buildDirectory('avatars')
 186
 187         const n1 = buildUUID() + '.png'
 188         const n2 = buildUUID() + '.jpg'
 189
 190         await createFile(join(base, n1))
 191         await createFile(join(base, n2))
 192
 193         badNames['avatars'] = [ n1, n2 ]
 194       }
 195
 196       {
 197         const directory = join('streaming-playlists', 'hls')
 198         const basePublic = servers[0].servers.buildDirectory(directory)
 199         const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private'))
 200
 201         const n1 = buildUUID()
 202         await createFile(join(basePublic, n1))
 203         await createFile(join(basePrivate, n1))
 204         badNames[directory] = [ n1 ]
 205       }
 206     }
 207   })
 208
 209   it('Should run prune storage', async function () {
 210     this.timeout(30000)
 211
 212     const env = servers[0].cli.getEnv()
 213     await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
 214   })
 215
 216   it('Should have removed files', async function () {
 217     await assertCountAreOkay(servers)
 218
 219     for (const directory of Object.keys(badNames)) {
 220       for (const name of badNames[directory]) {
 221         await assertNotExists(servers[0], directory, name)
 222       }
 223     }
 224   })
 225
 226   after(async function () {
 227     await cleanupTests(servers)
 228   })
 229 })