server/tests/cli/prune-storage.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { createFile, readdir } from 'fs-extra'
   6 import { join } from 'path'
   7 import { buildUUID } from '@server/helpers/uuid'
   8 import {
   9   cleanupTests,
  10   CLICommand,
  11   createMultipleServers,
  12   doubleFollow,
  13   killallServers,
  14   makeGetRequest,
  15   PeerTubeServer,
  16   setAccessTokensToServers,
  17   setDefaultVideoChannel,
  18   wait,
  19   waitJobs
  20 } from '@shared/extra-utils'
  21 import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models'
  22
  23 const expect = chai.expect
  24
  25 async function countFiles (server: PeerTubeServer, directory: string) {
  26   const files = await readdir(server.servers.buildDirectory(directory))
  27
  28   return files.length
  29 }
  30
  31 async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
  32   const files = await readdir(server.servers.buildDirectory(directory))
  33
  34   for (const f of files) {
  35     expect(f).to.not.contain(substring)
  36   }
  37 }
  38
  39 async function assertCountAreOkay (servers: PeerTubeServer[]) {
  40   for (const server of servers) {
  41     const videosCount = await countFiles(server, 'videos')
  42     expect(videosCount).to.equal(8)
  43
  44     const torrentsCount = await countFiles(server, 'torrents')
  45     expect(torrentsCount).to.equal(16)
  46
  47     const previewsCount = await countFiles(server, 'previews')
  48     expect(previewsCount).to.equal(2)
  49
  50     const thumbnailsCount = await countFiles(server, 'thumbnails')
  51     expect(thumbnailsCount).to.equal(6)
  52
  53     const avatarsCount = await countFiles(server, 'avatars')
  54     expect(avatarsCount).to.equal(2)
  55   }
  56 }
  57
  58 describe('Test prune storage scripts', function () {
  59   let servers: PeerTubeServer[]
  60   const badNames: { [directory: string]: string[] } = {}
  61
  62   before(async function () {
  63     this.timeout(120000)
  64
  65     servers = await createMultipleServers(2, { transcoding: { enabled: true } })
  66     await setAccessTokensToServers(servers)
  67     await setDefaultVideoChannel(servers)
  68
  69     for (const server of servers) {
  70       await server.videos.upload({ attributes: { name: 'video 1' } })
  71       await server.videos.upload({ attributes: { name: 'video 2' } })
  72
  73       await server.users.updateMyAvatar({ fixture: 'avatar.png' })
  74
  75       await server.playlists.create({
  76         attributes: {
  77           displayName: 'playlist',
  78           privacy: VideoPlaylistPrivacy.PUBLIC,
  79           videoChannelId: server.store.channel.id,
  80           thumbnailfile: 'thumbnail.jpg'
  81         }
  82       })
  83     }
  84
  85     await doubleFollow(servers[0], servers[1])
  86
  87     // Lazy load the remote avatar
  88     {
  89       const account = await servers[0].accounts.get({ accountName: 'root@localhost:' + servers[1].port })
  90       await makeGetRequest({
  91         url: servers[0].url,
  92         path: account.avatar.path,
  93         expectedStatus: HttpStatusCode.OK_200
  94       })
  95     }
  96
  97     {
  98       const account = await servers[1].accounts.get({ accountName: 'root@localhost:' + servers[0].port })
  99       await makeGetRequest({
 100         url: servers[1].url,
 101         path: account.avatar.path,
 102         expectedStatus: HttpStatusCode.OK_200
 103       })
 104     }
 105
 106     await wait(1000)
 107
 108     await waitJobs(servers)
 109     await killallServers(servers)
 110
 111     await wait(1000)
 112   })
 113
 114   it('Should have the files on the disk', async function () {
 115     await assertCountAreOkay(servers)
 116   })
 117
 118   it('Should create some dirty files', async function () {
 119     for (let i = 0; i < 2; i++) {
 120       {
 121         const base = servers[0].servers.buildDirectory('videos')
 122
 123         const n1 = buildUUID() + '.mp4'
 124         const n2 = buildUUID() + '.webm'
 125
 126         await createFile(join(base, n1))
 127         await createFile(join(base, n2))
 128
 129         badNames['videos'] = [ n1, n2 ]
 130       }
 131
 132       {
 133         const base = servers[0].servers.buildDirectory('torrents')
 134
 135         const n1 = buildUUID() + '-240.torrent'
 136         const n2 = buildUUID() + '-480.torrent'
 137
 138         await createFile(join(base, n1))
 139         await createFile(join(base, n2))
 140
 141         badNames['torrents'] = [ n1, n2 ]
 142       }
 143
 144       {
 145         const base = servers[0].servers.buildDirectory('thumbnails')
 146
 147         const n1 = buildUUID() + '.jpg'
 148         const n2 = buildUUID() + '.jpg'
 149
 150         await createFile(join(base, n1))
 151         await createFile(join(base, n2))
 152
 153         badNames['thumbnails'] = [ n1, n2 ]
 154       }
 155
 156       {
 157         const base = servers[0].servers.buildDirectory('previews')
 158
 159         const n1 = buildUUID() + '.jpg'
 160         const n2 = buildUUID() + '.jpg'
 161
 162         await createFile(join(base, n1))
 163         await createFile(join(base, n2))
 164
 165         badNames['previews'] = [ n1, n2 ]
 166       }
 167
 168       {
 169         const base = servers[0].servers.buildDirectory('avatars')
 170
 171         const n1 = buildUUID() + '.png'
 172         const n2 = buildUUID() + '.jpg'
 173
 174         await createFile(join(base, n1))
 175         await createFile(join(base, n2))
 176
 177         badNames['avatars'] = [ n1, n2 ]
 178       }
 179     }
 180   })
 181
 182   it('Should run prune storage', async function () {
 183     this.timeout(30000)
 184
 185     const env = servers[0].cli.getEnv()
 186     await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
 187   })
 188
 189   it('Should have removed files', async function () {
 190     await assertCountAreOkay(servers)
 191
 192     for (const directory of Object.keys(badNames)) {
 193       for (const name of badNames[directory]) {
 194         await assertNotExists(servers[0], directory, name)
 195       }
 196     }
 197   })
 198
 199   after(async function () {
 200     await cleanupTests(servers)
 201   })
 202 })