server/tests/cli/prune-storage.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { createFile, readdir } from 'fs-extra'
   6 import { join } from 'path'
   7 import { buildUUID } from '@server/helpers/uuid'
   8 import { HttpStatusCode } from '@shared/core-utils'
   9 import {
  10   cleanupTests,
  11   CLICommand,
  12   doubleFollow,
  13   createMultipleServers,
  14   killallServers,
  15   makeGetRequest,
  16   PeerTubeServer,
  17   setAccessTokensToServers,
  18   setDefaultVideoChannel,
  19   wait,
  20   waitJobs
  21 } from '@shared/extra-utils'
  22 import { VideoPlaylistPrivacy } from '@shared/models'
  23
  24 const expect = chai.expect
  25
  26 async function countFiles (server: PeerTubeServer, directory: string) {
  27   const files = await readdir(server.servers.buildDirectory(directory))
  28
  29   return files.length
  30 }
  31
  32 async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
  33   const files = await readdir(server.servers.buildDirectory(directory))
  34
  35   for (const f of files) {
  36     expect(f).to.not.contain(substring)
  37   }
  38 }
  39
  40 async function assertCountAreOkay (servers: PeerTubeServer[]) {
  41   for (const server of servers) {
  42     const videosCount = await countFiles(server, 'videos')
  43     expect(videosCount).to.equal(8)
  44
  45     const torrentsCount = await countFiles(server, 'torrents')
  46     expect(torrentsCount).to.equal(16)
  47
  48     const previewsCount = await countFiles(server, 'previews')
  49     expect(previewsCount).to.equal(2)
  50
  51     const thumbnailsCount = await countFiles(server, 'thumbnails')
  52     expect(thumbnailsCount).to.equal(6)
  53
  54     const avatarsCount = await countFiles(server, 'avatars')
  55     expect(avatarsCount).to.equal(2)
  56   }
  57 }
  58
  59 describe('Test prune storage scripts', function () {
  60   let servers: PeerTubeServer[]
  61   const badNames: { [directory: string]: string[] } = {}
  62
  63   before(async function () {
  64     this.timeout(120000)
  65
  66     servers = await createMultipleServers(2, { transcoding: { enabled: true } })
  67     await setAccessTokensToServers(servers)
  68     await setDefaultVideoChannel(servers)
  69
  70     for (const server of servers) {
  71       await server.videos.upload({ attributes: { name: 'video 1' } })
  72       await server.videos.upload({ attributes: { name: 'video 2' } })
  73
  74       await server.users.updateMyAvatar({ fixture: 'avatar.png' })
  75
  76       await server.playlists.create({
  77         attributes: {
  78           displayName: 'playlist',
  79           privacy: VideoPlaylistPrivacy.PUBLIC,
  80           videoChannelId: server.store.channel.id,
  81           thumbnailfile: 'thumbnail.jpg'
  82         }
  83       })
  84     }
  85
  86     await doubleFollow(servers[0], servers[1])
  87
  88     // Lazy load the remote avatar
  89     {
  90       const account = await servers[0].accounts.get({ accountName: 'root@localhost:' + servers[1].port })
  91       await makeGetRequest({
  92         url: servers[0].url,
  93         path: account.avatar.path,
  94         statusCodeExpected: HttpStatusCode.OK_200
  95       })
  96     }
  97
  98     {
  99       const account = await servers[1].accounts.get({ accountName: 'root@localhost:' + servers[0].port })
 100       await makeGetRequest({
 101         url: servers[1].url,
 102         path: account.avatar.path,
 103         statusCodeExpected: HttpStatusCode.OK_200
 104       })
 105     }
 106
 107     await wait(1000)
 108
 109     await waitJobs(servers)
 110     await killallServers(servers)
 111
 112     await wait(1000)
 113   })
 114
 115   it('Should have the files on the disk', async function () {
 116     await assertCountAreOkay(servers)
 117   })
 118
 119   it('Should create some dirty files', async function () {
 120     for (let i = 0; i < 2; i++) {
 121       {
 122         const base = servers[0].servers.buildDirectory('videos')
 123
 124         const n1 = buildUUID() + '.mp4'
 125         const n2 = buildUUID() + '.webm'
 126
 127         await createFile(join(base, n1))
 128         await createFile(join(base, n2))
 129
 130         badNames['videos'] = [ n1, n2 ]
 131       }
 132
 133       {
 134         const base = servers[0].servers.buildDirectory('torrents')
 135
 136         const n1 = buildUUID() + '-240.torrent'
 137         const n2 = buildUUID() + '-480.torrent'
 138
 139         await createFile(join(base, n1))
 140         await createFile(join(base, n2))
 141
 142         badNames['torrents'] = [ n1, n2 ]
 143       }
 144
 145       {
 146         const base = servers[0].servers.buildDirectory('thumbnails')
 147
 148         const n1 = buildUUID() + '.jpg'
 149         const n2 = buildUUID() + '.jpg'
 150
 151         await createFile(join(base, n1))
 152         await createFile(join(base, n2))
 153
 154         badNames['thumbnails'] = [ n1, n2 ]
 155       }
 156
 157       {
 158         const base = servers[0].servers.buildDirectory('previews')
 159
 160         const n1 = buildUUID() + '.jpg'
 161         const n2 = buildUUID() + '.jpg'
 162
 163         await createFile(join(base, n1))
 164         await createFile(join(base, n2))
 165
 166         badNames['previews'] = [ n1, n2 ]
 167       }
 168
 169       {
 170         const base = servers[0].servers.buildDirectory('avatars')
 171
 172         const n1 = buildUUID() + '.png'
 173         const n2 = buildUUID() + '.jpg'
 174
 175         await createFile(join(base, n1))
 176         await createFile(join(base, n2))
 177
 178         badNames['avatars'] = [ n1, n2 ]
 179       }
 180     }
 181   })
 182
 183   it('Should run prune storage', async function () {
 184     this.timeout(30000)
 185
 186     const env = servers[0].cli.getEnv()
 187     await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
 188   })
 189
 190   it('Should have removed files', async function () {
 191     await assertCountAreOkay(servers)
 192
 193     for (const directory of Object.keys(badNames)) {
 194       for (const name of badNames[directory]) {
 195         await assertNotExists(servers[0], directory, name)
 196       }
 197     }
 198   })
 199
 200   after(async function () {
 201     await cleanupTests(servers)
 202   })
 203 })