server/tests/cli/prune-storage.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { createFile, readdir } from 'fs-extra'
   6 import { join } from 'path'
   7 import { buildUUID } from '@server/helpers/uuid'
   8 import { HttpStatusCode } from '@shared/core-utils'
   9 import {
  10   cleanupTests,
  11   CLICommand,
  12   doubleFollow,
  13   flushAndRunMultipleServers,
  14   killallServers,
  15   makeGetRequest,
  16   ServerInfo,
  17   setAccessTokensToServers,
  18   setDefaultVideoChannel,
  19   uploadVideo,
  20   wait,
  21   waitJobs
  22 } from '@shared/extra-utils'
  23 import { VideoPlaylistPrivacy } from '@shared/models'
  24
  25 const expect = chai.expect
  26
  27 async function countFiles (server: ServerInfo, directory: string) {
  28   const files = await readdir(server.serversCommand.buildDirectory(directory))
  29
  30   return files.length
  31 }
  32
  33 async function assertNotExists (server: ServerInfo, directory: string, substring: string) {
  34   const files = await readdir(server.serversCommand.buildDirectory(directory))
  35
  36   for (const f of files) {
  37     expect(f).to.not.contain(substring)
  38   }
  39 }
  40
  41 async function assertCountAreOkay (servers: ServerInfo[]) {
  42   for (const server of servers) {
  43     const videosCount = await countFiles(server, 'videos')
  44     expect(videosCount).to.equal(8)
  45
  46     const torrentsCount = await countFiles(server, 'torrents')
  47     expect(torrentsCount).to.equal(16)
  48
  49     const previewsCount = await countFiles(server, 'previews')
  50     expect(previewsCount).to.equal(2)
  51
  52     const thumbnailsCount = await countFiles(server, 'thumbnails')
  53     expect(thumbnailsCount).to.equal(6)
  54
  55     const avatarsCount = await countFiles(server, 'avatars')
  56     expect(avatarsCount).to.equal(2)
  57   }
  58 }
  59
  60 describe('Test prune storage scripts', function () {
  61   let servers: ServerInfo[]
  62   const badNames: { [directory: string]: string[] } = {}
  63
  64   before(async function () {
  65     this.timeout(120000)
  66
  67     servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true } })
  68     await setAccessTokensToServers(servers)
  69     await setDefaultVideoChannel(servers)
  70
  71     for (const server of servers) {
  72       await uploadVideo(server.url, server.accessToken, { name: 'video 1' })
  73       await uploadVideo(server.url, server.accessToken, { name: 'video 2' })
  74
  75       await server.usersCommand.updateMyAvatar({ fixture: 'avatar.png' })
  76
  77       await server.playlistsCommand.create({
  78         attributes: {
  79           displayName: 'playlist',
  80           privacy: VideoPlaylistPrivacy.PUBLIC,
  81           videoChannelId: server.videoChannel.id,
  82           thumbnailfile: 'thumbnail.jpg'
  83         }
  84       })
  85     }
  86
  87     await doubleFollow(servers[0], servers[1])
  88
  89     // Lazy load the remote avatar
  90     {
  91       const account = await servers[0].accountsCommand.get({ accountName: 'root@localhost:' + servers[1].port })
  92       await makeGetRequest({
  93         url: servers[0].url,
  94         path: account.avatar.path,
  95         statusCodeExpected: HttpStatusCode.OK_200
  96       })
  97     }
  98
  99     {
 100       const account = await servers[1].accountsCommand.get({ accountName: 'root@localhost:' + servers[0].port })
 101       await makeGetRequest({
 102         url: servers[1].url,
 103         path: account.avatar.path,
 104         statusCodeExpected: HttpStatusCode.OK_200
 105       })
 106     }
 107
 108     await wait(1000)
 109
 110     await waitJobs(servers)
 111     await killallServers(servers)
 112
 113     await wait(1000)
 114   })
 115
 116   it('Should have the files on the disk', async function () {
 117     await assertCountAreOkay(servers)
 118   })
 119
 120   it('Should create some dirty files', async function () {
 121     for (let i = 0; i < 2; i++) {
 122       {
 123         const base = servers[0].serversCommand.buildDirectory('videos')
 124
 125         const n1 = buildUUID() + '.mp4'
 126         const n2 = buildUUID() + '.webm'
 127
 128         await createFile(join(base, n1))
 129         await createFile(join(base, n2))
 130
 131         badNames['videos'] = [ n1, n2 ]
 132       }
 133
 134       {
 135         const base = servers[0].serversCommand.buildDirectory('torrents')
 136
 137         const n1 = buildUUID() + '-240.torrent'
 138         const n2 = buildUUID() + '-480.torrent'
 139
 140         await createFile(join(base, n1))
 141         await createFile(join(base, n2))
 142
 143         badNames['torrents'] = [ n1, n2 ]
 144       }
 145
 146       {
 147         const base = servers[0].serversCommand.buildDirectory('thumbnails')
 148
 149         const n1 = buildUUID() + '.jpg'
 150         const n2 = buildUUID() + '.jpg'
 151
 152         await createFile(join(base, n1))
 153         await createFile(join(base, n2))
 154
 155         badNames['thumbnails'] = [ n1, n2 ]
 156       }
 157
 158       {
 159         const base = servers[0].serversCommand.buildDirectory('previews')
 160
 161         const n1 = buildUUID() + '.jpg'
 162         const n2 = buildUUID() + '.jpg'
 163
 164         await createFile(join(base, n1))
 165         await createFile(join(base, n2))
 166
 167         badNames['previews'] = [ n1, n2 ]
 168       }
 169
 170       {
 171         const base = servers[0].serversCommand.buildDirectory('avatars')
 172
 173         const n1 = buildUUID() + '.png'
 174         const n2 = buildUUID() + '.jpg'
 175
 176         await createFile(join(base, n1))
 177         await createFile(join(base, n2))
 178
 179         badNames['avatars'] = [ n1, n2 ]
 180       }
 181     }
 182   })
 183
 184   it('Should run prune storage', async function () {
 185     this.timeout(30000)
 186
 187     const env = servers[0].cliCommand.getEnv()
 188     await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
 189   })
 190
 191   it('Should have removed files', async function () {
 192     await assertCountAreOkay(servers)
 193
 194     for (const directory of Object.keys(badNames)) {
 195       for (const name of badNames[directory]) {
 196         await assertNotExists(servers[0], directory, name)
 197       }
 198     }
 199   })
 200
 201   after(async function () {
 202     await cleanupTests(servers)
 203   })
 204 })