server/tests/cli/prune-storage.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { createFile, readdir } from 'fs-extra'
   6 import { join } from 'path'
   7 import { buildUUID } from '@server/helpers/uuid'
   8 import { HttpStatusCode } from '@shared/core-utils'
   9 import {
  10   cleanupTests,
  11   CLICommand,
  12   doubleFollow,
  13   flushAndRunMultipleServers,
  14   killallServers,
  15   makeGetRequest,
  16   ServerInfo,
  17   setAccessTokensToServers,
  18   setDefaultVideoChannel,
  19   updateMyAvatar,
  20   uploadVideo,
  21   wait,
  22   waitJobs
  23 } from '@shared/extra-utils'
  24 import { VideoPlaylistPrivacy } from '@shared/models'
  25
  26 const expect = chai.expect
  27
  28 async function countFiles (server: ServerInfo, directory: string) {
  29   const files = await readdir(server.serversCommand.buildDirectory(directory))
  30
  31   return files.length
  32 }
  33
  34 async function assertNotExists (server: ServerInfo, directory: string, substring: string) {
  35   const files = await readdir(server.serversCommand.buildDirectory(directory))
  36
  37   for (const f of files) {
  38     expect(f).to.not.contain(substring)
  39   }
  40 }
  41
  42 async function assertCountAreOkay (servers: ServerInfo[]) {
  43   for (const server of servers) {
  44     const videosCount = await countFiles(server, 'videos')
  45     expect(videosCount).to.equal(8)
  46
  47     const torrentsCount = await countFiles(server, 'torrents')
  48     expect(torrentsCount).to.equal(16)
  49
  50     const previewsCount = await countFiles(server, 'previews')
  51     expect(previewsCount).to.equal(2)
  52
  53     const thumbnailsCount = await countFiles(server, 'thumbnails')
  54     expect(thumbnailsCount).to.equal(6)
  55
  56     const avatarsCount = await countFiles(server, 'avatars')
  57     expect(avatarsCount).to.equal(2)
  58   }
  59 }
  60
  61 describe('Test prune storage scripts', function () {
  62   let servers: ServerInfo[]
  63   const badNames: { [directory: string]: string[] } = {}
  64
  65   before(async function () {
  66     this.timeout(120000)
  67
  68     servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true } })
  69     await setAccessTokensToServers(servers)
  70     await setDefaultVideoChannel(servers)
  71
  72     for (const server of servers) {
  73       await uploadVideo(server.url, server.accessToken, { name: 'video 1' })
  74       await uploadVideo(server.url, server.accessToken, { name: 'video 2' })
  75
  76       await updateMyAvatar({ url: server.url, accessToken: server.accessToken, fixture: 'avatar.png' })
  77
  78       await server.playlistsCommand.create({
  79         attributes: {
  80           displayName: 'playlist',
  81           privacy: VideoPlaylistPrivacy.PUBLIC,
  82           videoChannelId: server.videoChannel.id,
  83           thumbnailfile: 'thumbnail.jpg'
  84         }
  85       })
  86     }
  87
  88     await doubleFollow(servers[0], servers[1])
  89
  90     // Lazy load the remote avatar
  91     {
  92       const account = await servers[0].accountsCommand.get({ accountName: 'root@localhost:' + servers[1].port })
  93       await makeGetRequest({
  94         url: servers[0].url,
  95         path: account.avatar.path,
  96         statusCodeExpected: HttpStatusCode.OK_200
  97       })
  98     }
  99
 100     {
 101       const account = await servers[1].accountsCommand.get({ accountName: 'root@localhost:' + servers[0].port })
 102       await makeGetRequest({
 103         url: servers[1].url,
 104         path: account.avatar.path,
 105         statusCodeExpected: HttpStatusCode.OK_200
 106       })
 107     }
 108
 109     await wait(1000)
 110
 111     await waitJobs(servers)
 112     await killallServers(servers)
 113
 114     await wait(1000)
 115   })
 116
 117   it('Should have the files on the disk', async function () {
 118     await assertCountAreOkay(servers)
 119   })
 120
 121   it('Should create some dirty files', async function () {
 122     for (let i = 0; i < 2; i++) {
 123       {
 124         const base = servers[0].serversCommand.buildDirectory('videos')
 125
 126         const n1 = buildUUID() + '.mp4'
 127         const n2 = buildUUID() + '.webm'
 128
 129         await createFile(join(base, n1))
 130         await createFile(join(base, n2))
 131
 132         badNames['videos'] = [ n1, n2 ]
 133       }
 134
 135       {
 136         const base = servers[0].serversCommand.buildDirectory('torrents')
 137
 138         const n1 = buildUUID() + '-240.torrent'
 139         const n2 = buildUUID() + '-480.torrent'
 140
 141         await createFile(join(base, n1))
 142         await createFile(join(base, n2))
 143
 144         badNames['torrents'] = [ n1, n2 ]
 145       }
 146
 147       {
 148         const base = servers[0].serversCommand.buildDirectory('thumbnails')
 149
 150         const n1 = buildUUID() + '.jpg'
 151         const n2 = buildUUID() + '.jpg'
 152
 153         await createFile(join(base, n1))
 154         await createFile(join(base, n2))
 155
 156         badNames['thumbnails'] = [ n1, n2 ]
 157       }
 158
 159       {
 160         const base = servers[0].serversCommand.buildDirectory('previews')
 161
 162         const n1 = buildUUID() + '.jpg'
 163         const n2 = buildUUID() + '.jpg'
 164
 165         await createFile(join(base, n1))
 166         await createFile(join(base, n2))
 167
 168         badNames['previews'] = [ n1, n2 ]
 169       }
 170
 171       {
 172         const base = servers[0].serversCommand.buildDirectory('avatars')
 173
 174         const n1 = buildUUID() + '.png'
 175         const n2 = buildUUID() + '.jpg'
 176
 177         await createFile(join(base, n1))
 178         await createFile(join(base, n2))
 179
 180         badNames['avatars'] = [ n1, n2 ]
 181       }
 182     }
 183   })
 184
 185   it('Should run prune storage', async function () {
 186     this.timeout(30000)
 187
 188     const env = servers[0].cliCommand.getEnv()
 189     await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
 190   })
 191
 192   it('Should have removed files', async function () {
 193     await assertCountAreOkay(servers)
 194
 195     for (const directory of Object.keys(badNames)) {
 196       for (const name of badNames[directory]) {
 197         await assertNotExists(servers[0], directory, name)
 198       }
 199     }
 200   })
 201
 202   after(async function () {
 203     await cleanupTests(servers)
 204   })
 205 })