aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/lazy-static.ts7
-rw-r--r--server/lib/files-cache/videos-preview-cache.ts2
-rw-r--r--server/models/video/thumbnail.ts10
-rw-r--r--server/models/video/video.ts23
-rw-r--r--server/tests/cli/index.ts1
-rw-r--r--server/tests/cli/prune-storage.ts199
6 files changed, 236 insertions, 6 deletions
diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts
index 4285fd727..28d2f862a 100644
--- a/server/controllers/lazy-static.ts
+++ b/server/controllers/lazy-static.ts
@@ -49,7 +49,12 @@ async function getAvatar (req: express.Request, res: express.Response) {
49 49
50 logger.info('Lazy serve remote avatar image %s.', avatar.fileUrl) 50 logger.info('Lazy serve remote avatar image %s.', avatar.fileUrl)
51 51
52 await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl }) 52 try {
53 await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl })
54 } catch (err) {
55 logger.warn('Cannot process remote avatar %s.', avatar.fileUrl, { err })
56 return res.sendStatus(404)
57 }
53 58
54 avatar.onDisk = true 59 avatar.onDisk = true
55 avatar.save() 60 avatar.save()
diff --git a/server/lib/files-cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts
index a68619d07..3da6bb138 100644
--- a/server/lib/files-cache/videos-preview-cache.ts
+++ b/server/lib/files-cache/videos-preview-cache.ts
@@ -18,7 +18,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
18 } 18 }
19 19
20 async getFilePathImpl (videoUUID: string) { 20 async getFilePathImpl (videoUUID: string) {
21 const video = await VideoModel.loadByUUIDWithFile(videoUUID) 21 const video = await VideoModel.loadByUUID(videoUUID)
22 if (!video) return undefined 22 if (!video) return undefined
23 23
24 if (video.isOwned()) return { isOwned: true, path: video.getPreview().getPath() } 24 if (video.isOwned()) return { isOwned: true, path: video.getPreview().getPath() }
diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts
index cf2040cbf..f1952dcc1 100644
--- a/server/models/video/thumbnail.ts
+++ b/server/models/video/thumbnail.ts
@@ -100,6 +100,16 @@ export class ThumbnailModel extends Model<ThumbnailModel> {
100 .catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, err)) 100 .catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, err))
101 } 101 }
102 102
103 static loadByName (filename: string) {
104 const query = {
105 where: {
106 filename
107 }
108 }
109
110 return ThumbnailModel.findOne(query)
111 }
112
103 static generateDefaultPreviewName (videoUUID: string) { 113 static generateDefaultPreviewName (videoUUID: string) {
104 return videoUUID + '.jpg' 114 return videoUUID + '.jpg'
105 } 115 }
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 1321337ff..b59df397d 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -119,6 +119,7 @@ import { CONFIG } from '../../initializers/config'
119import { ThumbnailModel } from './thumbnail' 119import { ThumbnailModel } from './thumbnail'
120import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' 120import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
121import { createTorrentPromise } from '../../helpers/webtorrent' 121import { createTorrentPromise } from '../../helpers/webtorrent'
122import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
122 123
123// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation 124// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
124const indexes: (ModelIndexesOptions & { where?: WhereOptions })[] = [ 125const indexes: (ModelIndexesOptions & { where?: WhereOptions })[] = [
@@ -1422,15 +1423,23 @@ export class VideoModel extends Model<VideoModel> {
1422 return VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findOne(options) 1423 return VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findOne(options)
1423 } 1424 }
1424 1425
1425 static loadWithFiles (id: number, t?: Transaction, logging?: boolean) { 1426 static loadWithFiles (id: number | string, t?: Transaction, logging?: boolean) {
1427 const where = buildWhereIdOrUUID(id)
1428
1429 const query = {
1430 where,
1431 transaction: t,
1432 logging
1433 }
1434
1426 return VideoModel.scope([ 1435 return VideoModel.scope([
1427 ScopeNames.WITH_FILES, 1436 ScopeNames.WITH_FILES,
1428 ScopeNames.WITH_STREAMING_PLAYLISTS, 1437 ScopeNames.WITH_STREAMING_PLAYLISTS,
1429 ScopeNames.WITH_THUMBNAILS 1438 ScopeNames.WITH_THUMBNAILS
1430 ]).findByPk(id, { transaction: t, logging }) 1439 ]).findOne(query)
1431 } 1440 }
1432 1441
1433 static loadByUUIDWithFile (uuid: string) { 1442 static loadByUUID (uuid: string) {
1434 const options = { 1443 const options = {
1435 where: { 1444 where: {
1436 uuid 1445 uuid
@@ -1754,7 +1763,7 @@ export class VideoModel extends Model<VideoModel> {
1754 return maxBy(this.VideoFiles, file => file.resolution) 1763 return maxBy(this.VideoFiles, file => file.resolution)
1755 } 1764 }
1756 1765
1757 getFile (resolution: VideoResolution) { 1766 getFile (resolution: number) {
1758 if (Array.isArray(this.VideoFiles) === false) return undefined 1767 if (Array.isArray(this.VideoFiles) === false) return undefined
1759 1768
1760 return this.VideoFiles.find(f => f.resolution === resolution) 1769 return this.VideoFiles.find(f => f.resolution === resolution)
@@ -1893,6 +1902,12 @@ export class VideoModel extends Model<VideoModel> {
1893 return `/api/${API_VERSION}/videos/${this.uuid}/description` 1902 return `/api/${API_VERSION}/videos/${this.uuid}/description`
1894 } 1903 }
1895 1904
1905 getHLSPlaylist () {
1906 if (!this.VideoStreamingPlaylists) return undefined
1907
1908 return this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
1909 }
1910
1896 removeFile (videoFile: VideoFileModel, isRedundancy = false) { 1911 removeFile (videoFile: VideoFileModel, isRedundancy = false) {
1897 const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR 1912 const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR
1898 1913
diff --git a/server/tests/cli/index.ts b/server/tests/cli/index.ts
index 5af286fe2..029cd5196 100644
--- a/server/tests/cli/index.ts
+++ b/server/tests/cli/index.ts
@@ -4,5 +4,6 @@ import './create-transcoding-job'
4import './optimize-old-videos' 4import './optimize-old-videos'
5import './peertube' 5import './peertube'
6import './plugins' 6import './plugins'
7import './prune-storage'
7import './reset-password' 8import './reset-password'
8import './update-host' 9import './update-host'
diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts
new file mode 100644
index 000000000..67a5c564e
--- /dev/null
+++ b/server/tests/cli/prune-storage.ts
@@ -0,0 +1,199 @@
1/* tslint:disable:no-unused-expression */
2
3import 'mocha'
4import * as chai from 'chai'
5import { waitJobs } from '../../../shared/extra-utils/server/jobs'
6import {
7 buildServerDirectory,
8 cleanupTests,
9 createVideoPlaylist,
10 doubleFollow,
11 execCLI,
12 flushAndRunMultipleServers,
13 getAccount,
14 getEnvCli,
15 ServerInfo,
16 setAccessTokensToServers, setDefaultVideoChannel,
17 updateMyAvatar,
18 uploadVideo,
19 wait
20} from '../../../shared/extra-utils'
21import { Account, VideoPlaylistPrivacy } from '../../../shared/models'
22import { createFile, readdir } from 'fs-extra'
23import * as uuidv4 from 'uuid/v4'
24import { join } from 'path'
25import * as request from 'supertest'
26
27const expect = chai.expect
28
29async function countFiles (internalServerNumber: number, directory: string) {
30 const files = await readdir(buildServerDirectory(internalServerNumber, directory))
31
32 return files.length
33}
34
35async function assertNotExists (internalServerNumber: number, directory: string, substring: string) {
36 const files = await readdir(buildServerDirectory(internalServerNumber, directory))
37
38 for (const f of files) {
39 expect(f).to.not.contain(substring)
40 }
41}
42
43async function assertCountAreOkay (servers: ServerInfo[]) {
44 for (const server of servers) {
45 const videosCount = await countFiles(server.internalServerNumber, 'videos')
46 expect(videosCount).to.equal(8)
47
48 const torrentsCount = await countFiles(server.internalServerNumber, 'torrents')
49 expect(torrentsCount).to.equal(8)
50
51 const previewsCount = await countFiles(server.internalServerNumber, 'previews')
52 expect(previewsCount).to.equal(2)
53
54 const thumbnailsCount = await countFiles(server.internalServerNumber, 'thumbnails')
55 expect(thumbnailsCount).to.equal(6)
56
57 const avatarsCount = await countFiles(server.internalServerNumber, 'avatars')
58 expect(avatarsCount).to.equal(2)
59 }
60}
61
62describe('Test prune storage scripts', function () {
63 let servers: ServerInfo[]
64 const badNames: { [ directory: string ]: string[] } = {}
65
66 before(async function () {
67 this.timeout(120000)
68
69 servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true } })
70 await setAccessTokensToServers(servers)
71 await setDefaultVideoChannel(servers)
72
73 for (const server of servers) {
74 await uploadVideo(server.url, server.accessToken, { name: 'video 1' })
75 await uploadVideo(server.url, server.accessToken, { name: 'video 2' })
76
77 await updateMyAvatar({ url: server.url, accessToken: server.accessToken, fixture: 'avatar.png' })
78
79 await createVideoPlaylist({
80 url: server.url,
81 token: server.accessToken,
82 playlistAttrs: {
83 displayName: 'playlist',
84 privacy: VideoPlaylistPrivacy.PUBLIC,
85 videoChannelId: server.videoChannel.id,
86 thumbnailfile: 'thumbnail.jpg'
87 }
88 })
89 }
90
91 await doubleFollow(servers[0], servers[1])
92
93 // Lazy load the remote avatar
94 {
95 const res = await getAccount(servers[ 0 ].url, 'root@localhost:' + servers[ 1 ].port)
96 const account: Account = res.body
97 await request('http://localhost:' + servers[ 0 ].port).get(account.avatar.path).expect(200)
98 }
99
100 {
101 const res = await getAccount(servers[ 1 ].url, 'root@localhost:' + servers[ 0 ].port)
102 const account: Account = res.body
103 await request('http://localhost:' + servers[ 1 ].port).get(account.avatar.path).expect(200)
104 }
105
106 await wait(1000)
107
108 await waitJobs(servers)
109 })
110
111 it('Should have the files on the disk', async function () {
112 await assertCountAreOkay(servers)
113 })
114
115 it('Should create some dirty files', async function () {
116 for (let i = 0; i < 2; i++) {
117 {
118 const base = buildServerDirectory(servers[0].internalServerNumber, 'videos')
119
120 const n1 = uuidv4() + '.mp4'
121 const n2 = uuidv4() + '.webm'
122
123 await createFile(join(base, n1))
124 await createFile(join(base, n2))
125
126 badNames['videos'] = [ n1, n2 ]
127 }
128
129 {
130 const base = buildServerDirectory(servers[0].internalServerNumber, 'torrents')
131
132 const n1 = uuidv4() + '-240.torrent'
133 const n2 = uuidv4() + '-480.torrent'
134
135 await createFile(join(base, n1))
136 await createFile(join(base, n2))
137
138 badNames['torrents'] = [ n1, n2 ]
139 }
140
141 {
142 const base = buildServerDirectory(servers[0].internalServerNumber, 'thumbnails')
143
144 const n1 = uuidv4() + '.jpg'
145 const n2 = uuidv4() + '.jpg'
146
147 await createFile(join(base, n1))
148 await createFile(join(base, n2))
149
150 badNames['thumbnails'] = [ n1, n2 ]
151 }
152
153 {
154 const base = buildServerDirectory(servers[0].internalServerNumber, 'previews')
155
156 const n1 = uuidv4() + '.jpg'
157 const n2 = uuidv4() + '.jpg'
158
159 await createFile(join(base, n1))
160 await createFile(join(base, n2))
161
162 badNames['previews'] = [ n1, n2 ]
163 }
164
165 {
166 const base = buildServerDirectory(servers[0].internalServerNumber, 'avatars')
167
168 const n1 = uuidv4() + '.png'
169 const n2 = uuidv4() + '.jpg'
170
171 await createFile(join(base, n1))
172 await createFile(join(base, n2))
173
174 badNames['avatars'] = [ n1, n2 ]
175 }
176 }
177 })
178
179 it('Should run prune storage', async function () {
180 this.timeout(30000)
181
182 const env = getEnvCli(servers[0])
183 await execCLI(`echo y | ${env} npm run prune-storage`)
184 })
185
186 it('Should have removed files', async function () {
187 await assertCountAreOkay(servers)
188
189 for (const directory of Object.keys(badNames)) {
190 for (const name of badNames[directory]) {
191 await assertNotExists(servers[0].internalServerNumber, directory, name)
192 }
193 }
194 })
195
196 after(async function () {
197 await cleanupTests(servers)
198 })
199})