UpdatedAt
} from 'sequelize-typescript'
import { ActorModel } from '../activitypub/actor'
-import { throwIfNotValid } from '../utils'
+import { getVideoSort, throwIfNotValid } from '../utils'
import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
-import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers'
+import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers'
import { VideoFileModel } from '../video/video-file'
-import { isDateValid } from '../../helpers/custom-validators/misc'
import { getServerActor } from '../../helpers/utils'
import { VideoModel } from '../video/video'
import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
return VideoRedundancyModel.findOne(query)
}
+ static getVideoSample (rows: { id: number }[]) {
+ const ids = rows.map(r => r.id)
+ const id = sample(ids)
+
+ return VideoModel.loadWithFile(id, undefined, !isTestInstance())
+ }
+
static async findMostViewToDuplicate (randomizedFactor: number) {
// On VideoModel!
const query = {
+ attributes: [ 'id', 'views' ],
logging: !isTestInstance(),
limit: randomizedFactor,
- order: [ [ 'views', 'DESC' ] ],
+ order: getVideoSort('-views'),
include: [
- {
- model: VideoFileModel.unscoped(),
- required: true,
- where: {
- id: {
- [ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn()
- }
- }
- },
- {
- attributes: [],
- model: VideoChannelModel.unscoped(),
- required: true,
- include: [
- {
- attributes: [],
- model: ActorModel.unscoped(),
- required: true,
- include: [
- {
- attributes: [],
- model: ServerModel.unscoped(),
- required: true,
- where: {
- redundancyAllowed: true
- }
- }
- ]
- }
- ]
- }
+ await VideoRedundancyModel.buildVideoFileForDuplication(),
+ VideoRedundancyModel.buildServerRedundancyInclude()
+ ]
+ }
+
+ const rows = await VideoModel.unscoped().findAll(query)
+
+ return VideoRedundancyModel.getVideoSample(rows as { id: number }[])
+ }
+
+ static async findTrendingToDuplicate (randomizedFactor: number) {
+ // On VideoModel!
+ const query = {
+ attributes: [ 'id', 'views' ],
+ subQuery: false,
+ logging: !isTestInstance(),
+ group: 'VideoModel.id',
+ limit: randomizedFactor,
+ order: getVideoSort('-trending'),
+ include: [
+ await VideoRedundancyModel.buildVideoFileForDuplication(),
+ VideoRedundancyModel.buildServerRedundancyInclude(),
+
+ VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
]
}
const rows = await VideoModel.unscoped().findAll(query)
- return sample(rows)
+ return VideoRedundancyModel.getVideoSample(rows as { id: number }[])
}
static async getVideoFiles (strategy: VideoRedundancyStrategy) {
logging: !isTestInstance(),
where: {
expiresOn: {
- [Sequelize.Op.lt]: new Date()
+ [ Sequelize.Op.lt ]: new Date()
}
}
}
}
}
- private static async buildExcludeIn () {
+ // Don't include video files we already duplicated
+ private static async buildVideoFileForDuplication () {
const actor = await getServerActor()
- return Sequelize.literal(
+ const notIn = Sequelize.literal(
'(' +
`SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` +
')'
)
+
+ return {
+ attributes: [],
+ model: VideoFileModel.unscoped(),
+ required: true,
+ where: {
+ id: {
+ [ Sequelize.Op.notIn ]: notIn
+ }
+ }
+ }
+ }
+
+ private static buildServerRedundancyInclude () {
+ return {
+ attributes: [],
+ model: VideoChannelModel.unscoped(),
+ required: true,
+ include: [
+ {
+ attributes: [],
+ model: ActorModel.unscoped(),
+ required: true,
+ include: [
+ {
+ attributes: [],
+ model: ServerModel.unscoped(),
+ required: true,
+ where: {
+ redundancyAllowed: true
+ }
+ }
+ ]
+ }
+ ]
+ }
}
}
import { ActorFollow } from '../../../../shared/models/actors'
import { readdir } from 'fs-extra'
import { join } from 'path'
+import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
const expect = chai.expect
+let servers: ServerInfo[] = []
+let video1Server2UUID: string
+let video2Server2UUID: string
+
function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) {
const parsed = magnetUtil.decode(file.magnetUri)
}
}
-describe('Test videos redundancy', function () {
- let servers: ServerInfo[] = []
- let video1Server2UUID: string
- let video2Server2UUID: string
+async function runServers (strategy: VideoRedundancyStrategy) {
+ const config = {
+ redundancy: {
+ videos: [
+ {
+ strategy: strategy,
+ size: '100KB'
+ }
+ ]
+ }
+ }
+ servers = await flushAndRunMultipleServers(3, config)
- before(async function () {
- this.timeout(120000)
+ // Get the access tokens
+ await setAccessTokensToServers(servers)
- servers = await flushAndRunMultipleServers(3)
+ {
+ const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
+ video1Server2UUID = res.body.video.uuid
- // Get the access tokens
- await setAccessTokensToServers(servers)
+ await viewVideo(servers[ 1 ].url, video1Server2UUID)
+ }
- {
- const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
- video1Server2UUID = res.body.video.uuid
+ {
+ const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
+ video2Server2UUID = res.body.video.uuid
+ }
- await viewVideo(servers[1].url, video1Server2UUID)
- }
+ await waitJobs(servers)
- {
- const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
- video2Server2UUID = res.body.video.uuid
- }
+ // Server 1 and server 2 follow each other
+ await doubleFollow(servers[ 0 ], servers[ 1 ])
+ // Server 1 and server 3 follow each other
+ await doubleFollow(servers[ 0 ], servers[ 2 ])
+ // Server 2 and server 3 follow each other
+ await doubleFollow(servers[ 1 ], servers[ 2 ])
+
+ await waitJobs(servers)
+}
- await waitJobs(servers)
+async function check1WebSeed () {
+ const webseeds = [
+ 'http://localhost:9002/static/webseed/' + video1Server2UUID
+ ]
- // Server 1 and server 2 follow each other
- await doubleFollow(servers[0], servers[1])
- // Server 1 and server 3 follow each other
- await doubleFollow(servers[0], servers[2])
- // Server 2 and server 3 follow each other
- await doubleFollow(servers[1], servers[2])
+ for (const server of servers) {
+ const res = await getVideo(server.url, video1Server2UUID)
- await waitJobs(servers)
- })
+ const video: VideoDetails = res.body
+ video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
+ }
+}
+
+async function enableRedundancy () {
+ await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
+
+ const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
+ const follows: ActorFollow[] = res.body.data
+ const server2 = follows.find(f => f.following.host === 'localhost:9002')
+ const server3 = follows.find(f => f.following.host === 'localhost:9003')
+
+ expect(server3).to.not.be.undefined
+ expect(server3.following.hostRedundancyAllowed).to.be.false
+
+ expect(server2).to.not.be.undefined
+ expect(server2.following.hostRedundancyAllowed).to.be.true
+}
- it('Should have 1 webseed on the first video', async function () {
- const webseeds = [
- 'http://localhost:9002/static/webseed/' + video1Server2UUID
- ]
+async function check2Webseeds () {
+ await waitJobs(servers)
+ await wait(15000)
+ await waitJobs(servers)
- for (const server of servers) {
- const res = await getVideo(server.url, video1Server2UUID)
+ const webseeds = [
+ 'http://localhost:9001/static/webseed/' + video1Server2UUID,
+ 'http://localhost:9002/static/webseed/' + video1Server2UUID
+ ]
- const video: VideoDetails = res.body
- video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
+ for (const server of servers) {
+ const res = await getVideo(server.url, video1Server2UUID)
+
+ const video: VideoDetails = res.body
+
+ for (const file of video.files) {
+ checkMagnetWebseeds(file, webseeds)
}
- })
+ }
- it('Should enable redundancy on server 1', async function () {
- await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
+ const files = await readdir(join(root(), 'test1', 'videos'))
+ expect(files).to.have.lengthOf(4)
- const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt')
- const follows: ActorFollow[] = res.body.data
- const server2 = follows.find(f => f.following.host === 'localhost:9002')
- const server3 = follows.find(f => f.following.host === 'localhost:9003')
+ for (const resolution of [ 240, 360, 480, 720 ]) {
+ expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
+ }
+}
- expect(server3).to.not.be.undefined
- expect(server3.following.hostRedundancyAllowed).to.be.false
+async function cleanServers () {
+ killallServers(servers)
+}
- expect(server2).to.not.be.undefined
- expect(server2.following.hostRedundancyAllowed).to.be.true
- })
+describe('Test videos redundancy', function () {
- it('Should have 2 webseed on the first video', async function () {
- this.timeout(40000)
+ describe('With most-views strategy', function () {
- await waitJobs(servers)
- await wait(15000)
- await waitJobs(servers)
+ before(function () {
+ this.timeout(120000)
- const webseeds = [
- 'http://localhost:9001/static/webseed/' + video1Server2UUID,
- 'http://localhost:9002/static/webseed/' + video1Server2UUID
- ]
+ return runServers('most-views')
+ })
- for (const server of servers) {
- const res = await getVideo(server.url, video1Server2UUID)
+ it('Should have 1 webseed on the first video', function () {
+ return check1WebSeed()
+ })
- const video: VideoDetails = res.body
+ it('Should enable redundancy on server 1', async function () {
+ return enableRedundancy()
+ })
- for (const file of video.files) {
- checkMagnetWebseeds(file, webseeds)
- }
- }
+ it('Should have 2 webseed on the first video', async function () {
+ this.timeout(40000)
- const files = await readdir(join(root(), 'test1', 'videos'))
- expect(files).to.have.lengthOf(4)
+ return check2Webseeds()
+ })
- for (const resolution of [ 240, 360, 480, 720 ]) {
- expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
- }
+ after(function () {
+ return cleanServers()
+ })
})
- after(async function () {
- killallServers(servers)
+ describe('With trending strategy', function () {
- // Keep the logs if the test failed
- if (this['ok']) {
- await flushTests()
- }
+ before(function () {
+ this.timeout(120000)
+
+ return runServers('trending')
+ })
+
+ it('Should have 1 webseed on the first video', function () {
+ return check1WebSeed()
+ })
+
+ it('Should enable redundancy on server 1', async function () {
+ return enableRedundancy()
+ })
+
+ it('Should have 2 webseed on the first video', async function () {
+ this.timeout(40000)
+
+ return check2Webseeds()
+ })
+
+ after(function () {
+ return cleanServers()
+ })
})
})
projects / github / Chocobozzz / PeerTube.git / commitdiff
