Videos overview page: first version

8 files changed, 272 insertions, 2 deletions

diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts
index e928a7478..8a58b5466 100644
--- a/server/controllers/api/index.ts
+++ b/server/controllers/api/index.ts
@@ -10,6 +10,7 @@ import { badRequest } from '../../helpers/express-utils'
 import { videoChannelRouter } from './video-channel'
 import * as cors from 'cors'
 import { searchRouter } from './search'
+import { overviewsRouter } from './overviews'
 
 const apiRouter = express.Router()
 
@@ -28,6 +29,7 @@ apiRouter.use('/video-channels', videoChannelRouter)
 apiRouter.use('/videos', videosRouter)
 apiRouter.use('/jobs', jobsRouter)
 apiRouter.use('/search', searchRouter)
+apiRouter.use('/overviews', overviewsRouter)
 apiRouter.use('/ping', pong)
 apiRouter.use('/*', badRequest)
 
diff --git a/server/controllers/api/overviews.ts b/server/controllers/api/overviews.ts
new file mode 100644
index 000000000..56f921ce5
--- /dev/null
+++ b/server/controllers/api/overviews.ts
@@ -0,0 +1,97 @@
+import * as express from 'express'
+import { buildNSFWFilter } from '../../helpers/express-utils'
+import { VideoModel } from '../../models/video/video'
+import { asyncMiddleware, executeIfActivityPub } from '../../middlewares'
+import { TagModel } from '../../models/video/tag'
+import { VideosOverview } from '../../../shared/models/overviews'
+import { OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers'
+import { cacheRoute } from '../../middlewares/cache'
+
+const overviewsRouter = express.Router()
+
+overviewsRouter.get('/videos',
+  executeIfActivityPub(asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS))),
+  asyncMiddleware(getVideosOverview)
+)
+
+// ---------------------------------------------------------------------------
+
+export { overviewsRouter }
+
+// ---------------------------------------------------------------------------
+
+// This endpoint could be quite long, but we cache it
+async function getVideosOverview (req: express.Request, res: express.Response) {
+  const attributes = await buildSamples()
+  const result: VideosOverview = {
+    categories: await Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))),
+    channels: await Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))),
+    tags: await Promise.all(attributes.tags.map(t => getVideosByTag(t, res)))
+  }
+
+  // Cleanup our object
+  for (const key of Object.keys(result)) {
+    result[key] = result[key].filter(v => v !== undefined)
+  }
+
+  return res.json(result)
+}
+
+async function buildSamples () {
+  const [ categories, channels, tags ] = await Promise.all([
+    VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
+    VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
+    TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
+  ])
+
+  return { categories, channels, tags }
+}
+
+async function getVideosByTag (tag: string, res: express.Response) {
+  const videos = await getVideos(res, { tagsOneOf: [ tag ] })
+
+  if (videos.length === 0) return undefined
+
+  return {
+    tag,
+    videos
+  }
+}
+
+async function getVideosByCategory (category: number, res: express.Response) {
+  const videos = await getVideos(res, { categoryOneOf: [ category ] })
+
+  if (videos.length === 0) return undefined
+
+  return {
+    category: videos[0].category,
+    videos
+  }
+}
+
+async function getVideosByChannel (channelId: number, res: express.Response) {
+  const videos = await getVideos(res, { videoChannelId: channelId })
+
+  if (videos.length === 0) return undefined
+
+  return {
+    channel: videos[0].channel,
+    videos
+  }
+}
+
+async function getVideos (
+  res: express.Response,
+  where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] }
+) {
+  const { data } = await VideoModel.listForApi(Object.assign({
+    start: 0,
+    count: 10,
+    sort: '-createdAt',
+    includeLocalVideos: true,
+    nsfw: buildNSFWFilter(res),
+    withFiles: false
+  }, where))
+
+  return data.map(d => d.toFormattedJSON())
+}
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 5d93c6b82..16d8dca68 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -58,6 +58,9 @@ const ROUTE_CACHE_LIFETIME = {
   ROBOTS: '2 hours',
   NODEINFO: '10 minutes',
   DNT_POLICY: '1 week',
+  OVERVIEWS: {
+    VIDEOS: '1 hour'
+  },
   ACTIVITY_PUB: {
     VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
   }
@@ -464,6 +467,15 @@ const TORRENT_MIMETYPE_EXT = {
 
 // ---------------------------------------------------------------------------
 
+const OVERVIEWS = {
+  VIDEOS: {
+    SAMPLE_THRESHOLD: 4,
+    SAMPLES_COUNT: 2
+  }
+}
+
+// ---------------------------------------------------------------------------
+
 const SERVER_ACTOR_NAME = 'peertube'
 
 const ACTIVITY_PUB = {
@@ -666,6 +678,7 @@ export {
   USER_PASSWORD_RESET_LIFETIME,
   USER_EMAIL_VERIFY_LIFETIME,
   IMAGE_MIMETYPE_EXT,
+  OVERVIEWS,
   SCHEDULER_INTERVALS_MS,
   REPEAT_JOBS,
   STATIC_DOWNLOAD_PATHS,
diff --git a/server/models/video/tag.ts b/server/models/video/tag.ts
index 6d79a5575..e39a418cd 100644
--- a/server/models/video/tag.ts
+++ b/server/models/video/tag.ts
@@ -1,10 +1,11 @@
 import * as Bluebird from 'bluebird'
-import { Transaction } from 'sequelize'
+import * as Sequelize from 'sequelize'
 import { AllowNull, BelongsToMany, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
 import { isVideoTagValid } from '../../helpers/custom-validators/videos'
 import { throwIfNotValid } from '../utils'
 import { VideoModel } from './video'
 import { VideoTagModel } from './video-tag'
+import { VideoPrivacy, VideoState } from '../../../shared/models/videos'
 
 @Table({
   tableName: 'tag',
@@ -36,7 +37,7 @@ export class TagModel extends Model<TagModel> {
   })
   Videos: VideoModel[]
 
-  static findOrCreateTags (tags: string[], transaction: Transaction) {
+  static findOrCreateTags (tags: string[], transaction: Sequelize.Transaction) {
     if (tags === null) return []
 
     const tasks: Bluebird<TagModel>[] = []
@@ -59,4 +60,23 @@ export class TagModel extends Model<TagModel> {
 
     return Promise.all(tasks)
   }
+
+  // threshold corresponds to how many video the field should have to be returned
+  static getRandomSamples (threshold: number, count: number): Bluebird<string[]> {
+    const query = 'SELECT tag.name FROM tag ' +
+      'INNER JOIN "videoTag" ON "videoTag"."tagId" = tag.id ' +
+      'INNER JOIN video ON video.id = "videoTag"."videoId" ' +
+      'WHERE video.privacy = $videoPrivacy AND video.state = $videoState ' +
+      'GROUP BY tag.name HAVING COUNT(tag.name) >= $threshold ' +
+      'ORDER BY random() ' +
+      'LIMIT $count'
+
+    const options = {
+      bind: { threshold, count, videoPrivacy: VideoPrivacy.PUBLIC, videoState: VideoState.PUBLISHED },
+      type: Sequelize.QueryTypes.SELECT
+    }
+
+    return TagModel.sequelize.query(query, options)
+                    .then(data => data.map(d => d.name))
+  }
 }
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 3410833c8..695990b17 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -1083,6 +1083,29 @@ export class VideoModel extends Model<VideoModel> {
     })
   }
 
+  // threshold corresponds to how many video the field should have to be returned
+  static getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) {
+    const query: IFindOptions<VideoModel> = {
+      attributes: [ field ],
+      limit: count,
+      group: field,
+      having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), {
+        [Sequelize.Op.gte]: threshold
+      }) as any, // FIXME: typings
+      where: {
+        [field]: {
+          [Sequelize.Op.not]: null,
+        },
+        privacy: VideoPrivacy.PUBLIC,
+        state: VideoState.PUBLISHED
+      },
+      order: [ this.sequelize.random() ]
+    }
+
+    return VideoModel.findAll(query)
+      .then(rows => rows.map(r => r[field]))
+  }
+
   private static buildActorWhereWithFilter (filter?: VideoFilter) {
     if (filter && filter === 'local') {
       return {
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index bc66a7824..8286ff356 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -13,3 +13,4 @@ import './video-nsfw'
 import './video-privacy'
 import './video-schedule-update'
 import './video-transcoder'
+import './videos-overview'
diff --git a/server/tests/api/videos/videos-overview.ts b/server/tests/api/videos/videos-overview.ts
new file mode 100644
index 000000000..1514d1bda
--- /dev/null
+++ b/server/tests/api/videos/videos-overview.ts
@@ -0,0 +1,96 @@
+/* tslint:disable:no-unused-expression */
+
+import * as chai from 'chai'
+import 'mocha'
+import { flushTests, killallServers, runServer, ServerInfo, setAccessTokensToServers, uploadVideo } from '../../utils'
+import { getVideosOverview } from '../../utils/overviews/overviews'
+import { VideosOverview } from '../../../../shared/models/overviews'
+
+const expect = chai.expect
+
+describe('Test a videos overview', function () {
+  let server: ServerInfo = null
+
+  before(async function () {
+    this.timeout(30000)
+
+    await flushTests()
+
+    server = await runServer(1)
+
+    await setAccessTokensToServers([ server ])
+  })
+
+  it('Should send empty overview', async function () {
+    const res = await getVideosOverview(server.url)
+
+    const overview: VideosOverview = res.body
+    expect(overview.tags).to.have.lengthOf(0)
+    expect(overview.categories).to.have.lengthOf(0)
+    expect(overview.channels).to.have.lengthOf(0)
+  })
+
+  it('Should upload 3 videos in a specific category, tag and channel but not include them in overview', async function () {
+    for (let i = 0; i < 3; i++) {
+      await uploadVideo(server.url, server.accessToken, {
+        name: 'video ' + i,
+        category: 3,
+        tags: [ 'coucou1', 'coucou2' ]
+      })
+    }
+
+    const res = await getVideosOverview(server.url)
+
+    const overview: VideosOverview = res.body
+    expect(overview.tags).to.have.lengthOf(0)
+    expect(overview.categories).to.have.lengthOf(0)
+    expect(overview.channels).to.have.lengthOf(0)
+  })
+
+  it('Should upload another video and include all videos in the overview', async function () {
+    await uploadVideo(server.url, server.accessToken, {
+      name: 'video 3',
+      category: 3,
+      tags: [ 'coucou1', 'coucou2' ]
+    })
+
+    const res = await getVideosOverview(server.url)
+
+    const overview: VideosOverview = res.body
+    expect(overview.tags).to.have.lengthOf(2)
+    expect(overview.categories).to.have.lengthOf(1)
+    expect(overview.channels).to.have.lengthOf(1)
+  })
+
+  it('Should have the correct overview', async function () {
+    const res = await getVideosOverview(server.url)
+
+    const overview: VideosOverview = res.body
+
+    for (const attr of [ 'tags', 'categories', 'channels' ]) {
+      const obj = overview[attr][0]
+
+      expect(obj.videos).to.have.lengthOf(4)
+      expect(obj.videos[0].name).to.equal('video 3')
+      expect(obj.videos[1].name).to.equal('video 2')
+      expect(obj.videos[2].name).to.equal('video 1')
+      expect(obj.videos[3].name).to.equal('video 0')
+    }
+
+    expect(overview.tags.find(t => t.tag === 'coucou1')).to.not.be.undefined
+    expect(overview.tags.find(t => t.tag === 'coucou2')).to.not.be.undefined
+
+    expect(overview.categories[0].category.id).to.equal(3)
+
+    expect(overview.channels[0].channel.name).to.equal('root_channel')
+  })
+
+  after(async function () {
+    killallServers([ server ])
+
+    // Keep the logs if the test failed
+    if (this['ok']) {
+      await flushTests()
+    }
+  })
+})
diff --git a/server/tests/utils/overviews/overviews.ts b/server/tests/utils/overviews/overviews.ts
new file mode 100644
index 000000000..23e3ceb1e
--- /dev/null
+++ b/server/tests/utils/overviews/overviews.ts
@@ -0,0 +1,18 @@
+import { makeGetRequest } from '../requests/requests'
+
+function getVideosOverview (url: string, useCache = false) {
+  const path = '/api/v1/overviews/videos'
+
+  const query = {
+    t: useCache ? undefined : new Date().getTime()
+  }
+
+  return makeGetRequest({
+    url,
+    path,
+    query,
+    statusCodeExpected: 200
+  })
+}
+
+export { getVideosOverview }