Add sitemap

11 files changed, 241 insertions, 4 deletions

diff --git a/package.json b/package.json
index c5e4c329c..aa4f447aa 100644
--- a/package.json
+++ b/package.json
@@ -148,6 +148,7 @@
     "sequelize": "4.41.2",
     "sequelize-typescript": "0.6.6",
     "sharp": "^0.21.0",
+    "sitemap": "^2.1.0",
     "srt-to-vtt": "^1.1.2",
     "summon-install": "^0.4.3",
     "useragent": "^2.3.0",
diff --git a/scripts/clean/server/test.sh b/scripts/clean/server/test.sh
index 235ff52cc..b897c30ba 100755
--- a/scripts/clean/server/test.sh
+++ b/scripts/clean/server/test.sh
@@ -18,6 +18,7 @@ removeFiles () {
 
 dropRedis () {
   redis-cli KEYS "bull-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
+  redis-cli KEYS "redis-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
 }
 
 for i in $(seq 1 6); do
diff --git a/server.ts b/server.ts
index 3025a6fd7..4a2a6ddf5 100644
--- a/server.ts
+++ b/server.ts
@@ -87,7 +87,7 @@ import {
   servicesRouter,
   webfingerRouter,
   trackerRouter,
-  createWebsocketServer
+  createWebsocketServer, botsRouter
 } from './server/controllers'
 import { advertiseDoNotTrack } from './server/middlewares/dnt'
 import { Redis } from './server/lib/redis'
@@ -156,6 +156,7 @@ app.use('/', activityPubRouter)
 app.use('/', feedsRouter)
 app.use('/', webfingerRouter)
 app.use('/', trackerRouter)
+app.use('/', botsRouter)
 
 // Static files
 app.use('/', staticRouter)
diff --git a/server/controllers/bots.ts b/server/controllers/bots.ts
new file mode 100644
index 000000000..b4eaccf9f
--- /dev/null
+++ b/server/controllers/bots.ts
@@ -0,0 +1,101 @@
+import * as express from 'express'
+import { asyncMiddleware } from '../middlewares'
+import { CONFIG, ROUTE_CACHE_LIFETIME } from '../initializers'
+import * as sitemapModule from 'sitemap'
+import { logger } from '../helpers/logger'
+import { VideoModel } from '../models/video/video'
+import { VideoChannelModel } from '../models/video/video-channel'
+import { AccountModel } from '../models/account/account'
+import { cacheRoute } from '../middlewares/cache'
+import { buildNSFWFilter } from '../helpers/express-utils'
+import { truncate } from 'lodash'
+
+const botsRouter = express.Router()
+
+// Special route that add OpenGraph and oEmbed tags
+// Do not use a template engine for a so little thing
+botsRouter.use('/sitemap.xml',
+  asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP)),
+  asyncMiddleware(getSitemap)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  botsRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function getSitemap (req: express.Request, res: express.Response) {
+  let urls = getSitemapBasicUrls()
+
+  urls = urls.concat(await getSitemapLocalVideoUrls())
+  urls = urls.concat(await getSitemapVideoChannelUrls())
+  urls = urls.concat(await getSitemapAccountUrls())
+
+  const sitemap = sitemapModule.createSitemap({
+    hostname: CONFIG.WEBSERVER.URL,
+    urls: urls
+  })
+
+  sitemap.toXML((err, xml) => {
+    if (err) {
+      logger.error('Cannot generate sitemap.', { err })
+      return res.sendStatus(500)
+    }
+
+    res.header('Content-Type', 'application/xml')
+    res.send(xml)
+  })
+}
+
+async function getSitemapVideoChannelUrls () {
+  const rows = await VideoChannelModel.listLocalsForSitemap('createdAt')
+
+  return rows.map(channel => ({
+    url: CONFIG.WEBSERVER.URL + '/video-channels/' + channel.Actor.preferredUsername
+  }))
+}
+
+async function getSitemapAccountUrls () {
+  const rows = await AccountModel.listLocalsForSitemap('createdAt')
+
+  return rows.map(channel => ({
+    url: CONFIG.WEBSERVER.URL + '/accounts/' + channel.Actor.preferredUsername
+  }))
+}
+
+async function getSitemapLocalVideoUrls () {
+  const resultList = await VideoModel.listForApi({
+    start: 0,
+    count: undefined,
+    sort: 'createdAt',
+    includeLocalVideos: true,
+    nsfw: buildNSFWFilter(),
+    filter: 'local',
+    withFiles: false
+  })
+
+  return resultList.data.map(v => ({
+    url: CONFIG.WEBSERVER.URL + '/videos/watch/' + v.uuid,
+    video: [
+      {
+        title: v.name,
+        // Sitemap description should be < 2000 characters
+        description: truncate(v.description || v.name, { length: 2000, omission: '...' }),
+        player_loc: CONFIG.WEBSERVER.URL + '/videos/embed/' + v.uuid,
+        thumbnail_loc: v.getThumbnailStaticPath()
+      }
+    ]
+  }))
+}
+
+function getSitemapBasicUrls () {
+  const paths = [
+    '/about/instance',
+    '/videos/local'
+  ]
+
+  return paths.map(p => ({ url: CONFIG.WEBSERVER.URL + p }))
+}
diff --git a/server/controllers/index.ts b/server/controllers/index.ts
index 197fa897a..a88a03c79 100644
--- a/server/controllers/index.ts
+++ b/server/controllers/index.ts
@@ -6,3 +6,4 @@ export * from './services'
 export * from './static'
 export * from './webfinger'
 export * from './tracker'
+export * from './bots'
diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts
index 162fe2244..9a72ee96d 100644
--- a/server/helpers/express-utils.ts
+++ b/server/helpers/express-utils.ts
@@ -7,12 +7,12 @@ import { extname } from 'path'
 import { isArray } from './custom-validators/misc'
 import { UserModel } from '../models/account/user'
 
-function buildNSFWFilter (res: express.Response, paramNSFW?: string) {
+function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
   if (paramNSFW === 'true') return true
   if (paramNSFW === 'false') return false
   if (paramNSFW === 'both') return undefined
 
-  if (res.locals.oauth) {
+  if (res && res.locals.oauth) {
     const user: UserModel = res.locals.oauth.token.User
 
     // User does not want NSFW videos
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 7195ae6c5..6b798875c 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -61,6 +61,7 @@ const OAUTH_LIFETIME = {
 const ROUTE_CACHE_LIFETIME = {
   FEEDS: '15 minutes',
   ROBOTS: '2 hours',
+  SITEMAP: '1 day',
   SECURITYTXT: '2 hours',
   NODEINFO: '10 minutes',
   DNT_POLICY: '1 week',
diff --git a/server/models/account/account.ts b/server/models/account/account.ts
index 5a237d733..a99e9b1ad 100644
--- a/server/models/account/account.ts
+++ b/server/models/account/account.ts
@@ -241,6 +241,27 @@ export class AccountModel extends Model<AccountModel> {
       })
   }
 
+  static listLocalsForSitemap (sort: string) {
+    const query = {
+      attributes: [ ],
+      offset: 0,
+      order: getSort(sort),
+      include: [
+        {
+          attributes: [ 'preferredUsername', 'serverId' ],
+          model: ActorModel.unscoped(),
+          where: {
+            serverId: null
+          }
+        }
+      ]
+    }
+
+    return AccountModel
+      .unscoped()
+      .findAll(query)
+  }
+
   toFormattedJSON (): Account {
     const actor = this.Actor.toFormattedJSON()
     const account = {
diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts
index f4586917e..86bf0461a 100644
--- a/server/models/video/video-channel.ts
+++ b/server/models/video/video-channel.ts
@@ -233,6 +233,27 @@ export class VideoChannelModel extends Model<VideoChannelModel> {
       })
   }
 
+  static listLocalsForSitemap (sort: string) {
+    const query = {
+      attributes: [ ],
+      offset: 0,
+      order: getSort(sort),
+      include: [
+        {
+          attributes: [ 'preferredUsername', 'serverId' ],
+          model: ActorModel.unscoped(),
+          where: {
+            serverId: null
+          }
+        }
+      ]
+    }
+
+    return VideoChannelModel
+      .unscoped()
+      .findAll(query)
+  }
+
   static searchForApi (options: {
     actorId: number
     search: string
diff --git a/server/tests/misc-endpoints.ts b/server/tests/misc-endpoints.ts
index 8fab20971..b53803ee1 100644
--- a/server/tests/misc-endpoints.ts
+++ b/server/tests/misc-endpoints.ts
@@ -2,7 +2,18 @@
 
 import 'mocha'
 import * as chai from 'chai'
-import { flushTests, killallServers, makeGetRequest, runServer, ServerInfo } from './utils'
+import {
+  addVideoChannel,
+  createUser,
+  flushTests,
+  killallServers,
+  makeGetRequest,
+  runServer,
+  ServerInfo,
+  setAccessTokensToServers,
+  uploadVideo
+} from './utils'
+import { VideoPrivacy } from '../../shared/models/videos'
 
 const expect = chai.expect
 
@@ -15,6 +26,7 @@ describe('Test misc endpoints', function () {
     await flushTests()
 
     server = await runServer(1)
+    await setAccessTokensToServers([ server ])
   })
 
   describe('Test a well known endpoints', function () {
@@ -93,6 +105,64 @@ describe('Test misc endpoints', function () {
     })
   })
 
+  describe('Test bots endpoints', function () {
+
+    it('Should get the empty sitemap', async function () {
+      const res = await makeGetRequest({
+        url: server.url,
+        path: '/sitemap.xml',
+        statusCodeExpected: 200
+      })
+
+      expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
+    })
+
+    it('Should get the empty cached sitemap', async function () {
+      const res = await makeGetRequest({
+        url: server.url,
+        path: '/sitemap.xml',
+        statusCodeExpected: 200
+      })
+
+      expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
+    })
+
+    it('Should add videos, channel and accounts and get sitemap', async function () {
+      this.timeout(35000)
+
+      await uploadVideo(server.url, server.accessToken, { name: 'video 1', nsfw: false })
+      await uploadVideo(server.url, server.accessToken, { name: 'video 2', nsfw: false })
+      await uploadVideo(server.url, server.accessToken, { name: 'video 3', privacy: VideoPrivacy.PRIVATE })
+
+      await addVideoChannel(server.url, server.accessToken, { name: 'channel1', displayName: 'channel 1' })
+      await addVideoChannel(server.url, server.accessToken, { name: 'channel2', displayName: 'channel 2' })
+
+      await createUser(server.url, server.accessToken, 'user1', 'password')
+      await createUser(server.url, server.accessToken, 'user2', 'password')
+
+      const res = await makeGetRequest({
+        url: server.url,
+        path: '/sitemap.xml?t=1', // avoid using cache
+        statusCodeExpected: 200
+      })
+
+      expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
+
+      expect(res.text).to.contain('<video:title><![CDATA[video 1]]></video:title>')
+      expect(res.text).to.contain('<video:title><![CDATA[video 2]]></video:title>')
+      expect(res.text).to.not.contain('<video:title><![CDATA[video 3]]></video:title>')
+
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/video-channels/channel1</loc></url>')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/video-channels/channel2</loc></url>')
+
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/accounts/user1</loc></url>')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/accounts/user2</loc></url>')
+    })
+  })
+
   after(async function () {
     killallServers([ server ])
   })
diff --git a/yarn.lock b/yarn.lock
index 1cbe6756d..8d74f9d55 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7457,6 +7457,15 @@ simple-websocket@^7.0.1:
     readable-stream "^2.0.5"
     ws "^6.0.0"
 
+sitemap@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-2.1.0.tgz#1633cb88c196d755ad94becfb1c1bcacc6d3425a"
+  integrity sha512-AkfA7RDVCITQo+j5CpXsMJlZ/8ENO2NtgMHYIh+YMvex2Hao/oe3MQgNa03p0aWY6srCfUA1Q02OgiWCAiuccA==
+  dependencies:
+    lodash "^4.17.10"
+    url-join "^4.0.0"
+    xmlbuilder "^10.0.0"
+
 slash@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55"
@@ -8592,6 +8601,11 @@ urix@^0.1.0:
   resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72"
   integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=
 
+url-join@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/url-join/-/url-join-4.0.0.tgz#4d3340e807d3773bda9991f8305acdcc2a665d2a"
+  integrity sha1-TTNA6AfTdzvamZH4MFrNzCpmXSo=
+
 url-parse-lax@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-1.0.0.tgz#7af8f303645e9bd79a272e7a14ac68bc0609da73"
@@ -9001,6 +9015,11 @@ xml@^1.0.1:
   resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
   integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=
 
+xmlbuilder@^10.0.0:
+  version "10.1.1"
+  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz#8cae6688cc9b38d850b7c8d3c0a4161dcaf475b0"
+  integrity sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==
+
 xmlbuilder@~9.0.1:
   version "9.0.7"
   resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"