server/controllers/bots.ts

   1 import * as express from 'express'
   2 import { asyncMiddleware } from '../middlewares'
   3 import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants'
   4 import { SitemapStream, streamToPromise } from 'sitemap'
   5 import { VideoModel } from '../models/video/video'
   6 import { VideoChannelModel } from '../models/video/video-channel'
   7 import { AccountModel } from '../models/account/account'
   8 import { cacheRoute } from '../middlewares/cache/cache'
   9 import { buildNSFWFilter } from '../helpers/express-utils'
  10 import { truncate } from 'lodash'
  11
  12 const botsRouter = express.Router()
  13
  14 // Special route that add OpenGraph and oEmbed tags
  15 // Do not use a template engine for a so little thing
  16 botsRouter.use('/sitemap.xml',
  17   cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP),
  18   asyncMiddleware(getSitemap)
  19 )
  20
  21 // ---------------------------------------------------------------------------
  22
  23 export {
  24   botsRouter
  25 }
  26
  27 // ---------------------------------------------------------------------------
  28
  29 async function getSitemap (req: express.Request, res: express.Response) {
  30   let urls = getSitemapBasicUrls()
  31
  32   urls = urls.concat(await getSitemapLocalVideoUrls())
  33   urls = urls.concat(await getSitemapVideoChannelUrls())
  34   urls = urls.concat(await getSitemapAccountUrls())
  35
  36   const sitemapStream = new SitemapStream({ hostname: WEBSERVER.URL })
  37
  38   for (const urlObj of urls) {
  39     sitemapStream.write(urlObj)
  40   }
  41   sitemapStream.end()
  42
  43   const xml = await streamToPromise(sitemapStream)
  44
  45   res.header('Content-Type', 'application/xml')
  46   res.send(xml)
  47 }
  48
  49 async function getSitemapVideoChannelUrls () {
  50   const rows = await VideoChannelModel.listLocalsForSitemap('createdAt')
  51
  52   return rows.map(channel => ({
  53     url: WEBSERVER.URL + '/video-channels/' + channel.Actor.preferredUsername
  54   }))
  55 }
  56
  57 async function getSitemapAccountUrls () {
  58   const rows = await AccountModel.listLocalsForSitemap('createdAt')
  59
  60   return rows.map(channel => ({
  61     url: WEBSERVER.URL + '/accounts/' + channel.Actor.preferredUsername
  62   }))
  63 }
  64
  65 async function getSitemapLocalVideoUrls () {
  66   const { data } = await VideoModel.listForApi({
  67     start: 0,
  68     count: undefined,
  69     sort: 'createdAt',
  70     includeLocalVideos: true,
  71     nsfw: buildNSFWFilter(),
  72     filter: 'local',
  73     withFiles: false,
  74     countVideos: false
  75   })
  76
  77   return data.map(v => ({
  78     url: WEBSERVER.URL + '/w/' + v.uuid,
  79     video: [
  80       {
  81         title: v.name,
  82         // Sitemap description should be < 2000 characters
  83         description: truncate(v.description || v.name, { length: 2000, omission: '...' }),
  84         player_loc: WEBSERVER.URL + '/videos/embed/' + v.uuid,
  85         thumbnail_loc: WEBSERVER.URL + v.getMiniatureStaticPath()
  86       }
  87     ]
  88   }))
  89 }
  90
  91 function getSitemapBasicUrls () {
  92   const paths = [
  93     '/about/instance',
  94     '/videos/local'
  95   ]
  96
  97   return paths.map(p => ({ url: WEBSERVER.URL + p }))
  98 }