server/controllers/bots.ts

   1 import express from 'express'
   2 import { truncate } from 'lodash'
   3 import { ErrorLevel, SitemapStream, streamToPromise } from 'sitemap'
   4 import { logger } from '@server/helpers/logger'
   5 import { getServerActor } from '@server/models/application/application'
   6 import { buildNSFWFilter } from '../helpers/express-utils'
   7 import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants'
   8 import { asyncMiddleware } from '../middlewares'
   9 import { cacheRoute } from '../middlewares/cache/cache'
  10 import { AccountModel } from '../models/account/account'
  11 import { VideoModel } from '../models/video/video'
  12 import { VideoChannelModel } from '../models/video/video-channel'
  13
  14 const botsRouter = express.Router()
  15
  16 // Special route that add OpenGraph and oEmbed tags
  17 // Do not use a template engine for a so little thing
  18 botsRouter.use('/sitemap.xml',
  19   cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP),
  20   asyncMiddleware(getSitemap)
  21 )
  22
  23 // ---------------------------------------------------------------------------
  24
  25 export {
  26   botsRouter
  27 }
  28
  29 // ---------------------------------------------------------------------------
  30
  31 async function getSitemap (req: express.Request, res: express.Response) {
  32   let urls = getSitemapBasicUrls()
  33
  34   urls = urls.concat(await getSitemapLocalVideoUrls())
  35   urls = urls.concat(await getSitemapVideoChannelUrls())
  36   urls = urls.concat(await getSitemapAccountUrls())
  37
  38   const sitemapStream = new SitemapStream({
  39     hostname: WEBSERVER.URL,
  40     errorHandler: (err: Error, level: ErrorLevel) => {
  41       if (level === 'warn') {
  42         logger.warn('Warning in sitemap generation.', { err })
  43       } else if (level === 'throw') {
  44         logger.error('Error in sitemap generation.', { err })
  45
  46         throw err
  47       }
  48     }
  49   })
  50
  51   for (const urlObj of urls) {
  52     sitemapStream.write(urlObj)
  53   }
  54   sitemapStream.end()
  55
  56   const xml = await streamToPromise(sitemapStream)
  57
  58   res.header('Content-Type', 'application/xml')
  59   res.send(xml)
  60 }
  61
  62 async function getSitemapVideoChannelUrls () {
  63   const rows = await VideoChannelModel.listLocalsForSitemap('createdAt')
  64
  65   return rows.map(channel => ({
  66     url: WEBSERVER.URL + '/video-channels/' + channel.Actor.preferredUsername
  67   }))
  68 }
  69
  70 async function getSitemapAccountUrls () {
  71   const rows = await AccountModel.listLocalsForSitemap('createdAt')
  72
  73   return rows.map(channel => ({
  74     url: WEBSERVER.URL + '/accounts/' + channel.Actor.preferredUsername
  75   }))
  76 }
  77
  78 async function getSitemapLocalVideoUrls () {
  79   const serverActor = await getServerActor()
  80
  81   const { data } = await VideoModel.listForApi({
  82     start: 0,
  83     count: undefined,
  84     sort: 'createdAt',
  85     displayOnlyForFollower: {
  86       actorId: serverActor.id,
  87       orLocalVideos: true
  88     },
  89     isLocal: true,
  90     nsfw: buildNSFWFilter(),
  91     countVideos: false
  92   })
  93
  94   return data.map(v => ({
  95     url: WEBSERVER.URL + v.getWatchStaticPath(),
  96     video: [
  97       {
  98         // Sitemap title should be < 100 characters
  99         title: truncate(v.name, { length: 100, omission: '...' }),
 100         // Sitemap description should be < 2000 characters
 101         description: truncate(v.description || v.name, { length: 2000, omission: '...' }),
 102         player_loc: WEBSERVER.URL + v.getEmbedStaticPath(),
 103         thumbnail_loc: WEBSERVER.URL + v.getMiniatureStaticPath()
 104       }
 105     ]
 106   }))
 107 }
 108
 109 function getSitemapBasicUrls () {
 110   const paths = [
 111     '/about/instance',
 112     '/videos/local'
 113   ]
 114
 115   return paths.map(p => ({ url: WEBSERVER.URL + p }))
 116 }