1 import { getServerActor } from '@server/models/application/application'
2 import { logger } from '@uploadx/core'
3 import express from 'express'
4 import { truncate } from 'lodash'
5 import { SitemapStream, streamToPromise, ErrorLevel } from 'sitemap'
6 import { buildNSFWFilter } from '../helpers/express-utils'
7 import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants'
8 import { asyncMiddleware } from '../middlewares'
9 import { cacheRoute } from '../middlewares/cache/cache'
10 import { AccountModel } from '../models/account/account'
11 import { VideoModel } from '../models/video/video'
12 import { VideoChannelModel } from '../models/video/video-channel'
14 const botsRouter = express.Router()
16 // Special route that add OpenGraph and oEmbed tags
17 // Do not use a template engine for a so little thing
18 botsRouter.use('/sitemap.xml',
19 cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP),
20 asyncMiddleware(getSitemap)
23 // ---------------------------------------------------------------------------
29 // ---------------------------------------------------------------------------
31 async function getSitemap (req: express.Request, res: express.Response) {
32 let urls = getSitemapBasicUrls()
34 urls = urls.concat(await getSitemapLocalVideoUrls())
35 urls = urls.concat(await getSitemapVideoChannelUrls())
36 urls = urls.concat(await getSitemapAccountUrls())
38 const sitemapStream = new SitemapStream({
39 hostname: WEBSERVER.URL,
40 errorHandler: (err: Error, level: ErrorLevel) => {
41 if (level === 'warn') {
42 logger.warn('Warning in sitemap generation.', { err })
43 } else if (level === 'throw') {
44 logger.error('Error in sitemap generation.', { err })
51 for (const urlObj of urls) {
52 sitemapStream.write(urlObj)
56 const xml = await streamToPromise(sitemapStream)
58 res.header('Content-Type', 'application/xml')
62 async function getSitemapVideoChannelUrls () {
63 const rows = await VideoChannelModel.listLocalsForSitemap('createdAt')
65 return rows.map(channel => ({
66 url: WEBSERVER.URL + '/video-channels/' + channel.Actor.preferredUsername
70 async function getSitemapAccountUrls () {
71 const rows = await AccountModel.listLocalsForSitemap('createdAt')
73 return rows.map(channel => ({
74 url: WEBSERVER.URL + '/accounts/' + channel.Actor.preferredUsername
78 async function getSitemapLocalVideoUrls () {
79 const serverActor = await getServerActor()
81 const { data } = await VideoModel.listForApi({
85 displayOnlyForFollower: {
86 actorId: serverActor.id,
90 nsfw: buildNSFWFilter(),
94 return data.map(v => ({
95 url: WEBSERVER.URL + v.getWatchStaticPath(),
98 // Sitemap title should be < 100 characters
99 title: truncate(v.name, { length: 100, omission: '...' }),
100 // Sitemap description should be < 2000 characters
101 description: truncate(v.description || v.name, { length: 2000, omission: '...' }),
102 player_loc: WEBSERVER.URL + v.getEmbedStaticPath(),
103 thumbnail_loc: WEBSERVER.URL + v.getMiniatureStaticPath()
109 function getSitemapBasicUrls () {
115 return paths.map(p => ({ url: WEBSERVER.URL + p }))