aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-12-05 17:27:24 +0100
committerChocobozzz <me@florianbigard.com>2018-12-05 17:44:34 +0100
commit2feebf3e6afaad9ab80976d1557d3a7bcf94de03 (patch)
tree47df940d7d600cec5e08eb7715bdb5bdae085e59
parent3b3b18203fe73e499bf8b49b15369710df95993e (diff)
downloadPeerTube-2feebf3e6afaad9ab80976d1557d3a7bcf94de03.tar.gz
PeerTube-2feebf3e6afaad9ab80976d1557d3a7bcf94de03.tar.zst
PeerTube-2feebf3e6afaad9ab80976d1557d3a7bcf94de03.zip
Add sitemap
-rw-r--r--package.json1
-rwxr-xr-xscripts/clean/server/test.sh1
-rw-r--r--server.ts3
-rw-r--r--server/controllers/bots.ts101
-rw-r--r--server/controllers/index.ts1
-rw-r--r--server/helpers/express-utils.ts4
-rw-r--r--server/initializers/constants.ts1
-rw-r--r--server/models/account/account.ts21
-rw-r--r--server/models/video/video-channel.ts21
-rw-r--r--server/tests/misc-endpoints.ts72
-rw-r--r--yarn.lock19
11 files changed, 241 insertions, 4 deletions
diff --git a/package.json b/package.json
index c5e4c329c..aa4f447aa 100644
--- a/package.json
+++ b/package.json
@@ -148,6 +148,7 @@
148 "sequelize": "4.41.2", 148 "sequelize": "4.41.2",
149 "sequelize-typescript": "0.6.6", 149 "sequelize-typescript": "0.6.6",
150 "sharp": "^0.21.0", 150 "sharp": "^0.21.0",
151 "sitemap": "^2.1.0",
151 "srt-to-vtt": "^1.1.2", 152 "srt-to-vtt": "^1.1.2",
152 "summon-install": "^0.4.3", 153 "summon-install": "^0.4.3",
153 "useragent": "^2.3.0", 154 "useragent": "^2.3.0",
diff --git a/scripts/clean/server/test.sh b/scripts/clean/server/test.sh
index 235ff52cc..b897c30ba 100755
--- a/scripts/clean/server/test.sh
+++ b/scripts/clean/server/test.sh
@@ -18,6 +18,7 @@ removeFiles () {
18 18
19dropRedis () { 19dropRedis () {
20 redis-cli KEYS "bull-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL 20 redis-cli KEYS "bull-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
21 redis-cli KEYS "redis-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
21} 22}
22 23
23for i in $(seq 1 6); do 24for i in $(seq 1 6); do
diff --git a/server.ts b/server.ts
index 3025a6fd7..4a2a6ddf5 100644
--- a/server.ts
+++ b/server.ts
@@ -87,7 +87,7 @@ import {
87 servicesRouter, 87 servicesRouter,
88 webfingerRouter, 88 webfingerRouter,
89 trackerRouter, 89 trackerRouter,
90 createWebsocketServer 90 createWebsocketServer, botsRouter
91} from './server/controllers' 91} from './server/controllers'
92import { advertiseDoNotTrack } from './server/middlewares/dnt' 92import { advertiseDoNotTrack } from './server/middlewares/dnt'
93import { Redis } from './server/lib/redis' 93import { Redis } from './server/lib/redis'
@@ -156,6 +156,7 @@ app.use('/', activityPubRouter)
156app.use('/', feedsRouter) 156app.use('/', feedsRouter)
157app.use('/', webfingerRouter) 157app.use('/', webfingerRouter)
158app.use('/', trackerRouter) 158app.use('/', trackerRouter)
159app.use('/', botsRouter)
159 160
160// Static files 161// Static files
161app.use('/', staticRouter) 162app.use('/', staticRouter)
diff --git a/server/controllers/bots.ts b/server/controllers/bots.ts
new file mode 100644
index 000000000..b4eaccf9f
--- /dev/null
+++ b/server/controllers/bots.ts
@@ -0,0 +1,101 @@
1import * as express from 'express'
2import { asyncMiddleware } from '../middlewares'
3import { CONFIG, ROUTE_CACHE_LIFETIME } from '../initializers'
4import * as sitemapModule from 'sitemap'
5import { logger } from '../helpers/logger'
6import { VideoModel } from '../models/video/video'
7import { VideoChannelModel } from '../models/video/video-channel'
8import { AccountModel } from '../models/account/account'
9import { cacheRoute } from '../middlewares/cache'
10import { buildNSFWFilter } from '../helpers/express-utils'
11import { truncate } from 'lodash'
12
13const botsRouter = express.Router()
14
15// Special route that add OpenGraph and oEmbed tags
16// Do not use a template engine for a so little thing
17botsRouter.use('/sitemap.xml',
18 asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP)),
19 asyncMiddleware(getSitemap)
20)
21
22// ---------------------------------------------------------------------------
23
24export {
25 botsRouter
26}
27
28// ---------------------------------------------------------------------------
29
30async function getSitemap (req: express.Request, res: express.Response) {
31 let urls = getSitemapBasicUrls()
32
33 urls = urls.concat(await getSitemapLocalVideoUrls())
34 urls = urls.concat(await getSitemapVideoChannelUrls())
35 urls = urls.concat(await getSitemapAccountUrls())
36
37 const sitemap = sitemapModule.createSitemap({
38 hostname: CONFIG.WEBSERVER.URL,
39 urls: urls
40 })
41
42 sitemap.toXML((err, xml) => {
43 if (err) {
44 logger.error('Cannot generate sitemap.', { err })
45 return res.sendStatus(500)
46 }
47
48 res.header('Content-Type', 'application/xml')
49 res.send(xml)
50 })
51}
52
53async function getSitemapVideoChannelUrls () {
54 const rows = await VideoChannelModel.listLocalsForSitemap('createdAt')
55
56 return rows.map(channel => ({
57 url: CONFIG.WEBSERVER.URL + '/video-channels/' + channel.Actor.preferredUsername
58 }))
59}
60
61async function getSitemapAccountUrls () {
62 const rows = await AccountModel.listLocalsForSitemap('createdAt')
63
64 return rows.map(channel => ({
65 url: CONFIG.WEBSERVER.URL + '/accounts/' + channel.Actor.preferredUsername
66 }))
67}
68
69async function getSitemapLocalVideoUrls () {
70 const resultList = await VideoModel.listForApi({
71 start: 0,
72 count: undefined,
73 sort: 'createdAt',
74 includeLocalVideos: true,
75 nsfw: buildNSFWFilter(),
76 filter: 'local',
77 withFiles: false
78 })
79
80 return resultList.data.map(v => ({
81 url: CONFIG.WEBSERVER.URL + '/videos/watch/' + v.uuid,
82 video: [
83 {
84 title: v.name,
85 // Sitemap description should be < 2000 characters
86 description: truncate(v.description || v.name, { length: 2000, omission: '...' }),
87 player_loc: CONFIG.WEBSERVER.URL + '/videos/embed/' + v.uuid,
88 thumbnail_loc: v.getThumbnailStaticPath()
89 }
90 ]
91 }))
92}
93
94function getSitemapBasicUrls () {
95 const paths = [
96 '/about/instance',
97 '/videos/local'
98 ]
99
100 return paths.map(p => ({ url: CONFIG.WEBSERVER.URL + p }))
101}
diff --git a/server/controllers/index.ts b/server/controllers/index.ts
index 197fa897a..a88a03c79 100644
--- a/server/controllers/index.ts
+++ b/server/controllers/index.ts
@@ -6,3 +6,4 @@ export * from './services'
6export * from './static' 6export * from './static'
7export * from './webfinger' 7export * from './webfinger'
8export * from './tracker' 8export * from './tracker'
9export * from './bots'
diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts
index 162fe2244..9a72ee96d 100644
--- a/server/helpers/express-utils.ts
+++ b/server/helpers/express-utils.ts
@@ -7,12 +7,12 @@ import { extname } from 'path'
7import { isArray } from './custom-validators/misc' 7import { isArray } from './custom-validators/misc'
8import { UserModel } from '../models/account/user' 8import { UserModel } from '../models/account/user'
9 9
10function buildNSFWFilter (res: express.Response, paramNSFW?: string) { 10function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
11 if (paramNSFW === 'true') return true 11 if (paramNSFW === 'true') return true
12 if (paramNSFW === 'false') return false 12 if (paramNSFW === 'false') return false
13 if (paramNSFW === 'both') return undefined 13 if (paramNSFW === 'both') return undefined
14 14
15 if (res.locals.oauth) { 15 if (res && res.locals.oauth) {
16 const user: UserModel = res.locals.oauth.token.User 16 const user: UserModel = res.locals.oauth.token.User
17 17
18 // User does not want NSFW videos 18 // User does not want NSFW videos
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 7195ae6c5..6b798875c 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -61,6 +61,7 @@ const OAUTH_LIFETIME = {
61const ROUTE_CACHE_LIFETIME = { 61const ROUTE_CACHE_LIFETIME = {
62 FEEDS: '15 minutes', 62 FEEDS: '15 minutes',
63 ROBOTS: '2 hours', 63 ROBOTS: '2 hours',
64 SITEMAP: '1 day',
64 SECURITYTXT: '2 hours', 65 SECURITYTXT: '2 hours',
65 NODEINFO: '10 minutes', 66 NODEINFO: '10 minutes',
66 DNT_POLICY: '1 week', 67 DNT_POLICY: '1 week',
diff --git a/server/models/account/account.ts b/server/models/account/account.ts
index 5a237d733..a99e9b1ad 100644
--- a/server/models/account/account.ts
+++ b/server/models/account/account.ts
@@ -241,6 +241,27 @@ export class AccountModel extends Model<AccountModel> {
241 }) 241 })
242 } 242 }
243 243
244 static listLocalsForSitemap (sort: string) {
245 const query = {
246 attributes: [ ],
247 offset: 0,
248 order: getSort(sort),
249 include: [
250 {
251 attributes: [ 'preferredUsername', 'serverId' ],
252 model: ActorModel.unscoped(),
253 where: {
254 serverId: null
255 }
256 }
257 ]
258 }
259
260 return AccountModel
261 .unscoped()
262 .findAll(query)
263 }
264
244 toFormattedJSON (): Account { 265 toFormattedJSON (): Account {
245 const actor = this.Actor.toFormattedJSON() 266 const actor = this.Actor.toFormattedJSON()
246 const account = { 267 const account = {
diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts
index f4586917e..86bf0461a 100644
--- a/server/models/video/video-channel.ts
+++ b/server/models/video/video-channel.ts
@@ -233,6 +233,27 @@ export class VideoChannelModel extends Model<VideoChannelModel> {
233 }) 233 })
234 } 234 }
235 235
236 static listLocalsForSitemap (sort: string) {
237 const query = {
238 attributes: [ ],
239 offset: 0,
240 order: getSort(sort),
241 include: [
242 {
243 attributes: [ 'preferredUsername', 'serverId' ],
244 model: ActorModel.unscoped(),
245 where: {
246 serverId: null
247 }
248 }
249 ]
250 }
251
252 return VideoChannelModel
253 .unscoped()
254 .findAll(query)
255 }
256
236 static searchForApi (options: { 257 static searchForApi (options: {
237 actorId: number 258 actorId: number
238 search: string 259 search: string
diff --git a/server/tests/misc-endpoints.ts b/server/tests/misc-endpoints.ts
index 8fab20971..b53803ee1 100644
--- a/server/tests/misc-endpoints.ts
+++ b/server/tests/misc-endpoints.ts
@@ -2,7 +2,18 @@
2 2
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { flushTests, killallServers, makeGetRequest, runServer, ServerInfo } from './utils' 5import {
6 addVideoChannel,
7 createUser,
8 flushTests,
9 killallServers,
10 makeGetRequest,
11 runServer,
12 ServerInfo,
13 setAccessTokensToServers,
14 uploadVideo
15} from './utils'
16import { VideoPrivacy } from '../../shared/models/videos'
6 17
7const expect = chai.expect 18const expect = chai.expect
8 19
@@ -15,6 +26,7 @@ describe('Test misc endpoints', function () {
15 await flushTests() 26 await flushTests()
16 27
17 server = await runServer(1) 28 server = await runServer(1)
29 await setAccessTokensToServers([ server ])
18 }) 30 })
19 31
20 describe('Test a well known endpoints', function () { 32 describe('Test a well known endpoints', function () {
@@ -93,6 +105,64 @@ describe('Test misc endpoints', function () {
93 }) 105 })
94 }) 106 })
95 107
108 describe('Test bots endpoints', function () {
109
110 it('Should get the empty sitemap', async function () {
111 const res = await makeGetRequest({
112 url: server.url,
113 path: '/sitemap.xml',
114 statusCodeExpected: 200
115 })
116
117 expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
118 expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
119 })
120
121 it('Should get the empty cached sitemap', async function () {
122 const res = await makeGetRequest({
123 url: server.url,
124 path: '/sitemap.xml',
125 statusCodeExpected: 200
126 })
127
128 expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
129 expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
130 })
131
132 it('Should add videos, channel and accounts and get sitemap', async function () {
133 this.timeout(35000)
134
135 await uploadVideo(server.url, server.accessToken, { name: 'video 1', nsfw: false })
136 await uploadVideo(server.url, server.accessToken, { name: 'video 2', nsfw: false })
137 await uploadVideo(server.url, server.accessToken, { name: 'video 3', privacy: VideoPrivacy.PRIVATE })
138
139 await addVideoChannel(server.url, server.accessToken, { name: 'channel1', displayName: 'channel 1' })
140 await addVideoChannel(server.url, server.accessToken, { name: 'channel2', displayName: 'channel 2' })
141
142 await createUser(server.url, server.accessToken, 'user1', 'password')
143 await createUser(server.url, server.accessToken, 'user2', 'password')
144
145 const res = await makeGetRequest({
146 url: server.url,
147 path: '/sitemap.xml?t=1', // avoid using cache
148 statusCodeExpected: 200
149 })
150
151 expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
152 expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
153
154 expect(res.text).to.contain('<video:title><![CDATA[video 1]]></video:title>')
155 expect(res.text).to.contain('<video:title><![CDATA[video 2]]></video:title>')
156 expect(res.text).to.not.contain('<video:title><![CDATA[video 3]]></video:title>')
157
158 expect(res.text).to.contain('<url><loc>http://localhost:9001/video-channels/channel1</loc></url>')
159 expect(res.text).to.contain('<url><loc>http://localhost:9001/video-channels/channel2</loc></url>')
160
161 expect(res.text).to.contain('<url><loc>http://localhost:9001/accounts/user1</loc></url>')
162 expect(res.text).to.contain('<url><loc>http://localhost:9001/accounts/user2</loc></url>')
163 })
164 })
165
96 after(async function () { 166 after(async function () {
97 killallServers([ server ]) 167 killallServers([ server ])
98 }) 168 })
diff --git a/yarn.lock b/yarn.lock
index 1cbe6756d..8d74f9d55 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7457,6 +7457,15 @@ simple-websocket@^7.0.1:
7457 readable-stream "^2.0.5" 7457 readable-stream "^2.0.5"
7458 ws "^6.0.0" 7458 ws "^6.0.0"
7459 7459
7460sitemap@^2.1.0:
7461 version "2.1.0"
7462 resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-2.1.0.tgz#1633cb88c196d755ad94becfb1c1bcacc6d3425a"
7463 integrity sha512-AkfA7RDVCITQo+j5CpXsMJlZ/8ENO2NtgMHYIh+YMvex2Hao/oe3MQgNa03p0aWY6srCfUA1Q02OgiWCAiuccA==
7464 dependencies:
7465 lodash "^4.17.10"
7466 url-join "^4.0.0"
7467 xmlbuilder "^10.0.0"
7468
7460slash@^1.0.0: 7469slash@^1.0.0:
7461 version "1.0.0" 7470 version "1.0.0"
7462 resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55" 7471 resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55"
@@ -8592,6 +8601,11 @@ urix@^0.1.0:
8592 resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" 8601 resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72"
8593 integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI= 8602 integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=
8594 8603
8604url-join@^4.0.0:
8605 version "4.0.0"
8606 resolved "https://registry.yarnpkg.com/url-join/-/url-join-4.0.0.tgz#4d3340e807d3773bda9991f8305acdcc2a665d2a"
8607 integrity sha1-TTNA6AfTdzvamZH4MFrNzCpmXSo=
8608
8595url-parse-lax@^1.0.0: 8609url-parse-lax@^1.0.0:
8596 version "1.0.0" 8610 version "1.0.0"
8597 resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-1.0.0.tgz#7af8f303645e9bd79a272e7a14ac68bc0609da73" 8611 resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-1.0.0.tgz#7af8f303645e9bd79a272e7a14ac68bc0609da73"
@@ -9001,6 +9015,11 @@ xml@^1.0.1:
9001 resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" 9015 resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
9002 integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU= 9016 integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=
9003 9017
9018xmlbuilder@^10.0.0:
9019 version "10.1.1"
9020 resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz#8cae6688cc9b38d850b7c8d3c0a4161dcaf475b0"
9021 integrity sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==
9022
9004xmlbuilder@~9.0.1: 9023xmlbuilder@~9.0.1:
9005 version "9.0.7" 9024 version "9.0.7"
9006 resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" 9025 resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"