34 files changed, 347 insertions, 96 deletions

diff --git a/server/controllers/api/accounts.ts b/server/controllers/api/accounts.ts
index 86ef2aed1..a69a83acf 100644
--- a/server/controllers/api/accounts.ts
+++ b/server/controllers/api/accounts.ts
@@ -74,10 +74,10 @@ async function listVideoAccountChannels (req: express.Request, res: express.Resp
 
 async function listAccountVideos (req: express.Request, res: express.Response, next: express.NextFunction) {
   const account: AccountModel = res.locals.account
-  const actorId = isUserAbleToSearchRemoteURI(res) ? null : undefined
+  const followerActorId = isUserAbleToSearchRemoteURI(res) ? null : undefined
 
   const resultList = await VideoModel.listForApi({
-    actorId,
+    followerActorId,
     start: req.query.start,
     count: req.query.count,
     sort: req.query.sort,
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 5233e9f68..d65e321e9 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -11,6 +11,7 @@ import { ClientHtml } from '../../lib/client-html'
 import { auditLoggerFactory, CustomConfigAuditView, getAuditIdFromRes } from '../../helpers/audit-logger'
 import { remove, writeJSON } from 'fs-extra'
 import { getServerCommit } from '../../helpers/utils'
+import { Emailer } from '../../lib/emailer'
 
 const packageJSON = require('../../../../package.json')
 const configRouter = express.Router()
@@ -61,6 +62,9 @@ async function getConfig (req: express.Request, res: express.Response) {
         css: CONFIG.INSTANCE.CUSTOMIZATIONS.CSS
       }
     },
+    email: {
+      enabled: Emailer.Instance.isEnabled()
+    },
     serverVersion: packageJSON.version,
     serverCommit,
     signup: {
diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts
index 82299747d..d2456346b 100644
--- a/server/controllers/api/users/me.ts
+++ b/server/controllers/api/users/me.ts
@@ -42,7 +42,7 @@ import { AccountModel } from '../../../models/account/account'
 
 const auditLogger = auditLoggerFactory('users-me')
 
-const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR })
+const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR })
 
 const meRouter = express.Router()
 
@@ -238,7 +238,7 @@ async function getUserSubscriptionVideos (req: express.Request, res: express.Res
     nsfw: buildNSFWFilter(res, req.query.nsfw),
     filter: req.query.filter as VideoFilter,
     withFiles: false,
-    actorId: user.Account.Actor.id,
+    followerActorId: user.Account.Actor.id,
     user
   })
 
@@ -348,7 +348,7 @@ async function updateMe (req: express.Request, res: express.Response, next: expr
   return res.sendStatus(204)
 }
 
-async function updateMyAvatar (req: express.Request, res: express.Response, next: express.NextFunction) {
+async function updateMyAvatar (req: express.Request, res: express.Response) {
   const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ]
   const user: UserModel = res.locals.oauth.token.user
   const oldUserAuditView = new UserAuditView(user.toFormattedJSON())
diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts
index 9bf3c5fd8..fd143a139 100644
--- a/server/controllers/api/video-channel.ts
+++ b/server/controllers/api/video-channel.ts
@@ -32,7 +32,7 @@ import { resetSequelizeInstance } from '../../helpers/database-utils'
 import { UserModel } from '../../models/account/user'
 
 const auditLogger = auditLoggerFactory('channels')
-const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR })
+const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR })
 
 const videoChannelRouter = express.Router()
 
@@ -202,10 +202,10 @@ async function getVideoChannel (req: express.Request, res: express.Response, nex
 
 async function listVideoChannelVideos (req: express.Request, res: express.Response, next: express.NextFunction) {
   const videoChannelInstance: VideoChannelModel = res.locals.videoChannel
-  const actorId = isUserAbleToSearchRemoteURI(res) ? null : undefined
+  const followerActorId = isUserAbleToSearchRemoteURI(res) ? null : undefined
 
   const resultList = await VideoModel.listForApi({
-    actorId,
+    followerActorId,
     start: req.query.start,
     count: req.query.count,
     sort: req.query.sort,
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 398fd5a7f..f27d648c7 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -37,9 +37,9 @@ const reqVideoFileImport = createReqFiles(
   [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
   Object.assign({}, TORRENT_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT),
   {
-    thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
-    previewfile: CONFIG.STORAGE.PREVIEWS_DIR,
-    torrentfile: CONFIG.STORAGE.TORRENTS_DIR
+    thumbnailfile: CONFIG.STORAGE.TMP_DIR,
+    previewfile: CONFIG.STORAGE.TMP_DIR,
+    torrentfile: CONFIG.STORAGE.TMP_DIR
   }
 )
 
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 3d1b2e1a2..4e4697ef4 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -67,17 +67,17 @@ const reqVideoFileAdd = createReqFiles(
   [ 'videofile', 'thumbnailfile', 'previewfile' ],
   Object.assign({}, VIDEO_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT),
   {
-    videofile: CONFIG.STORAGE.VIDEOS_DIR,
-    thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
-    previewfile: CONFIG.STORAGE.PREVIEWS_DIR
+    videofile: CONFIG.STORAGE.TMP_DIR,
+    thumbnailfile: CONFIG.STORAGE.TMP_DIR,
+    previewfile: CONFIG.STORAGE.TMP_DIR
   }
 )
 const reqVideoFileUpdate = createReqFiles(
   [ 'thumbnailfile', 'previewfile' ],
   IMAGE_MIMETYPE_EXT,
   {
-    thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
-    previewfile: CONFIG.STORAGE.PREVIEWS_DIR
+    thumbnailfile: CONFIG.STORAGE.TMP_DIR,
+    previewfile: CONFIG.STORAGE.TMP_DIR
   }
 )
 
diff --git a/server/controllers/bots.ts b/server/controllers/bots.ts
new file mode 100644
index 000000000..2db86a2d8
--- /dev/null
+++ b/server/controllers/bots.ts
@@ -0,0 +1,101 @@
+import * as express from 'express'
+import { asyncMiddleware } from '../middlewares'
+import { CONFIG, ROUTE_CACHE_LIFETIME } from '../initializers'
+import * as sitemapModule from 'sitemap'
+import { logger } from '../helpers/logger'
+import { VideoModel } from '../models/video/video'
+import { VideoChannelModel } from '../models/video/video-channel'
+import { AccountModel } from '../models/account/account'
+import { cacheRoute } from '../middlewares/cache'
+import { buildNSFWFilter } from '../helpers/express-utils'
+import { truncate } from 'lodash'
+
+const botsRouter = express.Router()
+
+// Special route that add OpenGraph and oEmbed tags
+// Do not use a template engine for a so little thing
+botsRouter.use('/sitemap.xml',
+  asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP)),
+  asyncMiddleware(getSitemap)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  botsRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function getSitemap (req: express.Request, res: express.Response) {
+  let urls = getSitemapBasicUrls()
+
+  urls = urls.concat(await getSitemapLocalVideoUrls())
+  urls = urls.concat(await getSitemapVideoChannelUrls())
+  urls = urls.concat(await getSitemapAccountUrls())
+
+  const sitemap = sitemapModule.createSitemap({
+    hostname: CONFIG.WEBSERVER.URL,
+    urls: urls
+  })
+
+  sitemap.toXML((err, xml) => {
+    if (err) {
+      logger.error('Cannot generate sitemap.', { err })
+      return res.sendStatus(500)
+    }
+
+    res.header('Content-Type', 'application/xml')
+    res.send(xml)
+  })
+}
+
+async function getSitemapVideoChannelUrls () {
+  const rows = await VideoChannelModel.listLocalsForSitemap('createdAt')
+
+  return rows.map(channel => ({
+    url: CONFIG.WEBSERVER.URL + '/video-channels/' + channel.Actor.preferredUsername
+  }))
+}
+
+async function getSitemapAccountUrls () {
+  const rows = await AccountModel.listLocalsForSitemap('createdAt')
+
+  return rows.map(channel => ({
+    url: CONFIG.WEBSERVER.URL + '/accounts/' + channel.Actor.preferredUsername
+  }))
+}
+
+async function getSitemapLocalVideoUrls () {
+  const resultList = await VideoModel.listForApi({
+    start: 0,
+    count: undefined,
+    sort: 'createdAt',
+    includeLocalVideos: true,
+    nsfw: buildNSFWFilter(),
+    filter: 'local',
+    withFiles: false
+  })
+
+  return resultList.data.map(v => ({
+    url: CONFIG.WEBSERVER.URL + '/videos/watch/' + v.uuid,
+    video: [
+      {
+        title: v.name,
+        // Sitemap description should be < 2000 characters
+        description: truncate(v.description || v.name, { length: 2000, omission: '...' }),
+        player_loc: CONFIG.WEBSERVER.URL + '/videos/embed/' + v.uuid,
+        thumbnail_loc: CONFIG.WEBSERVER.URL + v.getThumbnailStaticPath()
+      }
+    ]
+  }))
+}
+
+function getSitemapBasicUrls () {
+  const paths = [
+    '/about/instance',
+    '/videos/local'
+  ]
+
+  return paths.map(p => ({ url: CONFIG.WEBSERVER.URL + p }))
+}
diff --git a/server/controllers/index.ts b/server/controllers/index.ts
index 197fa897a..a88a03c79 100644
--- a/server/controllers/index.ts
+++ b/server/controllers/index.ts
@@ -6,3 +6,4 @@ export * from './services'
 export * from './static'
 export * from './webfinger'
 export * from './tracker'
+export * from './bots'
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index 75e30353c..55e7392a1 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -34,13 +34,18 @@ staticRouter.use(
 )
 
 // Videos path for webseeding
-const videosPhysicalPath = CONFIG.STORAGE.VIDEOS_DIR
 staticRouter.use(
   STATIC_PATHS.WEBSEED,
   cors(),
-  express.static(videosPhysicalPath)
+  express.static(CONFIG.STORAGE.VIDEOS_DIR, { fallthrough: false }) // 404 because we don't have this video
 )
 staticRouter.use(
+  STATIC_PATHS.REDUNDANCY,
+  cors(),
+  express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }) // 404 because we don't have this video
+)
+
+staticRouter.use(
   STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension',
   asyncMiddleware(videosGetValidator),
   asyncMiddleware(downloadVideoFile)
diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts
index 162fe2244..9a72ee96d 100644
--- a/server/helpers/express-utils.ts
+++ b/server/helpers/express-utils.ts
@@ -7,12 +7,12 @@ import { extname } from 'path'
 import { isArray } from './custom-validators/misc'
 import { UserModel } from '../models/account/user'
 
-function buildNSFWFilter (res: express.Response, paramNSFW?: string) {
+function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
   if (paramNSFW === 'true') return true
   if (paramNSFW === 'false') return false
   if (paramNSFW === 'both') return undefined
 
-  if (res.locals.oauth) {
+  if (res && res.locals.oauth) {
     const user: UserModel = res.locals.oauth.token.User
 
     // User does not want NSFW videos
diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts
index da3285b13..e43ea3f1d 100644
--- a/server/helpers/image-utils.ts
+++ b/server/helpers/image-utils.ts
@@ -1,6 +1,7 @@
 import 'multer'
 import * as sharp from 'sharp'
-import { move, remove } from 'fs-extra'
+import { readFile, remove } from 'fs-extra'
+import { logger } from './logger'
 
 async function processImage (
   physicalFile: { path: string },
@@ -11,14 +12,11 @@ async function processImage (
     throw new Error('Sharp needs an input path different that the output path.')
   }
 
-  const sharpInstance = sharp(physicalFile.path)
-  const metadata = await sharpInstance.metadata()
+  logger.debug('Processing image %s to %s.', physicalFile.path, destination)
 
-  // No need to resize
-  if (metadata.width === newSize.width && metadata.height === newSize.height) {
-    await move(physicalFile.path, destination, { overwrite: true })
-    return
-  }
+  // Avoid sharp cache
+  const buf = await readFile(physicalFile.path)
+  const sharpInstance = sharp(buf)
 
   await remove(destination)
 
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts
index 805930a9f..3fc776f1a 100644
--- a/server/helpers/requests.ts
+++ b/server/helpers/requests.ts
@@ -1,8 +1,9 @@
 import * as Bluebird from 'bluebird'
 import { createWriteStream } from 'fs-extra'
 import * as request from 'request'
-import { ACTIVITY_PUB } from '../initializers'
+import { ACTIVITY_PUB, CONFIG } from '../initializers'
 import { processImage } from './image-utils'
+import { join } from 'path'
 
 function doRequest <T> (
   requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean }
@@ -28,11 +29,11 @@ function doRequestAndSaveToFile (requestOptions: request.CoreOptions & request.U
   })
 }
 
-async function downloadImage (url: string, destPath: string, size: { width: number, height: number }) {
-  const tmpPath = destPath + '.tmp'
-
+async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
+  const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
   await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath)
 
+  const destPath = join(destDir, destName)
   await processImage({ path: tmpPath }, destPath, size)
 }
 
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index 5c9d6fe2f..9b89e3e61 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -46,11 +46,11 @@ const getServerActor = memoizee(async function () {
   return actor
 })
 
-function generateVideoTmpPath (target: string | ParseTorrent) {
+function generateVideoImportTmpPath (target: string | ParseTorrent) {
   const id = typeof target === 'string' ? target : target.infoHash
 
   const hash = sha256(id)
-  return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
+  return join(CONFIG.STORAGE.TMP_DIR, hash + '-import.mp4')
 }
 
 function getSecureTorrentName (originalName: string) {
@@ -103,6 +103,6 @@ export {
   getSecureTorrentName,
   getServerActor,
   getServerCommit,
-  generateVideoTmpPath,
+  generateVideoImportTmpPath,
   getUUIDFromFilename
 }
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index ce35b87da..3c9a0b96a 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -1,5 +1,5 @@
 import { logger } from './logger'
-import { generateVideoTmpPath } from './utils'
+import { generateVideoImportTmpPath } from './utils'
 import * as WebTorrent from 'webtorrent'
 import { createWriteStream, ensureDir, remove } from 'fs-extra'
 import { CONFIG } from '../initializers'
@@ -9,10 +9,10 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
   const id = target.magnetUri || target.torrentName
   let timer
 
-  const path = generateVideoTmpPath(id)
+  const path = generateVideoImportTmpPath(id)
   logger.info('Importing torrent video %s', id)
 
-  const directoryPath = join(CONFIG.STORAGE.VIDEOS_DIR, 'import')
+  const directoryPath = join(CONFIG.STORAGE.TMP_DIR, 'webtorrent')
   await ensureDir(directoryPath)
 
   return new Promise<string>((res, rej) => {
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 2a5663042..b74351b42 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -1,7 +1,7 @@
 import { truncate } from 'lodash'
 import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
 import { logger } from './logger'
-import { generateVideoTmpPath } from './utils'
+import { generateVideoImportTmpPath } from './utils'
 import { join } from 'path'
 import { root } from './core-utils'
 import { ensureDir, writeFile, remove } from 'fs-extra'
@@ -40,7 +40,7 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
 }
 
 function downloadYoutubeDLVideo (url: string, timeout: number) {
-  const path = generateVideoTmpPath(url)
+  const path = generateVideoImportTmpPath(url)
   let timer
 
   logger.info('Importing youtubeDL video %s', url)
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 9dfb5d68c..b51c7cfba 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -12,6 +12,7 @@ function checkMissedConfig () {
     'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max',
     'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
     'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
+    'storage.redundancy', 'storage.tmp',
     'log.level',
     'user.video_quota', 'user.video_quota_daily',
     'cache.previews.size', 'admin.email',
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index aa243859c..6b798875c 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -61,6 +61,7 @@ const OAUTH_LIFETIME = {
 const ROUTE_CACHE_LIFETIME = {
   FEEDS: '15 minutes',
   ROBOTS: '2 hours',
+  SITEMAP: '1 day',
   SECURITYTXT: '2 hours',
   NODEINFO: '10 minutes',
   DNT_POLICY: '1 week',
@@ -185,9 +186,11 @@ const CONFIG = {
     FROM_ADDRESS: config.get<string>('smtp.from_address')
   },
   STORAGE: {
+    TMP_DIR: buildPath(config.get<string>('storage.tmp')),
     AVATARS_DIR: buildPath(config.get<string>('storage.avatars')),
     LOG_DIR: buildPath(config.get<string>('storage.logs')),
     VIDEOS_DIR: buildPath(config.get<string>('storage.videos')),
+    REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
     THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
     PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
     CAPTIONS_DIR: buildPath(config.get<string>('storage.captions')),
@@ -569,6 +572,7 @@ const STATIC_PATHS = {
   THUMBNAILS: '/static/thumbnails/',
   TORRENTS: '/static/torrents/',
   WEBSEED: '/static/webseed/',
+  REDUNDANCY: '/static/redundancy/',
   AVATARS: '/static/avatars/',
   VIDEO_CAPTIONS: '/static/video-captions/'
 }
@@ -773,7 +777,7 @@ function buildVideosRedundancy (objs: any[]): VideosRedundancy[] {
   if (!objs) return []
 
   return objs.map(obj => {
-    return Object.assign(obj, {
+    return Object.assign({}, obj, {
       minLifetime: parseDuration(obj.min_lifetime),
       size: bytes.parse(obj.size),
       minViews: obj.min_views
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 504263c99..bbe48833d 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -178,9 +178,7 @@ async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
     const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType]
 
     const avatarName = uuidv4() + extension
-    const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
-
-    await downloadImage(actorJSON.icon.url, destPath, AVATARS_SIZE)
+    await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
 
     return avatarName
   }
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index 03831a00e..c6b42d846 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -51,7 +51,7 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
     return undefined
   }
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id })
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id, allowRefresh: false })
   const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
 
   const updateOptions = {
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 998f90330..3d17e6846 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -95,9 +95,8 @@ function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Fu
 
 function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
   const thumbnailName = video.getThumbnailName()
-  const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
 
-  return downloadImage(icon.url, thumbnailPath, THUMBNAILS_SIZE)
+  return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE)
 }
 
 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
@@ -158,25 +157,30 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid
 async function getOrCreateVideoAndAccountAndChannel (options: {
   videoObject: VideoTorrentObject | string,
   syncParam?: SyncParam,
-  fetchType?: VideoFetchByUrlType
+  fetchType?: VideoFetchByUrlType,
+  allowRefresh?: boolean // true by default
 }) {
   // Default params
   const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
   const fetchType = options.fetchType || 'all'
+  const allowRefresh = options.allowRefresh !== false
 
   // Get video url
   const videoUrl = getAPUrl(options.videoObject)
 
   let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
   if (videoFromDatabase) {
-    const refreshOptions = {
-      video: videoFromDatabase,
-      fetchedType: fetchType,
-      syncParam
-    }
 
-    if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
-    else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoFromDatabase.url } })
+    if (allowRefresh === true) {
+      const refreshOptions = {
+        video: videoFromDatabase,
+        fetchedType: fetchType,
+        syncParam
+      }
+
+      if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
+      else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoFromDatabase.url } })
+    }
 
     return { video: videoFromDatabase }
   }
diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts
index 9327792fb..074d4ad44 100644
--- a/server/lib/emailer.ts
+++ b/server/lib/emailer.ts
@@ -14,6 +14,7 @@ class Emailer {
   private static instance: Emailer
   private initialized = false
   private transporter: Transporter
+  private enabled = false
 
   private constructor () {}
 
@@ -50,6 +51,8 @@ class Emailer {
         tls,
         auth
       })
+
+      this.enabled = true
     } else {
       if (!isTestInstance()) {
         logger.error('Cannot use SMTP server because of lack of configuration. PeerTube will not be able to send mails!')
@@ -57,6 +60,10 @@ class Emailer {
     }
   }
 
+  isEnabled () {
+    return this.enabled
+  }
+
   async checkConnectionOrDie () {
     if (!this.transporter) return
 
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts
index 7752b3b40..671b0f487 100644
--- a/server/lib/job-queue/handlers/activitypub-refresher.ts
+++ b/server/lib/job-queue/handlers/activitypub-refresher.ts
@@ -10,7 +10,8 @@ export type RefreshPayload = {
 
 async function refreshAPObject (job: Bull.Job) {
   const payload = job.data as RefreshPayload
-  logger.info('Processing AP refresher in job %d.', job.id)
+
+  logger.info('Processing AP refresher in job %d for video %s.', job.id, payload.videoUrl)
 
   if (payload.type === 'video') return refreshAPVideo(payload.videoUrl)
 }
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts
index adc0a2a15..ddbf6d1c2 100644
--- a/server/lib/job-queue/handlers/video-file.ts
+++ b/server/lib/job-queue/handlers/video-file.ts
@@ -1,5 +1,5 @@
 import * as Bull from 'bull'
-import { VideoResolution, VideoState } from '../../../../shared'
+import { VideoResolution, VideoState, Job } from '../../../../shared'
 import { logger } from '../../../helpers/logger'
 import { VideoModel } from '../../../models/video/video'
 import { JobQueue } from '../job-queue'
@@ -111,7 +111,7 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole
     )
 
     if (resolutionsEnabled.length !== 0) {
-      const tasks: Bluebird<any>[] = []
+      const tasks: Bluebird<Bull.Job<any>>[] = []
 
       for (const resolution of resolutionsEnabled) {
         const dataInput = {
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 4de901c0c..51a0b5faf 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -7,7 +7,7 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro
 import { extname, join } from 'path'
 import { VideoFileModel } from '../../../models/video/video-file'
 import { CONFIG, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_IMPORT_TIMEOUT } from '../../../initializers'
-import { doRequestAndSaveToFile, downloadImage } from '../../../helpers/requests'
+import { downloadImage } from '../../../helpers/requests'
 import { VideoState } from '../../../../shared'
 import { JobQueue } from '../index'
 import { federateVideoIfNeeded } from '../../activitypub'
@@ -109,6 +109,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
   let tempVideoPath: string
   let videoDestFile: string
   let videoFile: VideoFileModel
+
   try {
     // Download video from youtubeDL
     tempVideoPath = await downloader()
@@ -144,8 +145,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
     // Process thumbnail
     if (options.downloadThumbnail) {
       if (options.thumbnailUrl) {
-        const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
-        await downloadImage(options.thumbnailUrl, destThumbnailPath, THUMBNAILS_SIZE)
+        await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName(), THUMBNAILS_SIZE)
       } else {
         await videoImport.Video.createThumbnail(videoFile)
       }
@@ -156,8 +156,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
     // Process preview
     if (options.downloadPreview) {
       if (options.thumbnailUrl) {
-        const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
-        await downloadImage(options.thumbnailUrl, destPreviewPath, PREVIEWS_SIZE)
+        await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName(), PREVIEWS_SIZE)
       } else {
         await videoImport.Video.createPreview(videoFile)
       }
diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts
index 038ef43e2..fa1fd13b3 100644
--- a/server/lib/job-queue/handlers/video-views.ts
+++ b/server/lib/job-queue/handlers/video-views.ts
@@ -23,9 +23,7 @@ async function processVideosViews () {
   for (const videoId of videoIds) {
     try {
       const views = await Redis.Instance.getVideoViews(videoId, hour)
-      if (isNaN(views)) {
-        logger.error('Cannot process videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, views)
-      } else {
+      if (views) {
         logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour)
 
         try {
diff --git a/server/lib/redis.ts b/server/lib/redis.ts
index abd75d512..3e25e6a2c 100644
--- a/server/lib/redis.ts
+++ b/server/lib/redis.ts
@@ -121,7 +121,14 @@ class Redis {
     const key = this.generateVideoViewKey(videoId, hour)
 
     const valueString = await this.getValue(key)
-    return parseInt(valueString, 10)
+    const valueInt = parseInt(valueString, 10)
+
+    if (isNaN(valueInt)) {
+      logger.error('Cannot get videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString)
+      return undefined
+    }
+
+    return valueInt
   }
 
   async getVideosIdViewed (hour: number) {
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 8b7f33539..2a99a665d 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -145,13 +145,13 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
 
       const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
 
-      const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file))
+      const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file))
       await rename(tmpPath, destPath)
 
       const createdModel = await VideoRedundancyModel.create({
         expiresOn: this.buildNewExpiration(redundancy.minLifetime),
         url: getVideoCacheFileActivityPubUrl(file),
-        fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL),
+        fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
         strategy: redundancy.strategy,
         videoFileId: file.id,
         actorId: serverActor.id
diff --git a/server/models/account/account.ts b/server/models/account/account.ts
index 5a237d733..a99e9b1ad 100644
--- a/server/models/account/account.ts
+++ b/server/models/account/account.ts
@@ -241,6 +241,27 @@ export class AccountModel extends Model<AccountModel> {
       })
   }
 
+  static listLocalsForSitemap (sort: string) {
+    const query = {
+      attributes: [ ],
+      offset: 0,
+      order: getSort(sort),
+      include: [
+        {
+          attributes: [ 'preferredUsername', 'serverId' ],
+          model: ActorModel.unscoped(),
+          where: {
+            serverId: null
+          }
+        }
+      ]
+    }
+
+    return AccountModel
+      .unscoped()
+      .findAll(query)
+  }
+
   toFormattedJSON (): Account {
     const actor = this.Actor.toFormattedJSON()
     const account = {
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index 9de4356b4..dd37dad22 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -15,7 +15,7 @@ import {
 import { ActorModel } from '../activitypub/actor'
 import { getVideoSort, throwIfNotValid } from '../utils'
 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
-import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers'
+import { CONFIG, CONSTRAINTS_FIELDS, STATIC_PATHS, VIDEO_EXT_MIMETYPE } from '../../initializers'
 import { VideoFileModel } from '../video/video-file'
 import { getServerActor } from '../../helpers/utils'
 import { VideoModel } from '../video/video'
@@ -124,7 +124,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
     const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
     logger.info('Removing duplicated video file %s.', logIdentifier)
 
-    videoFile.Video.removeFile(videoFile)
+    videoFile.Video.removeFile(videoFile, true)
              .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
 
     return undefined
diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts
index f4586917e..86bf0461a 100644
--- a/server/models/video/video-channel.ts
+++ b/server/models/video/video-channel.ts
@@ -233,6 +233,27 @@ export class VideoChannelModel extends Model<VideoChannelModel> {
       })
   }
 
+  static listLocalsForSitemap (sort: string) {
+    const query = {
+      attributes: [ ],
+      offset: 0,
+      order: getSort(sort),
+      include: [
+        {
+          attributes: [ 'preferredUsername', 'serverId' ],
+          model: ActorModel.unscoped(),
+          where: {
+            serverId: null
+          }
+        }
+      ]
+    }
+
+    return VideoChannelModel
+      .unscoped()
+      .findAll(query)
+  }
+
   static searchForApi (options: {
     actorId: number
     search: string
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 0f18d9f0c..adef37937 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -140,7 +140,7 @@ type ForAPIOptions = {
 
 type AvailableForListIDsOptions = {
   serverAccountId: number
-  actorId: number
+  followerActorId: number
   includeLocalVideos: boolean
   filter?: VideoFilter
   categoryOneOf?: number[]
@@ -315,7 +315,7 @@ type AvailableForListIDsOptions = {
       query.include.push(videoChannelInclude)
     }
 
-    if (options.actorId) {
+    if (options.followerActorId) {
       let localVideosReq = ''
       if (options.includeLocalVideos === true) {
         localVideosReq = ' UNION ALL ' +
@@ -327,7 +327,7 @@ type AvailableForListIDsOptions = {
       }
 
       // Force actorId to be a number to avoid SQL injections
-      const actorIdNumber = parseInt(options.actorId.toString(), 10)
+      const actorIdNumber = parseInt(options.followerActorId.toString(), 10)
       query.where[ 'id' ][ Sequelize.Op.and ].push({
         [ Sequelize.Op.in ]: Sequelize.literal(
           '(' +
@@ -985,7 +985,7 @@ export class VideoModel extends Model<VideoModel> {
     filter?: VideoFilter,
     accountId?: number,
     videoChannelId?: number,
-    actorId?: number
+    followerActorId?: number
     trendingDays?: number,
     user?: UserModel
   }, countVideos = true) {
@@ -1008,11 +1008,11 @@ export class VideoModel extends Model<VideoModel> {
 
     const serverActor = await getServerActor()
 
-    // actorId === null has a meaning, so just check undefined
-    const actorId = options.actorId !== undefined ? options.actorId : serverActor.id
+    // followerActorId === null has a meaning, so just check undefined
+    const followerActorId = options.followerActorId !== undefined ? options.followerActorId : serverActor.id
 
     const queryOptions = {
-      actorId,
+      followerActorId,
       serverAccountId: serverActor.Account.id,
       nsfw: options.nsfw,
       categoryOneOf: options.categoryOneOf,
@@ -1118,7 +1118,7 @@ export class VideoModel extends Model<VideoModel> {
 
     const serverActor = await getServerActor()
     const queryOptions = {
-      actorId: serverActor.id,
+      followerActorId: serverActor.id,
       serverAccountId: serverActor.Account.id,
       includeLocalVideos: options.includeLocalVideos,
       nsfw: options.nsfw,
@@ -1273,11 +1273,11 @@ export class VideoModel extends Model<VideoModel> {
   // threshold corresponds to how many video the field should have to be returned
   static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) {
     const serverActor = await getServerActor()
-    const actorId = serverActor.id
+    const followerActorId = serverActor.id
 
     const scopeOptions: AvailableForListIDsOptions = {
       serverAccountId: serverActor.Account.id,
-      actorId,
+      followerActorId,
       includeLocalVideos: true
     }
 
@@ -1538,8 +1538,10 @@ export class VideoModel extends Model<VideoModel> {
       .catch(err => logger.warn('Cannot delete preview %s.', previewPath, { err }))
   }
 
-  removeFile (videoFile: VideoFileModel) {
-    const filePath = join(CONFIG.STORAGE.VIDEOS_DIR, this.getVideoFilename(videoFile))
+  removeFile (videoFile: VideoFileModel, isRedundancy = false) {
+    const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR
+
+    const filePath = join(baseDir, this.getVideoFilename(videoFile))
     return remove(filePath)
       .catch(err => logger.warn('Cannot delete file %s.', filePath, { err }))
   }
@@ -1617,6 +1619,10 @@ export class VideoModel extends Model<VideoModel> {
     return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
   }
 
+  getVideoRedundancyUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+    return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getVideoFilename(videoFile)
+  }
+
   getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
     return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
   }
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index a8a2f305f..5b29a503a 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -136,7 +136,7 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st
   if (!videoUUID) videoUUID = video1Server2UUID
 
   const webseeds = [
-    'http://localhost:9001/static/webseed/' + videoUUID,
+    'http://localhost:9001/static/redundancy/' + videoUUID,
     'http://localhost:9002/static/webseed/' + videoUUID
   ]
 
@@ -148,20 +148,23 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st
     for (const file of video.files) {
       checkMagnetWebseeds(file, webseeds, server)
 
-      // Only servers 1 and 2 have the video
-      if (server.serverNumber !== 3) {
-        await makeGetRequest({
-          url: server.url,
-          statusCodeExpected: 200,
-          path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`,
-          contentType: null
-        })
-      }
+      await makeGetRequest({
+        url: servers[0].url,
+        statusCodeExpected: 200,
+        path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
+        contentType: null
+      })
+      await makeGetRequest({
+        url: servers[1].url,
+        statusCodeExpected: 200,
+        path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`,
+        contentType: null
+      })
     }
   }
 
-  for (const directory of [ 'test1', 'test2' ]) {
-    const files = await readdir(join(root(), directory, 'videos'))
+  for (const directory of [ 'test1/redundancy', 'test2/videos' ]) {
+    const files = await readdir(join(root(), directory))
     expect(files).to.have.length.at.least(4)
 
     for (const resolution of [ 240, 360, 480, 720 ]) {
diff --git a/server/tests/cli/index.ts b/server/tests/cli/index.ts
index 6201314ce..c6b7ec078 100644
--- a/server/tests/cli/index.ts
+++ b/server/tests/cli/index.ts
@@ -1,6 +1,7 @@
 // Order of the tests we want to execute
 import './create-import-video-file-job'
 import './create-transcoding-job'
+import './optimize-old-videos'
 import './peertube'
 import './reset-password'
 import './update-host'
diff --git a/server/tests/misc-endpoints.ts b/server/tests/misc-endpoints.ts
index 8fab20971..b53803ee1 100644
--- a/server/tests/misc-endpoints.ts
+++ b/server/tests/misc-endpoints.ts
@@ -2,7 +2,18 @@
 
 import 'mocha'
 import * as chai from 'chai'
-import { flushTests, killallServers, makeGetRequest, runServer, ServerInfo } from './utils'
+import {
+  addVideoChannel,
+  createUser,
+  flushTests,
+  killallServers,
+  makeGetRequest,
+  runServer,
+  ServerInfo,
+  setAccessTokensToServers,
+  uploadVideo
+} from './utils'
+import { VideoPrivacy } from '../../shared/models/videos'
 
 const expect = chai.expect
 
@@ -15,6 +26,7 @@ describe('Test misc endpoints', function () {
     await flushTests()
 
     server = await runServer(1)
+    await setAccessTokensToServers([ server ])
   })
 
   describe('Test a well known endpoints', function () {
@@ -93,6 +105,64 @@ describe('Test misc endpoints', function () {
     })
   })
 
+  describe('Test bots endpoints', function () {
+
+    it('Should get the empty sitemap', async function () {
+      const res = await makeGetRequest({
+        url: server.url,
+        path: '/sitemap.xml',
+        statusCodeExpected: 200
+      })
+
+      expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
+    })
+
+    it('Should get the empty cached sitemap', async function () {
+      const res = await makeGetRequest({
+        url: server.url,
+        path: '/sitemap.xml',
+        statusCodeExpected: 200
+      })
+
+      expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
+    })
+
+    it('Should add videos, channel and accounts and get sitemap', async function () {
+      this.timeout(35000)
+
+      await uploadVideo(server.url, server.accessToken, { name: 'video 1', nsfw: false })
+      await uploadVideo(server.url, server.accessToken, { name: 'video 2', nsfw: false })
+      await uploadVideo(server.url, server.accessToken, { name: 'video 3', privacy: VideoPrivacy.PRIVATE })
+
+      await addVideoChannel(server.url, server.accessToken, { name: 'channel1', displayName: 'channel 1' })
+      await addVideoChannel(server.url, server.accessToken, { name: 'channel2', displayName: 'channel 2' })
+
+      await createUser(server.url, server.accessToken, 'user1', 'password')
+      await createUser(server.url, server.accessToken, 'user2', 'password')
+
+      const res = await makeGetRequest({
+        url: server.url,
+        path: '/sitemap.xml?t=1', // avoid using cache
+        statusCodeExpected: 200
+      })
+
+      expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
+
+      expect(res.text).to.contain('<video:title><![CDATA[video 1]]></video:title>')
+      expect(res.text).to.contain('<video:title><![CDATA[video 2]]></video:title>')
+      expect(res.text).to.not.contain('<video:title><![CDATA[video 3]]></video:title>')
+
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/video-channels/channel1</loc></url>')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/video-channels/channel2</loc></url>')
+
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/accounts/user1</loc></url>')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/accounts/user2</loc></url>')
+    })
+  })
+
   after(async function () {
     killallServers([ server ])
   })