16 files changed, 123 insertions, 47 deletions

diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 5f90e4308..4265f3217 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -158,7 +158,11 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
 
   // Process video thumbnail from url if processing from request.files failed
   if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
-    thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
+    try {
+      thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
+    } catch (err) {
+      logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
+    }
   }
 
   // Process video preview from request.files
@@ -166,7 +170,11 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
 
   // Process video preview from url if processing from request.files failed
   if (!previewModel && youtubeDLInfo.thumbnailUrl) {
-    previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
+    try {
+      previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
+    } catch (err) {
+      logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
+    }
   }
 
   const videoImport = await insertIntoDB({
diff --git a/server/controllers/client.ts b/server/controllers/client.ts
index d81e35ec3..cdc556da2 100644
--- a/server/controllers/client.ts
+++ b/server/controllers/client.ts
@@ -5,12 +5,12 @@ import { join } from 'path'
 import { logger } from '@server/helpers/logger'
 import { CONFIG } from '@server/initializers/config'
 import { Hooks } from '@server/lib/plugins/hooks'
-import { HttpStatusCode } from '@shared/models'
 import { buildFileLocale, getCompleteLocale, is18nLocale, LOCALE_FILES } from '@shared/core-utils/i18n'
+import { HttpStatusCode } from '@shared/models'
 import { root } from '../helpers/core-utils'
 import { STATIC_MAX_AGE } from '../initializers/constants'
 import { ClientHtml, sendHTML, serveIndexHTML } from '../lib/client-html'
-import { asyncMiddleware, embedCSP } from '../middlewares'
+import { asyncMiddleware, disableRobots, embedCSP } from '../middlewares'
 
 const clientsRouter = express.Router()
 
@@ -81,6 +81,12 @@ clientsRouter.use('/client/*', (req: express.Request, res: express.Response) =>
   res.status(HttpStatusCode.NOT_FOUND_404).end()
 })
 
+// No index exceptions
+clientsRouter.all('/about/peertube',
+  disableRobots,
+  asyncMiddleware(serveIndexHTML)
+)
+
 // Always serve index client page (the client is a single page application, let it handle routing)
 // Try to provide the right language index.html
 clientsRouter.use('/(:language)?', asyncMiddleware(serveIndexHTML))
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index 1c04e4b11..fe1629910 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -69,6 +69,7 @@ staticRouter.get('/robots.txt',
   cacheRoute(ROUTE_CACHE_LIFETIME.ROBOTS),
   (_, res: express.Response) => {
     res.type('text/plain')
+
     return res.send(CONFIG.INSTANCE.ROBOTS)
   }
 )
diff --git a/server/helpers/regexp.ts b/server/helpers/regexp.ts
index cfc2be488..257054cea 100644
--- a/server/helpers/regexp.ts
+++ b/server/helpers/regexp.ts
@@ -4,7 +4,6 @@ function regexpCapture (str: string, regex: RegExp, maxIterations = 100) {
   let m: RegExpExecArray
   let i = 0
 
-  // tslint:disable:no-conditional-assignment
   while ((m = regex.exec(str)) !== null && i < maxIterations) {
     // This is necessary to avoid infinite loops with zero-width matches
     if (m.index === regex.lastIndex) {
diff --git a/server/initializers/migrations/0135-video-channel-actor.ts b/server/initializers/migrations/0135-video-channel-actor.ts
index 3f620dfa3..6989e1cbb 100644
--- a/server/initializers/migrations/0135-video-channel-actor.ts
+++ b/server/initializers/migrations/0135-video-channel-actor.ts
@@ -56,7 +56,6 @@ async function up (utils: {
   }
 
   {
-    // tslint:disable:no-trailing-whitespace
     const query1 =
       `
       INSERT INTO "actor"
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts
index 0613d94bf..9352a67d1 100644
--- a/server/lib/video-state.ts
+++ b/server/lib/video-state.ts
@@ -70,13 +70,13 @@ async function moveToPublishedState (video: MVideoFullLight, isNewVideo: boolean
   logger.info('Publishing video %s.', video.uuid, { tags: [ video.uuid ] })
 
   const previousState = video.state
-  await video.setNewState(VideoState.PUBLISHED, transaction)
+  await video.setNewState(VideoState.PUBLISHED, isNewVideo, transaction)
 
   // If the video was not published, we consider it is a new one for other instances
   // Live videos are always federated, so it's not a new video
   await federateVideoIfNeeded(video, isNewVideo, transaction)
 
-  Notifier.Instance.notifyOnNewVideoIfNeeded(video)
+  if (isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(video)
 
   if (previousState === VideoState.TO_TRANSCODE) {
     Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video)
@@ -90,7 +90,7 @@ async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: b
   // We want to wait all transcoding jobs before moving the video on an external storage
   if (pendingTranscode !== 0) return
 
-  await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, transaction)
+  await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, isNewVideo, transaction)
 
   logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] })
 
diff --git a/server/middlewares/index.ts b/server/middlewares/index.ts
index a0035f623..d2ed079b6 100644
--- a/server/middlewares/index.ts
+++ b/server/middlewares/index.ts
@@ -4,6 +4,7 @@ export * from './activitypub'
 export * from './async'
 export * from './auth'
 export * from './pagination'
+export * from './robots'
 export * from './servers'
 export * from './sort'
 export * from './user-right'
diff --git a/server/middlewares/robots.ts b/server/middlewares/robots.ts
new file mode 100644
index 000000000..b22b24a9f
--- /dev/null
+++ b/server/middlewares/robots.ts
@@ -0,0 +1,13 @@
+import express from 'express'
+
+function disableRobots (req: express.Request, res: express.Response, next: express.NextFunction) {
+  res.setHeader('X-Robots-Tag', 'noindex')
+
+  return next()
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  disableRobots
+}
diff --git a/server/middlewares/validators/oembed.ts b/server/middlewares/validators/oembed.ts
index 5e47211b5..96c8adc99 100644
--- a/server/middlewares/validators/oembed.ts
+++ b/server/middlewares/validators/oembed.ts
@@ -62,12 +62,26 @@ const oembedValidator = [
 
     const url = req.query.url as string
 
+    let urlPath: string
+
+    try {
+      urlPath = new URL(url).pathname
+    } catch (err) {
+      return res.fail({
+        status: HttpStatusCode.BAD_REQUEST_400,
+        message: err.message,
+        data: {
+          url
+        }
+      })
+    }
+
     const isPlaylist = startPlaylistURLs.some(u => url.startsWith(u))
     const isVideo = isPlaylist ? false : startVideoURLs.some(u => url.startsWith(u))
 
     const startIsOk = isVideo || isPlaylist
 
-    const matches = watchRegex.exec(url)
+    const matches = watchRegex.exec(urlPath)
 
     if (startIsOk === false || matches === null) {
       return res.fail({
diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts
index c04bd4355..8ccb818b3 100644
--- a/server/models/video/video-channel.ts
+++ b/server/models/video/video-channel.ts
@@ -121,7 +121,7 @@ export type SummaryOptions = {
       for (const handle of options.handles || []) {
         const [ preferredUsername, host ] = handle.split('@')
 
-        if (!host) {
+        if (!host || host === WEBSERVER.HOST) {
           or.push({
             '$Actor.preferredUsername$': preferredUsername,
             '$Actor.serverId$': null
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index e0c4dd2db..d2daf18ee 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -1766,12 +1766,12 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
       this.privacy === VideoPrivacy.INTERNAL
   }
 
-  async setNewState (newState: VideoState, transaction: Transaction) {
+  async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) {
     if (this.state === newState) throw new Error('Cannot use same state ' + newState)
 
     this.state = newState
 
-    if (this.state === VideoState.PUBLISHED) {
+    if (this.state === VideoState.PUBLISHED && isNewVideo) {
       this.publishedAt = new Date()
     }
 
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts
index ba952aff5..5cac3bc4e 100644
--- a/server/tests/api/live/live.ts
+++ b/server/tests/api/live/live.ts
@@ -439,7 +439,7 @@ describe('Test live', function () {
     })
 
     it('Should enable transcoding without additional resolutions', async function () {
-      this.timeout(60000)
+      this.timeout(120000)
 
       liveVideoId = await createLiveWrapper(false)
 
@@ -453,7 +453,7 @@ describe('Test live', function () {
     })
 
     it('Should enable transcoding with some resolutions', async function () {
-      this.timeout(60000)
+      this.timeout(120000)
 
       const resolutions = [ 240, 480 ]
       await updateConf(resolutions)
diff --git a/server/tests/api/search/search-channels.ts b/server/tests/api/search/search-channels.ts
index 8a01aff90..67612537c 100644
--- a/server/tests/api/search/search-channels.ts
+++ b/server/tests/api/search/search-channels.ts
@@ -129,6 +129,13 @@ describe('Test channels search', function () {
     }
 
     {
+      const body = await command.advancedChannelSearch({ search: { handles: [ 'squall_channel@' + server.host ] } })
+      expect(body.total).to.equal(1)
+      expect(body.data).to.have.lengthOf(1)
+      expect(body.data[0].displayName).to.equal('Squall channel')
+    }
+
+    {
       const body = await command.advancedChannelSearch({ search: { handles: [ 'chocobozzz_channel' ] } })
       expect(body.total).to.equal(0)
       expect(body.data).to.have.lengthOf(0)
diff --git a/server/tests/api/server/services.ts b/server/tests/api/server/services.ts
index 69d030dbb..3a87df981 100644
--- a/server/tests/api/server/services.ts
+++ b/server/tests/api/server/services.ts
@@ -52,42 +52,46 @@ describe('Test services', function () {
 
   it('Should have a valid oEmbed video response', async function () {
     for (const basePath of [ '/videos/watch/', '/w/' ]) {
-      const oembedUrl = 'http://localhost:' + server.port + basePath + video.uuid
-
-      const res = await server.services.getOEmbed({ oembedUrl })
-      const expectedHtml = '<iframe width="560" height="315" sandbox="allow-same-origin allow-scripts" ' +
-        `title="${video.name}" src="http://localhost:${server.port}/videos/embed/${video.uuid}" ` +
-        'frameborder="0" allowfullscreen></iframe>'
-      const expectedThumbnailUrl = 'http://localhost:' + server.port + video.previewPath
-
-      expect(res.body.html).to.equal(expectedHtml)
-      expect(res.body.title).to.equal(video.name)
-      expect(res.body.author_name).to.equal(server.store.channel.displayName)
-      expect(res.body.width).to.equal(560)
-      expect(res.body.height).to.equal(315)
-      expect(res.body.thumbnail_url).to.equal(expectedThumbnailUrl)
-      expect(res.body.thumbnail_width).to.equal(850)
-      expect(res.body.thumbnail_height).to.equal(480)
+      for (const suffix of [ '', '?param=1' ]) {
+        const oembedUrl = server.url + basePath + video.uuid + suffix
+
+        const res = await server.services.getOEmbed({ oembedUrl })
+        const expectedHtml = '<iframe width="560" height="315" sandbox="allow-same-origin allow-scripts" ' +
+          `title="${video.name}" src="http://localhost:${server.port}/videos/embed/${video.uuid}" ` +
+          'frameborder="0" allowfullscreen></iframe>'
+        const expectedThumbnailUrl = 'http://localhost:' + server.port + video.previewPath
+
+        expect(res.body.html).to.equal(expectedHtml)
+        expect(res.body.title).to.equal(video.name)
+        expect(res.body.author_name).to.equal(server.store.channel.displayName)
+        expect(res.body.width).to.equal(560)
+        expect(res.body.height).to.equal(315)
+        expect(res.body.thumbnail_url).to.equal(expectedThumbnailUrl)
+        expect(res.body.thumbnail_width).to.equal(850)
+        expect(res.body.thumbnail_height).to.equal(480)
+      }
     }
   })
 
   it('Should have a valid playlist oEmbed response', async function () {
     for (const basePath of [ '/videos/watch/playlist/', '/w/p/' ]) {
-      const oembedUrl = 'http://localhost:' + server.port + basePath + playlistUUID
-
-      const res = await server.services.getOEmbed({ oembedUrl })
-      const expectedHtml = '<iframe width="560" height="315" sandbox="allow-same-origin allow-scripts" ' +
-        `title="${playlistDisplayName}" src="http://localhost:${server.port}/video-playlists/embed/${playlistUUID}" ` +
-        'frameborder="0" allowfullscreen></iframe>'
-
-      expect(res.body.html).to.equal(expectedHtml)
-      expect(res.body.title).to.equal('The Life and Times of Scrooge McDuck')
-      expect(res.body.author_name).to.equal(server.store.channel.displayName)
-      expect(res.body.width).to.equal(560)
-      expect(res.body.height).to.equal(315)
-      expect(res.body.thumbnail_url).exist
-      expect(res.body.thumbnail_width).to.equal(280)
-      expect(res.body.thumbnail_height).to.equal(157)
+      for (const suffix of [ '', '?param=1' ]) {
+        const oembedUrl = server.url + basePath + playlistUUID + suffix
+
+        const res = await server.services.getOEmbed({ oembedUrl })
+        const expectedHtml = '<iframe width="560" height="315" sandbox="allow-same-origin allow-scripts" ' +
+          `title="${playlistDisplayName}" src="http://localhost:${server.port}/video-playlists/embed/${playlistUUID}" ` +
+          'frameborder="0" allowfullscreen></iframe>'
+
+        expect(res.body.html).to.equal(expectedHtml)
+        expect(res.body.title).to.equal('The Life and Times of Scrooge McDuck')
+        expect(res.body.author_name).to.equal(server.store.channel.displayName)
+        expect(res.body.width).to.equal(560)
+        expect(res.body.height).to.equal(315)
+        expect(res.body.thumbnail_url).exist
+        expect(res.body.thumbnail_width).to.equal(280)
+        expect(res.body.thumbnail_height).to.equal(157)
+      }
     }
   })
 
diff --git a/server/tests/cli/create-transcoding-job.ts b/server/tests/cli/create-transcoding-job.ts
index 3fd624091..2b388ab0c 100644
--- a/server/tests/cli/create-transcoding-job.ts
+++ b/server/tests/cli/create-transcoding-job.ts
@@ -33,9 +33,10 @@ async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent'
 function runTests (objectStorage: boolean) {
   let servers: PeerTubeServer[] = []
   const videosUUID: string[] = []
+  const publishedAt: string[] = []
 
   before(async function () {
-    this.timeout(60000)
+    this.timeout(120000)
 
     const config = objectStorage
       ? ObjectStorageCommand.getDefaultConfig()
@@ -54,6 +55,11 @@ function runTests (objectStorage: boolean) {
     for (let i = 1; i <= 5; i++) {
       const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'video' + i } })
 
+      await waitJobs(servers)
+
+      const video = await servers[0].videos.get({ id: uuid })
+      publishedAt.push(video.publishedAt as string)
+
       if (i > 2) {
         videosUUID.push(uuid)
       } else {
@@ -225,6 +231,14 @@ function runTests (objectStorage: boolean) {
     }
   })
 
+  it('Should not have updated published at attributes', async function () {
+    for (const id of videosUUID) {
+      const video = await servers[0].videos.get({ id })
+
+      expect(publishedAt.some(p => video.publishedAt === p)).to.be.true
+    }
+  })
+
   after(async function () {
     await cleanupTests(servers)
   })
diff --git a/server/tests/client.ts b/server/tests/client.ts
index a91bec906..6c32c81db 100644
--- a/server/tests/client.ts
+++ b/server/tests/client.ts
@@ -482,6 +482,16 @@ describe('Test a client controllers', function () {
         }
       }
     })
+
+    it('Should add noindex header for some paths', async function () {
+      const paths = [ '/about/peertube' ]
+
+      for (const path of paths) {
+        const { headers } = await makeHTMLRequest(servers[0].url, path)
+
+        expect(headers['x-robots-tag']).to.equal('noindex')
+      }
+    })
   })
 
   describe('Embed HTML', function () {