aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--package.json2
-rw-r--r--server.ts18
-rw-r--r--server/controllers/static.ts19
-rw-r--r--server/initializers/constants.ts1
-rw-r--r--server/middlewares/dnt.ts13
-rw-r--r--server/static/dnt-policy/dnt-policy-1.0.txt218
-rw-r--r--yarn.lock21
7 files changed, 287 insertions, 5 deletions
diff --git a/package.json b/package.json
index c7163d8bb..b5d695344 100644
--- a/package.json
+++ b/package.json
@@ -100,6 +100,7 @@
100 "flat": "^4.1.0", 100 "flat": "^4.1.0",
101 "fluent-ffmpeg": "^2.1.0", 101 "fluent-ffmpeg": "^2.1.0",
102 "helmet": "^3.12.1", 102 "helmet": "^3.12.1",
103 "ip-anonymize": "^0.0.6",
103 "ipaddr.js": "https://github.com/whitequark/ipaddr.js.git#8e69afeb4053ee32447a101845f860848280eca5", 104 "ipaddr.js": "https://github.com/whitequark/ipaddr.js.git#8e69afeb4053ee32447a101845f860848280eca5",
104 "is-cidr": "^2.0.5", 105 "is-cidr": "^2.0.5",
105 "iso-639-3": "^1.0.1", 106 "iso-639-3": "^1.0.1",
@@ -129,6 +130,7 @@
129 "sequelize-typescript": "0.6.6-beta.1", 130 "sequelize-typescript": "0.6.6-beta.1",
130 "sharp": "^0.20.0", 131 "sharp": "^0.20.0",
131 "srt-to-vtt": "^1.1.2", 132 "srt-to-vtt": "^1.1.2",
133 "useragent": "^2.3.0",
132 "uuid": "^3.1.0", 134 "uuid": "^3.1.0",
133 "validator": "^10.2.0", 135 "validator": "^10.2.0",
134 "webfinger.js": "^2.6.6", 136 "webfinger.js": "^2.6.6",
diff --git a/server.ts b/server.ts
index 9094ac943..eccd19919 100644
--- a/server.ts
+++ b/server.ts
@@ -16,6 +16,8 @@ import * as morgan from 'morgan'
16import * as cors from 'cors' 16import * as cors from 'cors'
17import * as cookieParser from 'cookie-parser' 17import * as cookieParser from 'cookie-parser'
18import * as helmet from 'helmet' 18import * as helmet from 'helmet'
19import * as useragent from 'useragent'
20import * as anonymise from 'ip-anonymize'
19 21
20process.title = 'peertube' 22process.title = 'peertube'
21 23
@@ -27,7 +29,7 @@ import { checkMissedConfig, checkFFmpeg, checkConfig, checkActivityPubUrls } fro
27 29
28// Do not use barrels because we don't want to load all modules here (we need to initialize database first) 30// Do not use barrels because we don't want to load all modules here (we need to initialize database first)
29import { logger } from './server/helpers/logger' 31import { logger } from './server/helpers/logger'
30import { API_VERSION, CONFIG, STATIC_PATHS, CACHE, REMOTE_SCHEME } from './server/initializers/constants' 32import { API_VERSION, CONFIG, CACHE } from './server/initializers/constants'
31 33
32const missed = checkMissedConfig() 34const missed = checkMissedConfig()
33if (missed.length !== 0) { 35if (missed.length !== 0) {
@@ -85,6 +87,7 @@ import {
85 trackerRouter, 87 trackerRouter,
86 createWebsocketServer 88 createWebsocketServer
87} from './server/controllers' 89} from './server/controllers'
90import { advertiseDoNotTrack } from './server/middlewares/dnt'
88import { Redis } from './server/lib/redis' 91import { Redis } from './server/lib/redis'
89import { BadActorFollowScheduler } from './server/lib/schedulers/bad-actor-follow-scheduler' 92import { BadActorFollowScheduler } from './server/lib/schedulers/bad-actor-follow-scheduler'
90import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler' 93import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler'
@@ -103,8 +106,17 @@ if (isTestInstance()) {
103 credentials: true 106 credentials: true
104 })) 107 }))
105} 108}
106
107// For the logger 109// For the logger
110morgan.token('remote-addr', req => {
111 return (req.get('DNT') === '1') ?
112 anonymise(req.ip || (req.connection && req.connection.remoteAddress) || undefined,
113 16, // bitmask for IPv4
114 16 // bitmask for IPv6
115 ) :
116 req.ip
117})
118morgan.token('user-agent', req => (req.get('DNT') === '1') ?
119 useragent.parse(req.get('user-agent')).family : req.get('user-agent'))
108app.use(morgan('combined', { 120app.use(morgan('combined', {
109 stream: { write: logger.info.bind(logger) } 121 stream: { write: logger.info.bind(logger) }
110})) 122}))
@@ -116,6 +128,8 @@ app.use(bodyParser.json({
116})) 128}))
117// Cookies 129// Cookies
118app.use(cookieParser()) 130app.use(cookieParser())
131// W3C DNT Tracking Status
132app.use(advertiseDoNotTrack)
119 133
120// ----------- Views, routes and static files ----------- 134// ----------- Views, routes and static files -----------
121 135
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index 3ccf624a7..2a92810f3 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -1,4 +1,5 @@
1import * as cors from 'cors' 1import * as cors from 'cors'
2import { createReadStream } from 'fs'
2import * as express from 'express' 3import * as express from 'express'
3import { CONFIG, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS, ROUTE_CACHE_LIFETIME } from '../initializers' 4import { CONFIG, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS, ROUTE_CACHE_LIFETIME } from '../initializers'
4import { VideosPreviewCache } from '../lib/cache' 5import { VideosPreviewCache } from '../lib/cache'
@@ -93,10 +94,26 @@ staticRouter.use('/.well-known/nodeinfo',
93 } 94 }
94) 95)
95staticRouter.use('/nodeinfo/:version.json', 96staticRouter.use('/nodeinfo/:version.json',
96 // asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.NODEINFO)), 97 asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.NODEINFO)),
97 asyncMiddleware(generateNodeinfo) 98 asyncMiddleware(generateNodeinfo)
98) 99)
99 100
101// dnt-policy.txt service (see https://www.eff.org/dnt-policy)
102staticRouter.use('/.well-known/dnt-policy.txt',
103 asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.DNT_POLICY)),
104 (_, res: express.Response) => {
105 res.type('text/plain')
106 createReadStream('./server/static/dnt-policy/dnt-policy-1.0.txt').pipe(res)
107 }
108)
109
110// dnt service (see https://www.w3.org/TR/tracking-dnt/#status-resource)
111staticRouter.use('/.well-known/dnt/',
112 (_, res: express.Response) => {
113 res.json({ tracking: 'N' })
114 }
115)
116
100// --------------------------------------------------------------------------- 117// ---------------------------------------------------------------------------
101 118
102export { 119export {
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 069d9b2e8..74fe7965d 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -55,6 +55,7 @@ const ROUTE_CACHE_LIFETIME = {
55 FEEDS: '15 minutes', 55 FEEDS: '15 minutes',
56 ROBOTS: '2 hours', 56 ROBOTS: '2 hours',
57 NODEINFO: '10 minutes', 57 NODEINFO: '10 minutes',
58 DNT_POLICY: '1 week',
58 ACTIVITY_PUB: { 59 ACTIVITY_PUB: {
59 VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example 60 VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
60 } 61 }
diff --git a/server/middlewares/dnt.ts b/server/middlewares/dnt.ts
new file mode 100644
index 000000000..cabad39c6
--- /dev/null
+++ b/server/middlewares/dnt.ts
@@ -0,0 +1,13 @@
1import * as ipaddr from 'ipaddr.js'
2import { format } from 'util'
3
4const advertiseDoNotTrack = (_, res, next) => {
5 res.setHeader('Tk', 'N')
6 return next()
7}
8
9// ---------------------------------------------------------------------------
10
11export {
12 advertiseDoNotTrack
13 }
diff --git a/server/static/dnt-policy/dnt-policy-1.0.txt b/server/static/dnt-policy/dnt-policy-1.0.txt
new file mode 100644
index 000000000..ad946d1f8
--- /dev/null
+++ b/server/static/dnt-policy/dnt-policy-1.0.txt
@@ -0,0 +1,218 @@
1Do Not Track Compliance Policy
2
3Version 1.0
4
5This domain complies with user opt-outs from tracking via the "Do Not Track"
6or "DNT" header [http://www.w3.org/TR/tracking-dnt/]. This file will always
7be posted via HTTPS at https://example-domain.com/.well-known/dnt-policy.txt
8to indicate this fact.
9
10SCOPE
11
12This policy document allows an operator of a Fully Qualified Domain Name
13("domain") to declare that it respects Do Not Track as a meaningful privacy
14opt-out of tracking, so that privacy-protecting software can better determine
15whether to block or anonymize communications with this domain. This policy is
16intended first and foremost to be posted on domains that publish ads, widgets,
17images, scripts and other third-party embedded hypertext (for instance on
18widgets.example.com), but it can be posted on any domain, including those users
19visit directly (such as www.example.com). The policy may be applied to some
20domains used by a company, site, or service, and not to others. Do Not Track
21may be sent by any client that uses the HTTP protocol, including websites,
22mobile apps, and smart devices like TVs. Do Not Track also works with all
23protocols able to read HTTP headers, including SPDY.
24
25NOTE: This policy contains both Requirements and Exceptions. Where possible
26terms are defined in the text, but a few additional definitions are included
27at the end.
28
29REQUIREMENTS
30
31When this domain receives Web requests from a user who enables DNT by actively
32choosing an opt-out setting in their browser or by installing software that is
33primarily designed to protect privacy ("DNT User"), we will take the following
34measures with respect to those users' data, subject to the Exceptions, also
35listed below:
36
371. END USER IDENTIFIERS:
38
39 a. If a DNT User has logged in to our service, all user identifiers, such as
40 unique or nearly unique cookies, "supercookies" and fingerprints are
41 discarded as soon as the HTTP(S) response is issued.
42
43 Data structures which associate user identifiers with accounts may be
44 employed to recognize logged in users per Exception 4 below, but may not
45 be associated with records of the user's activities unless otherwise
46 excepted.
47
48 b. If a DNT User is not logged in to our service, we will take steps to ensure
49 that no user identifiers are transmitted to us at all.
50
512. LOG RETENTION:
52
53 a. Logs with DNT Users' identifiers removed (but including IP addresses and
54 User Agent strings) may be retained for a period of 10 days or less,
55 unless an Exception (below) applies. This period of time balances privacy
56 concerns with the need to ensure that log processing systems have time to
57 operate; that operations engineers have time to monitor and fix technical
58 and performance problems; and that security and data aggregation systems
59 have time to operate.
60
61 b. These logs will not be used for any other purposes.
62
633. OTHER DOMAINS:
64
65 a. If this domain transfers identifiable user data about DNT Users to
66 contractors, affiliates or other parties, or embeds from or posts data to
67 other domains, we will either:
68
69 b. ensure that the operators of those domains abide by this policy overall
70 by posting it at /.well-known/dnt-policy.txt via HTTPS on the domains in
71 question,
72
73 OR
74
75 ensure that the recipient's policies and practices require the recipient
76 to respect the policy for our DNT Users' data.
77
78 OR
79
80 obtain a contractual commitment from the recipient to respect this policy
81 for our DNT Users' data.
82
83 NOTE: if an “Other Domain” does not receive identifiable user information
84 from the domain because such information has been removed, because the
85 Other Domain does not log that information, or for some other reason, these
86 requirements do not apply.
87
88 c. "Identifiable" means any records which are not Anonymized or otherwise
89 covered by the Exceptions below.
90
914. PERIODIC REASSERTION OF COMPLIANCE:
92
93 At least once every 12 months, we will take reasonable steps commensurate
94 with the size of our organization and the nature of our service to confirm
95 our ongoing compliance with this document, and we will publicly reassert our
96 compliance.
97
985. USER NOTIFICATION:
99
100 a. If we are required by law to retain or disclose user identifiers, we will
101 attempt to provide the users with notice (unless we are prohibited or it
102 would be futile) that a request for their information has been made in
103 order to give the users an opportunity to object to the retention or
104 disclosure.
105
106 b. We will attempt to provide this notice by email, if the users have given
107 us an email address, and by postal mail if the users have provided a
108 postal address.
109
110 c. If the users do not challenge the disclosure request, we may be legally
111 required to turn over their information.
112
113 d. We may delay notice if we, in good faith, believe that an emergency
114 involving danger of death or serious physical injury to any person
115 requires disclosure without delay of information relating to the
116 emergency.
117
118EXCEPTIONS
119
120Data from DNT Users collected by this domain may be logged or retained only in
121the following specific situations:
122
1231. CONSENT / "OPT BACK IN"
124
125 a. DNT Users are opting out from tracking across the Web. It is possible
126 that for some feature or functionality, we will need to ask a DNT User to
127 "opt back in" to be tracked by us across the entire Web.
128
129 b. If we do that, we will take reasonable steps to verify that the users who
130 select this option have genuinely intended to opt back in to tracking.
131 One way to do this is by performing scientifically reasonable user
132 studies with a representative sample of our users, but smaller
133 organizations can satisfy this requirement by other means.
134
135 c. Where we believe that we have opt back in consent, our server will
136 send a tracking value status header "Tk: C" as described in section 6.2
137 of the W3C Tracking Preference Expression draft:
138
139 http://www.w3.org/TR/tracking-dnt/#tracking-status-value
140
1412. TRANSACTIONS
142
143 If a DNT User actively and knowingly enters a transaction with our
144 services (for instance, clicking on a clearly-labeled advertisement,
145 posting content to a widget, or purchasing an item), we will retain
146 necessary data for as long as required to perform the transaction. This
147 may for example include keeping auditing information for clicks on
148 advertising links; keeping a copy of posted content and the name of the
149 posting user; keeping server-side session IDs to recognize logged in
150 users; or keeping a copy of the physical address to which a purchased
151 item will be shipped. By their nature, some transactions will require data
152 to be retained indefinitely.
153
1543. TECHNICAL AND SECURITY LOGGING:
155
156 a. If, during the processing of the initial request (for unique identifiers)
157 or during the subsequent 10 days (for IP addresses and User Agent strings),
158 we obtain specific information that causes our employees or systems to
159 believe that a request is, or is likely to be, part of a security attack,
160 spam submission, or fraudulent transaction, then logs of those requests
161 are not subject to this policy.
162
163 b. If we encounter technical problems with our site, then, in rare
164 circumstances, we may retain logs for longer than 10 days, if that is
165 necessary to diagnose and fix those problems, but this practice will not be
166 routinized and we will strive to delete such logs as soon as possible.
167
1684. AGGREGATION:
169
170 a. We may retain and share anonymized datasets, such as aggregate records of
171 readership patterns; statistical models of user behavior; graphs of system
172 variables; data structures to count active users on monthly or yearly
173 bases; database tables mapping authentication cookies to logged in
174 accounts; non-unique data structures constructed within browsers for tasks
175 such as ad frequency capping or conversion tracking; or logs with truncated
176 and/or encrypted IP addresses and simplified User Agent strings.
177
178 b. "Anonymized" means we have conducted risk mitigation to ensure
179 that the dataset, plus any additional information that is in our
180 possession or likely to be available to us, does not allow the
181 reconstruction of reading habits, online or offline activity of groups of
182 fewer than 5000 individuals or devices.
183
184 c. If we generate anonymized datasets under this exception we will publicly
185 document our anonymization methods in sufficient detail to allow outside
186 experts to evaluate the effectiveness of those methods.
187
1885. ERRORS:
189
190From time to time, there may be errors by which user data is temporarily
191logged or retained in violation of this policy. If such errors are
192inadvertent, rare, and made in good faith, they do not constitute a breach
193of this policy. We will delete such data as soon as practicable after we
194become aware of any error and take steps to ensure that it is deleted by any
195third-party who may have had access to the data.
196
197ADDITIONAL DEFINITIONS
198
199"Fully Qualified Domain Name" means a domain name that addresses a computer
200connected to the Internet. For instance, example1.com; www.example1.com;
201ads.example1.com; and widgets.example2.com are all distinct FQDNs.
202
203"Supercookie" means any technology other than an HTTP Cookie which can be used
204by a server to associate identifiers with the clients that visit it. Examples
205of supercookies include Flash LSO cookies, DOM storage, HTML5 storage, or
206tricks to store information in caches or etags.
207
208"Risk mitigation" means an engineering process that evaluates the possibility
209and likelihood of various adverse outcomes, considers the available methods of
210making those adverse outcomes less likely, and deploys sufficient mitigations
211to bring the probability and harm from adverse outcomes below an acceptable
212threshold.
213
214"Reading habits" includes amongst other things lists of visited DNS names, if
215those domains pertain to specific topics or activities, but records of visited
216DNS names are not reading habits if those domain names serve content of a very
217diverse and general nature, thereby revealing minimal information about the
218opinions, interests or activities of the user.
diff --git a/yarn.lock b/yarn.lock
index ec261e025..206700a87 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3531,6 +3531,10 @@ ioredis@^3.1.4:
3531 redis-commands "^1.2.0" 3531 redis-commands "^1.2.0"
3532 redis-parser "^2.4.0" 3532 redis-parser "^2.4.0"
3533 3533
3534ip-anonymize@^0.0.6:
3535 version "0.0.6"
3536 resolved "https://registry.yarnpkg.com/ip-anonymize/-/ip-anonymize-0.0.6.tgz#d2c513e448e874e8cc380d03404691b94b018e68"
3537
3534ip-regex@^2.1.0: 3538ip-regex@^2.1.0:
3535 version "2.1.0" 3539 version "2.1.0"
3536 resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" 3540 resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9"
@@ -4492,7 +4496,7 @@ lowercase-keys@^1.0.0:
4492 version "1.0.1" 4496 version "1.0.1"
4493 resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f" 4497 resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f"
4494 4498
4495lru-cache@^4.0.1: 4499lru-cache@4.1.x, lru-cache@^4.0.1:
4496 version "4.1.3" 4500 version "4.1.3"
4497 resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.3.tgz#a1175cf3496dfc8436c156c334b4955992bce69c" 4501 resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.3.tgz#a1175cf3496dfc8436c156c334b4955992bce69c"
4498 dependencies: 4502 dependencies:
@@ -5256,7 +5260,7 @@ os-locale@^1.4.0:
5256 dependencies: 5260 dependencies:
5257 lcid "^1.0.0" 5261 lcid "^1.0.0"
5258 5262
5259os-tmpdir@^1.0.0, os-tmpdir@^1.0.1, os-tmpdir@~1.0.1: 5263os-tmpdir@^1.0.0, os-tmpdir@^1.0.1, os-tmpdir@~1.0.1, os-tmpdir@~1.0.2:
5260 version "1.0.2" 5264 version "1.0.2"
5261 resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" 5265 resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
5262 5266
@@ -7201,6 +7205,12 @@ tmp@0.0.31:
7201 dependencies: 7205 dependencies:
7202 os-tmpdir "~1.0.1" 7206 os-tmpdir "~1.0.1"
7203 7207
7208tmp@0.0.x:
7209 version "0.0.33"
7210 resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9"
7211 dependencies:
7212 os-tmpdir "~1.0.2"
7213
7204to-array@0.1.4: 7214to-array@0.1.4:
7205 version "0.1.4" 7215 version "0.1.4"
7206 resolved "https://registry.yarnpkg.com/to-array/-/to-array-0.1.4.tgz#17e6c11f73dd4f3d74cda7a4ff3238e9ad9bf890" 7216 resolved "https://registry.yarnpkg.com/to-array/-/to-array-0.1.4.tgz#17e6c11f73dd4f3d74cda7a4ff3238e9ad9bf890"
@@ -7564,6 +7574,13 @@ user-home@^2.0.0:
7564 dependencies: 7574 dependencies:
7565 os-homedir "^1.0.0" 7575 os-homedir "^1.0.0"
7566 7576
7577useragent@^2.3.0:
7578 version "2.3.0"
7579 resolved "https://registry.yarnpkg.com/useragent/-/useragent-2.3.0.tgz#217f943ad540cb2128658ab23fc960f6a88c9972"
7580 dependencies:
7581 lru-cache "4.1.x"
7582 tmp "0.0.x"
7583
7567ut_metadata@^3.0.8: 7584ut_metadata@^3.0.8:
7568 version "3.2.2" 7585 version "3.2.2"
7569 resolved "https://registry.yarnpkg.com/ut_metadata/-/ut_metadata-3.2.2.tgz#189baf4ec690111ec242d7dfd954fa17242d4d70" 7586 resolved "https://registry.yarnpkg.com/ut_metadata/-/ut_metadata-3.2.2.tgz#189baf4ec690111ec242d7dfd954fa17242d4d70"