aboutsummaryrefslogtreecommitdiffhomepage
path: root/docs/client.html
diff options
context:
space:
mode:
Diffstat (limited to 'docs/client.html')
-rw-r--r--docs/client.html1201
1 files changed, 1201 insertions, 0 deletions
diff --git a/docs/client.html b/docs/client.html
new file mode 100644
index 0000000..4e69b89
--- /dev/null
+++ b/docs/client.html
@@ -0,0 +1,1201 @@
1<!DOCTYPE html>
2<html>
3<head>
4 <meta http-equiv="content-type" content="text/html;charset=utf-8">
5 <title>client.py</title>
6 <link rel="stylesheet" href="pycco.css">
7</head>
8<body>
9<div id='container'>
10 <div id="background"></div>
11 <div class='section'>
12 <div class='docs'><h1>client.py</h1></div>
13 </div>
14 <div class='clearall'>
15 <div class='section' id='section-0'>
16 <div class='docs'>
17 <div class='octowrap'>
18 <a class='octothorpe' href='#section-0'>#</a>
19 </div>
20
21 </div>
22 <div class='code'>
23 <div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
24<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
25<span class="kn">import</span> <span class="nn">backoff</span>
26<span class="kn">import</span> <span class="nn">requests</span>
27<span class="kn">import</span> <span class="nn">singer</span>
28<span class="kn">from</span> <span class="nn">singer</span> <span class="kn">import</span> <span class="n">metrics</span>
29<span class="kn">from</span> <span class="nn">singer</span> <span class="kn">import</span> <span class="n">utils</span>
30
31<span class="n">BASE_URL</span> <span class="o">=</span> <span class="s1">&#39;https://www.googleapis.com&#39;</span>
32<span class="n">GOOGLE_TOKEN_URI</span> <span class="o">=</span> <span class="s1">&#39;https://oauth2.googleapis.com/token&#39;</span>
33<span class="n">LOGGER</span> <span class="o">=</span> <span class="n">singer</span><span class="o">.</span><span class="n">get_logger</span><span class="p">()</span></pre></div>
34 </div>
35 </div>
36 <div class='clearall'></div>
37 <div class='section' id='section-1'>
38 <div class='docs'>
39 <div class='octowrap'>
40 <a class='octothorpe' href='#section-1'>#</a>
41 </div>
42
43 </div>
44 <div class='code'>
45 <div class="highlight"><pre><span class="k">class</span> <span class="nc">Server5xxError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
46 <span class="k">pass</span></pre></div>
47 </div>
48 </div>
49 <div class='clearall'></div>
50 <div class='section' id='section-2'>
51 <div class='docs'>
52 <div class='octowrap'>
53 <a class='octothorpe' href='#section-2'>#</a>
54 </div>
55
56 </div>
57 <div class='code'>
58 <div class="highlight"><pre><span class="k">class</span> <span class="nc">Server429Error</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
59 <span class="k">pass</span></pre></div>
60 </div>
61 </div>
62 <div class='clearall'></div>
63 <div class='section' id='section-3'>
64 <div class='docs'>
65 <div class='octowrap'>
66 <a class='octothorpe' href='#section-3'>#</a>
67 </div>
68
69 </div>
70 <div class='code'>
71 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
72 <span class="k">pass</span></pre></div>
73 </div>
74 </div>
75 <div class='clearall'></div>
76 <div class='section' id='section-4'>
77 <div class='docs'>
78 <div class='octowrap'>
79 <a class='octothorpe' href='#section-4'>#</a>
80 </div>
81
82 </div>
83 <div class='code'>
84 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleBadRequestError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
85 <span class="k">pass</span></pre></div>
86 </div>
87 </div>
88 <div class='clearall'></div>
89 <div class='section' id='section-5'>
90 <div class='docs'>
91 <div class='octowrap'>
92 <a class='octothorpe' href='#section-5'>#</a>
93 </div>
94
95 </div>
96 <div class='code'>
97 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleUnauthorizedError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
98 <span class="k">pass</span></pre></div>
99 </div>
100 </div>
101 <div class='clearall'></div>
102 <div class='section' id='section-6'>
103 <div class='docs'>
104 <div class='octowrap'>
105 <a class='octothorpe' href='#section-6'>#</a>
106 </div>
107
108 </div>
109 <div class='code'>
110 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GooglePaymentRequiredError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
111 <span class="k">pass</span></pre></div>
112 </div>
113 </div>
114 <div class='clearall'></div>
115 <div class='section' id='section-7'>
116 <div class='docs'>
117 <div class='octowrap'>
118 <a class='octothorpe' href='#section-7'>#</a>
119 </div>
120
121 </div>
122 <div class='code'>
123 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleNotFoundError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
124 <span class="k">pass</span></pre></div>
125 </div>
126 </div>
127 <div class='clearall'></div>
128 <div class='section' id='section-8'>
129 <div class='docs'>
130 <div class='octowrap'>
131 <a class='octothorpe' href='#section-8'>#</a>
132 </div>
133
134 </div>
135 <div class='code'>
136 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleMethodNotAllowedError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
137 <span class="k">pass</span></pre></div>
138 </div>
139 </div>
140 <div class='clearall'></div>
141 <div class='section' id='section-9'>
142 <div class='docs'>
143 <div class='octowrap'>
144 <a class='octothorpe' href='#section-9'>#</a>
145 </div>
146
147 </div>
148 <div class='code'>
149 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleConflictError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
150 <span class="k">pass</span></pre></div>
151 </div>
152 </div>
153 <div class='clearall'></div>
154 <div class='section' id='section-10'>
155 <div class='docs'>
156 <div class='octowrap'>
157 <a class='octothorpe' href='#section-10'>#</a>
158 </div>
159
160 </div>
161 <div class='code'>
162 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleGoneError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
163 <span class="k">pass</span></pre></div>
164 </div>
165 </div>
166 <div class='clearall'></div>
167 <div class='section' id='section-11'>
168 <div class='docs'>
169 <div class='octowrap'>
170 <a class='octothorpe' href='#section-11'>#</a>
171 </div>
172
173 </div>
174 <div class='code'>
175 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GooglePreconditionFailedError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
176 <span class="k">pass</span></pre></div>
177 </div>
178 </div>
179 <div class='clearall'></div>
180 <div class='section' id='section-12'>
181 <div class='docs'>
182 <div class='octowrap'>
183 <a class='octothorpe' href='#section-12'>#</a>
184 </div>
185
186 </div>
187 <div class='code'>
188 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleRequestEntityTooLargeError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
189 <span class="k">pass</span></pre></div>
190 </div>
191 </div>
192 <div class='clearall'></div>
193 <div class='section' id='section-13'>
194 <div class='docs'>
195 <div class='octowrap'>
196 <a class='octothorpe' href='#section-13'>#</a>
197 </div>
198
199 </div>
200 <div class='code'>
201 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleRequestedRangeNotSatisfiableError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
202 <span class="k">pass</span></pre></div>
203 </div>
204 </div>
205 <div class='clearall'></div>
206 <div class='section' id='section-14'>
207 <div class='docs'>
208 <div class='octowrap'>
209 <a class='octothorpe' href='#section-14'>#</a>
210 </div>
211
212 </div>
213 <div class='code'>
214 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleExpectationFailedError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
215 <span class="k">pass</span></pre></div>
216 </div>
217 </div>
218 <div class='clearall'></div>
219 <div class='section' id='section-15'>
220 <div class='docs'>
221 <div class='octowrap'>
222 <a class='octothorpe' href='#section-15'>#</a>
223 </div>
224
225 </div>
226 <div class='code'>
227 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleForbiddenError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
228 <span class="k">pass</span></pre></div>
229 </div>
230 </div>
231 <div class='clearall'></div>
232 <div class='section' id='section-16'>
233 <div class='docs'>
234 <div class='octowrap'>
235 <a class='octothorpe' href='#section-16'>#</a>
236 </div>
237
238 </div>
239 <div class='code'>
240 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleUnprocessableEntityError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
241 <span class="k">pass</span></pre></div>
242 </div>
243 </div>
244 <div class='clearall'></div>
245 <div class='section' id='section-17'>
246 <div class='docs'>
247 <div class='octowrap'>
248 <a class='octothorpe' href='#section-17'>#</a>
249 </div>
250
251 </div>
252 <div class='code'>
253 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GooglePreconditionRequiredError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
254 <span class="k">pass</span></pre></div>
255 </div>
256 </div>
257 <div class='clearall'></div>
258 <div class='section' id='section-18'>
259 <div class='docs'>
260 <div class='octowrap'>
261 <a class='octothorpe' href='#section-18'>#</a>
262 </div>
263
264 </div>
265 <div class='code'>
266 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleInternalServiceError</span><span class="p">(</span><span class="n">GoogleError</span><span class="p">):</span>
267 <span class="k">pass</span></pre></div>
268 </div>
269 </div>
270 <div class='clearall'></div>
271 <div class='section' id='section-19'>
272 <div class='docs'>
273 <div class='octowrap'>
274 <a class='octothorpe' href='#section-19'>#</a>
275 </div>
276 <p>Error Codes: https://developers.google.com/webmaster-tools/search-console-api-original/v3/errors</p>
277 </div>
278 <div class='code'>
279 <div class="highlight"><pre><span class="n">ERROR_CODE_EXCEPTION_MAPPING</span> <span class="o">=</span> <span class="p">{</span>
280 <span class="mi">400</span><span class="p">:</span> <span class="n">GoogleBadRequestError</span><span class="p">,</span>
281 <span class="mi">401</span><span class="p">:</span> <span class="n">GoogleUnauthorizedError</span><span class="p">,</span>
282 <span class="mi">402</span><span class="p">:</span> <span class="n">GooglePaymentRequiredError</span><span class="p">,</span>
283 <span class="mi">403</span><span class="p">:</span> <span class="n">GoogleForbiddenError</span><span class="p">,</span>
284 <span class="mi">404</span><span class="p">:</span> <span class="n">GoogleNotFoundError</span><span class="p">,</span>
285 <span class="mi">405</span><span class="p">:</span> <span class="n">GoogleMethodNotAllowedError</span><span class="p">,</span>
286 <span class="mi">409</span><span class="p">:</span> <span class="n">GoogleConflictError</span><span class="p">,</span>
287 <span class="mi">410</span><span class="p">:</span> <span class="n">GoogleGoneError</span><span class="p">,</span>
288 <span class="mi">412</span><span class="p">:</span> <span class="n">GooglePreconditionFailedError</span><span class="p">,</span>
289 <span class="mi">413</span><span class="p">:</span> <span class="n">GoogleRequestEntityTooLargeError</span><span class="p">,</span>
290 <span class="mi">416</span><span class="p">:</span> <span class="n">GoogleRequestedRangeNotSatisfiableError</span><span class="p">,</span>
291 <span class="mi">417</span><span class="p">:</span> <span class="n">GoogleExpectationFailedError</span><span class="p">,</span>
292 <span class="mi">422</span><span class="p">:</span> <span class="n">GoogleUnprocessableEntityError</span><span class="p">,</span>
293 <span class="mi">428</span><span class="p">:</span> <span class="n">GooglePreconditionRequiredError</span><span class="p">,</span>
294 <span class="mi">500</span><span class="p">:</span> <span class="n">GoogleInternalServiceError</span><span class="p">}</span></pre></div>
295 </div>
296 </div>
297 <div class='clearall'></div>
298 <div class='section' id='section-20'>
299 <div class='docs'>
300 <div class='octowrap'>
301 <a class='octothorpe' href='#section-20'>#</a>
302 </div>
303
304 </div>
305 <div class='code'>
306 <div class="highlight"><pre><span class="k">def</span> <span class="nf">get_exception_for_error_code</span><span class="p">(</span><span class="n">error_code</span><span class="p">):</span>
307 <span class="k">return</span> <span class="n">ERROR_CODE_EXCEPTION_MAPPING</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">error_code</span><span class="p">,</span> <span class="n">GoogleError</span><span class="p">)</span></pre></div>
308 </div>
309 </div>
310 <div class='clearall'></div>
311 <div class='section' id='section-21'>
312 <div class='docs'>
313 <div class='octowrap'>
314 <a class='octothorpe' href='#section-21'>#</a>
315 </div>
316 <hr />
317 </div>
318 <div class='code'>
319 <div class="highlight"><pre></pre></div>
320 </div>
321 </div>
322 <div class='clearall'></div>
323 <div class='section' id='section-22'>
324 <div class='docs'>
325 <div class='octowrap'>
326 <a class='octothorpe' href='#section-22'>#</a>
327 </div>
328 <p><code>client.py:raise_for_error()</code> calls the <code>raise_for_status()</code> function from the <code>requests</code> library.
329and catches all <code>requests.HTTPError</code> and <code>requests.ConnectionError</code>. Note the name difference.</p>
330 </div>
331 <div class='code'>
332 <div class="highlight"><pre></pre></div>
333 </div>
334 </div>
335 <div class='clearall'></div>
336 <div class='section' id='section-23'>
337 <div class='docs'>
338 <div class='octowrap'>
339 <a class='octothorpe' href='#section-23'>#</a>
340 </div>
341 <h5>Thoughts</h5>
342<p>I believe there are 5 ways to leave this function. It&rsquo;s worth skimming this just to understand the
343structure. I&rsquo;ll note below, but I think there&rsquo;s just two ways to leave this function.</p>
344 </div>
345 <div class='code'>
346 <div class="highlight"><pre></pre></div>
347 </div>
348 </div>
349 <div class='clearall'></div>
350 <div class='section' id='section-24'>
351 <div class='docs'>
352 <div class='octowrap'>
353 <a class='octothorpe' href='#section-24'>#</a>
354 </div>
355 <ol>
356<li>If the length of the response content is 0, then we just leave<ul>
357<li>I believe this results in us swallowing the <code>requests.HTTPError</code> and successfully returns to
358the calling function</li>
359<li>I believe it&rsquo;s possible to leave the function this way</li>
360</ul>
361</li>
362<li>If you can call <code>response.json()</code>, then we attempt to create a specific error message via
363 <code>client.py:get_exception_for_error_code()</code>, which just looks up a code found in
364 <code>response.json()</code><ul>
365<li>I am not convinced this ever works for this tap because my understanding of
366<code>raise_for_status()</code> is if you <code>raise_for_status()</code> is unsuccessful then <code>response.json()</code>
367will also be unsuccessful. So, because we are in the exception handling for
368<code>raise_for_status()</code> I think we never make it past <code>response = response.json()</code> on
369<code>client.py:118</code></li>
370<li>I believe it&rsquo;s possible to leave the function this way</li>
371</ul>
372</li>
373<li>Assuming <code>response.json()</code> does fail, then that function will raise a
374 <code>simplejson.scanner.JSONDecodeError</code> with an error message like <code>"Expecting value: line 1
375 column 1 (char 0)"</code></li>
376<li>Assuming <code>response.json()</code> worked, but it&rsquo;s lacks an <code>error</code> key and lacks an <code>errorCode</code> key,
377 we re-raise whatever was caught from <code>raise_for_status()</code></li>
378<li>We also re-raise whatever was caught from <code>raise_for_status()</code> if a <code>ValueError</code> or <code>TypeError</code>
379 occurs in trying to handle the <code>raise_for_status()</code> error</li>
380</ol>
381 </div>
382 <div class='code'>
383 <div class="highlight"><pre></pre></div>
384 </div>
385 </div>
386 <div class='clearall'></div>
387 <div class='section' id='section-25'>
388 <div class='docs'>
389 <div class='octowrap'>
390 <a class='octothorpe' href='#section-25'>#</a>
391 </div>
392 <hr />
393 </div>
394 <div class='code'>
395 <div class="highlight"><pre></pre></div>
396 </div>
397 </div>
398 <div class='clearall'></div>
399 <div class='section' id='section-26'>
400 <div class='docs'>
401 <div class='octowrap'>
402 <a class='octothorpe' href='#section-26'>#</a>
403 </div>
404 <p>Try to catch API errors to rethrow as tap specific errors</p>
405 </div>
406 <div class='code'>
407 <div class="highlight"><pre><span class="k">def</span> <span class="nf">raise_for_error</span><span class="p">(</span><span class="n">response</span><span class="p">):</span></pre></div>
408 </div>
409 </div>
410 <div class='clearall'></div>
411 <div class='section' id='section-27'>
412 <div class='docs'>
413 <div class='octowrap'>
414 <a class='octothorpe' href='#section-27'>#</a>
415 </div>
416 <p>Inputs:</p>
417<ul>
418<li><code>response</code>: A requests.Response object</li>
419</ul>
420<p>Returns:</p>
421<ul>
422<li>None</li>
423</ul>
424<p>Side Effects:</p>
425<ul>
426<li>Raises a GoogleError</li>
427</ul>
428 </div>
429 <div class='code'>
430 <div class="highlight"><pre> <span class="k">try</span><span class="p">:</span>
431 <span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
432 <span class="k">except</span> <span class="p">(</span><span class="n">requests</span><span class="o">.</span><span class="n">HTTPError</span><span class="p">,</span> <span class="n">requests</span><span class="o">.</span><span class="n">ConnectionError</span><span class="p">)</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
433 <span class="k">try</span><span class="p">:</span>
434 <span class="n">content_length</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
435 <span class="k">if</span> <span class="n">content_length</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
436 <span class="k">return</span>
437 <span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
438 <span class="k">if</span> <span class="p">(</span><span class="s1">&#39;error&#39;</span> <span class="ow">in</span> <span class="n">response</span><span class="p">)</span> <span class="ow">or</span> <span class="p">(</span><span class="s1">&#39;errorCode&#39;</span> <span class="ow">in</span> <span class="n">response</span><span class="p">):</span>
439 <span class="n">message</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="si">%s</span><span class="s1">: </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;error&#39;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">error</span><span class="p">)),</span>
440 <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;message&#39;</span><span class="p">,</span> <span class="s1">&#39;Unknown Error&#39;</span><span class="p">))</span>
441 <span class="n">error_code</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;error&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;code&#39;</span><span class="p">)</span>
442 <span class="n">ex</span> <span class="o">=</span> <span class="n">get_exception_for_error_code</span><span class="p">(</span><span class="n">error_code</span><span class="p">)</span>
443 <span class="k">raise</span> <span class="n">ex</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
444 <span class="k">raise</span> <span class="n">GoogleError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
445 <span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">):</span>
446 <span class="k">raise</span> <span class="n">GoogleError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span></pre></div>
447 </div>
448 </div>
449 <div class='clearall'></div>
450 <div class='section' id='section-28'>
451 <div class='docs'>
452 <div class='octowrap'>
453 <a class='octothorpe' href='#section-28'>#</a>
454 </div>
455 <hr />
456 </div>
457 <div class='code'>
458 <div class="highlight"><pre></pre></div>
459 </div>
460 </div>
461 <div class='clearall'></div>
462 <div class='section' id='section-29'>
463 <div class='docs'>
464 <div class='octowrap'>
465 <a class='octothorpe' href='#section-29'>#</a>
466 </div>
467 <h3>Handling a successful response</h3>
468 </div>
469 <div class='code'>
470 <div class="highlight"><pre></pre></div>
471 </div>
472 </div>
473 <div class='clearall'></div>
474 <div class='section' id='section-30'>
475 <div class='docs'>
476 <div class='octowrap'>
477 <a class='octothorpe' href='#section-30'>#</a>
478 </div>
479 <p>A successful response is defined as anything that returns a <code>HTTP 200</code>.</p>
480 </div>
481 <div class='code'>
482 <div class="highlight"><pre></pre></div>
483 </div>
484 </div>
485 <div class='clearall'></div>
486 <div class='section' id='section-31'>
487 <div class='docs'>
488 <div class='octowrap'>
489 <a class='octothorpe' href='#section-31'>#</a>
490 </div>
491 <p>On a successful response, we store the <code>access_token</code> returned on a private field,
492<code>GoogleClient.__access_token</code>, and we update <code>GoogleClient.__expires</code> to be the time this
493<code>access_token</code> expires. <code>GoogleClient.__expires</code> is a <code>datetime</code> object in UTC.</p>
494 </div>
495 <div class='code'>
496 <div class="highlight"><pre></pre></div>
497 </div>
498 </div>
499 <div class='clearall'></div>
500 <div class='section' id='section-32'>
501 <div class='docs'>
502 <div class='octowrap'>
503 <a class='octothorpe' href='#section-32'>#</a>
504 </div>
505 <h3>Handling an unsuccessful response</h3>
506 </div>
507 <div class='code'>
508 <div class="highlight"><pre></pre></div>
509 </div>
510 </div>
511 <div class='clearall'></div>
512 <div class='section' id='section-33'>
513 <div class='docs'>
514 <div class='octowrap'>
515 <a class='octothorpe' href='#section-33'>#</a>
516 </div>
517 <p>To handle unsuccessful requests, the tap has the following pattern</p>
518 </div>
519 <div class='code'>
520 <div class="highlight"><pre></pre></div>
521 </div>
522 </div>
523 <div class='clearall'></div>
524 <div class='section' id='section-34'>
525 <div class='docs'>
526 <div class='octowrap'>
527 <a class='octothorpe' href='#section-34'>#</a>
528 </div>
529 <pre><code class="language-Python">if response.status_code &gt;= 500:
530 raise Server5xxError()
531
532if response.status_code != 200:
533 raise_for_error(response)
534</code></pre>
535 </div>
536 <div class='code'>
537 <div class="highlight"><pre></pre></div>
538 </div>
539 </div>
540 <div class='clearall'></div>
541 <div class='section' id='section-35'>
542 <div class='docs'>
543 <div class='octowrap'>
544 <a class='octothorpe' href='#section-35'>#</a>
545 </div>
546 <p>The <code>client.py:Server5xxError</code> is caught by <code>backoff</code> and we exponentially backoff the request.</p>
547 </div>
548 <div class='code'>
549 <div class="highlight"><pre></pre></div>
550 </div>
551 </div>
552 <div class='clearall'></div>
553 <div class='section' id='section-36'>
554 <div class='docs'>
555 <div class='octowrap'>
556 <a class='octothorpe' href='#section-36'>#</a>
557 </div>
558 <hr />
559 </div>
560 <div class='code'>
561 <div class="highlight"><pre></pre></div>
562 </div>
563 </div>
564 <div class='clearall'></div>
565 <div class='section' id='section-37'>
566 <div class='docs'>
567 <div class='octowrap'>
568 <a class='octothorpe' href='#section-37'>#</a>
569 </div>
570 <p>This is a class implemented in the tap in <code>client.py</code>. We initialize it once in <code>__init__.py</code> as
571a context manager in <code>__init__.py:main()</code></p>
572 </div>
573 <div class='code'>
574 <div class="highlight"><pre><span class="k">class</span> <span class="nc">GoogleClient</span><span class="p">:</span> <span class="c1"># pylint: disable=too-many-instance-attributes</span></pre></div>
575 </div>
576 </div>
577 <div class='clearall'></div>
578 <div class='section' id='section-38'>
579 <div class='docs'>
580 <div class='octowrap'>
581 <a class='octothorpe' href='#section-38'>#</a>
582 </div>
583
584 </div>
585 <div class='code'>
586 <div class="highlight"><pre></pre></div>
587 </div>
588 </div>
589 <div class='clearall'></div>
590 <div class='section' id='section-39'>
591 <div class='docs'>
592 <div class='octowrap'>
593 <a class='octothorpe' href='#section-39'>#</a>
594 </div>
595 <p>To create the <code>GoogleClient</code> object, we have to pass in the three OAuth2 variables. Optionally we
596can include the <code>user_agent</code>.</p>
597<p>Side Effects:</p>
598<ul>
599<li>All of this gets stored in private fields by the constructor.</li>
600<li>The constructor also initializes a <code>requests.Session</code>.</li>
601</ul>
602 </div>
603 <div class='code'>
604 <div class="highlight"><pre> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">client_id</span><span class="p">,</span> <span class="n">client_secret</span><span class="p">,</span> <span class="n">refresh_token</span><span class="p">,</span> <span class="n">access_token</span><span class="p">,</span> <span class="n">user_agent</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span></pre></div>
605 </div>
606 </div>
607 <div class='clearall'></div>
608 <div class='section' id='section-40'>
609 <div class='docs'>
610 <div class='octowrap'>
611 <a class='octothorpe' href='#section-40'>#</a>
612 </div>
613
614 </div>
615 <div class='code'>
616 <div class="highlight"><pre> <span class="bp">self</span><span class="o">.</span><span class="n">__client_id</span> <span class="o">=</span> <span class="n">client_id</span>
617 <span class="bp">self</span><span class="o">.</span><span class="n">__client_secret</span> <span class="o">=</span> <span class="n">client_secret</span>
618 <span class="bp">self</span><span class="o">.</span><span class="n">__refresh_token</span> <span class="o">=</span> <span class="n">refresh_token</span>
619 <span class="bp">self</span><span class="o">.</span><span class="n">__user_agent</span> <span class="o">=</span> <span class="n">user_agent</span>
620 <span class="bp">self</span><span class="o">.</span><span class="n">__access_token</span> <span class="o">=</span> <span class="n">access_token</span>
621 <span class="bp">self</span><span class="o">.</span><span class="n">__session</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
622 <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="kc">None</span></pre></div>
623 </div>
624 </div>
625 <div class='clearall'></div>
626 <div class='section' id='section-41'>
627 <div class='docs'>
628 <div class='octowrap'>
629 <a class='octothorpe' href='#section-41'>#</a>
630 </div>
631 <p>On enter, get a new access token</p>
632 </div>
633 <div class='code'>
634 <div class="highlight"><pre> <span class="k">def</span> <span class="fm">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span></pre></div>
635 </div>
636 </div>
637 <div class='clearall'></div>
638 <div class='section' id='section-42'>
639 <div class='docs'>
640 <div class='octowrap'>
641 <a class='octothorpe' href='#section-42'>#</a>
642 </div>
643
644 </div>
645 <div class='code'>
646 <div class="highlight"><pre> <span class="bp">self</span><span class="o">.</span><span class="n">get_access_token</span><span class="p">()</span>
647 <span class="k">return</span> <span class="bp">self</span></pre></div>
648 </div>
649 </div>
650 <div class='clearall'></div>
651 <div class='section' id='section-43'>
652 <div class='docs'>
653 <div class='octowrap'>
654 <a class='octothorpe' href='#section-43'>#</a>
655 </div>
656 <p>On exit, close the Requests Session</p>
657 </div>
658 <div class='code'>
659 <div class="highlight"><pre> <span class="k">def</span> <span class="fm">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">exception_type</span><span class="p">,</span> <span class="n">exception_value</span><span class="p">,</span> <span class="n">traceback</span><span class="p">):</span></pre></div>
660 </div>
661 </div>
662 <div class='clearall'></div>
663 <div class='section' id='section-44'>
664 <div class='docs'>
665 <div class='octowrap'>
666 <a class='octothorpe' href='#section-44'>#</a>
667 </div>
668
669 </div>
670 <div class='code'>
671 <div class="highlight"><pre> <span class="bp">self</span><span class="o">.</span><span class="n">__session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></pre></div>
672 </div>
673 </div>
674 <div class='clearall'></div>
675 <div class='section' id='section-45'>
676 <div class='docs'>
677 <div class='octowrap'>
678 <a class='octothorpe' href='#section-45'>#</a>
679 </div>
680 <p><code>get_access_token()</code> will <code>POST</code> to <code>client.py:GOOGLE_TOKEN_URI</code> which is just
681<code>https://oauth2.googleapis.com/token</code>. The body of the <code>POST</code> looks like</p>
682<pre><code class="language-JSON">{
683 &quot;grant_type&quot;: &quot;refresh_token&quot;,
684 &quot;client_id&quot;: my_client_id,
685 &quot;client_secret&quot;: my_client_secret,
686 &quot;refresh_token&quot;: my_refresh_token
687}
688</code></pre>
689<p>Side Effects:</p>
690<ul>
691<li>Store the access token and time it expires in private fields on the Client object</li>
692</ul>
693 </div>
694 <div class='code'>
695 <div class="highlight"><pre> <span class="nd">@backoff</span><span class="o">.</span><span class="n">on_exception</span><span class="p">(</span><span class="n">backoff</span><span class="o">.</span><span class="n">expo</span><span class="p">,</span>
696 <span class="n">Server5xxError</span><span class="p">,</span>
697 <span class="n">max_tries</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span>
698 <span class="n">factor</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
699 <span class="k">def</span> <span class="nf">get_access_token</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span></pre></div>
700 </div>
701 </div>
702 <div class='clearall'></div>
703 <div class='section' id='section-46'>
704 <div class='docs'>
705 <div class='octowrap'>
706 <a class='octothorpe' href='#section-46'>#</a>
707 </div>
708
709 </div>
710 <div class='code'>
711 <div class="highlight"><pre> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">__access_token</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
712 <span class="k">return</span>
713
714 <span class="n">headers</span> <span class="o">=</span> <span class="p">{}</span>
715 <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">__user_agent</span><span class="p">:</span>
716 <span class="n">headers</span><span class="p">[</span><span class="s1">&#39;User-Agent&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__user_agent</span>
717
718 <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span>
719 <span class="n">url</span><span class="o">=</span><span class="n">GOOGLE_TOKEN_URI</span><span class="p">,</span>
720 <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span>
721 <span class="n">data</span><span class="o">=</span><span class="p">{</span>
722 <span class="s1">&#39;grant_type&#39;</span><span class="p">:</span> <span class="s1">&#39;refresh_token&#39;</span><span class="p">,</span>
723 <span class="s1">&#39;client_id&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">__client_id</span><span class="p">,</span>
724 <span class="s1">&#39;client_secret&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">__client_secret</span><span class="p">,</span>
725 <span class="s1">&#39;refresh_token&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">__refresh_token</span><span class="p">,</span>
726 <span class="p">})</span>
727
728 <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">&gt;=</span> <span class="mi">500</span><span class="p">:</span>
729 <span class="k">raise</span> <span class="n">Server5xxError</span><span class="p">()</span>
730
731 <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">!=</span> <span class="mi">200</span><span class="p">:</span>
732 <span class="n">raise_for_error</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
733
734 <span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
735 <span class="bp">self</span><span class="o">.</span><span class="n">__access_token</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;access_token&#39;</span><span class="p">]</span>
736 <span class="bp">self</span><span class="o">.</span><span class="n">__expires</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">+</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">seconds</span><span class="o">=</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;expires_in&#39;</span><span class="p">])</span>
737 <span class="n">LOGGER</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Authorized, token expires = </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__expires</span><span class="p">))</span></pre></div>
738 </div>
739 </div>
740 <div class='clearall'></div>
741 <div class='section' id='section-47'>
742 <div class='docs'>
743 <div class='octowrap'>
744 <a class='octothorpe' href='#section-47'>#</a>
745 </div>
746 <hr />
747 </div>
748 <div class='code'>
749 <div class="highlight"><pre></pre></div>
750 </div>
751 </div>
752 <div class='clearall'></div>
753 <div class='section' id='section-48'>
754 <div class='docs'>
755 <div class='octowrap'>
756 <a class='octothorpe' href='#section-48'>#</a>
757 </div>
758 <p>This function starts with a call to <code>GoogleClient.get_access_token()</code> which likely returns
759immediately most of the time.</p>
760 </div>
761 <div class='code'>
762 <div class="highlight"><pre></pre></div>
763 </div>
764 </div>
765 <div class='clearall'></div>
766 <div class='section' id='section-49'>
767 <div class='docs'>
768 <div class='octowrap'>
769 <a class='octothorpe' href='#section-49'>#</a>
770 </div>
771 <p>Then we decide what url we are sending the request to. Sometimes it&rsquo;s
772<code>https://sheets.googleapis.com/v4</code> and sometimes it&rsquo;s <code>https://www.googleapis.com/drive/v3</code>.</p>
773 </div>
774 <div class='code'>
775 <div class="highlight"><pre></pre></div>
776 </div>
777 </div>
778 <div class='clearall'></div>
779 <div class='section' id='section-50'>
780 <div class='docs'>
781 <div class='octowrap'>
782 <a class='octothorpe' href='#section-50'>#</a>
783 </div>
784 <ul>
785<li>It seems like a mistake to decide this so deep into the code. Why doesn&rsquo;t the caller decide
786 where the request goes?</li>
787</ul>
788 </div>
789 <div class='code'>
790 <div class="highlight"><pre></pre></div>
791 </div>
792 </div>
793 <div class='clearall'></div>
794 <div class='section' id='section-51'>
795 <div class='docs'>
796 <div class='octowrap'>
797 <a class='octothorpe' href='#section-51'>#</a>
798 </div>
799 <p>Then we set up the request headers. The <code>authorization</code>, <code>user-agent</code>, and <code>content-type</code> keys come
800into play here</p>
801 </div>
802 <div class='code'>
803 <div class="highlight"><pre></pre></div>
804 </div>
805 </div>
806 <div class='clearall'></div>
807 <div class='section' id='section-52'>
808 <div class='docs'>
809 <div class='octowrap'>
810 <a class='octothorpe' href='#section-52'>#</a>
811 </div>
812 <ul>
813<li>One benefit of a a <code>requests.Session</code> is that you can set the headers for the session. I&rsquo;m not
814sure why we don&rsquo;t do that here</li>
815<li>If we did that, we wouldn&rsquo;t have to think about the access_token making it into the headers
816here. They would just already be there</li>
817</ul>
818 </div>
819 <div class='code'>
820 <div class="highlight"><pre></pre></div>
821 </div>
822 </div>
823 <div class='clearall'></div>
824 <div class='section' id='section-53'>
825 <div class='docs'>
826 <div class='octowrap'>
827 <a class='octothorpe' href='#section-53'>#</a>
828 </div>
829 <p>Then we make the request, timing how long it takes with a <code>singer.metrics.http_request_timer</code>.</p>
830 </div>
831 <div class='code'>
832 <div class="highlight"><pre></pre></div>
833 </div>
834 </div>
835 <div class='clearall'></div>
836 <div class='section' id='section-54'>
837 <div class='docs'>
838 <div class='octowrap'>
839 <a class='octothorpe' href='#section-54'>#</a>
840 </div>
841 <p>The chunk of code after making the request handles an unsuccessful response. We will retry <code>HTTP
842500</code> and <code>HTTP 429</code> errors, and <code>client.py:raise_for_error</code> for everything else.</p>
843 </div>
844 <div class='code'>
845 <div class="highlight"><pre></pre></div>
846 </div>
847 </div>
848 <div class='clearall'></div>
849 <div class='section' id='section-55'>
850 <div class='docs'>
851 <div class='octowrap'>
852 <a class='octothorpe' href='#section-55'>#</a>
853 </div>
854 <p>The most unique thing of this tap happens here: we return an <code>OrderedDict</code> of the response with this
855line</p>
856 </div>
857 <div class='code'>
858 <div class="highlight"><pre></pre></div>
859 </div>
860 </div>
861 <div class='clearall'></div>
862 <div class='section' id='section-56'>
863 <div class='docs'>
864 <div class='octowrap'>
865 <a class='octothorpe' href='#section-56'>#</a>
866 </div>
867 <pre><code class="language-Python">return response.json(object_pairs_hook=OrderedDict)
868</code></pre>
869 </div>
870 <div class='code'>
871 <div class="highlight"><pre></pre></div>
872 </div>
873 </div>
874 <div class='clearall'></div>
875 <div class='section' id='section-57'>
876 <div class='docs'>
877 <div class='octowrap'>
878 <a class='octothorpe' href='#section-57'>#</a>
879 </div>
880 <p>where <code>object_pairs_hook</code> is a <code>kwarg</code> passed to the JSON parser used by <code>requests</code>.</p>
881 </div>
882 <div class='code'>
883 <div class="highlight"><pre></pre></div>
884 </div>
885 </div>
886 <div class='clearall'></div>
887 <div class='section' id='section-58'>
888 <div class='docs'>
889 <div class='octowrap'>
890 <a class='octothorpe' href='#section-58'>#</a>
891 </div>
892 <p>This turns every key-value pair in the JSON response into a <code>OrderedDict</code>.</p>
893 </div>
894 <div class='code'>
895 <div class="highlight"><pre></pre></div>
896 </div>
897 </div>
898 <div class='clearall'></div>
899 <div class='section' id='section-59'>
900 <div class='docs'>
901 <div class='octowrap'>
902 <a class='octothorpe' href='#section-59'>#</a>
903 </div>
904 <p>Why do we do this? I don&rsquo;t know. See the footnote for code examples</p>
905 </div>
906 <div class='code'>
907 <div class="highlight"><pre></pre></div>
908 </div>
909 </div>
910 <div class='clearall'></div>
911 <div class='section' id='section-60'>
912 <div class='docs'>
913 <div class='octowrap'>
914 <a class='octothorpe' href='#section-60'>#</a>
915 </div>
916 <hr />
917 </div>
918 <div class='code'>
919 <div class="highlight"><pre></pre></div>
920 </div>
921 </div>
922 <div class='clearall'></div>
923 <div class='section' id='section-61'>
924 <div class='docs'>
925 <div class='octowrap'>
926 <a class='octothorpe' href='#section-61'>#</a>
927 </div>
928 <p>Rate Limit: https://developers.google.com/sheets/api/limits
929 100 request per 100 seconds per User</p>
930 </div>
931 <div class='code'>
932 <div class="highlight"><pre> <span class="nd">@backoff</span><span class="o">.</span><span class="n">on_exception</span><span class="p">(</span><span class="n">backoff</span><span class="o">.</span><span class="n">expo</span><span class="p">,</span>
933 <span class="p">(</span><span class="n">Server5xxError</span><span class="p">,</span> <span class="ne">ConnectionError</span><span class="p">,</span> <span class="n">Server429Error</span><span class="p">),</span>
934 <span class="n">max_tries</span><span class="o">=</span><span class="mi">7</span><span class="p">,</span>
935 <span class="n">factor</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
936 <span class="nd">@utils</span><span class="o">.</span><span class="n">ratelimit</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="mi">100</span><span class="p">)</span>
937 <span class="k">def</span> <span class="nf">request</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">method</span><span class="p">,</span> <span class="n">path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">api</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span></pre></div>
938 </div>
939 </div>
940 <div class='clearall'></div>
941 <div class='section' id='section-62'>
942 <div class='docs'>
943 <div class='octowrap'>
944 <a class='octothorpe' href='#section-62'>#</a>
945 </div>
946 <p>Make a request to the API</p>
947<p>Inputs:</p>
948<ul>
949<li>method: &ldquo;GET&rdquo; or &ldquo;POST&rdquo;</li>
950<li>url: The start of the url to make the request to</li>
951<li>path:</li>
952</ul>
953<p>Returns:</p>
954<ul>
955<li>A requests.Reponse</li>
956</ul>
957<p>Side Effects:</p>
958<ul>
959<li>Might store a new access token</li>
960</ul>
961 </div>
962 <div class='code'>
963 <div class="highlight"><pre> <span class="bp">self</span><span class="o">.</span><span class="n">get_access_token</span><span class="p">()</span></pre></div>
964 </div>
965 </div>
966 <div class='clearall'></div>
967 <div class='section' id='section-63'>
968 <div class='docs'>
969 <div class='octowrap'>
970 <a class='octothorpe' href='#section-63'>#</a>
971 </div>
972 <p>Construct the URL to make a request to</p>
973 </div>
974 <div class='code'>
975 <div class="highlight"><pre> <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;https://sheets.googleapis.com/v4&#39;</span>
976 <span class="k">if</span> <span class="n">api</span> <span class="o">==</span> <span class="s1">&#39;files&#39;</span><span class="p">:</span>
977 <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;https://www.googleapis.com/drive/v3&#39;</span>
978
979 <span class="k">if</span> <span class="ow">not</span> <span class="n">url</span> <span class="ow">and</span> <span class="n">path</span><span class="p">:</span>
980 <span class="n">url</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="si">{}</span><span class="s1">/</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">base_url</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span>
981
982 <span class="k">if</span> <span class="s1">&#39;endpoint&#39;</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="p">:</span>
983 <span class="n">endpoint</span> <span class="o">=</span> <span class="n">kwargs</span><span class="p">[</span><span class="s1">&#39;endpoint&#39;</span><span class="p">]</span>
984 <span class="k">del</span> <span class="n">kwargs</span><span class="p">[</span><span class="s1">&#39;endpoint&#39;</span><span class="p">]</span>
985 <span class="k">else</span><span class="p">:</span>
986 <span class="n">endpoint</span> <span class="o">=</span> <span class="kc">None</span>
987 <span class="n">LOGGER</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{}</span><span class="s1"> URL = </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">endpoint</span><span class="p">,</span> <span class="n">url</span><span class="p">))</span></pre></div>
988 </div>
989 </div>
990 <div class='clearall'></div>
991 <div class='section' id='section-64'>
992 <div class='docs'>
993 <div class='octowrap'>
994 <a class='octothorpe' href='#section-64'>#</a>
995 </div>
996 <p>Contruct the <code>headers</code> arg for <code>requests.request()</code></p>
997 </div>
998 <div class='code'>
999 <div class="highlight"><pre> <span class="k">if</span> <span class="s1">&#39;headers&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="p">:</span>
1000 <span class="n">kwargs</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
1001 <span class="n">kwargs</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Authorization&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;Bearer </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__access_token</span><span class="p">)</span>
1002
1003 <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">__user_agent</span><span class="p">:</span>
1004 <span class="n">kwargs</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;User-Agent&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__user_agent</span>
1005
1006 <span class="k">if</span> <span class="n">method</span> <span class="o">==</span> <span class="s1">&#39;POST&#39;</span><span class="p">:</span>
1007 <span class="n">kwargs</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Content-Type&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;application/json&#39;</span></pre></div>
1008 </div>
1009 </div>
1010 <div class='clearall'></div>
1011 <div class='section' id='section-65'>
1012 <div class='docs'>
1013 <div class='octowrap'>
1014 <a class='octothorpe' href='#section-65'>#</a>
1015 </div>
1016 <p>Make request</p>
1017 </div>
1018 <div class='code'>
1019 <div class="highlight"><pre> <span class="k">with</span> <span class="n">metrics</span><span class="o">.</span><span class="n">http_request_timer</span><span class="p">(</span><span class="n">endpoint</span><span class="p">)</span> <span class="k">as</span> <span class="n">timer</span><span class="p">:</span>
1020 <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__session</span><span class="o">.</span><span class="n">request</span><span class="p">(</span><span class="n">method</span><span class="p">,</span> <span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
1021 <span class="n">timer</span><span class="o">.</span><span class="n">tags</span><span class="p">[</span><span class="n">metrics</span><span class="o">.</span><span class="n">Tag</span><span class="o">.</span><span class="n">http_status_code</span><span class="p">]</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span></pre></div>
1022 </div>
1023 </div>
1024 <div class='clearall'></div>
1025 <div class='section' id='section-66'>
1026 <div class='docs'>
1027 <div class='octowrap'>
1028 <a class='octothorpe' href='#section-66'>#</a>
1029 </div>
1030 <p>Start backoff logic</p>
1031 </div>
1032 <div class='code'>
1033 <div class="highlight"><pre> <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">&gt;=</span> <span class="mi">500</span><span class="p">:</span>
1034 <span class="k">raise</span> <span class="n">Server5xxError</span><span class="p">()</span>
1035
1036 <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">429</span><span class="p">:</span>
1037 <span class="k">raise</span> <span class="n">Server429Error</span><span class="p">()</span>
1038
1039 <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">!=</span> <span class="mi">200</span><span class="p">:</span>
1040 <span class="n">raise_for_error</span><span class="p">(</span><span class="n">response</span><span class="p">)</span></pre></div>
1041 </div>
1042 </div>
1043 <div class='clearall'></div>
1044 <div class='section' id='section-67'>
1045 <div class='docs'>
1046 <div class='octowrap'>
1047 <a class='octothorpe' href='#section-67'>#</a>
1048 </div>
1049 <p>Ensure keys and rows are ordered as received from API.
1050QUESITON: But why??</p>
1051 </div>
1052 <div class='code'>
1053 <div class="highlight"><pre> <span class="k">return</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">(</span><span class="n">object_pairs_hook</span><span class="o">=</span><span class="n">OrderedDict</span><span class="p">)</span></pre></div>
1054 </div>
1055 </div>
1056 <div class='clearall'></div>
1057 <div class='section' id='section-68'>
1058 <div class='docs'>
1059 <div class='octowrap'>
1060 <a class='octothorpe' href='#section-68'>#</a>
1061 </div>
1062 <h3>Syntactic Sugar</h3>
1063 </div>
1064 <div class='code'>
1065 <div class="highlight"><pre> <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">api</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
1066 <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">request</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">&#39;GET&#39;</span><span class="p">,</span> <span class="n">path</span><span class="o">=</span><span class="n">path</span><span class="p">,</span> <span class="n">api</span><span class="o">=</span><span class="n">api</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></pre></div>
1067 </div>
1068 </div>
1069 <div class='clearall'></div>
1070 <div class='section' id='section-69'>
1071 <div class='docs'>
1072 <div class='octowrap'>
1073 <a class='octothorpe' href='#section-69'>#</a>
1074 </div>
1075
1076 </div>
1077 <div class='code'>
1078 <div class="highlight"><pre> <span class="k">def</span> <span class="nf">post</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">api</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
1079 <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">request</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">&#39;POST&#39;</span><span class="p">,</span> <span class="n">path</span><span class="o">=</span><span class="n">path</span><span class="p">,</span> <span class="n">api</span><span class="o">=</span><span class="n">api</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></pre></div>
1080 </div>
1081 </div>
1082 <div class='clearall'></div>
1083 <div class='section' id='section-70'>
1084 <div class='docs'>
1085 <div class='octowrap'>
1086 <a class='octothorpe' href='#section-70'>#</a>
1087 </div>
1088 <hr />
1089 </div>
1090 <div class='code'>
1091 <div class="highlight"><pre></pre></div>
1092 </div>
1093 </div>
1094 <div class='clearall'></div>
1095 <div class='section' id='section-71'>
1096 <div class='docs'>
1097 <div class='octowrap'>
1098 <a class='octothorpe' href='#section-71'>#</a>
1099 </div>
1100 <h1>Footnotes</h1>
1101 </div>
1102 <div class='code'>
1103 <div class="highlight"><pre></pre></div>
1104 </div>
1105 </div>
1106 <div class='clearall'></div>
1107 <div class='section' id='section-72'>
1108 <div class='docs'>
1109 <div class='octowrap'>
1110 <a class='octothorpe' href='#section-72'>#</a>
1111 </div>
1112 <p>Here&rsquo;s a normal <code>.json()</code>&lsquo;s output</p>
1113<pre><code class="language-Python">{&quot;file&quot;: &quot;this is my file&quot;}
1114</code></pre>
1115 </div>
1116 <div class='code'>
1117 <div class="highlight"><pre></pre></div>
1118 </div>
1119 </div>
1120 <div class='clearall'></div>
1121 <div class='section' id='section-73'>
1122 <div class='docs'>
1123 <div class='octowrap'>
1124 <a class='octothorpe' href='#section-73'>#</a>
1125 </div>
1126 <p>Here&rsquo;s the weird one&rsquo;s <code>.json(object_pairs_hook=OrderedDict)</code> output</p>
1127<pre><code class="language-Python">OrderedDict([('file', 'this is my file')])
1128</code></pre>
1129 </div>
1130 <div class='code'>
1131 <div class="highlight"><pre></pre></div>
1132 </div>
1133 </div>
1134 <div class='clearall'></div>
1135 <div class='section' id='section-74'>
1136 <div class='docs'>
1137 <div class='octowrap'>
1138 <a class='octothorpe' href='#section-74'>#</a>
1139 </div>
1140 <p>Here&rsquo;s a more complex example:</p>
1141 </div>
1142 <div class='code'>
1143 <div class="highlight"><pre></pre></div>
1144 </div>
1145 </div>
1146 <div class='clearall'></div>
1147 <div class='section' id='section-75'>
1148 <div class='docs'>
1149 <div class='octowrap'>
1150 <a class='octothorpe' href='#section-75'>#</a>
1151 </div>
1152 <pre><code class="language-python">{ &quot;deleted&quot;: false,
1153 &quot;__v&quot;: 0,
1154 &quot;_id&quot;: &quot;5887e1d85c873e0011036889&quot;,
1155 &quot;text&quot;: &quot;Cats make about 100 different sounds. Dogs make only about 10.&quot;,
1156 &quot;createdAt&quot;: &quot;2018-01-15T21:20:00.003Z&quot;,
1157 &quot;updatedAt&quot;: &quot;2020-09-03T16:39:39.578Z&quot;,
1158 &quot;used&quot;: true,
1159 &quot;status&quot;: {
1160 &quot;sentCount&quot;: 1,
1161 &quot;feedback&quot;: &quot;&quot;,
1162 &quot;verified&quot;: true
1163 },
1164 &quot;type&quot;: &quot;cat&quot;,
1165 &quot;user&quot;: &quot;5a9ac18c7478810ea6c06381&quot;,
1166 &quot;source&quot;: &quot;user&quot;}
1167</code></pre>
1168 </div>
1169 <div class='code'>
1170 <div class="highlight"><pre></pre></div>
1171 </div>
1172 </div>
1173 <div class='clearall'></div>
1174 <div class='section' id='section-76'>
1175 <div class='docs'>
1176 <div class='octowrap'>
1177 <a class='octothorpe' href='#section-76'>#</a>
1178 </div>
1179 <p>Versus <code>.json(object_pairs_hook=OrderedDict)</code></p>
1180<pre><code class="language-python">OrderedDict([('status', OrderedDict([('verified', True),
1181 ('sentCount', 1),
1182 ('feedback', '')])),
1183 ('type', 'cat'),
1184 ('deleted', False),
1185 ('_id', '5887e1d85c873e0011036889'),
1186 ('user', '5a9ac18c7478810ea6c06381'),
1187 ('text', 'Cats make about 100 different sounds. Dogs make only about 10.'),
1188 ('__v', 0),
1189 ('source', 'user'),
1190 ('updatedAt', '2020-09-03T16:39:39.578Z'),
1191 ('createdAt', '2018-01-15T21:20:00.003Z'),
1192 ('used', True)])
1193</code></pre>
1194 </div>
1195 <div class='code'>
1196 <div class="highlight"><pre></pre></div>
1197 </div>
1198 </div>
1199 <div class='clearall'></div>
1200</div>
1201</body>
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-09 22:41:33 +0000