]>
Commit | Line | Data |
---|---|---|
89643ba6 | 1 | from datetime import datetime, timedelta |
99424fee | 2 | from collections import OrderedDict |
89643ba6 JH |
3 | import backoff |
4 | import requests | |
89643ba6 JH |
5 | import singer |
6 | from singer import metrics | |
7 | from singer import utils | |
8 | ||
9 | BASE_URL = 'https://www.googleapis.com' | |
10 | GOOGLE_TOKEN_URI = 'https://oauth2.googleapis.com/token' | |
11 | LOGGER = singer.get_logger() | |
12 | ||
13 | ||
14 | class Server5xxError(Exception): | |
15 | pass | |
16 | ||
17 | ||
18 | class Server429Error(Exception): | |
19 | pass | |
20 | ||
21 | ||
22 | class GoogleError(Exception): | |
23 | pass | |
24 | ||
25 | ||
26 | class GoogleBadRequestError(GoogleError): | |
27 | pass | |
28 | ||
29 | ||
30 | class GoogleUnauthorizedError(GoogleError): | |
31 | pass | |
32 | ||
33 | ||
34 | class GooglePaymentRequiredError(GoogleError): | |
35 | pass | |
36 | ||
37 | ||
38 | class GoogleNotFoundError(GoogleError): | |
39 | pass | |
40 | ||
41 | ||
42 | class GoogleMethodNotAllowedError(GoogleError): | |
43 | pass | |
44 | ||
45 | ||
46 | class GoogleConflictError(GoogleError): | |
47 | pass | |
48 | ||
49 | ||
50 | class GoogleGoneError(GoogleError): | |
51 | pass | |
52 | ||
53 | ||
54 | class GooglePreconditionFailedError(GoogleError): | |
55 | pass | |
56 | ||
57 | ||
58 | class GoogleRequestEntityTooLargeError(GoogleError): | |
59 | pass | |
60 | ||
61 | ||
62 | class GoogleRequestedRangeNotSatisfiableError(GoogleError): | |
63 | pass | |
64 | ||
65 | ||
66 | class GoogleExpectationFailedError(GoogleError): | |
67 | pass | |
68 | ||
69 | ||
70 | class GoogleForbiddenError(GoogleError): | |
71 | pass | |
72 | ||
73 | ||
74 | class GoogleUnprocessableEntityError(GoogleError): | |
75 | pass | |
76 | ||
77 | ||
78 | class GooglePreconditionRequiredError(GoogleError): | |
79 | pass | |
80 | ||
81 | ||
82 | class GoogleInternalServiceError(GoogleError): | |
83 | pass | |
84 | ||
85 | ||
86 | # Error Codes: https://developers.google.com/webmaster-tools/search-console-api-original/v3/errors | |
87 | ERROR_CODE_EXCEPTION_MAPPING = { | |
88 | 400: GoogleBadRequestError, | |
89 | 401: GoogleUnauthorizedError, | |
90 | 402: GooglePaymentRequiredError, | |
91 | 403: GoogleForbiddenError, | |
92 | 404: GoogleNotFoundError, | |
93 | 405: GoogleMethodNotAllowedError, | |
94 | 409: GoogleConflictError, | |
95 | 410: GoogleGoneError, | |
96 | 412: GooglePreconditionFailedError, | |
97 | 413: GoogleRequestEntityTooLargeError, | |
98 | 416: GoogleRequestedRangeNotSatisfiableError, | |
99 | 417: GoogleExpectationFailedError, | |
100 | 422: GoogleUnprocessableEntityError, | |
101 | 428: GooglePreconditionRequiredError, | |
102 | 500: GoogleInternalServiceError} | |
103 | ||
104 | ||
105 | def get_exception_for_error_code(error_code): | |
106 | return ERROR_CODE_EXCEPTION_MAPPING.get(error_code, GoogleError) | |
107 | ||
108 | def raise_for_error(response): | |
109 | try: | |
110 | response.raise_for_status() | |
111 | except (requests.HTTPError, requests.ConnectionError) as error: | |
112 | try: | |
113 | content_length = len(response.content) | |
114 | if content_length == 0: | |
115 | # There is nothing we can do here since Google has neither sent | |
116 | # us a 2xx response nor a response content. | |
117 | return | |
118 | response = response.json() | |
119 | if ('error' in response) or ('errorCode' in response): | |
120 | message = '%s: %s' % (response.get('error', str(error)), | |
121 | response.get('message', 'Unknown Error')) | |
122 | error_code = response.get('error', {}).get('code') | |
123 | ex = get_exception_for_error_code(error_code) | |
124 | raise ex(message) | |
99424fee | 125 | raise GoogleError(error) |
89643ba6 JH |
126 | except (ValueError, TypeError): |
127 | raise GoogleError(error) | |
128 | ||
129 | class GoogleClient: # pylint: disable=too-many-instance-attributes | |
130 | def __init__(self, | |
131 | client_id, | |
132 | client_secret, | |
133 | refresh_token, | |
134 | user_agent=None): | |
135 | self.__client_id = client_id | |
136 | self.__client_secret = client_secret | |
137 | self.__refresh_token = refresh_token | |
138 | self.__user_agent = user_agent | |
139 | self.__access_token = None | |
140 | self.__expires = None | |
141 | self.__session = requests.Session() | |
142 | self.base_url = None | |
143 | ||
144 | ||
145 | def __enter__(self): | |
146 | self.get_access_token() | |
147 | return self | |
148 | ||
149 | def __exit__(self, exception_type, exception_value, traceback): | |
150 | self.__session.close() | |
151 | ||
da690bda | 152 | |
89643ba6 JH |
153 | @backoff.on_exception(backoff.expo, |
154 | Server5xxError, | |
155 | max_tries=5, | |
156 | factor=2) | |
157 | def get_access_token(self): | |
158 | # The refresh_token never expires and may be used many times to generate each access_token | |
159 | # Since the refresh_token does not expire, it is not included in get access_token response | |
160 | if self.__access_token is not None and self.__expires > datetime.utcnow(): | |
161 | return | |
162 | ||
163 | headers = {} | |
164 | if self.__user_agent: | |
165 | headers['User-Agent'] = self.__user_agent | |
166 | ||
167 | response = self.__session.post( | |
168 | url=GOOGLE_TOKEN_URI, | |
169 | headers=headers, | |
170 | data={ | |
171 | 'grant_type': 'refresh_token', | |
172 | 'client_id': self.__client_id, | |
173 | 'client_secret': self.__client_secret, | |
174 | 'refresh_token': self.__refresh_token, | |
175 | }) | |
176 | ||
177 | if response.status_code >= 500: | |
178 | raise Server5xxError() | |
179 | ||
180 | if response.status_code != 200: | |
181 | raise_for_error(response) | |
182 | ||
183 | data = response.json() | |
184 | self.__access_token = data['access_token'] | |
185 | self.__expires = datetime.utcnow() + timedelta(seconds=data['expires_in']) | |
186 | LOGGER.info('Authorized, token expires = {}'.format(self.__expires)) | |
187 | ||
188 | ||
da690bda JH |
189 | # Rate Limit: https://developers.google.com/sheets/api/limits |
190 | # 100 request per 100 seconds per User | |
89643ba6 JH |
191 | @backoff.on_exception(backoff.expo, |
192 | (Server5xxError, ConnectionError, Server429Error), | |
193 | max_tries=7, | |
194 | factor=3) | |
da690bda | 195 | @utils.ratelimit(100, 100) |
89643ba6 | 196 | def request(self, method, path=None, url=None, api=None, **kwargs): |
89643ba6 | 197 | self.get_access_token() |
89643ba6 JH |
198 | self.base_url = 'https://sheets.googleapis.com/v4' |
199 | if api == 'files': | |
200 | self.base_url = 'https://www.googleapis.com/drive/v3' | |
201 | ||
202 | if not url and path: | |
203 | url = '{}/{}'.format(self.base_url, path) | |
204 | ||
205 | # endpoint = stream_name (from sync.py API call) | |
206 | if 'endpoint' in kwargs: | |
207 | endpoint = kwargs['endpoint'] | |
208 | del kwargs['endpoint'] | |
209 | else: | |
210 | endpoint = None | |
da690bda | 211 | LOGGER.info('{} URL = {}'.format(endpoint, url)) |
89643ba6 JH |
212 | |
213 | if 'headers' not in kwargs: | |
214 | kwargs['headers'] = {} | |
215 | kwargs['headers']['Authorization'] = 'Bearer {}'.format(self.__access_token) | |
216 | ||
217 | if self.__user_agent: | |
218 | kwargs['headers']['User-Agent'] = self.__user_agent | |
219 | ||
220 | if method == 'POST': | |
221 | kwargs['headers']['Content-Type'] = 'application/json' | |
222 | ||
223 | with metrics.http_request_timer(endpoint) as timer: | |
224 | response = self.__session.request(method, url, **kwargs) | |
225 | timer.tags[metrics.Tag.http_status_code] = response.status_code | |
226 | ||
227 | if response.status_code >= 500: | |
228 | raise Server5xxError() | |
229 | ||
230 | #Use retry functionality in backoff to wait and retry if | |
231 | #response code equals 429 because rate limit has been exceeded | |
232 | if response.status_code == 429: | |
5fd44182 | 233 | raise Server429Error(response.json().get("error",{}).get("message", "Rate limit exceeded")) |
89643ba6 JH |
234 | |
235 | if response.status_code != 200: | |
236 | raise_for_error(response) | |
237 | ||
238 | # Ensure keys and rows are ordered as received from API | |
239 | return response.json(object_pairs_hook=OrderedDict) | |
240 | ||
241 | def get(self, path, api, **kwargs): | |
242 | return self.request(method='GET', path=path, api=api, **kwargs) | |
243 | ||
244 | def post(self, path, api, **kwargs): | |
245 | return self.request(method='POST', path=path, api=api, **kwargs) |