1 from datetime
import datetime
, timedelta
4 from collections
import OrderedDict
7 from singer
import metrics
8 from singer
import utils
10 BASE_URL
= 'https://www.googleapis.com'
11 GOOGLE_TOKEN_URI
= 'https://oauth2.googleapis.com/token'
12 LOGGER
= singer
.get_logger()
15 class Server5xxError(Exception):
19 class Server429Error(Exception):
23 class GoogleError(Exception):
27 class GoogleBadRequestError(GoogleError
):
31 class GoogleUnauthorizedError(GoogleError
):
35 class GooglePaymentRequiredError(GoogleError
):
39 class GoogleNotFoundError(GoogleError
):
43 class GoogleMethodNotAllowedError(GoogleError
):
47 class GoogleConflictError(GoogleError
):
51 class GoogleGoneError(GoogleError
):
55 class GooglePreconditionFailedError(GoogleError
):
59 class GoogleRequestEntityTooLargeError(GoogleError
):
63 class GoogleRequestedRangeNotSatisfiableError(GoogleError
):
67 class GoogleExpectationFailedError(GoogleError
):
71 class GoogleForbiddenError(GoogleError
):
75 class GoogleUnprocessableEntityError(GoogleError
):
79 class GooglePreconditionRequiredError(GoogleError
):
83 class GoogleInternalServiceError(GoogleError
):
87 # Error Codes: https://developers.google.com/webmaster-tools/search-console-api-original/v3/errors
88 ERROR_CODE_EXCEPTION_MAPPING
= {
89 400: GoogleBadRequestError
,
90 401: GoogleUnauthorizedError
,
91 402: GooglePaymentRequiredError
,
92 403: GoogleForbiddenError
,
93 404: GoogleNotFoundError
,
94 405: GoogleMethodNotAllowedError
,
95 409: GoogleConflictError
,
97 412: GooglePreconditionFailedError
,
98 413: GoogleRequestEntityTooLargeError
,
99 416: GoogleRequestedRangeNotSatisfiableError
,
100 417: GoogleExpectationFailedError
,
101 422: GoogleUnprocessableEntityError
,
102 428: GooglePreconditionRequiredError
,
103 500: GoogleInternalServiceError
}
106 def get_exception_for_error_code(error_code
):
107 return ERROR_CODE_EXCEPTION_MAPPING
.get(error_code
, GoogleError
)
109 def raise_for_error(response
):
111 response
.raise_for_status()
112 except (requests
.HTTPError
, requests
.ConnectionError
) as error
:
114 content_length
= len(response
.content
)
115 if content_length
== 0:
116 # There is nothing we can do here since Google has neither sent
117 # us a 2xx response nor a response content.
119 response
= response
.json()
120 if ('error' in response
) or ('errorCode' in response
):
121 message
= '%s: %s' % (response
.get('error', str(error
)),
122 response
.get('message', 'Unknown Error'))
123 error_code
= response
.get('error', {}).get('code')
124 ex
= get_exception_for_error_code(error_code
)
127 raise GoogleError(error
)
128 except (ValueError, TypeError):
129 raise GoogleError(error
)
131 class GoogleClient
: # pylint: disable=too-many-instance-attributes
137 self
.__client
_id
= client_id
138 self
.__client
_secret
= client_secret
139 self
.__refresh
_token
= refresh_token
140 self
.__user
_agent
= user_agent
141 self
.__access
_token
= None
142 self
.__expires
= None
143 self
.__session
= requests
.Session()
148 self
.get_access_token()
151 def __exit__(self
, exception_type
, exception_value
, traceback
):
152 self
.__session
.close()
155 @backoff.on_exception(backoff
.expo
,
159 def get_access_token(self
):
160 # The refresh_token never expires and may be used many times to generate each access_token
161 # Since the refresh_token does not expire, it is not included in get access_token response
162 if self
.__access
_token
is not None and self
.__expires
> datetime
.utcnow():
166 if self
.__user
_agent
:
167 headers
['User-Agent'] = self
.__user
_agent
169 response
= self
.__session
.post(
170 url
=GOOGLE_TOKEN_URI
,
173 'grant_type': 'refresh_token',
174 'client_id': self
.__client
_id
,
175 'client_secret': self
.__client
_secret
,
176 'refresh_token': self
.__refresh
_token
,
179 if response
.status_code
>= 500:
180 raise Server5xxError()
182 if response
.status_code
!= 200:
183 raise_for_error(response
)
185 data
= response
.json()
186 self
.__access
_token
= data
['access_token']
187 self
.__expires
= datetime
.utcnow() + timedelta(seconds
=data
['expires_in'])
188 LOGGER
.info('Authorized, token expires = {}'.format(self
.__expires
))
191 # Rate Limit: https://developers.google.com/sheets/api/limits
192 # 100 request per 100 seconds per User
193 @backoff.on_exception(backoff
.expo
,
194 (Server5xxError
, ConnectionError
, Server429Error
),
197 @utils.ratelimit(100, 100)
198 def request(self
, method
, path
=None, url
=None, api
=None, **kwargs
):
200 self
.get_access_token()
202 self
.base_url
= 'https://sheets.googleapis.com/v4'
204 self
.base_url
= 'https://www.googleapis.com/drive/v3'
207 url
= '{}/{}'.format(self
.base_url
, path
)
209 # endpoint = stream_name (from sync.py API call)
210 if 'endpoint' in kwargs
:
211 endpoint
= kwargs
['endpoint']
212 del kwargs
['endpoint']
215 LOGGER
.info('{} URL = {}'.format(endpoint
, url
))
217 if 'headers' not in kwargs
:
218 kwargs
['headers'] = {}
219 kwargs
['headers']['Authorization'] = 'Bearer {}'.format(self
.__access
_token
)
221 if self
.__user
_agent
:
222 kwargs
['headers']['User-Agent'] = self
.__user
_agent
225 kwargs
['headers']['Content-Type'] = 'application/json'
227 with metrics
.http_request_timer(endpoint
) as timer
:
228 response
= self
.__session
.request(method
, url
, **kwargs
)
229 timer
.tags
[metrics
.Tag
.http_status_code
] = response
.status_code
231 if response
.status_code
>= 500:
232 raise Server5xxError()
234 #Use retry functionality in backoff to wait and retry if
235 #response code equals 429 because rate limit has been exceeded
236 if response
.status_code
== 429:
237 raise Server429Error()
239 if response
.status_code
!= 200:
240 raise_for_error(response
)
242 # Ensure keys and rows are ordered as received from API
243 return response
.json(object_pairs_hook
=OrderedDict
)
245 def get(self
, path
, api
, **kwargs
):
246 return self
.request(method
='GET', path
=path
, api
=api
, **kwargs
)
248 def post(self
, path
, api
, **kwargs
):
249 return self
.request(method
='POST', path
=path
, api
=api
, **kwargs
)