1 from datetime
import datetime
, timedelta
2 from collections
import OrderedDict
6 from singer
import metrics
7 from singer
import utils
9 BASE_URL
= 'https://www.googleapis.com'
10 GOOGLE_TOKEN_URI
= 'https://oauth2.googleapis.com/token'
11 LOGGER
= singer
.get_logger()
14 class Server5xxError(Exception):
18 class Server429Error(Exception):
22 class GoogleError(Exception):
26 class GoogleBadRequestError(GoogleError
):
30 class GoogleUnauthorizedError(GoogleError
):
34 class GooglePaymentRequiredError(GoogleError
):
38 class GoogleNotFoundError(GoogleError
):
42 class GoogleMethodNotAllowedError(GoogleError
):
46 class GoogleConflictError(GoogleError
):
50 class GoogleGoneError(GoogleError
):
54 class GooglePreconditionFailedError(GoogleError
):
58 class GoogleRequestEntityTooLargeError(GoogleError
):
62 class GoogleRequestedRangeNotSatisfiableError(GoogleError
):
66 class GoogleExpectationFailedError(GoogleError
):
70 class GoogleForbiddenError(GoogleError
):
74 class GoogleUnprocessableEntityError(GoogleError
):
78 class GooglePreconditionRequiredError(GoogleError
):
82 class GoogleInternalServiceError(GoogleError
):
86 # Error Codes: https://developers.google.com/webmaster-tools/search-console-api-original/v3/errors
87 ERROR_CODE_EXCEPTION_MAPPING
= {
88 400: GoogleBadRequestError
,
89 401: GoogleUnauthorizedError
,
90 402: GooglePaymentRequiredError
,
91 403: GoogleForbiddenError
,
92 404: GoogleNotFoundError
,
93 405: GoogleMethodNotAllowedError
,
94 409: GoogleConflictError
,
96 412: GooglePreconditionFailedError
,
97 413: GoogleRequestEntityTooLargeError
,
98 416: GoogleRequestedRangeNotSatisfiableError
,
99 417: GoogleExpectationFailedError
,
100 422: GoogleUnprocessableEntityError
,
101 428: GooglePreconditionRequiredError
,
102 500: GoogleInternalServiceError
}
105 def get_exception_for_error_code(error_code
):
106 return ERROR_CODE_EXCEPTION_MAPPING
.get(error_code
, GoogleError
)
108 def raise_for_error(response
):
110 response
.raise_for_status()
111 except (requests
.HTTPError
, requests
.ConnectionError
) as error
:
113 content_length
= len(response
.content
)
114 if content_length
== 0:
115 # There is nothing we can do here since Google has neither sent
116 # us a 2xx response nor a response content.
118 response
= response
.json()
119 if ('error' in response
) or ('errorCode' in response
):
120 message
= '%s: %s' % (response
.get('error', str(error
)),
121 response
.get('message', 'Unknown Error'))
122 error_code
= response
.get('error', {}).get('code')
123 ex
= get_exception_for_error_code(error_code
)
125 raise GoogleError(error
)
126 except (ValueError, TypeError):
127 raise GoogleError(error
)
129 class GoogleClient
: # pylint: disable=too-many-instance-attributes
135 self
.__client
_id
= client_id
136 self
.__client
_secret
= client_secret
137 self
.__refresh
_token
= refresh_token
138 self
.__user
_agent
= user_agent
139 self
.__access
_token
= None
140 self
.__expires
= None
141 self
.__session
= requests
.Session()
146 self
.get_access_token()
149 def __exit__(self
, exception_type
, exception_value
, traceback
):
150 self
.__session
.close()
153 @backoff.on_exception(backoff
.expo
,
157 def get_access_token(self
):
158 # The refresh_token never expires and may be used many times to generate each access_token
159 # Since the refresh_token does not expire, it is not included in get access_token response
160 if self
.__access
_token
is not None and self
.__expires
> datetime
.utcnow():
164 if self
.__user
_agent
:
165 headers
['User-Agent'] = self
.__user
_agent
167 response
= self
.__session
.post(
168 url
=GOOGLE_TOKEN_URI
,
171 'grant_type': 'refresh_token',
172 'client_id': self
.__client
_id
,
173 'client_secret': self
.__client
_secret
,
174 'refresh_token': self
.__refresh
_token
,
177 if response
.status_code
>= 500:
178 raise Server5xxError()
180 if response
.status_code
!= 200:
181 raise_for_error(response
)
183 data
= response
.json()
184 self
.__access
_token
= data
['access_token']
185 self
.__expires
= datetime
.utcnow() + timedelta(seconds
=data
['expires_in'])
186 LOGGER
.info('Authorized, token expires = {}'.format(self
.__expires
))
189 # Rate Limit: https://developers.google.com/sheets/api/limits
190 # 100 request per 100 seconds per User
191 @backoff.on_exception(backoff
.expo
,
192 (Server5xxError
, ConnectionError
, Server429Error
),
195 @utils.ratelimit(100, 100)
196 def request(self
, method
, path
=None, url
=None, api
=None, **kwargs
):
197 self
.get_access_token()
198 self
.base_url
= 'https://sheets.googleapis.com/v4'
200 self
.base_url
= 'https://www.googleapis.com/drive/v3'
203 url
= '{}/{}'.format(self
.base_url
, path
)
205 # endpoint = stream_name (from sync.py API call)
206 if 'endpoint' in kwargs
:
207 endpoint
= kwargs
['endpoint']
208 del kwargs
['endpoint']
211 LOGGER
.info('{} URL = {}'.format(endpoint
, url
))
213 if 'headers' not in kwargs
:
214 kwargs
['headers'] = {}
215 kwargs
['headers']['Authorization'] = 'Bearer {}'.format(self
.__access
_token
)
217 if self
.__user
_agent
:
218 kwargs
['headers']['User-Agent'] = self
.__user
_agent
221 kwargs
['headers']['Content-Type'] = 'application/json'
223 with metrics
.http_request_timer(endpoint
) as timer
:
224 response
= self
.__session
.request(method
, url
, **kwargs
)
225 timer
.tags
[metrics
.Tag
.http_status_code
] = response
.status_code
227 if response
.status_code
>= 500:
228 raise Server5xxError()
230 #Use retry functionality in backoff to wait and retry if
231 #response code equals 429 because rate limit has been exceeded
232 if response
.status_code
== 429:
233 raise Server429Error(response
.json().get("error",{}).get("message", "Rate limit exceeded"))
235 if response
.status_code
!= 200:
236 raise_for_error(response
)
238 # Ensure keys and rows are ordered as received from API
239 return response
.json(object_pairs_hook
=OrderedDict
)
241 def get(self
, path
, api
, **kwargs
):
242 return self
.request(method
='GET', path
=path
, api
=api
, **kwargs
)
244 def post(self
, path
, api
, **kwargs
):
245 return self
.request(method
='POST', path
=path
, api
=api
, **kwargs
)