diff options
Diffstat (limited to 'tap_google_sheets/sync.py')
-rw-r--r-- | tap_google_sheets/sync.py | 24 |
1 files changed, 17 insertions, 7 deletions
diff --git a/tap_google_sheets/sync.py b/tap_google_sheets/sync.py index 689be27..3e26bc4 100644 --- a/tap_google_sheets/sync.py +++ b/tap_google_sheets/sync.py | |||
@@ -1,6 +1,8 @@ | |||
1 | import time | 1 | import time |
2 | import math | 2 | import math |
3 | import json | 3 | import json |
4 | import re | ||
5 | import urllib.parse | ||
4 | from datetime import datetime, timedelta | 6 | from datetime import datetime, timedelta |
5 | import pytz | 7 | import pytz |
6 | import singer | 8 | import singer |
@@ -75,10 +77,14 @@ def process_records(catalog, | |||
75 | for record in records: | 77 | for record in records: |
76 | # Transform record for Singer.io | 78 | # Transform record for Singer.io |
77 | with Transformer() as transformer: | 79 | with Transformer() as transformer: |
78 | transformed_record = transformer.transform( | 80 | try: |
79 | record, | 81 | transformed_record = transformer.transform( |
80 | schema, | 82 | record, |
81 | stream_metadata) | 83 | schema, |
84 | stream_metadata) | ||
85 | except Exception as err: | ||
86 | LOGGER.error('{}'.format(err)) | ||
87 | raise RuntimeError(err) | ||
82 | write_record( | 88 | write_record( |
83 | stream_name=stream_name, | 89 | stream_name=stream_name, |
84 | record=transformed_record, | 90 | record=transformed_record, |
@@ -144,22 +150,26 @@ def get_data(stream_name, | |||
144 | if not range_rows: | 150 | if not range_rows: |
145 | range_rows = '' | 151 | range_rows = '' |
146 | # Replace {placeholder} variables in path | 152 | # Replace {placeholder} variables in path |
153 | # Encode stream_name: fixes issue w/ special characters in sheet name | ||
154 | stream_name_escaped = re.escape(stream_name) | ||
155 | stream_name_encoded = urllib.parse.quote_plus(stream_name) | ||
147 | path = endpoint_config.get('path', stream_name).replace( | 156 | path = endpoint_config.get('path', stream_name).replace( |
148 | '{spreadsheet_id}', spreadsheet_id).replace('{sheet_title}', stream_name).replace( | 157 | '{spreadsheet_id}', spreadsheet_id).replace('{sheet_title}', stream_name_encoded).replace( |
149 | '{range_rows}', range_rows) | 158 | '{range_rows}', range_rows) |
150 | params = endpoint_config.get('params', {}) | 159 | params = endpoint_config.get('params', {}) |
151 | api = endpoint_config.get('api', 'sheets') | 160 | api = endpoint_config.get('api', 'sheets') |
152 | # Add in querystring parameters and replace {placeholder} variables | 161 | # Add in querystring parameters and replace {placeholder} variables |
153 | # querystring function ensures parameters are added but not encoded causing API errors | 162 | # querystring function ensures parameters are added but not encoded causing API errors |
154 | querystring = '&'.join(['%s=%s' % (key, value) for (key, value) in params.items()]).replace( | 163 | querystring = '&'.join(['%s=%s' % (key, value) for (key, value) in params.items()]).replace( |
155 | '{sheet_title}', stream_name) | 164 | '{sheet_title}', stream_name_encoded) |
165 | LOGGER.info('URL: {}/{}?{}'.format(client.base_url, path, querystring)) | ||
156 | data = {} | 166 | data = {} |
157 | time_extracted = utils.now() | 167 | time_extracted = utils.now() |
158 | data = client.get( | 168 | data = client.get( |
159 | path=path, | 169 | path=path, |
160 | api=api, | 170 | api=api, |
161 | params=querystring, | 171 | params=querystring, |
162 | endpoint=stream_name) | 172 | endpoint=stream_name_escaped) |
163 | return data, time_extracted | 173 | return data, time_extracted |
164 | 174 | ||
165 | 175 | ||