aboutsummaryrefslogtreecommitdiffhomepage
path: root/tap_google_sheets/sync.py
diff options
context:
space:
mode:
Diffstat (limited to 'tap_google_sheets/sync.py')
-rw-r--r--tap_google_sheets/sync.py24
1 files changed, 17 insertions, 7 deletions
diff --git a/tap_google_sheets/sync.py b/tap_google_sheets/sync.py
index 689be27..3e26bc4 100644
--- a/tap_google_sheets/sync.py
+++ b/tap_google_sheets/sync.py
@@ -1,6 +1,8 @@
1import time 1import time
2import math 2import math
3import json 3import json
4import re
5import urllib.parse
4from datetime import datetime, timedelta 6from datetime import datetime, timedelta
5import pytz 7import pytz
6import singer 8import singer
@@ -75,10 +77,14 @@ def process_records(catalog,
75 for record in records: 77 for record in records:
76 # Transform record for Singer.io 78 # Transform record for Singer.io
77 with Transformer() as transformer: 79 with Transformer() as transformer:
78 transformed_record = transformer.transform( 80 try:
79 record, 81 transformed_record = transformer.transform(
80 schema, 82 record,
81 stream_metadata) 83 schema,
84 stream_metadata)
85 except Exception as err:
86 LOGGER.error('{}'.format(err))
87 raise RuntimeError(err)
82 write_record( 88 write_record(
83 stream_name=stream_name, 89 stream_name=stream_name,
84 record=transformed_record, 90 record=transformed_record,
@@ -144,22 +150,26 @@ def get_data(stream_name,
144 if not range_rows: 150 if not range_rows:
145 range_rows = '' 151 range_rows = ''
146 # Replace {placeholder} variables in path 152 # Replace {placeholder} variables in path
153 # Encode stream_name: fixes issue w/ special characters in sheet name
154 stream_name_escaped = re.escape(stream_name)
155 stream_name_encoded = urllib.parse.quote_plus(stream_name)
147 path = endpoint_config.get('path', stream_name).replace( 156 path = endpoint_config.get('path', stream_name).replace(
148 '{spreadsheet_id}', spreadsheet_id).replace('{sheet_title}', stream_name).replace( 157 '{spreadsheet_id}', spreadsheet_id).replace('{sheet_title}', stream_name_encoded).replace(
149 '{range_rows}', range_rows) 158 '{range_rows}', range_rows)
150 params = endpoint_config.get('params', {}) 159 params = endpoint_config.get('params', {})
151 api = endpoint_config.get('api', 'sheets') 160 api = endpoint_config.get('api', 'sheets')
152 # Add in querystring parameters and replace {placeholder} variables 161 # Add in querystring parameters and replace {placeholder} variables
153 # querystring function ensures parameters are added but not encoded causing API errors 162 # querystring function ensures parameters are added but not encoded causing API errors
154 querystring = '&'.join(['%s=%s' % (key, value) for (key, value) in params.items()]).replace( 163 querystring = '&'.join(['%s=%s' % (key, value) for (key, value) in params.items()]).replace(
155 '{sheet_title}', stream_name) 164 '{sheet_title}', stream_name_encoded)
165 LOGGER.info('URL: {}/{}?{}'.format(client.base_url, path, querystring))
156 data = {} 166 data = {}
157 time_extracted = utils.now() 167 time_extracted = utils.now()
158 data = client.get( 168 data = client.get(
159 path=path, 169 path=path,
160 api=api, 170 api=api,
161 params=querystring, 171 params=querystring,
162 endpoint=stream_name) 172 endpoint=stream_name_escaped)
163 return data, time_extracted 173 return data, time_extracted
164 174
165 175