aboutsummaryrefslogtreecommitdiffhomepage
path: root/tap_google_sheets/streams.py
diff options
context:
space:
mode:
authorJeff Huth <jeff.huth@bytecode.io>2019-11-13 17:03:56 -0800
committerJeff Huth <jeff.huth@bytecode.io>2019-11-13 17:03:56 -0800
commit89643ba6fa98db82efd3246805ef801a8bfb5c81 (patch)
tree739027b4e827def2db81631c9d6ed58ec2b97809 /tap_google_sheets/streams.py
parent5f8005471d3affaaf23489df93a58ca64c3da3ca (diff)
downloadtap-google-sheets-89643ba6fa98db82efd3246805ef801a8bfb5c81.tar.gz
tap-google-sheets-89643ba6fa98db82efd3246805ef801a8bfb5c81.tar.zst
tap-google-sheets-89643ba6fa98db82efd3246805ef801a8bfb5c81.zip
Initial commit
Discovery mode works. Still working on normal sync.
Diffstat (limited to 'tap_google_sheets/streams.py')
-rw-r--r--tap_google_sheets/streams.py66
1 files changed, 66 insertions, 0 deletions
diff --git a/tap_google_sheets/streams.py b/tap_google_sheets/streams.py
new file mode 100644
index 0000000..299326a
--- /dev/null
+++ b/tap_google_sheets/streams.py
@@ -0,0 +1,66 @@
1from collections import OrderedDict
2
3# streams: API URL endpoints to be called
4# properties:
5# <root node>: Plural stream name for the endpoint
6# path: API endpoint relative path, when added to the base URL, creates the full path,
7# default = stream_name
8# key_properties: Primary key fields for identifying an endpoint record.
9# replication_method: INCREMENTAL or FULL_TABLE
10# replication_keys: bookmark_field(s), typically a date-time, used for filtering the results
11# and setting the state
12# params: Query, sort, and other endpoint specific parameters; default = {}
13# data_key: JSON element containing the results list for the endpoint; default = root (no data_key)
14# bookmark_query_field: From date-time field used for filtering the query
15# bookmark_type: Data type for bookmark, integer or datetime
16
17FILE_METADATA = {
18 "api": "files",
19 "path": "files/{spreadsheet_id}",
20 "key_properties": ["id"],
21 "replication_method": "FULL_TABLE",
22 "params": {
23 "fields": "id,name,createdTime,modifiedTime,version,teamDriveId,driveId,lastModifyingUser"
24 }
25}
26
27SPREADSHEET_METADATA = {
28 "api": "sheets",
29 "path": "spreadsheets/{spreadsheet_id}",
30 "key_properties": ["spreadsheetId"],
31 "replication_method": "FULL_TABLE",
32 "params": {
33 "includeGridData": "false"
34 }
35}
36
37SHEET_METADATA = {
38 "api": "sheets",
39 "path": "spreadsheets/{spreadsheet_id}",
40 "key_properties": ["sheetId"],
41 "replication_method": "FULL_TABLE",
42 "params": {
43 "includeGridData": "true",
44 "ranges": "'{sheet_title}'!1:2"
45 }
46}
47
48SHEETS_LOADED = {
49 "api": "sheets",
50 "path": "spreadsheets/{spreadsheet_id}/values/'{sheet_title}'!{range_rows}",
51 "data_key": "values",
52 "key_properties": ["spreadsheetId", "sheetId", "loadDate"],
53 "replication_method": "FULL_TABLE",
54 "params": {
55 "dateTimeRenderOption": "SERIAL_NUMBER",
56 "valueRenderOption": "UNFORMATTED_VALUE",
57 "majorDimension": "ROWS"
58 }
59}
60
61# Ensure streams are ordered logically
62STREAMS = OrderedDict()
63STREAMS['file_metadata'] = FILE_METADATA
64STREAMS['spreadsheet_metadata'] = SPREADSHEET_METADATA
65STREAMS['sheet_metadata'] = SHEET_METADATA
66STREAMS['sheets_loaded'] = SHEETS_LOADED