tap_google_sheets/streams.py

   1 from collections import OrderedDict
   2
   3 # streams: API URL endpoints to be called
   4 # properties:
   5 #   <root node>: Plural stream name for the endpoint
   6 #   path: API endpoint relative path, when added to the base URL, creates the full path,
   7 #       default = stream_name
   8 #   key_properties: Primary key fields for identifying an endpoint record.
   9 #   replication_method: INCREMENTAL or FULL_TABLE
  10 #   replication_keys: bookmark_field(s), typically a date-time, used for filtering the results
  11 #       and setting the state
  12 #   params: Query, sort, and other endpoint specific parameters; default = {}
  13 #   data_key: JSON element containing the results list for the endpoint;
  14 #       default = root (no data_key)
  15
  16 FILE_METADATA = {
  17     "api": "files",
  18     "path": "files/{spreadsheet_id}",
  19     "key_properties": ["id"],
  20     "replication_method": "INCREMENTAL",
  21     "replication_keys": ["modifiedTime"],
  22     "params": {
  23         "fields": "id,name,createdTime,modifiedTime,version,teamDriveId,driveId,lastModifyingUser"
  24     }
  25 }
  26
  27 SPREADSHEET_METADATA = {
  28     "api": "sheets",
  29     "path": "spreadsheets/{spreadsheet_id}",
  30     "key_properties": ["spreadsheetId"],
  31     "replication_method": "FULL_TABLE",
  32     "params": {
  33         "includeGridData": "false"
  34     }
  35 }
  36
  37 SHEET_METADATA = {
  38     "api": "sheets",
  39     "path": "spreadsheets/{spreadsheet_id}",
  40     "key_properties": ["sheetId"],
  41     "replication_method": "FULL_TABLE",
  42     "params": {
  43         "includeGridData": "true",
  44         "ranges": "'{sheet_title}'!1:2"
  45     }
  46 }
  47
  48 SHEETS_LOADED = {
  49     "api": "sheets",
  50     "path": "spreadsheets/{spreadsheet_id}/values/'{sheet_title}'!{range_rows}",
  51     "data_key": "values",
  52     "key_properties": ["spreadsheetId", "sheetId", "loadDate"],
  53     "replication_method": "FULL_TABLE",
  54     "params": {
  55         "dateTimeRenderOption": "SERIAL_NUMBER",
  56         "valueRenderOption": "UNFORMATTED_VALUE",
  57         "majorDimension": "ROWS"
  58     }
  59 }
  60
  61 # Ensure streams are ordered logically
  62 STREAMS = OrderedDict()
  63 STREAMS['file_metadata'] = FILE_METADATA
  64 STREAMS['spreadsheet_metadata'] = SPREADSHEET_METADATA
  65 STREAMS['sheet_metadata'] = SHEET_METADATA
  66 STREAMS['sheets_loaded'] = SHEETS_LOADED