tap_google_sheets/streams.py

   1 from collections import OrderedDict
   2
   3 # streams: API URL endpoints to be called
   4 # properties:
   5 #   <root node>: Plural stream name for the endpoint
   6 #   path: API endpoint relative path, when added to the base URL, creates the full path,
   7 #       default = stream_name
   8 #   key_properties: Primary key fields for identifying an endpoint record.
   9 #   replication_method: INCREMENTAL or FULL_TABLE
  10 #   replication_keys: bookmark_field(s), typically a date-time, used for filtering the results
  11 #        and setting the state
  12 #   params: Query, sort, and other endpoint specific parameters; default = {}
  13 #   data_key: JSON element containing the results list for the endpoint; default = root (no data_key)
  14 #   bookmark_query_field: From date-time field used for filtering the query
  15 #   bookmark_type: Data type for bookmark, integer or datetime
  16
  17 FILE_METADATA = {
  18     "api": "files",
  19     "path": "files/{spreadsheet_id}",
  20     "key_properties": ["id"],
  21     "replication_method": "INCREMENTAL",
  22     "replication_keys": ["modifiedTime"],
  23     "params": {
  24         "fields": "id,name,createdTime,modifiedTime,version,teamDriveId,driveId,lastModifyingUser"
  25     }
  26 }
  27
  28 SPREADSHEET_METADATA = {
  29     "api": "sheets",
  30     "path": "spreadsheets/{spreadsheet_id}",
  31     "key_properties": ["spreadsheetId"],
  32     "replication_method": "FULL_TABLE",
  33     "params": {
  34         "includeGridData": "false"
  35     }
  36 }
  37
  38 SHEET_METADATA = {
  39     "api": "sheets",
  40     "path": "spreadsheets/{spreadsheet_id}",
  41     "key_properties": ["sheetId"],
  42     "replication_method": "FULL_TABLE",
  43     "params": {
  44         "includeGridData": "true",
  45         "ranges": "'{sheet_title}'!1:2"
  46     }
  47 }
  48
  49 SHEETS_LOADED = {
  50     "api": "sheets",
  51     "path": "spreadsheets/{spreadsheet_id}/values/'{sheet_title}'!{range_rows}",
  52     "data_key": "values",
  53     "key_properties": ["spreadsheetId", "sheetId", "loadDate"],
  54     "replication_method": "FULL_TABLE",
  55     "params": {
  56         "dateTimeRenderOption": "SERIAL_NUMBER",
  57         "valueRenderOption": "UNFORMATTED_VALUE",
  58         "majorDimension": "ROWS"
  59     }
  60 }
  61
  62 # Ensure streams are ordered logically
  63 STREAMS = OrderedDict()
  64 STREAMS['file_metadata'] = FILE_METADATA
  65 STREAMS['spreadsheet_metadata'] = SPREADSHEET_METADATA
  66 STREAMS['sheet_metadata'] = SHEET_METADATA
  67 STREAMS['sheets_loaded'] = SHEETS_LOADED