aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--.circleci/config.yml20
-rw-r--r--.gitignore1
-rw-r--r--tests/__init__.py0
-rw-r--r--tests/tap_combined_test.py122
4 files changed, 129 insertions, 14 deletions
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 577428f..4a51c9f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -17,12 +17,6 @@ jobs:
17 pip install pylint 17 pip install pylint
18 pylint tap_google_sheets -d C,R,W 18 pylint tap_google_sheets -d C,R,W
19 - run: 19 - run:
20 name: 'Unit Tests'
21 command: |
22 source ~/.virtualenvs/tap-google-sheets/bin/activate
23 pip install nose
24 nosetests
25 - run:
26 name: 'JSON Validator' 20 name: 'JSON Validator'
27 command: | 21 command: |
28 source /usr/local/share/virtualenvs/tap-tester/bin/activate 22 source /usr/local/share/virtualenvs/tap-tester/bin/activate
@@ -34,13 +28,13 @@ jobs:
34 aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/sandbox dev_env.sh 28 aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/sandbox dev_env.sh
35 source dev_env.sh 29 source dev_env.sh
36 source /usr/local/share/virtualenvs/tap-tester/bin/activate 30 source /usr/local/share/virtualenvs/tap-tester/bin/activate
37 run-a-test --tap=tap-google-sheets \ 31 run-test --tap=tap-google-sheets \
38 --target=target-stitch \ 32 --target=target-stitch \
39 --orchestrator=stitch-orchestrator \ 33 --orchestrator=stitch-orchestrator \
40 --email=harrison+sandboxtest@stitchdata.com \ 34 --email=harrison+sandboxtest@stitchdata.com \
41 --password=$SANDBOX_PASSWORD \ 35 --password=$SANDBOX_PASSWORD \
42 --client-id=50 \ 36 --client-id=50 \
43 tap_tester.suites.google_sheets 37 tests/tap_combined_test.py
44 - slack/notify-on-failure: 38 - slack/notify-on-failure:
45 only_for_branches: master 39 only_for_branches: master
46 40
diff --git a/.gitignore b/.gitignore
index d9633da..fc5cd0f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,5 +18,4 @@ build/
18tap_google_sheets/.vscode/settings.json 18tap_google_sheets/.vscode/settings.json
19*.ipynb 19*.ipynb
20.DS_Store 20.DS_Store
21test_configuration.py
22tap_target_commands.sh 21tap_target_commands.sh
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/__init__.py
diff --git a/tests/tap_combined_test.py b/tests/tap_combined_test.py
new file mode 100644
index 0000000..5e2a3c5
--- /dev/null
+++ b/tests/tap_combined_test.py
@@ -0,0 +1,122 @@
1import unittest
2import os
3from datetime import datetime as dt
4from datetime import timedelta
5
6from tap_tester import menagerie
7import tap_tester.runner as runner
8import tap_tester.connections as connections
9from tap_tester.scenario import SCENARIOS
10
11
12class TapCombinedTest(unittest.TestCase):
13 START_DATE_FORMAT = "%Y-%m-%dT00:00:00Z"
14
15 @staticmethod
16 def name():
17 return "tap_google_sheets_combined_test"
18
19 @staticmethod
20 def tap_name():
21 return "tap-google-sheets"
22
23 @staticmethod
24 def get_type():
25 return "platform.google-sheets"
26
27 def expected_check_streams(self):
28 return set(self.expected_pks().keys())
29
30 def expected_sync_streams(self):
31 return set(self.expected_pks().keys())
32
33 @staticmethod
34 def expected_pks():
35 return {
36 "file_metadata": {"id"},
37 "sheet_metadata": {"sheetId"},
38 "sheets_loaded": {"spreadsheetId", "sheetId", "loadDate"},
39 "spreadsheet_metadata": {"spreadsheetId"},
40 "Test-1": {"__sdc_row"},
41 "Test 2": {"__sdc_row"},
42 "SKU COGS": {"__sdc_row"},
43 "Item Master": {"__sdc_row"},
44 "Retail Price": {"__sdc_row"},
45 "Retail Price NEW": {"__sdc_row"},
46 "Forecast Scenarios": {"__sdc_row"},
47 "Promo Type": {"__sdc_row"},
48 "Shipping Method": {"__sdc_row"}
49 }
50
51
52 def get_properties(self):
53 return_value = {
54 'start_date': dt.strftime(dt.utcnow() - timedelta(days=3), self.START_DATE_FORMAT),
55 'spreadsheet_id': os.getenv("TAP_GOOGLE_SHEETS_SPREADSHEET_ID")
56 }
57
58 return return_value
59
60 @staticmethod
61 def get_credentials():
62 return {
63 "client_id": os.getenv("TAP_GOOGLE_SHEETS_CLIENT_ID"),
64 "client_secret": os.getenv("TAP_GOOGLE_SHEETS_CLIENT_SECRET"),
65 "refresh_token": os.getenv("TAP_GOOGLE_SHEETS_REFRESH_TOKEN"),
66 }
67
68 def setUp(self):
69 missing_envs = [x for x in [
70 "TAP_GOOGLE_SHEETS_SPREADSHEET_ID",
71 "TAP_GOOGLE_SHEETS_START_DATE",
72 "TAP_GOOGLE_SHEETS_CLIENT_ID",
73 "TAP_GOOGLE_SHEETS_CLIENT_SECRET",
74 "TAP_GOOGLE_SHEETS_REFRESH_TOKEN",
75 ] if os.getenv(x) is None]
76
77 if missing_envs:
78 raise Exception("Missing environment variables: {}".format(missing_envs))
79
80 def test_run(self):
81
82 conn_id = connections.ensure_connection(self, payload_hook=None)
83
84 # Run the tap in check mode
85 check_job_name = runner.run_check_mode(self, conn_id)
86
87 # Verify the check's exit status
88 exit_status = menagerie.get_exit_status(conn_id, check_job_name)
89 menagerie.verify_check_exit_status(self, exit_status, check_job_name)
90
91 # Verify that there are catalogs found
92 found_catalogs = menagerie.get_catalogs(conn_id)
93 self.assertGreater(len(found_catalogs), 0, msg="unable to locate schemas for connection {}".format(conn_id))
94
95 found_catalog_names = set(map(lambda c: c['tap_stream_id'], found_catalogs))
96 subset = self.expected_check_streams().issubset(found_catalog_names)
97 self.assertTrue(subset, msg="Expected check streams are not subset of discovered catalog")
98 #
99 # # Select some catalogs
100 our_catalogs = [c for c in found_catalogs if c.get('tap_stream_id') in self.expected_sync_streams()]
101 for catalog in our_catalogs:
102 schema = menagerie.get_annotated_schema(conn_id, catalog['stream_id'])
103 connections.select_catalog_and_fields_via_metadata(conn_id, catalog, schema, [], [])
104
105 # # Verify that all streams sync at least one row for initial sync
106 # # This test is also verifying access token expiration handling. If test fails with
107 # # authentication error, refresh token was not replaced after expiring.
108 menagerie.set_state(conn_id, {})
109 sync_job_name = runner.run_sync_mode(self, conn_id)
110
111 # # Verify tap and target exit codes
112 exit_status = menagerie.get_exit_status(conn_id, sync_job_name)
113 menagerie.verify_sync_exit_status(self, exit_status, sync_job_name)
114 record_count_by_stream = runner.examine_target_output_file(self, conn_id, self.expected_sync_streams(),
115 self.expected_pks())
116 zero_count_streams = {k for k, v in record_count_by_stream.items() if v == 0}
117 self.assertFalse(zero_count_streams,
118 msg="The following streams did not sync any rows {}".format(zero_count_streams))
119
120
121
122SCENARIOS.add(TapCombinedTest)