chore(tap_suiteql): Add filter to MonthlyRecurringRevenue (#43)

* chore(tap_suiteql): Add filter to MonthlyRecurringRevenue

* fix(tap-suiteql): Lint adjustments

* fix(tap-suiteql): Remove print

meltano.yml

@@ -30,3 +30,5 @@ plugins:
   - name: target-jsonl
     variant: andyh1203
     pip_url: target-jsonl
+    config:
+      destination_path: ''

tap_suiteql/client.py

@@ -26,6 +26,7 @@ def __init__(
     rest_method = "POST"
     stream_type = ""
     entity_name = ""
+    year_date_field = ""
 
     @property
     def url_base(self) -> str:

tap_suiteql/query_builder.py

@@ -37,6 +37,13 @@ def _build_where_statement(self):
         where_clauses = ["1=1"]
         where_statement = "where "
 
+        if (
+            self.stream.year_date_field
+        ):  # Filter added due to the limit of 100 thousand records that the Sensedata API returns # noqa:E501
+            where_clauses.append(
+                f"TO_CHAR({self.stream.year_date_field}, 'YYYY') >= 2023 and {self.stream.year_date_field} < ADD_MONTHS(SYSDATE, 3)"  # noqa:E501
+            )
+
         if self.stream.replication_key:
             where_clauses.append(
                 f"{self.stream.replication_key} >= TO_DATE(:{self.stream.replication_key}, 'YYYY-MM-DD\"T\"HH24:MI:SS')"  # noqa:E501

tap_suiteql/streams.py

@@ -264,6 +264,7 @@ class MonthlyRecurringRevenueStream(suiteqlStream):
     name = "MonthlyRecurringRevenue"
     path = "/query/v1/suiteql"
     primary_keys = ["id"]
+    year_date_field = "yearmonth"
     schema = th.PropertiesList(
         th.Property("id", th.StringType),
         th.Property("newchurnrevenue", th.StringType),

tap_suiteql/tests/test_query_builder.py

@@ -4,6 +4,7 @@
 class DummyStream:
     name = "dummy"
     entity_name = ""
+    year_date_field = None
     primary_keys = ["col_id"]
     replication_key = "replication_key_col"
     skip_attributes = []
@@ -23,6 +24,7 @@ class DummyStream:
 class DummyStreamWithoutReplicationKey:
     name = "dummy_without_replication_key"
     entity_name = ""
+    year_date_field = None
     primary_keys = ["col_id"]
     skip_attributes = []
     replication_key = None
@@ -41,6 +43,7 @@ class DummyStreamWithoutReplicationKey:
 class DummyStreamWithoutPrimaryKeys:
     name = "dummy_without_primary_keys"
     entity_name = ""
+    year_date_field = None
     skip_attributes = []
     primary_keys = None
     replication_key = None
@@ -58,6 +61,7 @@ class DummyStreamWithoutPrimaryKeys:
 class DummyStreamTransaction:
     name = "dummy"
     entity_name = "dummy_transaction"
+    year_date_field = None
     primary_keys = ["col_id"]
     replication_key = "replication_key_col"
     stream_type = "CustDummy"
@@ -74,6 +78,25 @@ class DummyStreamTransaction:
     }
 
 
+class DummyStreamWithFilter:
+    name = "dummy_with_stream"
+    entity_name = ""
+    primary_keys = ["col_id"]
+    year_date_field = "year_date_field"
+    replication_key = None
+    skip_attributes = []
+    stream_type = None
+    schema = {
+        "type": "object",
+        "properties": {
+            "col_id": {},
+            "col_1": {},
+            "col_2": {},
+            "year_date_field": {"format": "date-time"},
+        },
+    }
+
+
 def test_sql_builder_with_replication_key():
     expected = """select col_id,col_1,col_2,TO_CHAR(date_col, 'YYYY-MM-DD\"T\"HH24:MI:SS') date_col,TO_CHAR(replication_key_col, 'YYYY-MM-DD\"T\"HH24:MI:SS') replication_key_col
             from dummy
@@ -111,3 +134,13 @@ def test_sql_builder_from_transaction():
     query = QueryBuilder(DummyStreamTransaction).query()
 
     assert expected == query
+
+
+def test_sql_builder_with_filter():
+    expected = """select col_id,col_1,col_2,TO_CHAR(year_date_field, 'YYYY-MM-DD\"T\"HH24:MI:SS') year_date_field
+            from dummy_with_stream
+            where 1=1 and TO_CHAR(year_date_field, 'YYYY') >= 2023 and year_date_field < ADD_MONTHS(SYSDATE, 3)
+            order by col_id"""  # noqa:E501
+
+    query = QueryBuilder(DummyStreamWithFilter).query()
+    assert expected == query