Bugfix/nachimehta subscription history

.circleci/config.yml

@@ -31,7 +31,7 @@ jobs:
             dbt deps
             dbt seed --target redshift --full-refresh
             dbt run --target redshift --full-refresh
-            dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York"}' --target redshift
+            dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York", stripe__subscription_history: true}' --target redshift
             dbt test --target redshift
       - run:
           name: "Run Tests - Snowflake"
@@ -42,7 +42,7 @@ jobs:
             dbt deps
             dbt seed --target snowflake --full-refresh
             dbt run --target snowflake --full-refresh
-            dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York"}' --target snowflake
+            dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York", stripe__subscription_history: true}' --target snowflake
             dbt test --target snowflake
       - run:
           name: "Run Tests - BigQuery"
@@ -56,7 +56,7 @@ jobs:
             dbt deps
             dbt seed --target bigquery --full-refresh
             dbt run --target bigquery --full-refresh
-            dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York"}' --target bigquery
+            dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York", stripe__subscription_historys: true}' --target bigquery
             dbt test --target bigquery
       - save_cache:
           key: deps2-{{ .Branch }}

CHANGELOG.md

@@ -1,3 +1,13 @@
+# dbt_stripe v0.7.0
+## 🚨 Breaking Changes 🚨
+- Stripe connectors set up after February 09, 2022 no longer sync the `subscription` table; however, a newer `subscription_history` table is synced. To account for this change a variable `stripe__subscription_history` has been added to the package project to allow for users to define if their source contains the `subscription_history` table. ([#37](https://github.com/fivetran/dbt_stripe_source/pull/37))
+  - By default this variable is set to `false`. If you still have the `subscription` table, then there is no adjustment needed on your end. If you do have the `subscription_history` table then you will want to set the variable to `true`. 
+  - Similarly, if you have both tables, then I highly encourage you start leveraging the `subscription_history` source table in your package.
+  - This package now points to the latest `dbt_stripe_source` package version which accounts for the above update. ([#33](https://github.com/fivetran/dbt_stripe/pull/33) and [#34](https://github.com/fivetran/dbt_stripe/pull/34))
+
+## Contributors
+- [nachimehta](https://github.com/nachimehta) ([#37](https://github.com/fivetran/dbt_stripe_source/pull/37))
+
 # dbt_stripe v0.6.1
 ## 🐞 Bug Fixes 🐞
 - [#24](https://github.com/fivetran/dbt_stripe/issues/24): Updating docs to add `dbt_stripe` documentation in addition to `dbt_stripe_source` documentation.

README.md

@@ -30,7 +30,7 @@ Include in your `packages.yml`
 ```yaml
 packages:
   - package: fivetran/stripe
-    version: [">=0.6.0", "<0.7.0"]
+    version: [">=0.7.0", "<0.8.0"]
 ```
 
 ## Configuration
@@ -62,7 +62,16 @@ vars:
     using_payment_method:  False  #Disable if you are not using the payment_method and payment_method_card tables
     using_subscriptions:   False  #Disable if you are not using the subscription and plan tables.
 ```
+### Leveraging Subscription Vs Subscription History Sources
+For Stripe connectors set up after February 09, 2022 the `subscription` table has been replaced with the new `subscription_history` table. By default this package will look for your subscription data within the `subscription` source table. However, if you have a newer connector then you must leverage the `stripe__subscription_history` to have the package use the `subscription_history` source rather than the `subscription` table.
+> **Please note that if you have `stripe__subscription_history` enabled then the package will filter for only active records.**
+```yml
+# dbt_project.yml
 
+...
+vars:
+    stripe__subscription_history: True  # False by default. Set to True if your connector syncs the `subscription_history` table. 
+```
 ### Changing the Build Schema
 By default this package will build the Stripe staging models within a schema titled (<target_schema> + `_stg_stripe`) and the Stripe final models within a schema titled (<target_schema> + `_stripe`) in your target database. If this is not where you would like your modeled Stripe data to be written to, add the following configuration to your `dbt_project.yml` file:
 

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'stripe'
-version: '0.6.1'
+version: '0.7.0'
 require-dbt-version: [">=1.0.0", "<2.0.0"]
 models:
   stripe:
@@ -26,4 +26,4 @@ vars:
     subscription: "{{ ref('stg_stripe__subscription') }}"
     using_invoices: true
     using_payment_method: true
-    using_subscriptions: true
+    using_subscriptions: true

integration_tests/dbt_project.yml

@@ -1,27 +1,11 @@
 config-version: 2
 
 name: 'stripe_integration_tests'
-version: '0.6.1'
+version: '0.7.0'
 profile: 'integration_tests'
 
 vars:
   stripe_schema: stripe_integration_tests
-  stripe:
-    group:                  "{{ ref('stg_stripe__group') }}"
-    balance_transaction:    "{{ ref('stg_stripe__balance_transaction') }}"
-    card:                   "{{ ref('stg_stripe__card') }}"
-    charge:                 "{{ ref('stg_stripe__charge') }}"
-    customer:               "{{ ref('stg_stripe__customer') }}"
-    fee:                    "{{ ref('stg_stripe__fee') }}"
-    invoice:                "{{ ref('stg_stripe__invoice') }}"
-    invoice_line_item:      "{{ ref('stg_stripe__invoice_line_item') }}"
-    payment_intent:         "{{ ref('stg_stripe__payment_intent') }}"
-    payment_method_card:    "{{ ref('stg_stripe__payment_method_card') }}"
-    payment_method:         "{{ ref('stg_stripe__payment_method') }}"
-    payout:                 "{{ ref('stg_stripe__payout') }}"
-    plan:                   "{{ ref('stg_stripe__plan') }}"
-    refund:                 "{{ ref('stg_stripe__refund') }}"
-    subscription:           "{{ ref('stg_stripe__subscription') }}"
   stripe_source:
     group:                  "{{ ref('group_data') }}"
     balance_transaction:    "{{ ref('balance_transaction_data') }}"
@@ -37,8 +21,11 @@ vars:
     payout:                 "{{ ref('payout_data') }}"
     plan:                   "{{ ref('plan_data') }}"
     refund:                 "{{ ref('refund_data') }}"
+    subscription_history:   "{{ ref('subscription_history_data') }}"
     subscription:           "{{ ref('subscription_data') }}"
-
+
+    stripe__subscription_history: false
+
 seeds:
   stripe_integration_tests:
     +quote_columns: "{{ true if target.type == 'redshift' else false }}"

integration_tests/seeds/subscription_data.csv

@@ -1,3 +1,3 @@
-id,_fivetran_synced,application_fee_percent,billing,billing_cycle_anchor,billing_threshold_amount_gte,billing_threshold_reset_billing_cycle_anchor,cancel_at,cancel_at_period_end,canceled_at,created,current_period_end,current_period_start,customer_id,days_until_due,default_source_id,ended_at,livemode,quantity,start_date,status,tax_percent,trial_end,trial_start
-sdgfs4536,2021-02-18 14:16:37,5,fdfjk,2021-02-18 14:16:37,9,FALSE,2021-02-19 14:16:37,TRUE,2021-02-18 14:16:37,2021-02-18 14:16:37,2021-02-19 14:16:37,2021-02-18 14:16:37,sdf6547,4,dfa543,2021-02-19 14:16:37,TRUE,4,2021-02-18 14:16:37,active,5,2021-02-19 14:16:37,2021-02-18 14:16:37
-sfjytkyu876,2021-02-18 14:16:37,6,fdsiew,2021-02-18 14:16:37,5,FALSE,2021-02-19 14:16:37,TRUE,2021-02-18 14:16:37,2021-02-18 14:16:37,2021-02-19 14:16:37,2021-02-18 14:16:37,123fsdf,5,764rhgf,2021-02-19 14:16:37,TRUE,5,2021-02-18 14:16:37,active,6,2021-02-19 14:16:37,2021-02-18 14:16:37
+id,_fivetran_synced,application_fee_percent,billing,billing_cycle_anchor,billing_threshold_amount_gte,billing_threshold_reset_billing_cycle_anchor,cancel_at,cancel_at_period_end,canceled_at,created,current_period_end,current_period_start,customer_id,days_until_due,default_source_id,ended_at,livemode,metadata,quantity,start_date,status,tax_percent,trial_end,trial_start
+sdgfs4536,2021-02-18 14:16:37,5,fdfjk,2021-02-18 14:16:37,9,FALSE,2021-02-19 14:16:37,TRUE,2021-02-18 14:16:37,2021-02-18 14:16:37,2021-02-19 14:16:37,2021-02-18 14:16:37,sdf6547,4,dfa543,2021-02-19 14:16:37,TRUE,"{""external_id"":""80755622-45b3-40a6-9544-25af39e0e50c""}",4,2021-02-18 14:16:37,active,5,2021-02-19 14:16:37,2021-02-18 14:16:37
+sfjytkyu876,2021-02-18 14:16:37,6,fdsiew,2021-02-18 14:16:37,5,FALSE,2021-02-19 14:16:37,TRUE,2021-02-18 14:16:37,2021-02-18 14:16:37,2021-02-19 14:16:37,2021-02-18 14:16:37,123fsdf,5,764rhgf,2021-02-19 14:16:37,TRUE,"{""external_id"":""80755622-45b3-40a6-9544-25af39e0e50c""}",5,2021-02-18 14:16:37,active,6,2021-02-19 14:16:37,2021-02-18 14:16:37

integration_tests/seeds/subscription_history_data.csv

@@ -0,0 +1,3 @@
+id,_fivetran_synced,_fivetran_active,application_fee_percent,billing,billing_cycle_anchor,billing_threshold_amount_gte,billing_threshold_reset_billing_cycle_anchor,cancel_at,cancel_at_period_end,canceled_at,created,current_period_end,current_period_start,customer_id,days_until_due,default_source_id,ended_at,livemode,quantity,start_date,status,tax_percent,trial_end,trial_start
+sdgfs4536,2021-02-18 14:16:37,true,5,fdfjk,2021-02-18 14:16:37,9,FALSE,2021-02-19 14:16:37,TRUE,2021-02-18 14:16:37,2021-02-18 14:16:37,2021-02-19 14:16:37,2021-02-18 14:16:37,sdf6547,4,dfa543,2021-02-19 14:16:37,TRUE,4,2021-02-18 14:16:37,active,5,2021-02-19 14:16:37,2021-02-18 14:16:37
+sfjytkyu876,2021-02-18 14:16:37,false,6,fdsiew,2021-02-18 14:16:37,5,FALSE,2021-02-19 14:16:37,TRUE,2021-02-18 14:16:37,2021-02-18 14:16:37,2021-02-19 14:16:37,2021-02-18 14:16:37,123fsdf,5,764rhgf,2021-02-19 14:16:37,TRUE,5,2021-02-18 14:16:37,active,6,2021-02-19 14:16:37,2021-02-18 14:16:37

packages.yml

@@ -1,3 +1,6 @@
 packages:
-- package: fivetran/stripe_source
-  version: [">=0.5.0", "<0.6.0"]
+# - package: fivetran/stripe_source
+#   version: [">=0.5.0", "<0.6.0"] 
+  - git: https://github.com/fivetran/dbt_stripe_source.git
+    revision: bugfix/nachimehta-subscription-history
+    warn-unpinned: false