From 5887f2e90286c8bca787ec470203cd13c59a3b56 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 16 Jul 2025 13:30:53 -0500 Subject: [PATCH 1/4] Q2 FY26: Apply automated update. --- .../maintainer_pull_request_template.md | 55 ++++++++------ .github/workflows/auto-release.yml | 13 ++-- .github/workflows/generate-docs.yml | 13 ++++ .gitignore | 74 ++++++++++++++++++- CHANGELOG.md | 19 +++++ LICENSE | 2 +- README.md | 23 +++--- integration_tests/ci/sample.profiles.yml | 1 - integration_tests/dbt_project.yml | 4 + 9 files changed, 160 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/generate-docs.yml diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md index 3220674..0899dc6 100644 --- a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md @@ -1,29 +1,38 @@ -## PR Overview -**This PR will address the following Issue/Feature:** + +- `dbt run --full-refresh && dbt test` +- `dbt run` && `dbt test` (if incremental models are present) +- The related issue is linked, tagged, and appropriately assigned +- Documentation and version updates are included, if applicable +- `docs` have been regenerated (unless there are no code or YAML changes) +- BuildKite integration tests are passing +--> -**Please provide the finalized CHANGELOG entry which details the relevant changes included in this PR:** - +## PR Overview +**Package version introduced in this PR:** +- -## PR Checklist -### Basic Validation -Please acknowledge that you have successfully performed the following commands locally: -- [ ] dbt run –full-refresh && dbt test -- [ ] dbt run (if incremental models are present) && dbt test +**This PR addresses the following Issue/Feature(s):** + +- -Before marking this PR as "ready for review" the following have been applied: -- [ ] The appropriate issue has been linked, tagged, and properly assigned -- [ ] All necessary documentation and version upgrades have been applied -- [ ] docs were regenerated (unless this PR does not include any code or yml updates) -- [ ] BuildKite integration tests are passing -- [ ] Detailed validation steps have been provided below +**Summary of changes:** + +- -### Detailed Validation -Please share any and all of your validation steps: - +### Submission Checklist +- [ ] Alignment meeting with the reviewer (if needed) + - [ ] Timeline and validation requirements discussed +- [ ] Provide validation details: + - [ ] **Validation Steps:** Check for unintentional effects (e.g., add/run consistency & integrity tests) + - [ ] **Testing Instructions:** Confirm the change addresses the issue(s) + - [ ] **Focus Areas:** Complex logic or queries that need extra attention +- [ ] Merge any relevant open PRs into this PR -### If you had to summarize this PR in an emoji, which would it be? - -:dancer: \ No newline at end of file +### Changelog + +- [ ] Draft changelog for PR +- [ ] Final changelog for release review diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 8ed5853..76d6944 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -3,11 +3,12 @@ on: pull_request: types: - closed - branches: - - main + - labeled jobs: - call-workflow-passing-data: - if: github.event.pull_request.merged - uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main - secrets: inherit \ No newline at end of file + release: + if: | + (github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') || + github.event.label.name == 'pre-release' + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@feature/pre-release-support + secrets: inherit diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml new file mode 100644 index 0000000..5eb9b8b --- /dev/null +++ b/.github/workflows/generate-docs.yml @@ -0,0 +1,13 @@ +name: 'generate dbt docs' +on: + pull_request: + types: + - labeled + +jobs: + generate-docs: + if: github.event.label.name == 'docs:ready' + uses: fivetran/dbt_package_automations/.github/workflows/generate-docs.yml@main + secrets: inherit + with: + schema_var_name: app_reporting_schema \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2f4710d..06f4e77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,73 @@ - -target/ +# dbt +**/package-lock.yml +package-lock.yml +.dbt/ +dbt_modules/ dbt_packages/ logs/ +profiles.yml +target/ +*.log + +# IDE files +.idea/ +.vscode/ +*~ +*.swp +*.swo + +# Jupyter Notebook +.ipynb_checkpoints + +# OS generated files +**/.DS_Store .DS_Store -package-lock.yml -env/ \ No newline at end of file +.Spotlight-V100 +.Trashes +._* +Thumbs.db +ehthumbs.db + +# Python +*.egg +*.egg-info/ +*.py[cod] +*.so +*$py.class +.Python +__pycache__/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.env +.installed.cfg +lib/ +lib64/ +MANIFEST +parts/ +sdist/ +var/ +wheels/ + +# Secrets and credentials +.env.* +.secrets +credentials.json +service-account.json + +# Temporary files +.cache/ +*.temp +*.tmp + +# Virtual environments +.conda/ +.env +.venv +ENV/ +env/ +env.bak/ +venv/ +venv.bak/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 099a8a6..339c023 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +[PR #29](https://github.com/fivetran/dbt_app_reporting/pull/29) includes the following updates: + +### Under the Hood - July 2025 Updates + +- Updated conditions in `.github/workflows/auto-release.yml`. +- Added `.github/workflows/generate-docs.yml`. +- Added `+docs: show: False` to `integration_tests/dbt_project.yml`. +- Migrated `flags` (e.g., `send_anonymous_usage_stats`, `use_colors`) from `sample.profiles.yml` to `integration_tests/dbt_project.yml`. +- Updated `maintainer_pull_request_template.md` with improved checklist. +- Refreshed README tag block: + - Standardized Quickstart-compatible badge set + - Left-aligned and positioned below the H1 title. +- Updated Python image version to `3.10.13` in `pipeline.yml`. +- Added `CI_DATABRICKS_DBT_CATALOG` to: + - `.buildkite/hooks/pre-command` (as an export) + - `pipeline.yml` (under the `environment` block, after `CI_DATABRICKS_DBT_TOKEN`) +- Added `certifi==2025.1.31` to `requirements.txt` (if missing). +- Updated `.gitignore` to exclude additional DBT, Python, and system artifacts. + # dbt_app_reporting v0.6.0 [PR #28](https://github.com/fivetran/dbt_app_reporting/pull/28) includes the following updates: diff --git a/LICENSE b/LICENSE index 261eeb9..f9554a9 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright © 2025 Fivetran Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 4356aeb..58bbf0d 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,21 @@ -

+ +# App Reporting dbt Package ([Docs](https://fivetran.github.io/dbt_app_reporting/)) + +

+ href="https://github.com/fivetran/dbt_app_reporting/blob/main/LICENSE"> - + + +

-# App Reporting dbt Package ([Docs](https://fivetran.github.io/dbt_app_reporting/)) ## What does this dbt package do? - Standardizes schemas from various app platform connectors and creates reporting models for all activity aggregated to the device, country, OS version, app version, traffic source and subscription levels. - Currently supports the following Fivetran app platform connectors: @@ -70,7 +75,7 @@ By default, this package also looks for your connection data in specific schemas vars: apple_store_schema: itunes_connect apple_store_database: your_database_name - + google_play_schema: google_play google_play_database: your_database_name ``` @@ -115,7 +120,7 @@ models: +schema: apple_store # default schema suffix apple_store_source: +schema: apple_store_source # default schema suffix - + google_play: +schema: google_play # default schema suffix google_play_source: @@ -146,13 +151,13 @@ To connect your multiple schema/database sources to the package models, follow t If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable. This is not available when running the package on multiple unioned connections. > IMPORTANT: See the Apple Store [`dbt_project.yml`](https://github.com/fivetran/dbt_apple_store_source/blob/main/dbt_project.yml) and Google Play [`dbt_project.yml`](https://github.com/fivetran/dbt_google_play_source/blob/main/dbt_project.yml) variable declarations to see the expected names. - + ```yml vars: apple_store__identifier: your_table_name google_play__identifier: your_table_name ``` - +
@@ -178,7 +183,7 @@ packages: - package: fivetran/google_play version: [">=0.5.0", "<0.6.0"] - + - package: fivetran/google_play_source version: [">=0.5.0", "<0.6.0"] diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 4b9abfd..cda89da 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -2,7 +2,6 @@ # HEY! This file is used in the dbt package integrations tests with Buildkite. # You should __NEVER__ check credentials into version control. Thanks for reading :) - integration_tests: target: redshift outputs: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 123ee6e..d5546ea 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -54,6 +54,8 @@ models: +schema: "app_reporting_{{ var('directed_schema','dev') }}" ## To be used for validation testing seeds: + +docs: + show: False app_reporting_integration_tests: +quote_columns: "{{ true if target.type == 'redshift' else false }}" +column_types: @@ -66,3 +68,5 @@ seeds: app_version_code: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" flags: send_anonymous_usage_stats: False + + use_colors: True \ No newline at end of file From ded6ffe52141c7a951ddcdfa5f38b984fbbf4a1c Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 16 Jul 2025 13:33:04 -0500 Subject: [PATCH 2/4] Update generate-docs.yml --- .github/workflows/generate-docs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index 5eb9b8b..6b92a7f 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -10,4 +10,5 @@ jobs: uses: fivetran/dbt_package_automations/.github/workflows/generate-docs.yml@main secrets: inherit with: - schema_var_name: app_reporting_schema \ No newline at end of file + # This needs to be kept a long list like this, and ': public' should not be included in the last var of the list. + schema_var_name: "apple_store_schema: 'public', google_play_schema" From 29174197b381871426b78c50c2e5efebedb49243 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 22 Jul 2025 11:03:17 -0500 Subject: [PATCH 3/4] Q2 FY26: Update auto-release workflow only. --- .github/workflows/auto-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 76d6944..e0a7100 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -10,5 +10,5 @@ jobs: if: | (github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') || github.event.label.name == 'pre-release' - uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@feature/pre-release-support + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main secrets: inherit From 31b2f2a4fa6347b68d927e013b2d2e9180e90f1c Mon Sep 17 00:00:00 2001 From: Avinash Kunnath Date: Thu, 14 Aug 2025 22:46:35 -0700 Subject: [PATCH 4/4] @docs-include --- integration_tests/dbt_project.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index d5546ea..578fa44 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -3,9 +3,9 @@ version: '0.6.0' profile: 'integration_tests' config-version: 2 vars: - # apple_store__using_subscriptions: true # uncomment this line when generating docs! - # google_play__using_subscriptions: true # uncomment this line when regenerating docs! - # google_play__using_earnings: true # uncomment this line when regenerating docs! + # apple_store__using_subscriptions: true # @docs-include + # google_play__using_subscriptions: true # @docs-include + # google_play__using_earnings: true # @docs-include google_play_schema: app_reporting_integrations_test_9 apple_store_schema: app_reporting_integrations_test_9