From 28780b5612b7cf1d4375b016b39d7e6535b77ddf Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Thu, 16 Oct 2025 19:34:00 -0700 Subject: [PATCH 01/25] adding EC2 instructions --- docs/Infrastructure_Guide/aws.md | 81 +++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/docs/Infrastructure_Guide/aws.md b/docs/Infrastructure_Guide/aws.md index 676886e4..884c70c1 100644 --- a/docs/Infrastructure_Guide/aws.md +++ b/docs/Infrastructure_Guide/aws.md @@ -20,6 +20,7 @@ Amazon Web Services was the chosen cloud vendor for hosting this project's infra * [IAM](#iam) * [VPC](#vpc) * [RDS](#rds) +* [EC2](#ec2) !!! note @@ -146,4 +147,82 @@ Feel free to choose any other options that could be more suitable._ 10. Under **Tags**, create a new tag if desired for resource organization. 11. Under **Database Authentication**, choose _password authentication_. 12. Under **Monitoring**, choose the standard version of _Database Insights_. All other options in this section can be left as default. -13. Review the **Estimated Monthly Costs**, make any changes if necessary, they click on _create database_. \ No newline at end of file +13. Review the **Estimated Monthly Costs**, make any changes if necessary, they click on _create database_. + +--- + +## EC2 +_Elastic Compute Cloud_ + +AWS EC2 (Elastic Compute Cloud) is a cloud service that provides resizable virtual servers to run applications and workloads on demand. + +### Launch Instance +1. Visit the [EC2 console](https://console.aws.amazon.com/ec2). +2. Click on **Launch Instance**. +3. Provide a **name** for the virtual machine. +4. Under **Application and OS Images**, choose _Ubuntu 24.04 (HVM), SSD Volume Type 64-bit ARM_ or a different image if preferred. +5. Under **Instance Type**, choose `t2.small`. +6. Under **Key Pair (login)**, select a key pair or create a new one. If a new one is created, check for the `.pem` file in the downloads folder. +7. Under **Network Settings**: + * Select the VPC created earlier. + * Switch to a _public_ subnet to allow connection to the virtual machine. + * Enable _Auto-assign public IP_. + * For the _Firewall_, select the default security group that should've been created when setting up the VPC. +8. Under **Configure Storage**, leave as default. +9. Under **Advanced Details**, lease as default. + +### Connect to Instance +1. First, configure a trusted connection to the previously created RDS instance. + * Visit the [RDS console](https://console.aws.amazon.com/rds/home). + * Click on the RDS instance previously created. + * Scroll down to the **Connected Compute Resources** section, in the **Actions** drop-down, click **Set up EC2 Connection**. + * On the next screen, select the created EC2 instance from the drop-down. Then, select **Continue**. + * On the **Review and Confirm** screen, review all information then click **Continue**. +2. SSH into machine. + * Back in the [EC2 console](https://console.aws.amazon.com/ec2), click on the created EC2 instance. + * In the top-right of the **Summary** section, click on the **Connect** button. + * On the next page, click on the **SSH Client** tab. + * Instructions on how to connect will be provided and `ssh` command will be provided. For example: + * `ssh -i "dagster-vm-key-pair.pem" ubuntu@ec2-..compute.amazonaws.com` + * **Note:** Run this command in the directory of the `.pem` file. + * **Note:** Since the virtual machine was created with the default VPC security group, make sure the **Inbound Rules** of the security allows your IP address to connect. + * The terminal should show an Ubuntu welcome screen once connected. + +### Configure Instance +Once connected to the virtual machine, run the following commands to get everything set up: +1. Clone repository + * Create a new directory: `git init ` + * `cd ` + * `git remote add -f origin https://github.com/digitalghost-dev/poke-cli/` + * `git config core.sparseCheckout true` + * `echo "card_data/" >> .git/info/sparse-checkout` + * `git pull origin main` + * `ls` - verify that `card_data/` directory was created. +2. Install tools + * Install `uv` for Python: `curl -LsSf https://astral.sh/uv/0.7.21/install.sh | sh` + * Add to `PATH`: `source $HOME/.local/bin/env` + * Install libraries from `pyproject.toml` file: `uv sync` + * Activate virtual environment: `source .venv/bin/activate` + * Create `dagster.yaml` file: + ```bash + mkdir -p ~/.dagster && cat > ~/.dagster/dagster.yaml << 'EOF' + storage: + postgres: + postgres_db: + username: postgres + password: "rds-password" + hostname: "rds-hostname" + db_name: postgres + port: 5432 + params: + sslmode: require + EOF + ``` + * Set environment variables: + * `echo 'export DAGSTER_HOME="$HOME/.dagster"' >> ~/.bashrc` + * `echo 'export SUPABASE_USER="supabase_user"' >> ~/.bashrc` + * `echo 'export SUPABASE_PASSWORD="supabase_password"' >> ~/.bashrc` + * `source ~/.bashrc` - to load variables in current session. +3. Verify Dagster and Connectivity + * `dg dev --host 0.0.0.0 --port 3000` + * In the browser, visit `http://:3000` \ No newline at end of file From ba8fb1ae27b554497bb09ac277f2d95c41bbb7ee Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Fri, 17 Oct 2025 08:38:43 -0700 Subject: [PATCH 02/25] updating subnet groups --- card_data/infrastructure/aws/rds/db_instance.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/card_data/infrastructure/aws/rds/db_instance.tf b/card_data/infrastructure/aws/rds/db_instance.tf index 6736ab74..013579ec 100755 --- a/card_data/infrastructure/aws/rds/db_instance.tf +++ b/card_data/infrastructure/aws/rds/db_instance.tf @@ -50,5 +50,5 @@ resource "aws_db_instance" "tfer--dagster-db" { } username = "postgres" - vpc_security_group_ids = ["sg-026cc204887184c98"] + vpc_security_group_ids = ["sg-026cc204887184c98", "sg-09ff0b46e3dd7a843"] } From 9d8fe156a669c45f7a9dca436c93811661e28018 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Fri, 17 Oct 2025 08:42:26 -0700 Subject: [PATCH 03/25] initial commit --- .../aws/vpc/.terraform.lock.hcl | 25 +++++++++++++++++++ card_data/infrastructure/aws/vpc/outputs.tf | 3 +++ card_data/infrastructure/aws/vpc/provider.tf | 13 ++++++++++ card_data/infrastructure/aws/vpc/vpc.tf | 19 ++++++++++++++ 4 files changed, 60 insertions(+) create mode 100644 card_data/infrastructure/aws/vpc/.terraform.lock.hcl create mode 100755 card_data/infrastructure/aws/vpc/outputs.tf create mode 100755 card_data/infrastructure/aws/vpc/provider.tf create mode 100755 card_data/infrastructure/aws/vpc/vpc.tf diff --git a/card_data/infrastructure/aws/vpc/.terraform.lock.hcl b/card_data/infrastructure/aws/vpc/.terraform.lock.hcl new file mode 100644 index 00000000..ee20480f --- /dev/null +++ b/card_data/infrastructure/aws/vpc/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.7.0" + constraints = "~> 6.7.0" + hashes = [ + "h1:vISrEI1xUh0w7NXTQ9m6ZEnQ1dv02yy+EJvxW78DAoI=", + "zh:3c0a256f813e5e2c1e1aa137204ad9168ebe487f6cee874af9e9c78eb300568e", + "zh:3c49dd75ea28395b29ba259988826b956c8adf6c0b59dd8874feb4f47bad976a", + "zh:3e6e3e3bfc6594f4f9e2c017ee588c5fcad394b87dd0b68a3f37cd66001f3c8c", + "zh:3f9b55826eeebf9b2ed448fc111d772c703e1edc6678e1bb646e66f3c3f9308f", + "zh:44e4ced936045ddc42d22c653a6427e7eb2b7aee918dff8438da0cb40996beb4", + "zh:474ab4d63918f41e8ea1cef43aeb1c719629dbf289db175c95de1431a8853ae7", + "zh:71b9e1d82c5ccc8d9bf72b3712c2b90722fc1f35a0f0f7a9557b9ee01971e6e2", + "zh:7723256d6ccc55f4000d1df8db202b02b30a7d917f5d31624c717e14ba15ea95", + "zh:82174836faa830aff0e47ea61d4cfbb5c97e1e944b1978f1d933acd37f584c88", + "zh:8e62fdc10206ba7232eec991e5a387378f2fbe47cc717b7f60eeb1df2c974514", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:be24dd2d53b224d7098e75ca432746e3420ce071189eea100aa8cbcd2498d389", + "zh:d27651d0e458933127ddca35a833e1a0f0ff0c131391288b3239763a2fd8f96f", + "zh:d33c181fff1b96bf8366e6c3d92408370b21649291e8f4d1f7e9a3fbb920fc9d", + "zh:edc0a0a84f85036c6d3df29d09557bd43206d9ee57b10542b484050f0f34d242", + ] +} diff --git a/card_data/infrastructure/aws/vpc/outputs.tf b/card_data/infrastructure/aws/vpc/outputs.tf new file mode 100755 index 00000000..a203fa74 --- /dev/null +++ b/card_data/infrastructure/aws/vpc/outputs.tf @@ -0,0 +1,3 @@ +output "aws_vpc_tfer--vpc-0f9d6a37031fa6597_id" { + value = "${aws_vpc.tfer--vpc-0f9d6a37031fa6597.id}" +} diff --git a/card_data/infrastructure/aws/vpc/provider.tf b/card_data/infrastructure/aws/vpc/provider.tf new file mode 100755 index 00000000..ac059ca9 --- /dev/null +++ b/card_data/infrastructure/aws/vpc/provider.tf @@ -0,0 +1,13 @@ +provider "aws" { + profile = "terraform-user" + region = "us-west-2" +} + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.7.0" + } + } +} \ No newline at end of file diff --git a/card_data/infrastructure/aws/vpc/vpc.tf b/card_data/infrastructure/aws/vpc/vpc.tf new file mode 100755 index 00000000..456cd73e --- /dev/null +++ b/card_data/infrastructure/aws/vpc/vpc.tf @@ -0,0 +1,19 @@ +resource "aws_vpc" "tfer--vpc-0f9d6a37031fa6597" { + assign_generated_ipv6_cidr_block = "false" + cidr_block = "10.0.0.0/20" + enable_dns_hostnames = "true" + enable_dns_support = "true" + enable_network_address_usage_metrics = "false" + instance_tenancy = "default" + region = "us-west-2" + + tags = { + Name = "poke-cli-vpc" + project = "poke-cli" + } + + tags_all = { + Name = "poke-cli-vpc" + project = "poke-cli" + } +} From 7c764b1edbd58f91dfd2f279d15bebc4bede512c Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sat, 18 Oct 2025 23:50:01 -0700 Subject: [PATCH 04/25] initial commit --- .../aws/ec2/.terraform.lock.hcl | 25 +++++++++++++++++++ card_data/infrastructure/aws/ec2/instance.tf | 21 ++++++++++++++++ card_data/infrastructure/aws/ec2/outputs.tf | 3 +++ card_data/infrastructure/aws/ec2/provider.tf | 20 +++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 card_data/infrastructure/aws/ec2/.terraform.lock.hcl create mode 100755 card_data/infrastructure/aws/ec2/instance.tf create mode 100755 card_data/infrastructure/aws/ec2/outputs.tf create mode 100755 card_data/infrastructure/aws/ec2/provider.tf diff --git a/card_data/infrastructure/aws/ec2/.terraform.lock.hcl b/card_data/infrastructure/aws/ec2/.terraform.lock.hcl new file mode 100644 index 00000000..ee20480f --- /dev/null +++ b/card_data/infrastructure/aws/ec2/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.7.0" + constraints = "~> 6.7.0" + hashes = [ + "h1:vISrEI1xUh0w7NXTQ9m6ZEnQ1dv02yy+EJvxW78DAoI=", + "zh:3c0a256f813e5e2c1e1aa137204ad9168ebe487f6cee874af9e9c78eb300568e", + "zh:3c49dd75ea28395b29ba259988826b956c8adf6c0b59dd8874feb4f47bad976a", + "zh:3e6e3e3bfc6594f4f9e2c017ee588c5fcad394b87dd0b68a3f37cd66001f3c8c", + "zh:3f9b55826eeebf9b2ed448fc111d772c703e1edc6678e1bb646e66f3c3f9308f", + "zh:44e4ced936045ddc42d22c653a6427e7eb2b7aee918dff8438da0cb40996beb4", + "zh:474ab4d63918f41e8ea1cef43aeb1c719629dbf289db175c95de1431a8853ae7", + "zh:71b9e1d82c5ccc8d9bf72b3712c2b90722fc1f35a0f0f7a9557b9ee01971e6e2", + "zh:7723256d6ccc55f4000d1df8db202b02b30a7d917f5d31624c717e14ba15ea95", + "zh:82174836faa830aff0e47ea61d4cfbb5c97e1e944b1978f1d933acd37f584c88", + "zh:8e62fdc10206ba7232eec991e5a387378f2fbe47cc717b7f60eeb1df2c974514", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:be24dd2d53b224d7098e75ca432746e3420ce071189eea100aa8cbcd2498d389", + "zh:d27651d0e458933127ddca35a833e1a0f0ff0c131391288b3239763a2fd8f96f", + "zh:d33c181fff1b96bf8366e6c3d92408370b21649291e8f4d1f7e9a3fbb920fc9d", + "zh:edc0a0a84f85036c6d3df29d09557bd43206d9ee57b10542b484050f0f34d242", + ] +} diff --git a/card_data/infrastructure/aws/ec2/instance.tf b/card_data/infrastructure/aws/ec2/instance.tf new file mode 100755 index 00000000..da0a71a0 --- /dev/null +++ b/card_data/infrastructure/aws/ec2/instance.tf @@ -0,0 +1,21 @@ +resource "aws_instance" "tfer--i-01dbf82e21c0da38f_dagster-webserver" { + ami = "ami-0326baaa98cf958ed" + instance_type = "t4g.small" + key_name = "dagster-vm-key-pair" + subnet_id = "subnet-04fe6e100221b27d4" + vpc_security_group_ids = ["sg-04c2a30cb05044ad6", "sg-026cc204887184c98"] + ebs_optimized = true + monitoring = false + + root_block_device { + delete_on_termination = true + } + + tags = { + Name = "dagster-webserver" + } + + tags_all = { + Name = "dagster-webserver" + } +} diff --git a/card_data/infrastructure/aws/ec2/outputs.tf b/card_data/infrastructure/aws/ec2/outputs.tf new file mode 100755 index 00000000..4d1c9793 --- /dev/null +++ b/card_data/infrastructure/aws/ec2/outputs.tf @@ -0,0 +1,3 @@ +output "aws_instance_tfer--i-01dbf82e21c0da38f_dagster-webserver_id" { + value = "${aws_instance.tfer--i-01dbf82e21c0da38f_dagster-webserver.id}" +} diff --git a/card_data/infrastructure/aws/ec2/provider.tf b/card_data/infrastructure/aws/ec2/provider.tf new file mode 100755 index 00000000..0ce5f9f0 --- /dev/null +++ b/card_data/infrastructure/aws/ec2/provider.tf @@ -0,0 +1,20 @@ +provider "aws" { + region = "us-west-2" +} + +terraform { + cloud { + organization = "digitalghost-dev" + + workspaces { + project = "poke-cli" + name = "ec2" + } + } + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.7.0" + } + } +} \ No newline at end of file From 1bd01ec3c23b5014ee1dbba397a4c27b3c8325fd Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sat, 18 Oct 2025 23:50:33 -0700 Subject: [PATCH 05/25] adding HCP cloud --- card_data/infrastructure/aws/vpc/provider.tf | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/card_data/infrastructure/aws/vpc/provider.tf b/card_data/infrastructure/aws/vpc/provider.tf index ac059ca9..b5a90a67 100755 --- a/card_data/infrastructure/aws/vpc/provider.tf +++ b/card_data/infrastructure/aws/vpc/provider.tf @@ -1,9 +1,16 @@ provider "aws" { - profile = "terraform-user" region = "us-west-2" } terraform { + cloud { + organization = "digitalghost-dev" + + workspaces { + project = "poke-cli" + name = "vpc" + } + } required_providers { aws = { source = "hashicorp/aws" From 08527a48355ca7e2f3d6eee7cf575e0de9bdb720 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sat, 18 Oct 2025 23:51:35 -0700 Subject: [PATCH 06/25] updating workspace name --- card_data/infrastructure/aws/rds/provider.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/card_data/infrastructure/aws/rds/provider.tf b/card_data/infrastructure/aws/rds/provider.tf index 56b44533..80b8e940 100755 --- a/card_data/infrastructure/aws/rds/provider.tf +++ b/card_data/infrastructure/aws/rds/provider.tf @@ -8,7 +8,7 @@ terraform { workspaces { project = "poke-cli" - name = "poke-cli" + name = "rds" } } required_providers { From fe3c34ec1ea28f91d010e3159733f512671bc3b3 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sat, 18 Oct 2025 23:51:53 -0700 Subject: [PATCH 07/25] updating variable name --- card_data/infrastructure/aws/rds/db_instance.tf | 4 ++-- card_data/infrastructure/aws/rds/variables.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/card_data/infrastructure/aws/rds/db_instance.tf b/card_data/infrastructure/aws/rds/db_instance.tf index 013579ec..8e04b6f9 100755 --- a/card_data/infrastructure/aws/rds/db_instance.tf +++ b/card_data/infrastructure/aws/rds/db_instance.tf @@ -20,7 +20,7 @@ resource "aws_db_instance" "tfer--dagster-db" { iam_database_authentication_enabled = "false" identifier = "dagster-db" instance_class = "db.t3.micro" - kms_key_id = var.kms-arn-key + kms_key_id = var.kms_arn_key license_model = "postgresql-license" maintenance_window = "wed:08:28-wed:08:58" manage_master_user_password = true @@ -32,7 +32,7 @@ resource "aws_db_instance" "tfer--dagster-db" { option_group_name = "default:postgres-17" parameter_group_name = "default.postgres17" performance_insights_enabled = "true" - performance_insights_kms_key_id = var.kms-arn-key + performance_insights_kms_key_id = var.kms_arn_key performance_insights_retention_period = "7" port = "5432" publicly_accessible = "true" diff --git a/card_data/infrastructure/aws/rds/variables.tf b/card_data/infrastructure/aws/rds/variables.tf index c6463500..6f56a005 100644 --- a/card_data/infrastructure/aws/rds/variables.tf +++ b/card_data/infrastructure/aws/rds/variables.tf @@ -1,4 +1,4 @@ -variable "kms-arn-key" { +variable "kms_arn_key" { description = "KMS key for RDS" type = string sensitive = true From 118e479c24b778b549cfabb2957009c687c28045 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sat, 18 Oct 2025 23:54:01 -0700 Subject: [PATCH 08/25] adding non-root user --- docs/Dockerfile | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/Dockerfile b/docs/Dockerfile index 4f3715a9..f62b43a4 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,22 +1,34 @@ # Dockerfile.prod FROM python:3.12-slim AS builder +RUN groupadd -r -g 10001 docsuser && \ + useradd -r -u 10001 -g docsuser -m -s /sbin/nologin docsuser + WORKDIR /build -RUN pip install mkdocs mkdocs-material mkdocs-nav-weight +RUN pip install --no-cache-dir mkdocs mkdocs-material mkdocs-nav-weight + +COPY --chown=docsuser:docsuser mkdocs.yml /build/mkdocs.yml +COPY --chown=docsuser:docsuser docs/ /build/docs/ -COPY mkdocs.yml /build/mkdocs.yml +RUN chown -R docsuser:docsuser /build -COPY docs/ /build/docs/ +USER docsuser RUN mkdocs build # --- Serve with lightweight HTTP server --- FROM python:3.12-slim +# Create non-root user for runtime +RUN groupadd -r -g 10001 docsuser && \ + useradd -r -u 10001 -g docsuser -m -s /sbin/nologin docsuser + WORKDIR /site -COPY --from=builder /build/site /site +COPY --from=builder --chown=docsuser:docsuser /build/site /site + +USER docsuser EXPOSE 8080 From ac4418f7e6ba6e592fb57ae1983ab8b4e8b1786c Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sat, 18 Oct 2025 23:57:24 -0700 Subject: [PATCH 09/25] removing private subnets --- card_data/infrastructure/aws/rds/db_subnet_group.tf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/card_data/infrastructure/aws/rds/db_subnet_group.tf b/card_data/infrastructure/aws/rds/db_subnet_group.tf index 04f24e76..81e9a6a8 100755 --- a/card_data/infrastructure/aws/rds/db_subnet_group.tf +++ b/card_data/infrastructure/aws/rds/db_subnet_group.tf @@ -2,5 +2,8 @@ resource "aws_db_subnet_group" "tfer--poke-cli-db-subnet-group" { description = "Subnet group for RDS databases" name = "poke-cli-db-subnet-group" region = "us-west-2" - subnet_ids = ["subnet-08dae4b7aede93128", "subnet-0eeb519cf23a763bf", "subnet-04fe6e100221b27d4", "subnet-0be3aac807720c1d6"] + subnet_ids = [ + "subnet-04fe6e100221b27d4", + "subnet-0be3aac807720c1d6" + ] } From 0bce206820b6a1f7cc3cc5567c34e3088c7fbcfe Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 00:01:36 -0700 Subject: [PATCH 10/25] adding schedule for pricing_data --- card_data/pipelines/definitions.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/card_data/pipelines/definitions.py b/card_data/pipelines/definitions.py index c238565b..e7c5e692 100644 --- a/card_data/pipelines/definitions.py +++ b/card_data/pipelines/definitions.py @@ -5,10 +5,14 @@ import dagster as dg +from .defs.extract.extract_pricing_data import build_dataframe +from .defs.load.load_pricing_data import load_pricing_data + @definitions def defs(): - return load_from_defs_folder(project_root=Path(__file__).parent.parent) + folder_defs = load_from_defs_folder(project_root=Path(__file__).parent.parent) + return dg.Definitions.merge(folder_defs, defs_pricing) dbt_project_directory = Path(__file__).absolute().parent / "poke_cli_dbt" dbt_project = DbtProject(project_dir=dbt_project_directory) @@ -23,5 +27,13 @@ def defs(): def dbt_models(context: dg.AssetExecutionContext, dbt: DbtCliResource): yield from dbt.cli(["build"], context=context).stream() -# Dagster object that contains the dbt assets and resource -defs_dbt = dg.Definitions(assets=[dbt_models], resources={"dbt": dbt_resource}) \ No newline at end of file +price_schedule = dg.ScheduleDefinition( + name="price_schedule", + cron_schedule="10 10 * * *", + target=[build_dataframe, load_pricing_data], + execution_timezone="America/Los_Angeles", +) + +defs_pricing = dg.Definitions( + schedules=[price_schedule], +) \ No newline at end of file From e4f477c6212e5d45c3b0f554f948850bf1807bbe Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 01:53:22 -0700 Subject: [PATCH 11/25] updating test data --- .github/workflows/ci.yml | 2 +- .goreleaser.yml | 2 +- Dockerfile | 2 +- cmd/pokemon/pokemon.go | 13 +++++++++++-- nfpm.yaml | 2 +- testdata/main_latest_flag.golden | 2 +- testdata/pokemon_defense.golden | 2 +- testdata/pokemon_defense_ability_immunities.golden | 2 +- testdata/pokemon_image.golden | 2 +- testdata/pokemon_image_flag_non-valid_size.golden | 2 +- testdata/pokemon_no_flags_dual_type.golden | 2 +- testdata/pokemon_regional_form.golden | 2 +- testdata/pokemon_stats.golden | 2 +- 13 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb76d1dd..f20639bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ on: - main env: - VERSION_NUMBER: 'v1.7.1' + VERSION_NUMBER: 'v1.7.2' DOCKERHUB_REGISTRY_NAME: 'digitalghostdev/poke-cli' AWS_REGION: 'us-west-2' diff --git a/.goreleaser.yml b/.goreleaser.yml index e1710718..da490082 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -14,7 +14,7 @@ builds: - windows - darwin ldflags: - - -s -w -X main.version=v1.7.1 + - -s -w -X main.version=v1.7.2 archives: - formats: [ 'zip' ] diff --git a/Dockerfile b/Dockerfile index e26296e5..b7599edd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN go mod download COPY . . -RUN go build -ldflags "-X main.version=v1.7.1" -o poke-cli . +RUN go build -ldflags "-X main.version=v1.7.2" -o poke-cli . # build 2 FROM --platform=$BUILDPLATFORM alpine:3.22 diff --git a/cmd/pokemon/pokemon.go b/cmd/pokemon/pokemon.go index f9ca07b8..6f2eafa3 100644 --- a/cmd/pokemon/pokemon.go +++ b/cmd/pokemon/pokemon.go @@ -102,8 +102,17 @@ func PokemonCommand() (string, error) { var eggGroupSlice []string for _, entry := range pokemonSpeciesStruct.EggGroups { - capitalizedEggGroup := cases.Title(language.English).String(entry.Name) - eggGroupSlice = append(eggGroupSlice, capitalizedEggGroup) + modernEggGroupNames := map[string]string{ + "indeterminate": "Amorphous", + } + + if name, exists := modernEggGroupNames[entry.Name]; exists { + eggGroupSlice = append(eggGroupSlice, name) + } else { + capitalizedEggGroup := cases.Title(language.English).String(entry.Name) + eggGroupSlice = append(eggGroupSlice, capitalizedEggGroup) + } + } sort.Strings(eggGroupSlice) diff --git a/nfpm.yaml b/nfpm.yaml index a468f1f0..8957c785 100644 --- a/nfpm.yaml +++ b/nfpm.yaml @@ -1,7 +1,7 @@ name: "poke-cli" arch: "arm64" platform: "linux" -version: "v1.7.1" +version: "v1.7.2" section: "default" version_schema: semver maintainer: "Christian S" diff --git a/testdata/main_latest_flag.golden b/testdata/main_latest_flag.golden index c0f9ffd0..35897396 100644 --- a/testdata/main_latest_flag.golden +++ b/testdata/main_latest_flag.golden @@ -1,6 +1,6 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ ┃ ┃ ┃ Latest available version: ┃ -┃ • v1.7.0 ┃ +┃ • v1.7.1 ┃ ┃ ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ diff --git a/testdata/pokemon_defense.golden b/testdata/pokemon_defense.golden index a7554600..3bb066f0 100644 --- a/testdata/pokemon_defense.golden +++ b/testdata/pokemon_defense.golden @@ -9,7 +9,7 @@ look forward to being launched out at Mach speeds. • Weight: 50.0kg (110.2 lbs) • Height: 3.0m (9′10″) • Evolves from: Drakloak -• Egg Group(s): Dragon, Indeterminate +• Egg Group(s): Amorphous, Dragon ───────────── Type Defenses Immune: Fighting, Normal diff --git a/testdata/pokemon_defense_ability_immunities.golden b/testdata/pokemon_defense_ability_immunities.golden index 4695fd75..5d86513d 100644 --- a/testdata/pokemon_defense_ability_immunities.golden +++ b/testdata/pokemon_defense_ability_immunities.golden @@ -9,7 +9,7 @@ escapes. • Weight: 29.9kg (65.9 lbs) • Height: 0.9m (2′11″) • Evolves from: Shellos -• Egg Group(s): Indeterminate, Water1 +• Egg Group(s): Amorphous, Water1 ───────────── Type Defenses Immune: Electric diff --git a/testdata/pokemon_image.golden b/testdata/pokemon_image.golden index 7e713a66..7921d0db 100644 --- a/testdata/pokemon_image.golden +++ b/testdata/pokemon_image.golden @@ -10,7 +10,7 @@ Skeledirge’s head gained a soul. • Weight: 326.5kg (719.8 lbs) • Height: 1.6m (5′03″) • Evolves from: Crocalor -• Egg Group(s): Ground +• Egg Group(s): Field ───── Image ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ diff --git a/testdata/pokemon_image_flag_non-valid_size.golden b/testdata/pokemon_image_flag_non-valid_size.golden index 7292e67b..7018d2ed 100644 --- a/testdata/pokemon_image_flag_non-valid_size.golden +++ b/testdata/pokemon_image_flag_non-valid_size.golden @@ -9,7 +9,7 @@ in the rescues of drowning people. • Weight: 33.5kg (73.9 lbs) • Height: 1.1m (3′07″) • Evolves from: Buizel -• Egg Group(s): Ground, Water1 +• Egg Group(s): Field, Water1 ───── Image ╭───────────────────────────╮ diff --git a/testdata/pokemon_no_flags_dual_type.golden b/testdata/pokemon_no_flags_dual_type.golden index 1355d0a5..0dd5cf1b 100644 --- a/testdata/pokemon_no_flags_dual_type.golden +++ b/testdata/pokemon_no_flags_dual_type.golden @@ -9,4 +9,4 @@ Victini always win, regardless of the type of encounter. • Weight: 4.0kg (8.8 lbs) • Height: 0.4m (1′04″) • Basic Pokémon -• Egg Group(s): No-Eggs +• Egg Group(s): Undiscovered diff --git a/testdata/pokemon_regional_form.golden b/testdata/pokemon_regional_form.golden index ded2cdfb..f4d976c0 100644 --- a/testdata/pokemon_regional_form.golden +++ b/testdata/pokemon_regional_form.golden @@ -9,7 +9,7 @@ friendly and never appear to squabble. • Weight: 415.6kg (916.2 lbs) • Height: 10.9m (35′09″) • Evolves from: Exeggcute -• Egg Group(s): Plant +• Egg Group(s): Grass ────────── Base Stats HP ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 95 diff --git a/testdata/pokemon_stats.golden b/testdata/pokemon_stats.golden index 493af74e..a7862632 100644 --- a/testdata/pokemon_stats.golden +++ b/testdata/pokemon_stats.golden @@ -9,7 +9,7 @@ stored poison is churned for greater potency. • Weight: 44.4kg (97.9 lbs) • Height: 1.3m (4′03″) • Evolves from: Croagunk -• Egg Group(s): Humanshape +• Egg Group(s): Human-Like ────────── Base Stats HP ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 83 From 321c4d50959fd0c9c507485045261832dd57add9 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 02:10:15 -0700 Subject: [PATCH 12/25] updating test data --- .github/workflows/ci.yml | 2 +- .goreleaser.yml | 2 +- Dockerfile | 2 +- nfpm.yaml | 2 +- testdata/main_latest_flag.golden | 2 +- testdata/pokemon_defense.golden | 2 +- testdata/pokemon_defense_ability_immunities.golden | 2 +- testdata/pokemon_image.golden | 2 +- testdata/pokemon_image_flag_non-valid_size.golden | 2 +- testdata/pokemon_no_flags_dual_type.golden | 2 +- testdata/pokemon_regional_form.golden | 2 +- testdata/pokemon_stats.golden | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb76d1dd..f20639bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ on: - main env: - VERSION_NUMBER: 'v1.7.1' + VERSION_NUMBER: 'v1.7.2' DOCKERHUB_REGISTRY_NAME: 'digitalghostdev/poke-cli' AWS_REGION: 'us-west-2' diff --git a/.goreleaser.yml b/.goreleaser.yml index e1710718..da490082 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -14,7 +14,7 @@ builds: - windows - darwin ldflags: - - -s -w -X main.version=v1.7.1 + - -s -w -X main.version=v1.7.2 archives: - formats: [ 'zip' ] diff --git a/Dockerfile b/Dockerfile index e26296e5..b7599edd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN go mod download COPY . . -RUN go build -ldflags "-X main.version=v1.7.1" -o poke-cli . +RUN go build -ldflags "-X main.version=v1.7.2" -o poke-cli . # build 2 FROM --platform=$BUILDPLATFORM alpine:3.22 diff --git a/nfpm.yaml b/nfpm.yaml index a468f1f0..8957c785 100644 --- a/nfpm.yaml +++ b/nfpm.yaml @@ -1,7 +1,7 @@ name: "poke-cli" arch: "arm64" platform: "linux" -version: "v1.7.1" +version: "v1.7.2" section: "default" version_schema: semver maintainer: "Christian S" diff --git a/testdata/main_latest_flag.golden b/testdata/main_latest_flag.golden index c0f9ffd0..35897396 100644 --- a/testdata/main_latest_flag.golden +++ b/testdata/main_latest_flag.golden @@ -1,6 +1,6 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ ┃ ┃ ┃ Latest available version: ┃ -┃ • v1.7.0 ┃ +┃ • v1.7.1 ┃ ┃ ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ diff --git a/testdata/pokemon_defense.golden b/testdata/pokemon_defense.golden index a7554600..3bb066f0 100644 --- a/testdata/pokemon_defense.golden +++ b/testdata/pokemon_defense.golden @@ -9,7 +9,7 @@ look forward to being launched out at Mach speeds. • Weight: 50.0kg (110.2 lbs) • Height: 3.0m (9′10″) • Evolves from: Drakloak -• Egg Group(s): Dragon, Indeterminate +• Egg Group(s): Amorphous, Dragon ───────────── Type Defenses Immune: Fighting, Normal diff --git a/testdata/pokemon_defense_ability_immunities.golden b/testdata/pokemon_defense_ability_immunities.golden index 4695fd75..5d86513d 100644 --- a/testdata/pokemon_defense_ability_immunities.golden +++ b/testdata/pokemon_defense_ability_immunities.golden @@ -9,7 +9,7 @@ escapes. • Weight: 29.9kg (65.9 lbs) • Height: 0.9m (2′11″) • Evolves from: Shellos -• Egg Group(s): Indeterminate, Water1 +• Egg Group(s): Amorphous, Water1 ───────────── Type Defenses Immune: Electric diff --git a/testdata/pokemon_image.golden b/testdata/pokemon_image.golden index 7e713a66..7921d0db 100644 --- a/testdata/pokemon_image.golden +++ b/testdata/pokemon_image.golden @@ -10,7 +10,7 @@ Skeledirge’s head gained a soul. • Weight: 326.5kg (719.8 lbs) • Height: 1.6m (5′03″) • Evolves from: Crocalor -• Egg Group(s): Ground +• Egg Group(s): Field ───── Image ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ diff --git a/testdata/pokemon_image_flag_non-valid_size.golden b/testdata/pokemon_image_flag_non-valid_size.golden index 7292e67b..7018d2ed 100644 --- a/testdata/pokemon_image_flag_non-valid_size.golden +++ b/testdata/pokemon_image_flag_non-valid_size.golden @@ -9,7 +9,7 @@ in the rescues of drowning people. • Weight: 33.5kg (73.9 lbs) • Height: 1.1m (3′07″) • Evolves from: Buizel -• Egg Group(s): Ground, Water1 +• Egg Group(s): Field, Water1 ───── Image ╭───────────────────────────╮ diff --git a/testdata/pokemon_no_flags_dual_type.golden b/testdata/pokemon_no_flags_dual_type.golden index 1355d0a5..0dd5cf1b 100644 --- a/testdata/pokemon_no_flags_dual_type.golden +++ b/testdata/pokemon_no_flags_dual_type.golden @@ -9,4 +9,4 @@ Victini always win, regardless of the type of encounter. • Weight: 4.0kg (8.8 lbs) • Height: 0.4m (1′04″) • Basic Pokémon -• Egg Group(s): No-Eggs +• Egg Group(s): Undiscovered diff --git a/testdata/pokemon_regional_form.golden b/testdata/pokemon_regional_form.golden index ded2cdfb..f4d976c0 100644 --- a/testdata/pokemon_regional_form.golden +++ b/testdata/pokemon_regional_form.golden @@ -9,7 +9,7 @@ friendly and never appear to squabble. • Weight: 415.6kg (916.2 lbs) • Height: 10.9m (35′09″) • Evolves from: Exeggcute -• Egg Group(s): Plant +• Egg Group(s): Grass ────────── Base Stats HP ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 95 diff --git a/testdata/pokemon_stats.golden b/testdata/pokemon_stats.golden index 493af74e..a7862632 100644 --- a/testdata/pokemon_stats.golden +++ b/testdata/pokemon_stats.golden @@ -9,7 +9,7 @@ stored poison is churned for greater potency. • Weight: 44.4kg (97.9 lbs) • Height: 1.3m (4′03″) • Evolves from: Croagunk -• Egg Group(s): Humanshape +• Egg Group(s): Human-Like ────────── Base Stats HP ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 83 From 6ee6a869db2c3f620530153bddc5f9dc958d7270 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 02:14:12 -0700 Subject: [PATCH 13/25] updating egg group names to modern equivalents (#190) --- cmd/pokemon/pokemon.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cmd/pokemon/pokemon.go b/cmd/pokemon/pokemon.go index 6f2eafa3..9df67a8a 100644 --- a/cmd/pokemon/pokemon.go +++ b/cmd/pokemon/pokemon.go @@ -112,6 +112,20 @@ func PokemonCommand() (string, error) { capitalizedEggGroup := cases.Title(language.English).String(entry.Name) eggGroupSlice = append(eggGroupSlice, capitalizedEggGroup) } + modernEggGroupNames = map[string]string{ + "indeterminate": "Amorphous", + "ground": "Field", + "humanshape": "Human-Like", + "plant": "Grass", + "no-eggs": "Undiscovered", + } + + if name, exists := modernEggGroupNames[entry.Name]; exists { + eggGroupSlice = append(eggGroupSlice, name) + } else { + capitalizedEggGroup := cases.Title(language.English).String(entry.Name) + eggGroupSlice = append(eggGroupSlice, capitalizedEggGroup) + } } From adf217e2adbf5ac433c7f1dddcca635464b65af8 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 02:18:29 -0700 Subject: [PATCH 14/25] fixing accidental duplicate code (#190) --- cmd/pokemon/pokemon.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/cmd/pokemon/pokemon.go b/cmd/pokemon/pokemon.go index 9df67a8a..e67c1126 100644 --- a/cmd/pokemon/pokemon.go +++ b/cmd/pokemon/pokemon.go @@ -104,16 +104,6 @@ func PokemonCommand() (string, error) { for _, entry := range pokemonSpeciesStruct.EggGroups { modernEggGroupNames := map[string]string{ "indeterminate": "Amorphous", - } - - if name, exists := modernEggGroupNames[entry.Name]; exists { - eggGroupSlice = append(eggGroupSlice, name) - } else { - capitalizedEggGroup := cases.Title(language.English).String(entry.Name) - eggGroupSlice = append(eggGroupSlice, capitalizedEggGroup) - } - modernEggGroupNames = map[string]string{ - "indeterminate": "Amorphous", "ground": "Field", "humanshape": "Human-Like", "plant": "Grass", @@ -126,7 +116,6 @@ func PokemonCommand() (string, error) { capitalizedEggGroup := cases.Title(language.English).String(entry.Name) eggGroupSlice = append(eggGroupSlice, capitalizedEggGroup) } - } sort.Strings(eggGroupSlice) From 7592556f58360da23a37aea7854a5d080fb11433 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 12:31:23 -0700 Subject: [PATCH 15/25] updating version numbers --- card_data/pipelines/poke_cli_dbt/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/card_data/pipelines/poke_cli_dbt/dbt_project.yml b/card_data/pipelines/poke_cli_dbt/dbt_project.yml index 6c02b90a..4c7f2162 100644 --- a/card_data/pipelines/poke_cli_dbt/dbt_project.yml +++ b/card_data/pipelines/poke_cli_dbt/dbt_project.yml @@ -1,5 +1,5 @@ name: 'poke_cli_dbt' -version: '1.7.0' +version: '1.7.2' profile: 'poke_cli_dbt' From 724ab7f7571102e7d855158f0ce244aa9db965f3 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 13:53:25 -0700 Subject: [PATCH 16/25] fixing hyperlink --- docs/Infrastructure_Guide/dbt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Infrastructure_Guide/dbt.md b/docs/Infrastructure_Guide/dbt.md index 4cd17ec7..e78dcb8e 100644 --- a/docs/Infrastructure_Guide/dbt.md +++ b/docs/Infrastructure_Guide/dbt.md @@ -12,7 +12,7 @@ weight: 5 of data transformations. dbt compiles SQL models into executable queries and runs them in the proper order, turning raw data into analysis-ready datasets. - View more [about dbt](www.getdbt.com/product/what-is-dbt) + View more [about dbt](https://www.getdbt.com/product/what-is-dbt) ## Installation & Initialization From 3c7fde0eb7949ba4401df4015a4bb9b414947f39 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 13:56:24 -0700 Subject: [PATCH 17/25] updating version numbers --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7f19f28f..c952c057 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,12 @@ pokemon-logo

Pokémon CLI

version-label - docker-image-size + docker-image-size ci-status-badge +
+ coderabbit-review-count-badge +
tests-label go-version @@ -91,11 +94,11 @@ Cloudsmith is a fully cloud-based service that lets you easily create, store, an 3. Choose how to interact with the container: * Run a single command and exit: ```bash - docker run --rm -it digitalghostdev/poke-cli:v1.7.1 [subcommand] flag] + docker run --rm -it digitalghostdev/poke-cli:v1.7.2 [subcommand] flag] ``` * Enter the container and use its shell: ```bash - docker run --rm -it --name poke-cli --entrypoint /bin/sh digitalghostdev/poke-cli:v1.7.1 -c "cd /app && exec sh" + docker run --rm -it --name poke-cli --entrypoint /bin/sh digitalghostdev/poke-cli:v1.7.2 -c "cd /app && exec sh" # placed into the /app directory, run the program with './poke-cli' # example: ./poke-cli ability swift-swim ``` From 463f283dbc21d21bcbc8a9fb07a66fc5937d45a5 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 14:16:11 -0700 Subject: [PATCH 18/25] fixing formatting --- docs/Infrastructure_Guide/aws.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Infrastructure_Guide/aws.md b/docs/Infrastructure_Guide/aws.md index 884c70c1..37a21ad7 100644 --- a/docs/Infrastructure_Guide/aws.md +++ b/docs/Infrastructure_Guide/aws.md @@ -190,6 +190,7 @@ AWS EC2 (Elastic Compute Cloud) is a cloud service that provides resizable virtu ### Configure Instance Once connected to the virtual machine, run the following commands to get everything set up: + 1. Clone repository * Create a new directory: `git init ` * `cd ` From 488d96e68279830f099a657488911e9f67ac0df4 Mon Sep 17 00:00:00 2001 From: Christian <86637723+digitalghost-dev@users.noreply.github.com> Date: Sun, 19 Oct 2025 14:48:54 -0700 Subject: [PATCH 19/25] fixing typo Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- docs/Infrastructure_Guide/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Infrastructure_Guide/aws.md b/docs/Infrastructure_Guide/aws.md index 37a21ad7..d64cb273 100644 --- a/docs/Infrastructure_Guide/aws.md +++ b/docs/Infrastructure_Guide/aws.md @@ -147,7 +147,7 @@ Feel free to choose any other options that could be more suitable._ 10. Under **Tags**, create a new tag if desired for resource organization. 11. Under **Database Authentication**, choose _password authentication_. 12. Under **Monitoring**, choose the standard version of _Database Insights_. All other options in this section can be left as default. -13. Review the **Estimated Monthly Costs**, make any changes if necessary, they click on _create database_. +13. Review the **Estimated Monthly Costs**, make any changes if necessary, then click on _create database_. --- From 7210fc855979352ff44d67e1b22c597c738b1660 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 15:21:39 -0700 Subject: [PATCH 20/25] refactor load_pricing_data for explicit asset dependency --- card_data/pipelines/defs/load/load_pricing_data.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/card_data/pipelines/defs/load/load_pricing_data.py b/card_data/pipelines/defs/load/load_pricing_data.py index c9b989dd..3f20be64 100644 --- a/card_data/pipelines/defs/load/load_pricing_data.py +++ b/card_data/pipelines/defs/load/load_pricing_data.py @@ -1,4 +1,5 @@ import dagster as dg +import polars as pl from dagster import RetryPolicy, Backoff from sqlalchemy.exc import OperationalError from ..extract.extract_pricing_data import build_dataframe @@ -7,17 +8,15 @@ @dg.asset( - deps=[build_dataframe], kinds={"Supabase", "Postgres"}, retry_policy=RetryPolicy(max_retries=3, delay=2, backoff=Backoff.EXPONENTIAL), ) -def load_pricing_data() -> None: +def load_pricing_data(build_pricing_dataframe: pl.DataFrame) -> None: database_url: str = fetch_secret() table_name: str = "staging.pricing_data" - df = build_dataframe() try: - df.write_database( + build_pricing_dataframe.write_database( table_name=table_name, connection=database_url, if_table_exists="replace" ) print(colored(" ✓", "green"), f"Data loaded into {table_name}") From 9f0af93a914d87501baf561d587da386cf37c6eb Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 15:21:39 -0700 Subject: [PATCH 21/25] introduce pricing_pipeline_job and update schedule --- card_data/pipelines/definitions.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/card_data/pipelines/definitions.py b/card_data/pipelines/definitions.py index e7c5e692..c35b808b 100644 --- a/card_data/pipelines/definitions.py +++ b/card_data/pipelines/definitions.py @@ -27,13 +27,21 @@ def defs(): def dbt_models(context: dg.AssetExecutionContext, dbt: DbtCliResource): yield from dbt.cli(["build"], context=context).stream() +# Define the pricing pipeline job that materializes both assets +pricing_pipeline_job = dg.define_asset_job( + name="pricing_pipeline_job", + selection=dg.AssetSelection.assets(build_dataframe, load_pricing_data), +) + price_schedule = dg.ScheduleDefinition( name="price_schedule", cron_schedule="10 10 * * *", - target=[build_dataframe, load_pricing_data], + target=pricing_pipeline_job, execution_timezone="America/Los_Angeles", ) defs_pricing = dg.Definitions( + assets=[build_dataframe, load_pricing_data], + jobs=[pricing_pipeline_job], schedules=[price_schedule], ) \ No newline at end of file From 703329753f20a274c951ce72a6ee0f0af232e6a1 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 21:32:41 -0700 Subject: [PATCH 22/25] fixing imports --- card_data/pipelines/defs/load/load_pricing_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/card_data/pipelines/defs/load/load_pricing_data.py b/card_data/pipelines/defs/load/load_pricing_data.py index 3f20be64..5f15f859 100644 --- a/card_data/pipelines/defs/load/load_pricing_data.py +++ b/card_data/pipelines/defs/load/load_pricing_data.py @@ -2,10 +2,10 @@ import polars as pl from dagster import RetryPolicy, Backoff from sqlalchemy.exc import OperationalError -from ..extract.extract_pricing_data import build_dataframe -from ...utils.secret_retriever import fetch_secret from termcolor import colored +from ...utils.secret_retriever import fetch_secret + @dg.asset( kinds={"Supabase", "Postgres"}, From ec9d28c07e1d30e678dfbd573621d8782deb4783 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 21:33:06 -0700 Subject: [PATCH 23/25] renaming function --- card_data/pipelines/defs/transformation/transform_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/card_data/pipelines/defs/transformation/transform_data.py b/card_data/pipelines/defs/transformation/transform_data.py index b67bb473..c47c8fb3 100644 --- a/card_data/pipelines/defs/transformation/transform_data.py +++ b/card_data/pipelines/defs/transformation/transform_data.py @@ -28,7 +28,7 @@ def get_asset_key(self, dbt_resource_props): manifest=DBT_PROJECT_PATH / "target" / "manifest.json", dagster_dbt_translator=CustomDbtTranslator() ) -def poke_cli_dbt_assets(context: dg.AssetExecutionContext, dbt: DbtCliResource): +def dbt_load_pricing_data(context: dg.AssetExecutionContext, dbt: DbtCliResource): """ dbt assets that transform staging data into final models. """ @@ -36,6 +36,6 @@ def poke_cli_dbt_assets(context: dg.AssetExecutionContext, dbt: DbtCliResource): dbt_resource = DbtCliResource(project_dir=DBT_PROJECT_PATH) defs = dg.Definitions( - assets=[poke_cli_dbt_assets], + assets=[dbt_load_pricing_data], resources={"dbt": dbt_resource} ) From fccbd45fbe03537631d78353f9af056e42025155 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 21:35:32 -0700 Subject: [PATCH 24/25] adding asset key to pricing_data source --- card_data/pipelines/poke_cli_dbt/models/sources.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/card_data/pipelines/poke_cli_dbt/models/sources.yml b/card_data/pipelines/poke_cli_dbt/models/sources.yml index dac332ac..c70b131e 100644 --- a/card_data/pipelines/poke_cli_dbt/models/sources.yml +++ b/card_data/pipelines/poke_cli_dbt/models/sources.yml @@ -96,6 +96,9 @@ sources: - name: pricing_data description: "Card pricing data" + meta: + dagster: + asset_key: ["load_pricing_data"] columns: - name: product_id description: "Product ID" From bd9cf976ebacea97eeae5566e6cdee72dbf4ac96 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 19 Oct 2025 21:36:24 -0700 Subject: [PATCH 25/25] simplifying file, reducing code --- card_data/pipelines/definitions.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/card_data/pipelines/definitions.py b/card_data/pipelines/definitions.py index c35b808b..dc7a2a64 100644 --- a/card_data/pipelines/definitions.py +++ b/card_data/pipelines/definitions.py @@ -1,7 +1,6 @@ from pathlib import Path from dagster import definitions, load_from_defs_folder -from dagster_dbt import DbtCliResource, DbtProject, dbt_assets import dagster as dg @@ -11,31 +10,19 @@ @definitions def defs(): + # load_from_defs_folder discovers dbt assets from transform_data.py folder_defs = load_from_defs_folder(project_root=Path(__file__).parent.parent) return dg.Definitions.merge(folder_defs, defs_pricing) -dbt_project_directory = Path(__file__).absolute().parent / "poke_cli_dbt" -dbt_project = DbtProject(project_dir=dbt_project_directory) - -dbt_resource = DbtCliResource(project_dir=dbt_project) - -# Compiles the dbt project & allow Dagster to build an asset graph -dbt_project.prepare_if_dev() - -# Yields Dagster events streamed from the dbt CLI -@dbt_assets(manifest=dbt_project.manifest_path) -def dbt_models(context: dg.AssetExecutionContext, dbt: DbtCliResource): - yield from dbt.cli(["build"], context=context).stream() - -# Define the pricing pipeline job that materializes both assets +# Define the pricing pipeline job that materializes the assets and downstream dbt model pricing_pipeline_job = dg.define_asset_job( name="pricing_pipeline_job", - selection=dg.AssetSelection.assets(build_dataframe, load_pricing_data), + selection=dg.AssetSelection.assets(build_dataframe, load_pricing_data).downstream(include_self=True), ) price_schedule = dg.ScheduleDefinition( name="price_schedule", - cron_schedule="10 10 * * *", + cron_schedule="31 21 * * *", target=pricing_pipeline_job, execution_timezone="America/Los_Angeles", )