diff --git a/README.md b/README.md index 50204b0a60b..2d5ee89b000 100644 --- a/README.md +++ b/README.md @@ -651,6 +651,7 @@ Platform usage. The framework uses RIOT to bridge current product gaps with Memorystore Clusters * [ML Project Generator](tools/ml-project-generator) - A utility to create a Production grade ML project template with the best productivity tools installed like auto-formatting, license checks, linting, etc. * [Policy Tags Engine](tools/policy-tags-engine) - The tool allows developers to automatically apply BigQuery column-level security (Policy Tags) based on metadata files uploaded to a Google Cloud Storage (GCS) bucket. +* [Gemini CLI Health Dashboard](tools/gemini-cli-health-dashboard) - The tool allows developers to build a custom dashboard for tracking the Gemini CLI health in terms of success vs error rate. ## Contributing diff --git a/examples/ccai-agentassist-five9-grpc/requirements.txt b/examples/ccai-agentassist-five9-grpc/requirements.txt index d848dd43b9b..4cdb60f9245 100644 --- a/examples/ccai-agentassist-five9-grpc/requirements.txt +++ b/examples/ccai-agentassist-five9-grpc/requirements.txt @@ -28,4 +28,4 @@ rsa==4.9 six==1.16.0 sounddevice==0.4.7 tomlkit==0.13.0 -urllib3==2.5.0 +urllib3==2.6.0 diff --git a/examples/cloudrun-crl-monitor/crl-monitor/main.tf b/examples/cloudrun-crl-monitor/crl-monitor/main.tf index 46bd0f0ae07..d411aae7757 100644 --- a/examples/cloudrun-crl-monitor/crl-monitor/main.tf +++ b/examples/cloudrun-crl-monitor/crl-monitor/main.tf @@ -61,6 +61,10 @@ resource "google_cloud_run_v2_job" "crl_validation" { name = "CRL_EXPIRY_BUFFER" value = var.crl_expiration_buffer } + env { + name = "PROXY_URL" + value = var.proxy_url + } command = ["/bin/bash", "-c"] args = [ @@ -71,7 +75,13 @@ resource "google_cloud_run_v2_job" "crl_validation" { CRL_URL="$TARGET_DOMAIN" echo "Checking CRL at $CRL_URL..." - if curl -s -f -o crl.file "$CRL_URL"; then + PROXY_ARGS="" + if [ -n "$PROXY_URL" ]; then + echo "Using proxy: $PROXY_URL" + PROXY_ARGS="-x $PROXY_URL" + fi + + if curl -s -f $PROXY_ARGS -o crl.file "$CRL_URL"; then echo "CRL downloaded successfully." # Try to parse as DER first, then PEM diff --git a/examples/cloudrun-crl-monitor/crl-monitor/variables.tf b/examples/cloudrun-crl-monitor/crl-monitor/variables.tf index a0634b2700c..d338a2b8a44 100644 --- a/examples/cloudrun-crl-monitor/crl-monitor/variables.tf +++ b/examples/cloudrun-crl-monitor/crl-monitor/variables.tf @@ -44,3 +44,9 @@ variable "alert_autoclose" { type = string default = "1800s" } + +variable "proxy_url" { + description = "Optional proxy URL to use for curl" + type = string + default = "" +} diff --git a/examples/cloudrun-crl-monitor/main.tf b/examples/cloudrun-crl-monitor/main.tf index 7e6b38c2ccd..6c9f864ba8f 100644 --- a/examples/cloudrun-crl-monitor/main.tf +++ b/examples/cloudrun-crl-monitor/main.tf @@ -31,13 +31,14 @@ resource "google_project_iam_member" "crl_job_metric_writer" { } module "crl_monitor" { - source = "./crl-monitor" + source = "./crl-monitor" for_each = { for m in var.crl_monitors : m.target_url => m } - project_id = var.project_id - region = each.value.region - name = each.value.name - target_url = each.value.target_url - schedule = each.value.schedule + project_id = var.project_id + region = each.value.region + name = each.value.name + target_url = each.value.target_url + schedule = each.value.schedule crl_expiration_buffer = each.value.crl_expiration_buffer -} \ No newline at end of file + proxy_url = each.value.proxy_url +} diff --git a/examples/cloudrun-crl-monitor/terraform.tfvars b/examples/cloudrun-crl-monitor/terraform.tfvars index 829df773079..8a4f1b85032 100644 --- a/examples/cloudrun-crl-monitor/terraform.tfvars +++ b/examples/cloudrun-crl-monitor/terraform.tfvars @@ -14,22 +14,22 @@ * limitations under the License. */ -project_id = "sshsergey-argolis" +project_id = "sshsergey-argolis" crl_monitors = [ { - name = "bsi-ca" - region = "europe-west3" - target_url = "http://download.gsb.bund.de/BSI/crl/DE_CRL.crl" - schedule = "* * * * *" - crl_expiration_buffer = "3600s" + name = "bsi-ca" + region = "europe-west3" + target_url = "http://download.gsb.bund.de/BSI/crl/DE_CRL.crl" + schedule = "* * * * *" + crl_expiration_buffer = "3600s" }, { - name = "google-ca" - region = "europe-west3" - target_url = "http://c.pki.goog/we2/64OUIVzpZV4.crl" - schedule = "* * * * *" - crl_expiration_buffer = "3600s" + name = "google-ca" + region = "europe-west3" + target_url = "http://c.pki.goog/we2/64OUIVzpZV4.crl" + schedule = "* * * * *" + crl_expiration_buffer = "3600s" } ] diff --git a/examples/cloudrun-crl-monitor/variables.tf b/examples/cloudrun-crl-monitor/variables.tf index 00825e7b37c..387c1caa181 100644 --- a/examples/cloudrun-crl-monitor/variables.tf +++ b/examples/cloudrun-crl-monitor/variables.tf @@ -21,11 +21,12 @@ variable "project_id" { variable "crl_monitors" { description = "List of CRL monitors to configure" type = list(object({ - name = string - region = string - target_url = string - schedule = optional(string, "* * * * *") - crl_expiration_buffer = optional(string, "3600s") + name = string + region = string + target_url = string + schedule = optional(string, "* * * * *") + crl_expiration_buffer = optional(string, "3600s") + proxy_url = optional(string, "") })) } diff --git a/examples/dataflow-flex-python/requirements.txt b/examples/dataflow-flex-python/requirements.txt index 9c08f0abca7..d72c8599aca 100644 --- a/examples/dataflow-flex-python/requirements.txt +++ b/examples/dataflow-flex-python/requirements.txt @@ -75,6 +75,6 @@ requests==2.28.1 rsa==4.9 six==1.16.0 typing-extensions==4.4.0 -urllib3==2.5.0 +urllib3==2.6.0 xsdata==22.12 xsdata-pydantic==22.10 diff --git a/examples/dataflow-xml-pubsub-to-gcs/python/requirements.txt b/examples/dataflow-xml-pubsub-to-gcs/python/requirements.txt index 9e0aca1776f..70320777644 100644 --- a/examples/dataflow-xml-pubsub-to-gcs/python/requirements.txt +++ b/examples/dataflow-xml-pubsub-to-gcs/python/requirements.txt @@ -56,5 +56,5 @@ rsa==4.9 six==1.16.0 sqlparse==0.5.0 typing_extensions==4.5.0 -urllib3==2.5.0 +urllib3==2.6.0 zstandard==0.20.0 diff --git a/examples/iap-user-profile/package-lock.json b/examples/iap-user-profile/package-lock.json index a06008b6c6e..a9a6ed000ed 100644 --- a/examples/iap-user-profile/package-lock.json +++ b/examples/iap-user-profile/package-lock.json @@ -64,6 +64,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.24.7.tgz", "integrity": "sha512-nykK+LEK86ahTkX/3TgauT0ikKoNCfKHEaZYTUVupJdTLzGNvrblu4u6fa7DhZONAltdf8e662t/abY8idrd/g==", "dev": true, + "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.24.7", @@ -1932,6 +1933,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001629", "electron-to-chromium": "^1.4.796", @@ -1957,7 +1959,8 @@ "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" }, "node_modules/buffer-from": { "version": "1.1.2", @@ -4578,21 +4581,23 @@ } }, "node_modules/jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", "dependencies": { - "buffer-equal-constant-time": "1.0.1", + "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", "dependencies": { - "jwa": "^2.0.0", + "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, @@ -6238,6 +6243,7 @@ "version": "3.2.1", "resolved": "https://registry.npmjs.org/winston/-/winston-3.2.1.tgz", "integrity": "sha512-zU6vgnS9dAWCEKg/QYigd6cgMVVNwyTzKs81XZtTFuRwJOcDdBg7AU0mXVyNbs7O5RH2zdv+BdNZUlx7mXPuOw==", + "peer": true, "dependencies": { "async": "^2.6.1", "diagnostics": "^1.1.1", diff --git a/examples/risk-analysis-asset/frontend/package-lock.json b/examples/risk-analysis-asset/frontend/package-lock.json index a7bc2def31c..247972b0944 100644 --- a/examples/risk-analysis-asset/frontend/package-lock.json +++ b/examples/risk-analysis-asset/frontend/package-lock.json @@ -107,6 +107,7 @@ "version": "7.25.2", "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.25.2.tgz", "integrity": "sha512-BBt3opiCOxUr9euZ5/ro/Xv8/V7yJ5bjYMqG/C1YAo8MIKAnumZalCN+msbci3Pigy4lIQfPUpfMM27HMGaYEA==", + "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.24.7", @@ -752,6 +753,7 @@ "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.24.7.tgz", "integrity": "sha512-9G8GYT/dxn/D1IIKOUBmGX0mnmj46mGH9NnZyJLwtCpgh5f7D2VbuKodb+2s9m1Yavh1s7ASQN8lf0eqrb1LTw==", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.24.7" }, @@ -1573,6 +1575,7 @@ "version": "7.25.2", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.25.2.tgz", "integrity": "sha512-KQsqEAVBpU82NM/B/N9j9WOdphom1SZH3R+2V7INrQUH+V9EBFwZsEJl8eBIVeQE62FxJCc70jzEZwqU7RcVqA==", + "peer": true, "dependencies": { "@babel/helper-annotate-as-pure": "^7.24.7", "@babel/helper-module-imports": "^7.24.7", @@ -3461,6 +3464,7 @@ "version": "2.11.8", "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==", + "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/popperjs" @@ -4367,6 +4371,7 @@ "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.62.0.tgz", "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -4720,6 +4725,7 @@ "version": "8.12.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4806,6 +4812,7 @@ "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -5793,6 +5800,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001663", "electron-to-chromium": "^1.5.28", @@ -5841,7 +5849,8 @@ "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" }, "node_modules/buffer-from": { "version": "1.1.2", @@ -7860,6 +7869,7 @@ "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -10581,6 +10591,7 @@ "version": "27.5.1", "resolved": "https://registry.npmjs.org/jest/-/jest-27.5.1.tgz", "integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==", + "peer": true, "dependencies": { "@jest/core": "^27.5.1", "import-local": "^3.0.2", @@ -12657,21 +12668,23 @@ } }, "node_modules/jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", "dependencies": { - "buffer-equal-constant-time": "1.0.1", + "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", "dependencies": { - "jwa": "^2.0.0", + "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, @@ -13919,6 +13932,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "nanoid": "^3.3.7", "picocolors": "^1.1.0", @@ -15022,6 +15036,7 @@ "version": "6.1.2", "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -15414,6 +15429,7 @@ "version": "18.3.1", "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -15560,6 +15576,7 @@ "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -15582,6 +15599,7 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.11.0.tgz", "integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -16086,6 +16104,7 @@ "version": "2.79.2", "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.79.2.tgz", "integrity": "sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ==", + "peer": true, "bin": { "rollup": "dist/bin/rollup" }, @@ -16234,6 +16253,7 @@ "resolved": "https://registry.npmjs.org/sass/-/sass-1.79.3.tgz", "integrity": "sha512-m7dZxh0W9EZ3cw50Me5GOuYm/tVAJAn91SUnohLRo9cXBixGUOdvmryN+dXpwR831bhoY3Zv7rEFt85PUwTmzA==", "devOptional": true, + "peer": true, "dependencies": { "chokidar": "^4.0.0", "immutable": "^4.0.0", @@ -16357,6 +16377,7 @@ "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -17904,6 +17925,7 @@ "version": "0.21.3", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "peer": true, "engines": { "node": ">=10" }, @@ -18328,6 +18350,7 @@ "version": "5.95.0", "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.95.0.tgz", "integrity": "sha512-2t3XstrKULz41MNMBF+cJ97TyHdyQ8HCt//pqErqDvNjU9YQBnZxIHa11VXsi7F3mb5/aO2tuDxdeTPdU7xu9Q==", + "peer": true, "dependencies": { "@types/estree": "^1.0.5", "@webassemblyjs/ast": "^1.12.1", @@ -18395,6 +18418,7 @@ "version": "4.15.2", "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.15.2.tgz", "integrity": "sha512-0XavAZbNJ5sDrCbkpWL8mia0o5WPOd2YGtxrEiZkBK9FjLppIUK2TgxK6qGD2P3hUXTJNNPVibrerKcx5WkR1g==", + "peer": true, "dependencies": { "@types/bonjour": "^3.5.9", "@types/connect-history-api-fallback": "^1.3.5", @@ -18802,6 +18826,7 @@ "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", diff --git a/tools/bq-visualizer/package-lock.json b/tools/bq-visualizer/package-lock.json index 4863d426ef7..237553faed8 100644 --- a/tools/bq-visualizer/package-lock.json +++ b/tools/bq-visualizer/package-lock.json @@ -12,7 +12,7 @@ "@angular/animations": "~16.0.3", "@angular/cdk": "~16.0.2", "@angular/common": "^16.0.3", - "@angular/compiler": "^16", + "@angular/compiler": "^19", "@angular/core": "~16.0.3", "@angular/flex-layout": "^15.0.0-beta.42", "@angular/forms": "~16.0.3", @@ -1474,22 +1474,14 @@ } }, "node_modules/@angular/compiler": { - "version": "16.0.3", - "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-16.0.3.tgz", - "integrity": "sha512-LF/AS0bFXQ+qn6a8Ogx5nNHTYxf+OUYLXQYWECrKCJ4HSsouKDmQ/k8UPlh0gWt9NqQ4SPp9mNpzQhQ4Hq+rXw==", + "version": "19.2.17", + "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-19.2.17.tgz", + "integrity": "sha512-qo8psYASAlDiQ8fAL8i/E2JfWH2nPTpZDKKZxSWvgBVA8o+zUEjYAJu6/k6btnu+4Qcb425T0rmM/zao6EU9Aw==", "dependencies": { "tslib": "^2.3.0" }, "engines": { - "node": "^16.14.0 || >=18.10.0" - }, - "peerDependencies": { - "@angular/core": "16.0.3" - }, - "peerDependenciesMeta": { - "@angular/core": { - "optional": true - } + "node": "^18.19.1 || ^20.11.1 || >=22.0.0" } }, "node_modules/@angular/compiler-cli": { @@ -14125,21 +14117,21 @@ } }, "node_modules/jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", "dependencies": { - "buffer-equal-constant-time": "1.0.1", + "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", "dependencies": { - "jwa": "^2.0.0", + "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, @@ -22320,9 +22312,9 @@ } }, "@angular/compiler": { - "version": "16.0.3", - "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-16.0.3.tgz", - "integrity": "sha512-LF/AS0bFXQ+qn6a8Ogx5nNHTYxf+OUYLXQYWECrKCJ4HSsouKDmQ/k8UPlh0gWt9NqQ4SPp9mNpzQhQ4Hq+rXw==", + "version": "19.2.17", + "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-19.2.17.tgz", + "integrity": "sha512-qo8psYASAlDiQ8fAL8i/E2JfWH2nPTpZDKKZxSWvgBVA8o+zUEjYAJu6/k6btnu+4Qcb425T0rmM/zao6EU9Aw==", "requires": { "tslib": "^2.3.0" } @@ -31287,21 +31279,21 @@ } }, "jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", "requires": { - "buffer-equal-constant-time": "1.0.1", + "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", "requires": { - "jwa": "^2.0.0", + "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, diff --git a/tools/bq-visualizer/package.json b/tools/bq-visualizer/package.json index 8f68cb1bf5a..298e316cbd2 100644 --- a/tools/bq-visualizer/package.json +++ b/tools/bq-visualizer/package.json @@ -14,7 +14,7 @@ "@angular/animations": "~16.0.3", "@angular/cdk": "~16.0.2", "@angular/common": "^16.0.3", - "@angular/compiler": "^16", + "@angular/compiler": "^19", "@angular/core": "~16.0.3", "@angular/flex-layout": "^15.0.0-beta.42", "@angular/forms": "~16.0.3", diff --git a/tools/gcs-bucket-mover/requirements.txt b/tools/gcs-bucket-mover/requirements.txt index 1a2a49cfda2..777ae6ec9fd 100644 --- a/tools/gcs-bucket-mover/requirements.txt +++ b/tools/gcs-bucket-mover/requirements.txt @@ -142,7 +142,7 @@ text-unidecode==1.3 # via faker uritemplate==3.0.1 # via google-api-python-client -urllib3==2.5.0 +urllib3==2.6.0 # via requests yaspin==1.5.0 # via gcs_bucket_mover (setup.py) diff --git a/tools/google-cloud-support-slackbot/requirements.txt b/tools/google-cloud-support-slackbot/requirements.txt index 3dfda03ad7c..3507a343341 100644 --- a/tools/google-cloud-support-slackbot/requirements.txt +++ b/tools/google-cloud-support-slackbot/requirements.txt @@ -18,6 +18,6 @@ requests==2.32.4 six==1.16.0 slackclient==2.9.3 slackeventsapi==2.2.1 -Werkzeug==3.0.6 +Werkzeug==3.1.4 google.cloud.logging gevent==23.9.0 diff --git a/tools/k8s-discovery/README.md b/tools/k8s-discovery/README.md index 39d64a1cd14..be6bbf2f3a1 100644 --- a/tools/k8s-discovery/README.md +++ b/tools/k8s-discovery/README.md @@ -1,127 +1,136 @@ -# Kubescan | Multi-Cloud Kubernetes Cluster Discovery +# Kubernetes Cluster Discovery Tool -A command-line interface (CLI) tool to discover Kubernetes clusters on AWS (EKS), Azure (AKS), and Google Cloud (GKE), to aid in migration planning to Google Kubernetes Engine. -## Features +A comprehensive Python-based CLI tool designed to discover and inventory Kubernetes clusters across multiple cloud providers (AWS EKS, Azure AKS, Google GKE) as well as generic or local clusters (MicroK8s, Minikube, K3s, On-Premises). -- **Unified CLI**: A single, easy-to-use interface to discover Kubernetes resources. -- **Multi-Cloud**: Supports AWS (EKS), Azure (AKS), and GKE. -- **Rich Output**: Displays data in clean, colorful tables in your terminal. -- **CSV Export**: Save all discovered data to CSV files for further analysis. -- **Graceful Authentication**: Automatically uses your existing cloud CLI credentials. +This tool connects to clusters, fetches detailed inventories of resources (Nodes, Workloads, Storage, Networking), and exports the data into structured JSON and flat CSV files suitable for reporting and analysis. -## Development Setup -You can set up the development environment in two ways: using [uv](https://docs.astral.sh/uv/getting-started/installation/) or a Python virtual environment. +--- -### Option 1: Setup with uv +## 🚀 Features +* **Multi-Cloud Support:** Native discovery for **AWS EKS**, **Azure AKS**, and **Google GKE** using cloud-specific SDKs. +* **Generic Cluster Support:** Works with **any** Kubernetes cluster (MicroK8s, Minikube, On-prem, Rancher) via standard `kubeconfig`. +* **Detailed Inventory:** Captures configuration and status for: + * Nodes & Nodepools (Machine types, Kernel versions, OS images) + * Workloads (Deployments, StatefulSets, DaemonSets, Pods) + * Networking (Services, Ingresses) + * Storage (PVCs, PVs) + * Config (ConfigMaps, Secrets, Resource Quotas) +* **Data Export:** + * **JSON:** Hierarchical dump of all raw data. + * **CSV:** Relational, flattened files (e.g., `workloads.csv`, `nodes.csv`) ready for Excel or SQL import. +* **Robust Error Handling:** Includes null-safety checks for container resources to prevent crashes on misconfigured pods. -1. **Clone the repository**: - ```bash - git clone - cd - ``` +--- -2. **Install dependencies**: - ```bash - uv sync - ``` +## 📋 Prerequisites -3. **Run program in the virtual environment**: - ```bash - # Run the discovery tool for AWS (will output to ./discovery_output) - uv run -- kraw aws --region us-west-2 +### 1. System Requirements +* **Python 3.9+** +* `pip` (Python package manager) - # Run the discovery tool for Azure - uv run -- kraw azure --subscription-id "your-subscription-id" +### 2. Credentials +Depending on which clusters you intend to scan, you need the following: - # Run the discovery tool for GKE - uv run -- kraw gke --project-id "your-gcp-project-id" - ``` +* **Generic / Local:** A valid `kubeconfig` file (e.g., `~/.kube/config` or `microk8s config`). +* **AWS EKS:** AWS CLI configured with permissions (`eks:ListClusters`, `eks:DescribeCluster`). +* **Azure AKS:** Azure CLI logged in with Reader permissions on subscriptions. +* **Google GKE:** Google Cloud SDK (`gcloud`) authenticated with Application Default Credentials. -### Option 2: Setup with Python Virtual Environment -1. **Clone the repository**: - ```bash - git clone - cd - ``` +--- -2. **Create and activate a virtual environment**: -```bash -python3 -m venv .venv -source .venv/bin/activate # On Linux/Mac -.venv\Scripts\activate # On Windows -``` -3. **Install dependencies:**: -```bash -pip3 install -r requirements.txt -``` - -4. **Run the CLI:**: -```bash -python main.py aws --region us-west-2 -python main.py azure --subscription-id "your-subscription-id" -python main.py gke --project-id "your-gcp-project-id" -``` +## 🛠️ Installation +It is **highly recommended** to run this tool in a virtual environment to avoid conflicts with system packages (especially on modern Linux distros like Ubuntu 24.04). -## Authentication +1. **Create a virtual environment:** + ```bash + python3 -m venv venv + ``` -The tool uses the default credential chains for each cloud provider. +2. **Activate the environment:** + * **Linux/macOS:** + ```bash + source venv/bin/activate + ``` + * **Windows:** + ```bash + .\venv\Scripts\activate + ``` + +3. **Install dependencies:** + ```bash + pip install -r requirements.txt + ``` -- **AWS**: Make sure you have the AWS CLI installed and configured. You can do this by running `aws configure`. -- **Azure**: Make sure you have the Azure CLI installed. Log in by running `az login`. -- **GKE**: Make sure you have the Google Cloud CLI installed. Log in by running `gcloud auth application-default login`. +--- -## Usage +## 📖 Usage Guide -### Discover AWS EKS Clusters +The tool is divided into sub-commands based on the provider. -You can scan specific AWS regions using the `--region` option. If no region is specified, the tool will automatically scan all AWS regions where EKS is available. +### 1. Generic / Local Clusters (MicroK8s, Minikube, On-Prem) +Use this for clusters that do not require Cloud API access, such as local development environments. -**Scan a specific region:** +**Basic Usage (uses default `~/.kube/config`):** ```bash -kraw aws --region us-west-2 --output-dir ./aws-discovery-output +python main.py generic ``` -**Scan multiple regions:** -```bash -kraw aws --region us-east-1 --region us-west-2 +### Specific Kubeconfig (e.g., MicroK8s): +If you are running MicroK8s, you must export the config first: ``` +# 1. Export config to a file +microk8s config > microk8s.yaml -### Discover Azure AKS Clusters - -To scan specific subscriptions, use the `--subscription-id` option. You can use it multiple times. If no subscriptions are specified, all accessible subscriptions are scanned. -```bash -kraw azure --subscription-id "your-subscription-id" --output-dir ./azure-discovery-output +# 2. Run discovery using that file +python main.py generic --kubeconfig ./microk8s.yaml ``` -### Discover GKE Resources - -For GKE discovery, it is mandatory to provide a scope. You must specify either one or more project IDs or a single organization ID. - -Scan by Project ID: -Use the `--project-id` option to scan one or more specific GCP projects. - -```bash -kraw gke --project-id "your-gcp-project-id" --output-dir ./gke-discovery-output +### Filter by Context: +If your kubeconfig contains many contexts, scan only specific ones: ``` - -```bash -kraw gke --project-id "project-a" --project-id "project-b" +python main.py generic --context my-dev-cluster --context my-prod-cluster ``` -Scan by Organization ID: - -Use the `--gcp-organization-id` option to discover and scan all projects within a specific GCP organization. +## 2. AWS EKS (Elastic Kubernetes Service) +Scans AWS regions for EKS clusters. +``` +# Scan ALL available AWS regions +python main.py aws -```bash -kraw gke --gcp-organization-id "123456789012" --output-dir ./gke-discovery-output +# Scan specific regions only +python main.py aws --region us-east-1 --region eu-west-1 ``` +## 3. Azure AKS (Azure Kubernetes Service) +Scans Azure subscriptions for AKS clusters. +``` +# Scan ALL accessible subscriptions +python main.py azure -## Packaging the Application +# Scan specific subscription IDs +python main.py azure --subscription-id "subscription-guid-1" --subscription-id "subscription-guid-2" +``` +## 4. Google GKE (Google Kubernetes Engine) +Scans Google Cloud projects for GKE clusters. +``` +# Scan specific projects (Required) +python main.py gke --project-id "my-gcp-project-dev" --project-id "my-gcp-project-prod" -### Install PyInstaller as a dev dependency -`uv add -D pyinstaller` - -### Run PyInstaller -`uv run pyinstaller --onefile --name kraw main.py` \ No newline at end of file +# Scan an entire GCP Organization (Requires Org-level permissions) +python main.py gke --gcp-organization-id "123456789012" +``` +# 📂 Output +By default, results are saved to the ./discovery_output directory. You can change this using the --output-dir flag. + +## Key Output Files: +| File Name | Description | +| :--- | :--- | +| `[provider]_clusters.csv` | High-level cluster summary (Name, Version, Region, Status). | +| `nodes.csv` | Node details including CPU/Memory capacity, Kernel version, OS Image. | +| `nodepools.csv` | (Cloud Only) Details about node groups/agent pools. | +| `workloads.csv` | Flattened list of Deployments, StatefulSets, DaemonSets with resource requests/limits. | +| `pods.csv` | Individual Pod statuses, IPs, and Node assignment. | +| `services.csv` | Network services, ClusterIPs, and Ports. | +| `persistent_volume_claims.csv` | Storage requests and binding status. | +| `generic_data.json` | Complete hierarchical JSON dump of the raw discovery data. | diff --git a/tools/k8s-discovery/common.py b/tools/k8s-discovery/common.py index 493eb2e5709..2bdb92f4553 100644 --- a/tools/k8s-discovery/common.py +++ b/tools/k8s-discovery/common.py @@ -39,18 +39,23 @@ def _flatten_workloads(cluster_id, k8s_details): # Extract container info (simplified for CSV) containers = item.get("containers", []) if containers: - workload_info["image"] = containers[0].get("image") - resources = containers[0].get("resources", {}) - workload_info["cpu_request"] = resources.get("requests", {}).get( - "cpu" - ) - workload_info["memory_request"] = resources.get("requests", {}).get( - "memory" - ) - workload_info["cpu_limit"] = resources.get("limits", {}).get("cpu") - workload_info["memory_limit"] = resources.get("limits", {}).get( - "memory" - ) + # Take the first container for simplicity in this CSV view + container = containers[0] + workload_info["image"] = container.get("image") + + # --- FIX START: Safely handle None types in resources --- + # resources might be None, or 'requests'/'limits' keys inside might be None + resources = container.get("resources") or {} + + # explicit check: if resources.get("requests") returns None, default to {} + requests = resources.get("requests") or {} + limits = resources.get("limits") or {} + + workload_info["cpu_request"] = requests.get("cpu") + workload_info["memory_request"] = requests.get("memory") + workload_info["cpu_limit"] = limits.get("cpu") + workload_info["memory_limit"] = limits.get("memory") + # --- FIX END --- workloads_list.append(workload_info) return workloads_list @@ -178,6 +183,10 @@ def save_to_csvs(all_cluster_data, output_dir, provider): "labels": json.dumps(np.get("config", {}).get("labels", {})), } ) + elif provider == "generic": + # Generic clusters (standard K8s) do not have a standard "NodePool" resource + # to query via the core API. Nodes are typically standalone. + pass # --- Node Info --- if k8s_details and "nodes" in k8s_details: @@ -262,7 +271,12 @@ def to_csv(data, filename): df.to_csv(filepath, index=False) logging.info(f"Successfully saved data to {filepath}") - provider_name_map = {"aws": "eks", "azure": "aks", "gke": "gke"} + provider_name_map = { + "aws": "eks", + "azure": "aks", + "gke": "gke", + "generic": "generic" + } cluster_csv_filename = f"{provider_name_map.get(provider, provider)}_clusters.csv" to_csv(provider_clusters_list, cluster_csv_filename) diff --git a/tools/k8s-discovery/main.py b/tools/k8s-discovery/main.py index 48fdfe47917..bf08904e68d 100644 --- a/tools/k8s-discovery/main.py +++ b/tools/k8s-discovery/main.py @@ -2,6 +2,7 @@ import eks_discovery import aks_discovery import gke_discovery +import generic_discovery import common import boto3 import logging @@ -156,5 +157,39 @@ def gke( logging.info("No GKE data found across scanned projects.") +@app.command() +def generic( + contexts: list[str] = typer.Option( + [], + "--context", + help="Specific kubeconfig context names to scan. If empty, all contexts in kubeconfig will be scanned.", + ), + kubeconfig: str = typer.Option( + None, + "--kubeconfig", + help="Path to the kubeconfig file. Defaults to ~/.kube/config", + ), + output_dir: str = typer.Option( + "./discovery_output", help="The directory to save the output CSV files." + ), +): + """ + Discover generic Kubernetes clusters (k3s, microk8s, minikube, etc.) using local kubeconfig. + """ + logging.info("Starting Generic Kubernetes discovery...") + + # Run the discovery + all_data = generic_discovery.run_generic_discovery(kubeconfig, contexts) + + if all_data: + logging.info(f"Found a total of {len(all_data)} clusters.") + if output_dir: + json_filepath = os.path.join(output_dir, "generic_data.json") + common.save_to_json(all_data, json_filepath) + common.save_to_csvs(all_data, output_dir, "generic") + else: + logging.info("No data found.") + + if __name__ == "__main__": app() diff --git a/tools/rag-application-using-vector-search/backend/backend_service/requirements.txt b/tools/rag-application-using-vector-search/backend/backend_service/requirements.txt index 25368dd0ef2..e40d133c339 100644 --- a/tools/rag-application-using-vector-search/backend/backend_service/requirements.txt +++ b/tools/rag-application-using-vector-search/backend/backend_service/requirements.txt @@ -58,7 +58,7 @@ starlette==0.49.1 typer==0.15.2 typing-inspection==0.4.0 typing_extensions==4.13.2 -urllib3==2.4.0 +urllib3==2.6.0 uvicorn==0.34.1 uvloop==0.21.0 watchfiles==1.0.5 diff --git a/tools/rag-application-using-vector-search/backend/poller_job/requirements.txt b/tools/rag-application-using-vector-search/backend/poller_job/requirements.txt index eb072c74259..4255aaf19af 100644 --- a/tools/rag-application-using-vector-search/backend/poller_job/requirements.txt +++ b/tools/rag-application-using-vector-search/backend/poller_job/requirements.txt @@ -33,4 +33,4 @@ shapely==2.1.0 six==1.17.0 typing-inspection==0.4.0 typing_extensions==4.13.2 -urllib3==2.5.0 +urllib3==2.6.0 diff --git a/tools/rag-application-using-vector-search/frontend/requirements.txt b/tools/rag-application-using-vector-search/frontend/requirements.txt index 350d055788a..969faea51cb 100644 --- a/tools/rag-application-using-vector-search/frontend/requirements.txt +++ b/tools/rag-application-using-vector-search/frontend/requirements.txt @@ -38,5 +38,5 @@ toml==0.10.2 tornado==6.5.1 typing_extensions==4.13.1 tzdata==2025.2 -urllib3==2.5.0 +urllib3==2.6.0 watchdog==6.0.0