forked from GSA/datagov-harvester-airflow
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy path.profile
More file actions
55 lines (42 loc) · 2.25 KB
/
.profile
File metadata and controls
55 lines (42 loc) · 2.25 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
#!/bin/bash
##############################################################################
# NOTE: When adding commands to this file, be mindful of sensitive output.
# Since these logs are publicly available in github actions, we don't want
# to leak anything.
##############################################################################
set -o errexit
set -o pipefail
echo "airflow config setup..."
function vcap_get_service () {
local path name
name="$1"
path="$2"
#TODO FIX THIS
service_name=airflow-test-${name}
echo $VCAP_SERVICES | jq --raw-output --arg service_name "$service_name" ".[][] | select(.name == \$service_name) | $path"
}
export APP_NAME=$(echo $VCAP_APPLICATION | jq -r '.application_name')
# # Create a staging area for secrets and files
# CONFIG_DIR=$(mktemp -d)
# SHARED_DIR=$(mktemp -d)
# Extract credentials from VCAP_SERVICES
# export REDIS_HOST=$(vcap_get_service redis .credentials.host)
# export REDIS_PASSWORD=$(vcap_get_service redis .credentials.password)
# export REDIS_PORT=$(vcap_get_service redis .credentials.port)
# GET VCAP VALS FOR DB & ELASTICACHE REDIS
POSTGRES_DB_URI="$(vcap_get_service db .credentials.uri)"
REDIS_URI="$(vcap_get_service redis .credentials.uri)/0?ssl_cert_reqs=CERT_OPTIONAL"
export AIRFLOW__CELERY__BROKER_URL=${REDIS_URI/'redis'/'rediss'}
export AIRFLOW__CELERY__RESULT_BACKEND=${POSTGRES_DB_URI/'postgres'/'db+postgresql'}
# export FLOWER_PORT="$PORT"
# export SAML2_PRIVATE_KEY=$(vcap_get_service secrets .credentials.SAML2_PRIVATE_KEY)
# remote s3 for logs
# export AIRFLOW__LOGGING__REMOTE_LOGGING="true"
# export AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID="s3conn" # name of conn id in web ui?
# export AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID=$(vcap_get_service s3 .credentials.uri)
# export AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER="s3://$(vcap_get_service s3 .credentials.endpoing)/$(vcap_get_service s3 .credentials.bucket)/logs"
# export AIRFLOW__LOGGING__ENCRYPT_S3_LOGS="false"
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="${POSTGRES_DB_URI}"
export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN/'postgres'/'postgresql+psycopg2'}
# TODO connections can be provided here:
# https://airflow.apache.org/docs/apache-airflow/stable/howto/connection.html#storing-connections-in-environment-variables