diff --git a/.github/workflows/django_tests_against_emulator.yml b/.github/workflows/django_tests_against_emulator.yml new file mode 100644 index 0000000000..d8268d4ab0 --- /dev/null +++ b/.github/workflows/django_tests_against_emulator.yml @@ -0,0 +1,33 @@ +on: + push: + branches: + - master + pull_request: +name: Run Django backend tests against emulator +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + - 9020:9020 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run system tests + run: sh build.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + DJANGO_WORKER_INDEX: 0 + DJANGO_WORKER_COUNT: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 8d7113079a..976a11d281 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -eo pipefail - cd github/python-spanner-django # Disable buffering, so that the logs stream through. diff --git a/build.sh b/build.sh new file mode 100644 index 0000000000..8bcfab564a --- /dev/null +++ b/build.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x pipefail + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +# Setup service account credentials. +# export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json + +# Setup project id. +# export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") + +# Export essential environment variables for Django tests. +export RUNNING_SPANNER_BACKEND_TESTS=1 + +# The emulator is currently unusable for our tests because: +# a) It doesn't support INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE +# b) Cannot accept parameters whose types aren't known, so can't be used for +# Python and other dynamic languages. +# export USE_SPANNER_EMULATOR=0 + +pip3 install . +# Create a unique DJANGO_TESTS_DIR per worker to avoid +# any clashes with configured tests by other workers. +export DJANGO_TESTS_DIR="django_tests_$DJANGO_WORKER_INDEX" +mkdir -p $DJANGO_TESTS_DIR && git clone --depth 1 --single-branch --branch spanner-2.2.x https://github.com/timgraham/django.git $DJANGO_TESTS_DIR/django + +# Install dependencies for Django tests. +sudo apt-get update +apt-get install -y libffi-dev libjpeg-dev zlib1g-dev libmemcached-dev +cd $DJANGO_TESTS_DIR/django && pip3 install -e . && pip3 install -r tests/requirements/py3.txt; cd ../../ + +python3 create_test_instance.py +bash django_test_suite.sh \ No newline at end of file diff --git a/create_test_instance.py b/create_test_instance.py new file mode 100644 index 0000000000..df59738895 --- /dev/null +++ b/create_test_instance.py @@ -0,0 +1,26 @@ +# Copyright 2016 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.cloud.spanner_v1 import Client + + +client = Client( + project=os.getenv("GOOGLE_CLOUD_PROJECT", "emulator-test-project") +) + +instance = client.instance("google-cloud-django-backend-tests") +created_op = instance.create() +created_op.result(30) # block until completion diff --git a/django_spanner/base.py b/django_spanner/base.py index 044f8ddd75..e3fc611ac3 100644 --- a/django_spanner/base.py +++ b/django_spanner/base.py @@ -4,6 +4,8 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +import os + from django.db.backends.base.base import BaseDatabaseWrapper from google.cloud import spanner_dbapi as Database, spanner_v1 as spanner @@ -103,7 +105,9 @@ class DatabaseWrapper(BaseDatabaseWrapper): @property def instance(self): - return spanner.Client().instance(self.settings_dict["INSTANCE"]) + return spanner.Client( + project=os.environ["GOOGLE_CLOUD_PROJECT"] + ).instance(self.settings_dict["INSTANCE"]) @property def _nodb_connection(self): @@ -113,7 +117,7 @@ def _nodb_connection(self): def get_connection_params(self): return { - "project": self.settings_dict["PROJECT"], + "project": os.environ["GOOGLE_CLOUD_PROJECT"], "instance_id": self.settings_dict["INSTANCE"], "database_id": self.settings_dict["NAME"], "user_agent": "django_spanner/2.2.0a1", diff --git a/django_spanner/creation.py b/django_spanner/creation.py index 66bd531170..956fbee17f 100644 --- a/django_spanner/creation.py +++ b/django_spanner/creation.py @@ -51,6 +51,7 @@ def _create_test_db(self, verbosity, autoclobber, keepdb=False): # just return and skip it all. if keepdb: return test_database_name + self.log("Got an error creating the test database: %s" % e) if not autoclobber: confirm = input( @@ -81,6 +82,9 @@ def _create_test_db(self, verbosity, autoclobber, keepdb=False): return test_database_name def _execute_create_test_db(self, cursor, parameters, keepdb=False): + self.log( + "emulator: " + str(self.connection.instance._client._emulator_host) + ) self.connection.instance.database(parameters["dbname"]).create() def _destroy_test_db(self, test_database_name, verbosity): diff --git a/django_test_suite.sh b/django_test_suite.sh index e87cd6167f..d3d270eb00 100755 --- a/django_test_suite.sh +++ b/django_test_suite.sh @@ -5,7 +5,7 @@ # license that can be found in the LICENSE file. # exit when any command fails -set -e +set -x pipefail # If no SPANNER_TEST_DB is set, generate a unique one # so that we can have multiple tests running without @@ -15,7 +15,9 @@ TEST_DBNAME=${SPANNER_TEST_DB:-$(python3 -c 'import os, time; print(chr(ord("a") TEST_DBNAME_OTHER="$TEST_DBNAME-ot" TEST_APPS=${DJANGO_TEST_APPS:-basic} INSTANCE=${SPANNER_TEST_INSTANCE:-django-tests} -PROJECT=${PROJECT_ID:-appdev-soda-spanner-staging} +PROJECT=${PROJECT_ID} +SPANNER_EMULATOR_HOST=${SPANNER_EMULATOR_HOST} +GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT} SETTINGS_FILE="$TEST_DBNAME-settings" TESTS_DIR=${DJANGO_TESTS_DIR:-django_tests} @@ -42,21 +44,11 @@ PASSWORD_HASHERS = [ ! } -setup_emulator_if_needed() { - if [[ $SPANNER_EMULATOR_HOST != "" ]] - then - echo "Running the emulator at: $SPANNER_EMULATOR_HOST" - ./emulator_main --host_port "$SPANNER_EMULATOR_HOST" & - SPANNER_INSTANCE=$INSTANCE python3 .kokoro/ensure_instance_exists.py - fi -} - run_django_tests() { cd $TESTS_DIR/django/tests create_settings echo -e "\033[32mRunning Django tests: $TEST_APPS\033[00m" - python3 runtests.py $TEST_APPS --verbosity=2 --noinput --settings $SETTINGS_FILE + python3 runtests.py $TEST_APPS --verbosity=3 --noinput --settings $SETTINGS_FILE } -setup_emulator_if_needed run_django_tests diff --git a/parallelize_tests.go b/parallelize_tests.go index 49e6882dc7..c5b7ab52e1 100644 --- a/parallelize_tests.go +++ b/parallelize_tests.go @@ -31,19 +31,10 @@ import ( ) func main() { - workerIndex, err := strconv.ParseInt(os.Getenv("DJANGO_WORKER_INDEX"), 10, 64) - if err != nil { - log.Fatalf("Failed to parse DJANGO_WORKER_INDEX as an integer: %v", err) - } workerCount, err := strconv.ParseInt(os.Getenv("DJANGO_WORKER_COUNT"), 10, 64) if err != nil { log.Fatalf("Failed to parse DJANGO_WORKER_COUNT as an integer: %v", err) } - if workerIndex >= workerCount { - // Re-enable when we figure out how to deal with Cloud Spanner's very low quotas. - fmt.Printf("workerIndex (%d) >= workerCount (%d)", workerIndex, workerCount) - return - } allAppsBlob, err := ioutil.ReadFile("django_test_apps.txt") if err != nil { @@ -51,10 +42,7 @@ func main() { } allApps := strings.Split(string(allAppsBlob), "\n") appsPerWorker := int(math.Ceil(float64(len(allApps)) / float64(workerCount))) - startIndex := int(workerIndex) * appsPerWorker - if startIndex >= len(allApps) { - startIndex = int(workerIndex) - } + startIndex := 0 endIndex := startIndex + appsPerWorker if endIndex >= len(allApps) { endIndex = len(allApps)