Skip to content

Commit bce3f9b

Browse files
author
Nikhil Thorat
authored
Add cloud build scripts to run individual cloudbuild.yml's (#331)
DEV This mirrors the monorepo. No functional changes except parallelizing the build and only running the subset of tests necessary.
1 parent 6823914 commit bce3f9b

12 files changed

Lines changed: 448 additions & 12 deletions

File tree

‎body-pix/cloudbuild.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
steps:
2+
3+
# Install common dependencies.
4+
- name: 'node:10'
5+
id: 'yarn-common'
6+
entrypoint: 'yarn'
7+
args: ['install']
8+
9+
# Install tfjs dependencies.
10+
- name: 'node:10'
11+
dir: 'body-pix'
12+
entrypoint: 'yarn'
13+
id: 'yarn'
14+
args: ['install']
15+
waitFor: ['yarn-common']
16+
17+
# Build.
18+
- name: 'node:10'
19+
dir: 'body-pix'
20+
entrypoint: 'yarn'
21+
id: 'build'
22+
args: ['build']
23+
waitFor: ['yarn']
24+
25+
# Run tests.
26+
- name: 'node:10'
27+
dir: 'body-pix'
28+
entrypoint: 'yarn'
29+
id: 'test'
30+
args: ['test']
31+
waitFor: ['yarn']
32+
33+
# General configuration
34+
timeout: 1800s
35+
logsBucket: 'gs://tfjs-build-logs'
36+
substitutions:
37+
_NIGHTLY: ''
38+
options:
39+
logStreamingOption: 'STREAM_ON'
40+
substitution_option: 'ALLOW_LOOSE'

‎cloudbuild.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,70 @@ steps:
1414
- 'COMMIT_SHA=$COMMIT_SHA'
1515
- 'BRANCH_NAME=$BRANCH_NAME'
1616

17+
# Body pix.
18+
- name: 'gcr.io/cloud-builders/gcloud'
19+
entrypoint: 'bash'
20+
id: 'body-pix'
21+
args: ['./scripts/run-build.sh', 'body-pix']
22+
waitFor: ['diff']
23+
24+
# Coco SSD.
25+
- name: 'gcr.io/cloud-builders/gcloud'
26+
entrypoint: 'bash'
27+
id: 'coco-ssd'
28+
args: ['./scripts/run-build.sh', 'coco-ssd']
29+
waitFor: ['diff']
30+
31+
# Deeplab.
32+
- name: 'gcr.io/cloud-builders/gcloud'
33+
entrypoint: 'bash'
34+
id: 'deeplab'
35+
args: ['./scripts/run-build.sh', 'deep-lab']
36+
waitFor: ['diff']
37+
38+
# KNN Classifier.
39+
- name: 'gcr.io/cloud-builders/gcloud'
40+
entrypoint: 'bash'
41+
id: 'knn-classifier'
42+
args: ['./scripts/run-build.sh', 'knn-classifier']
43+
waitFor: ['diff']
44+
45+
# Mobilenet.
46+
- name: 'gcr.io/cloud-builders/gcloud'
47+
entrypoint: 'bash'
48+
id: 'mobilenet'
49+
args: ['./scripts/run-build.sh', 'mobilenet']
50+
waitFor: ['diff']
51+
52+
# Posenet.
53+
- name: 'gcr.io/cloud-builders/gcloud'
54+
entrypoint: 'bash'
55+
id: 'posenet'
56+
args: ['./scripts/run-build.sh', 'posenet']
57+
waitFor: ['diff']
58+
59+
# Speech commands.
60+
- name: 'gcr.io/cloud-builders/gcloud'
61+
entrypoint: 'bash'
62+
id: 'speech-commands'
63+
args: ['./scripts/run-build.sh', 'speech-commands']
64+
waitFor: ['diff']
65+
66+
# Toxicity.
67+
- name: 'gcr.io/cloud-builders/gcloud'
68+
entrypoint: 'bash'
69+
id: 'toxicity'
70+
args: ['./scripts/run-build.sh', 'toxicity']
71+
waitFor: ['diff']
72+
73+
# Universal sentence encoder.
74+
- name: 'gcr.io/cloud-builders/gcloud'
75+
entrypoint: 'bash'
76+
id: 'universal-sentence-encoder'
77+
args: ['./scripts/run-build.sh', 'universal-sentence-encoder']
78+
waitFor: ['diff']
79+
80+
# Run the top-level sanity checks.
1781
- name: 'node:10'
1882
entrypoint: 'yarn'
1983
id: 'test'

‎coco-ssd/cloudbuild.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
steps:
2+
3+
# Install common dependencies.
4+
- name: 'node:10'
5+
id: 'yarn-common'
6+
entrypoint: 'yarn'
7+
args: ['install']
8+
9+
# Install tfjs dependencies.
10+
- name: 'node:10'
11+
dir: 'coco-ssd'
12+
entrypoint: 'yarn'
13+
id: 'yarn'
14+
args: ['install']
15+
waitFor: ['yarn-common']
16+
17+
# Build.
18+
- name: 'node:10'
19+
dir: 'coco-ssd'
20+
entrypoint: 'yarn'
21+
id: 'build'
22+
args: ['build']
23+
waitFor: ['yarn']
24+
25+
# Run tests.
26+
- name: 'node:10'
27+
dir: 'coco-ssd'
28+
entrypoint: 'yarn'
29+
id: 'test'
30+
args: ['test']
31+
waitFor: ['yarn']
32+
33+
# General configuration
34+
timeout: 1800s
35+
logsBucket: 'gs://tfjs-build-logs'
36+
substitutions:
37+
_NIGHTLY: ''
38+
options:
39+
logStreamingOption: 'STREAM_ON'
40+
substitution_option: 'ALLOW_LOOSE'

‎deeplab/cloudbuild.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
steps:
2+
3+
# Install common dependencies.
4+
- name: 'node:10'
5+
id: 'yarn-common'
6+
entrypoint: 'yarn'
7+
args: ['install']
8+
9+
# Install tfjs dependencies.
10+
- name: 'node:10'
11+
dir: 'universal-sentence-encoder'
12+
entrypoint: 'yarn'
13+
id: 'yarn'
14+
args: ['install']
15+
waitFor: ['yarn-common']
16+
17+
# Build.
18+
- name: 'node:10'
19+
dir: 'universal-sentence-encoder'
20+
entrypoint: 'yarn'
21+
id: 'build'
22+
args: ['build']
23+
waitFor: ['yarn']
24+
25+
# Run tests.
26+
- name: 'node:10'
27+
dir: 'universal-sentence-encoder'
28+
entrypoint: 'yarn'
29+
id: 'test'
30+
args: ['test']
31+
waitFor: ['yarn']
32+
33+
# General configuration
34+
timeout: 1800s
35+
logsBucket: 'gs://tfjs-build-logs'
36+
substitutions:
37+
_NIGHTLY: ''
38+
options:
39+
logStreamingOption: 'STREAM_ON'
40+
substitution_option: 'ALLOW_LOOSE'

‎knn-classifier/cloudbuild.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
steps:
2+
3+
# Install common dependencies.
4+
- name: 'node:10'
5+
id: 'yarn-common'
6+
entrypoint: 'yarn'
7+
args: ['install']
8+
9+
# Install tfjs dependencies.
10+
- name: 'node:10'
11+
dir: 'knn-classifier'
12+
entrypoint: 'yarn'
13+
id: 'yarn'
14+
args: ['install']
15+
waitFor: ['yarn-common']
16+
17+
# Build.
18+
- name: 'node:10'
19+
dir: 'knn-classifier'
20+
entrypoint: 'yarn'
21+
id: 'build'
22+
args: ['build']
23+
waitFor: ['yarn']
24+
25+
# Run tests.
26+
- name: 'node:10'
27+
dir: 'knn-classifier'
28+
entrypoint: 'yarn'
29+
id: 'test'
30+
args: ['test']
31+
waitFor: ['yarn']
32+
33+
# General configuration
34+
timeout: 1800s
35+
logsBucket: 'gs://tfjs-build-logs'
36+
substitutions:
37+
_NIGHTLY: ''
38+
options:
39+
logStreamingOption: 'STREAM_ON'
40+
substitution_option: 'ALLOW_LOOSE'

‎mobilenet/cloudbuild.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
steps:
2+
3+
# Install common dependencies.
4+
- name: 'node:10'
5+
id: 'yarn-common'
6+
entrypoint: 'yarn'
7+
args: ['install']
8+
9+
# Install tfjs dependencies.
10+
- name: 'node:10'
11+
dir: 'mobilenet'
12+
entrypoint: 'yarn'
13+
id: 'yarn'
14+
args: ['install']
15+
waitFor: ['yarn-common']
16+
17+
# Build.
18+
- name: 'node:10'
19+
dir: 'mobilenet'
20+
entrypoint: 'yarn'
21+
id: 'build'
22+
args: ['build']
23+
waitFor: ['yarn']
24+
25+
# Run tests.
26+
- name: 'node:10'
27+
dir: 'mobilenet'
28+
entrypoint: 'yarn'
29+
id: 'test'
30+
args: ['test']
31+
waitFor: ['yarn']
32+
33+
# General configuration
34+
timeout: 1800s
35+
logsBucket: 'gs://tfjs-build-logs'
36+
substitutions:
37+
_NIGHTLY: ''
38+
options:
39+
logStreamingOption: 'STREAM_ON'
40+
substitution_option: 'ALLOW_LOOSE'

‎posenet/cloudbuild.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
steps:
2+
3+
# Install common dependencies.
4+
- name: 'node:10'
5+
id: 'yarn-common'
6+
entrypoint: 'yarn'
7+
args: ['install']
8+
9+
# Install tfjs dependencies.
10+
- name: 'node:10'
11+
dir: 'posenet'
12+
entrypoint: 'yarn'
13+
id: 'yarn'
14+
args: ['install']
15+
waitFor: ['yarn-common']
16+
17+
# Build.
18+
- name: 'node:10'
19+
dir: 'posenet'
20+
entrypoint: 'yarn'
21+
id: 'build'
22+
args: ['build']
23+
waitFor: ['yarn']
24+
25+
# Run tests.
26+
- name: 'node:10'
27+
dir: 'posenet'
28+
entrypoint: 'yarn'
29+
id: 'test'
30+
args: ['test']
31+
waitFor: ['yarn']
32+
33+
# General configuration
34+
timeout: 1800s
35+
logsBucket: 'gs://tfjs-build-logs'
36+
substitutions:
37+
_NIGHTLY: ''
38+
options:
39+
logStreamingOption: 'STREAM_ON'
40+
substitution_option: 'ALLOW_LOOSE'

‎presubmit.ts‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,25 +31,15 @@ const dirs = fs.readdirSync(dir)
3131
.filter(f => !f.startsWith('.') && f !== 'node_modules');
3232

3333
dirs.forEach(dir => {
34-
if (dir === 'scripts' || dir === 'clone') {
34+
if (!fs.existsSync(`${dir}/package.json`) || dir === 'clone') {
3535
return;
3636
}
37+
3738
console.log(`~~~~~~~~~~~~ Building ${dir} ~~~~~~~~~~~~`);
3839

3940
shell.cd(dir);
40-
shell.exec('yarn');
41-
shell.exec('yarn build');
4241

4342
const pkg = JSON.parse(fs.readFileSync('package.json').toString());
44-
if (pkg['scripts']['test'] != null) {
45-
console.log(`************ Testing ${dir} ************`);
46-
shell.exec('yarn test');
47-
} else {
48-
console.warn(
49-
`WARNING: ${dir} has no unit tests! ` +
50-
`Please consider adding unit tests to this model directory.`);
51-
}
52-
5343
// Make sure peer dependencies and dev dependencies of tfjs match, and make
5444
// sure the version uses ^.
5545
const peerDeps = pkg.peerDependencies;

‎scripts/run-build.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2019 Google LLC. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# =============================================================================
16+
17+
set -e
18+
19+
DIR=$1
20+
if test -f "$DIR/diff"; then
21+
gcloud builds submit . --config=$DIR/cloudbuild.yml
22+
fi

0 commit comments

Comments
 (0)