From 5fc3ef962587c6e1ed69b163f9c9e5027f5b89a7 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Mon, 19 Nov 2018 21:20:33 +0000 Subject: [PATCH 1/4] Bug 1508383 - set TASKCLUSTER_{ROOT,PROXY}_URL for tasks --- src/lib/features/taskcluster_proxy.js | 4 +++- src/lib/task.js | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/features/taskcluster_proxy.js b/src/lib/features/taskcluster_proxy.js index 2013b57d..172b7809 100644 --- a/src/lib/features/taskcluster_proxy.js +++ b/src/lib/features/taskcluster_proxy.js @@ -120,7 +120,9 @@ class TaskclusterProxy { return { links: [{name, alias: ALIAS}], - env: {} + env: { + TASKCLUSTER_PROXY_URL: `http://${ALIAS}`, + } }; } diff --git a/src/lib/task.js b/src/lib/task.js index af6dcdb1..c720dabf 100644 --- a/src/lib/task.js +++ b/src/lib/task.js @@ -376,6 +376,7 @@ class Task extends EventEmitter { env.TASKCLUSTER_INSTANCE_TYPE = this.runtime.workerNodeType; env.TASKCLUSTER_WORKER_GROUP = this.runtime.workerGroup; env.TASKCLUSTER_PUBLIC_IP = this.runtime.publicIp; + env.TASKCLUSTER_ROOT_URL = this.runtime.rootUrl; let privilegedTask = runAsPrivileged( this.task, this.runtime.dockerConfig.allowPrivileged From 0f8016f9f41ab5837f9e7923b4948b1d44dc5746 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Mon, 19 Nov 2018 22:02:55 +0000 Subject: [PATCH 2/4] Bug 1508383 - pass --root-url to taskcluster-proxy --- config.yml | 2 +- deploy/packer/app/scripts/deploy.sh | 4 ++-- src/lib/features/taskcluster_proxy.js | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/config.yml b/config.yml index 7b136e1d..ac959c40 100644 --- a/config.yml +++ b/config.yml @@ -53,7 +53,7 @@ defaults: restrictCPU: false # Image used to create the taskcluster proxy container. - taskclusterProxyImage: 'taskcluster/taskcluster-proxy:4.0.1' + taskclusterProxyImage: 'taskcluster/taskcluster-proxy:5.0.0' taskclusterLogImage: 'taskcluster/livelog:v4' testdroidProxyImage: 'taskcluster/testdroid-proxy:0.0.7' balrogVPNProxyImage: 'taskclusterprivate/taskcluster-vpn-proxy:0.0.3' diff --git a/deploy/packer/app/scripts/deploy.sh b/deploy/packer/app/scripts/deploy.sh index 0550b28d..96a111ab 100644 --- a/deploy/packer/app/scripts/deploy.sh +++ b/deploy/packer/app/scripts/deploy.sh @@ -43,7 +43,7 @@ sudo depmod relengapi_proxy_version=2.3.1 # Pull images used for sidecar containers -docker pull taskcluster/taskcluster-proxy:4.0.1 +docker pull taskcluster/taskcluster-proxy:5.0.0 docker pull taskcluster/livelog:v4 docker pull taskcluster/dind-service:v4.0 docker pull taskcluster/relengapi-proxy:$relengapi_proxy_version @@ -54,7 +54,7 @@ sudo sh -c 'echo "vm.panic_on_oom=1" >> /etc/sysctl.conf' sudo sh -c 'echo "kernel.panic=1" >> /etc/sysctl.conf' # Export the images as a tarball to load when insances are initialized -docker save taskcluster/taskcluster-proxy:4.0.1 taskcluster/livelog:v4 taskcluster/dind-service:v4.0 taskcluster/relengapi-proxy:$relengapi_proxy_version > /home/ubuntu/docker_worker/docker_worker_images.tar +docker save taskcluster/taskcluster-proxy:5.0.0 taskcluster/livelog:v4 taskcluster/dind-service:v4.0 taskcluster/relengapi-proxy:$relengapi_proxy_version > /home/ubuntu/docker_worker/docker_worker_images.tar # Blow away local docker state because it is never used. On actual workers # per-instance storage is initialized and Docker state goes there. diff --git a/src/lib/features/taskcluster_proxy.js b/src/lib/features/taskcluster_proxy.js index 172b7809..8db94ee8 100644 --- a/src/lib/features/taskcluster_proxy.js +++ b/src/lib/features/taskcluster_proxy.js @@ -38,6 +38,8 @@ class TaskclusterProxy { cmd.push('--certificate=' + task.claim.credentials.certificate); } + cmd.push('--root-url=' + task.runtime.rootUrl); + // supply the task's scopes, limiting what can be done via the proxy cmd = cmd.concat(task.task.scopes); From e27ee553976bfd0c3be567516267cd2f4671d758 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Thu, 27 Dec 2018 16:08:24 +0000 Subject: [PATCH 3/4] Bug 1508383 - allow proxy to create artifacts, too --- docs/features.md | 4 +++- src/lib/features/taskcluster_proxy.js | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/features.md b/docs/features.md index a823c749..42601cd0 100644 --- a/docs/features.md +++ b/docs/features.md @@ -95,7 +95,9 @@ References: #### Feature: `taskclusterProxy` The taskcluster proxy provides an easy and safe way to make authenticated -taskcluster requests within the scope(s) of a particular task. +taskcluster requests within the scope(s) of a particular task. The proxy +accepts un-authenticated requests and attaches credentials to them +corresponding to `task.scopes` as well as scopes to upload artifacts. For example lets say we have a task like this: diff --git a/src/lib/features/taskcluster_proxy.js b/src/lib/features/taskcluster_proxy.js index 8db94ee8..11e0f200 100644 --- a/src/lib/features/taskcluster_proxy.js +++ b/src/lib/features/taskcluster_proxy.js @@ -43,6 +43,10 @@ class TaskclusterProxy { // supply the task's scopes, limiting what can be done via the proxy cmd = cmd.concat(task.task.scopes); + // ..and include the scope to create artifacts on this task, which cannot + // be represented in task.scopes (since it contains a taskId) + cmd.push(`queue:create-artifact:${task.status.taskId}/${task.runId}`); + // create the container. this.container = await docker.createContainer({ Image: imageId, From 415273af4820e926cc861cb115d8e88d13291f30 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Thu, 27 Dec 2018 16:16:13 +0000 Subject: [PATCH 4/4] Bug 1508383 - update docs --- docs/features.md | 50 +++++++++++++++--------------------------------- 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/docs/features.md b/docs/features.md index 42601cd0..f8479c35 100644 --- a/docs/features.md +++ b/docs/features.md @@ -99,51 +99,31 @@ taskcluster requests within the scope(s) of a particular task. The proxy accepts un-authenticated requests and attaches credentials to them corresponding to `task.scopes` as well as scopes to upload artifacts. -For example lets say we have a task like this: +The proxy's rootUrl is available to tasks in the environment variable +`TASKCLUSTER_PROXY_URL`. It can be used with a client like this: ```js -{ - "scopes": ["a", "b"], - "payload": { - "features": { - "taskclusterProxy": true - } - } -} +var taskcluster = require('taskcluster-client'); +var queue = new taskcluster.Queue({ + rootUrl: process.env.TASKCLUSTER_PROXY_URL, +}); +queue.createTask(..); ``` -A special docker container is linked to your task contained named "taskcluster" -with this container linked you can make requests to various taskcluster services -with _only_ the scopes listed in the task (in this case ["a", "b"]) +This request would require that `task.scopes` contain the appropriate +`queue:create-task:..` scope for the `createTask` API call. -| Host | Service | -|---------------------------------|-------------------------------| -| queue.taskcluster.net | taskcluster/queue/ | -| index.taskcluster.net | taskcluster/index/ | -| aws-provisioner.taskcluster.net | taskcluster/aws-provisioner/ | -| secrets.taskcluster.net | taskcluster/secrets/ | -| auth.taskcluster.net | taskcluster/auth/ | -| hooks.taskcluster.net | taskcluster/hooks/ | -| purge-cache.taskcluster.net | taskcluster/purge-cache/ | +*NOTE*: as a special case, the scopes required to call +`queue.createArtifact(, , ..)` are automatically included, +regardless of `task.scopes`. -and maybe more - see [the source](https://github.com/taskcluster/taskcluster-proxy/blob/master/taskcluster/services.go). - -For example (using curl) inside a task container. +The proxy is easy to use within a shell command, too: ```sh -curl taskcluster/queue/v1/ +curl $TASKCLUSTER_PROXY_URL/api/secrets/v1/secret/my-top-secret-secret ``` -You can also use the `baseUrl` parameter in the taskcluster-client - -```js -var taskcluster = require('taskcluster-client'); -var queue = new taskcluster.Queue({ - baseUrl: 'taskcluster/queue' - }); - -queue.getTask(''); -``` +This invocation would require `secrets:get:my-top-secret-secret` in `task.scopes`. References: