Bug 1516458 - take rootURL as config and set TASKCLUSTER_ROOT_URL and TASKCLUSTER_PROXY_URL

.taskcluster.yml

@@ -68,6 +68,9 @@ tasks:
       - generic-worker:cache:generic-worker-checkout
     payload:
       maxRunTime: 3600
+      env:
+        # once bug 1508383 is landed, this won't be required (the worker will supply it)
+        TASKCLUSTER_ROOT_URL: https://taskcluster.net
       command:
         - set CGO_ENABLED=0
         - set GOPATH=%CD%\gopath1.10.3
@@ -150,6 +153,9 @@ tasks:
       - generic-worker:cache:generic-worker-checkout
     payload:
       maxRunTime: 3600
+      env:
+        # once bug 1508383 is landed, this won't be required (the worker will supply it)
+        TASKCLUSTER_ROOT_URL: https://taskcluster.net
       command:
         - set CGO_ENABLED=0
         - set GOPATH=%CD%\gopath1.10.3
@@ -221,6 +227,9 @@ tasks:
       - generic-worker:cache:generic-worker-checkout
     payload:
       maxRunTime: 3600
+      env:
+        # once bug 1508383 is landed, this won't be required (the worker will supply it)
+        TASKCLUSTER_ROOT_URL: https://taskcluster.net
       command:
         - set CGO_ENABLED=0
         - set GOPATH=%CD%\gopath1.10.3
@@ -303,6 +312,9 @@ tasks:
       - generic-worker:cache:generic-worker-checkout
     payload:
       maxRunTime: 3600
+      env:
+        # once bug 1508383 is landed, this won't be required (the worker will supply it)
+        TASKCLUSTER_ROOT_URL: https://taskcluster.net
       command:
         - - /bin/bash
           - -vxec
@@ -445,6 +457,9 @@ tasks:
         taskclusterProxy: true
       maxRunTime: 3600
       image: golang
+      env:
+        # once bug 1508383 is landed, this won't be required (the worker will supply it)
+        TASKCLUSTER_ROOT_URL: https://taskcluster.net
       command:
         - /bin/bash
         - -vxec

aws.go

@@ -53,7 +53,7 @@ func queryMetaData(url string) (string, error) {
 	return string(content), err
 }
 
-// taken from https://github.com/taskcluster/aws-provisioner/blob/5a01a94141c38447968ec75232fd86a86cca366a/src/worker-type.js#L601-L615
+// taken from https://github.com/taskcluster/aws-provisioner/blob/5a2bc7c57b20df00f9c4357e0daeb7967e6f5ee8/lib/worker-type.js#L607-L624
 type UserData struct {
 	Data                interface{} `json:"data"`
 	Capacity            int         `json:"capacity"`
@@ -67,6 +67,7 @@ type UserData struct {
 	LaunchSpecGenerated time.Time   `json:"launchSpecGenerated"`
 	LastModified        time.Time   `json:"lastModified"`
 	ProvisionerBaseURL  string      `json:"provisionerBaseUrl"`
+	TaskclusterRootURL  string      `json:"taskclusterRootUrl"`
 	SecurityToken       string      `json:"securityToken"`
 }
 
@@ -209,9 +210,11 @@ func updateConfigWithAmazonSettings(c *gwconfig.Config) error {
 	if removeErr != nil {
 		return removeErr
 	}
+
 	c.AccessToken = secToken.Credentials.AccessToken
-	c.ClientID = secToken.Credentials.ClientID
 	c.Certificate = secToken.Credentials.Certificate
+	c.ClientID = secToken.Credentials.ClientID
+	c.RootURL = userData.TaskclusterRootURL
 	c.WorkerGroup = userData.Region
 	c.WorkerType = userData.WorkerType
 

aws_helper_test.go

@@ -93,6 +93,7 @@ func (m *MockAWSProvisionedEnvironment) Setup(t *testing.T) func() {
 				"instanceType":        "p3.teenyweeny",
 				"spotBid":             3.5,
 				"price":               3.02,
+				"taskclusterRootUrl":  "http://localhost:13243",
 				"launchSpecGenerated": time.Now(),
 				"lastModified":        time.Now().Add(time.Minute * -30),
 				"provisionerBaseUrl":  "http://localhost:13243/provisioner",

docs/features.md

@@ -59,53 +59,37 @@ References:
 #### Since: generic-worker 10.6.0
 
 The taskcluster proxy provides an easy and safe way to make authenticated
-taskcluster requests within the scope(s) of a particular task.
+taskcluster requests within the scope(s) of a particular task.  The proxy
+accepts un-authenticated requests and attaches credentials to them
+corresponding to `task.scopes` as well as scopes to upload artifacts.
 
-For example lets say we have a task like this:
+The proxy's rootUrl is available to tasks in the environment variable
+`TASKCLUSTER_PROXY_URL`.  It can be used with a client like this:
 
 ```js
-{
-  "scopes": ["a", "b"],
-  "payload": {
-    "features": {
-      "taskclusterProxy": true
-    }
-  }
-}
+var taskcluster = require('taskcluster-client');
+var queue = new taskcluster.Queue({
+  rootUrl: process.env.TASKCLUSTER_PROXY_URL,
+});
+queue.createTask(..);
 ```
 
-A web service will execute (typically on port 80) of the local machine for the
-duration of the task, with which you can proxy unauthenticated requests to
-various taskcluster services. The proxy will inject the Authorization http
-header for you and proxy the request to the target service, granting the
-request the scopes of the task (in this case ["a", "b"]).
+This request would require that `task.scopes` contain the appropriate
+`queue:create-task:..` scope for the `createTask` API call.
 
-| Target Destination                             | Proxy Address                            |
-|------------------------------------------------|------------------------------------------|
-| https://queue.taskcluster.net/<PATH>           | http://localhost/queue/<PATH>            |
-| https://index.taskcluster.net/<PATH>           | http://localhost/index/<PATH>            |
-| https://aws-provisioner.taskcluster.net/<PATH> | http://localhost/aws-provisioner/<PATH>  |
-| https://secrets.taskcluster.net/<PATH>         | http://localhost/secrets/<PATH>          |
-| https://auth.taskcluster.net/<PATH>            | http://localhost/auth/<PATH>             |
-| https://hooks.taskcluster.net/<PATH>           | http://localhost/hooks/<PATH>            |
-| https://purge-cache.taskcluster.net/<PATH>     | http://localhost/purge-cache/<PATH>      |
+*NOTE*: as a special case, the scopes required to call
+`queue.createArtifact(<taskId>, <runId>, ..)` are automatically included,
+regardless of `task.scopes`.
 
-For example (using curl) inside a task container.
+The proxy is easy to use within a shell command, too:
 
 ```sh
-cat secret | curl --header 'Content-Type: application/json' --request PUT --data @- http://localhost/secrets/v1/secret/<secretName>
+curl $TASKCLUSTER_PROXY_URL/api/secrets/v1/secret/my-top-secret-secret
+# ..or
+cat secret | curl --header 'Content-Type: application/json' --request PUT --data @- $TASKCLUSTER_PROXY_URL/api/secrets/v1/secret/my-top-secret-secret
 ```
 
-You can also use the `baseUrl` parameter in the taskcluster-client
-
-```js
-var taskcluster = require('taskcluster-client');
-var queue = new taskcluster.Queue({
- baseUrl: 'http://localhost/queue'
- });
-
-queue.createTask(...);
-```
+These invocations would require `secrets:get:my-top-secret-secret` or `secrets:put:my-top-secret-secret`, respectively, in `task.scopes`.
 
 References:
 

gwconfig/config.go

@@ -43,6 +43,7 @@ type (
 		QueueBaseURL                   string                 `json:"queueBaseURL"`
 		Region                         string                 `json:"region"`
 		RequiredDiskSpaceMegabytes     uint                   `json:"requiredDiskSpaceMegabytes"`
+		RootURL                        string                 `json:"rootURL"`
 		RunAfterUserCreation           string                 `json:"runAfterUserCreation"`
 		RunTasksAsCurrentUser          bool                   `json:"runTasksAsCurrentUser"`
 		SentryProject                  string                 `json:"sentryProject"`
@@ -100,6 +101,7 @@ func (c *Config) Validate() error {
 		{value: c.LiveLogSecret, name: "livelogSecret", disallowed: ""},
 		{value: c.ProvisionerID, name: "provisionerId", disallowed: ""},
 		{value: c.PublicIP, name: "publicIP", disallowed: net.IP(nil)},
+		{value: c.RootURL, name: "rootURL", disallowed: ""},
 		{value: c.SigningKeyLocation, name: "signingKeyLocation", disallowed: ""},
 		{value: c.Subdomain, name: "subdomain", disallowed: ""},
 		{value: c.TasksDir, name: "tasksDir", disallowed: ""},

helper_test.go

@@ -129,6 +129,7 @@ func setup(t *testing.T) (teardown func()) {
 		PurgeCacheBaseURL:  tcpurgecache.DefaultBaseURL,
 		QueueBaseURL:       tcqueue.DefaultBaseURL,
 		Region:             "test-worker-group",
+		RootURL:            os.Getenv("TASKCLUSTER_ROOT_URL"),
 		// should be enough for tests, and travis-ci.org CI environments don't
 		// have a lot of free disk
 		RequiredDiskSpaceMegabytes:     16,
@@ -172,8 +173,10 @@ func setup(t *testing.T) (teardown func()) {
 
 func NewQueue(t *testing.T) *tcqueue.Queue {
 	// check we have all the env vars we need to run this test
-	if os.Getenv("TASKCLUSTER_CLIENT_ID") == "" || os.Getenv("TASKCLUSTER_ACCESS_TOKEN") == "" {
-		t.Skip("Skipping test since TASKCLUSTER_CLIENT_ID and/or TASKCLUSTER_ACCESS_TOKEN env vars not set")
+	if os.Getenv("TASKCLUSTER_CLIENT_ID") == "" ||
+		os.Getenv("TASKCLUSTER_ACCESS_TOKEN") == "" ||
+		os.Getenv("TASKCLUSTER_ROOT_URL") == "" {
+		t.Skip("Skipping test since TASKCLUSTER_{CLIENT_ID,ACCESS_TOKEN,ROOT_URL} env vars not set")
 	}
 	return tcqueue.NewFromEnv()
 }

main.go

@@ -170,6 +170,9 @@ and reports back results to the queue.
                                             for serving live logs; see
                                             https://github.com/taskcluster/livelog and
                                             https://github.com/taskcluster/stateless-dns-server
+          rootURL                           The root URL of the Taskcluster deploment to which
+                                            clientId and accessToken grant access. For example,
+                                            'https://taskcluster.net'.
           signingKeyLocation                The PGP signing key for signing artifacts with.
           workerId                          A name to uniquely identify your worker.
           workerType                        This should match a worker_type managed by the
@@ -487,6 +490,7 @@ func loadConfig(filename string, queryUserData bool) (*gwconfig.Config, error) {
 		PurgeCacheBaseURL:              tcpurgecache.DefaultBaseURL,
 		QueueBaseURL:                   tcqueue.DefaultBaseURL,
 		RequiredDiskSpaceMegabytes:     10240,
+		RootURL:                        "",
 		RunAfterUserCreation:           "",
 		RunTasksAsCurrentUser:          runtime.GOOS != "windows",
 		SentryProject:                  "",

main_test.go

@@ -92,8 +92,8 @@ func TestAbortAfterMaxRunTime(t *testing.T) {
 
 func TestIdleWithoutCrash(t *testing.T) {
 	defer setup(t)()
-	if config.ClientID == "" || config.AccessToken == "" {
-		t.Skip("Skipping test since TASKCLUSTER_CLIENT_ID and/or TASKCLUSTER_ACCESS_TOKEN env vars not set")
+	if config.ClientID == "" || config.AccessToken == "" || config.RootURL == "" {
+		t.Skip("Skipping test since TASKCLUSTER_{CLIENT_ID,ACCESS_TOKEN,ROOT_URL} env vars not set")
 	}
 	start := time.Now()
 	config.IdleTimeoutSecs = 7

plat_all-unix-style.go

@@ -104,6 +104,15 @@ func install(arguments map[string]interface{}) (err error) {
 	return nil
 }
 
+// Set an environment variable in each command.  This can be called from a feature's
+// NewTaskFeature method to set variables for the task.
+func (task *TaskRun) setVariable(variable string, value string) error {
+	for i := range task.Commands {
+		task.Commands[i].Cmd.Env = append(task.Commands[i].Cmd.Env, fmt.Sprintf("%s=%s", variable, value))
+	}
+	return nil
+}
+
 func (task *TaskRun) EnvVars() []string {
 	workerEnv := os.Environ()
 	taskEnv := map[string]string{}
@@ -121,6 +130,8 @@ func (task *TaskRun) EnvVars() []string {
 		taskEnv[k] = v
 	}
 	taskEnv["TASK_ID"] = task.TaskID
+	taskEnv["TASKCLUSTER_ROOT_URL"] = config.RootURL
+
 	for i, j := range taskEnv {
 		taskEnvArray = append(taskEnvArray, i+"="+j)
 	}

plat_windows.go

@@ -311,6 +311,7 @@ func (task *TaskRun) prepareCommand(index int) *CommandExecutionError {
 			contents += "set " + envVar + "=" + envValue + "\r\n"
 		}
 		contents += "set TASK_ID=" + task.TaskID + "\r\n"
+		contents += "set TASKCLUSTER_ROOT_URL=" + config.RootURL + "\r\n"
 		contents += "cd \"" + taskContext.TaskDir + "\"" + "\r\n"
 
 		// Otherwise get the env from the previous command
@@ -396,6 +397,20 @@ func (task *TaskRun) prepareCommand(index int) *CommandExecutionError {
 	return nil
 }
 
+// Set an environment variable in each command.  This can be called from a feature's
+// NewTaskFeature method to set variables for the task.
+func (task *TaskRun) setVariable(variable string, value string) error {
+	for i := range task.Commands {
+		newEnv := []string{fmt.Sprintf("%s=%s", variable, value)}
+		combined, err := win32.MergeEnvLists(&task.Commands[i].Cmd.Env, &newEnv)
+		if err != nil {
+			return err
+		}
+		task.Commands[i].Cmd.Env = *combined
+	}
+	return nil
+}
+
 // Only return critical errors
 func purgeOldTasks() error {
 	if config.CleanUpTaskDirs {

process/process_windows.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"os"
 	"os/exec"
 	"strconv"
 	"sync"
@@ -53,15 +54,22 @@ func (r *Result) Crashed() bool {
 }
 
 func NewCommand(commandLine []string, workingDirectory string, env []string, accessToken syscall.Token) (*Command, error) {
+	var err error
+	var combined *[]string
 	if accessToken != 0 {
-		environment, err := win32.CreateEnvironment(&env, accessToken)
-		if err != nil {
-			return nil, err
-		}
-		env = *environment
+		// in task-user mode, we must merge env with the task user's environment
+		combined, err = win32.CreateEnvironment(&env, accessToken)
+	} else {
+		// in current-user mode, we merge env with the *current* environment
+		parentEnv := os.Environ()
+		combined, err = win32.MergeEnvLists(&parentEnv, &env)
+
+	}
+	if err != nil {
+		return nil, err
 	}
 	cmd := exec.Command(commandLine[0], commandLine[1:]...)
-	cmd.Env = env
+	cmd.Env = *combined
 	cmd.Dir = workingDirectory
 	isWindows8OrGreater := win32.IsWindows8OrGreater()
 	creationFlags := uint32(win32.CREATE_NEW_PROCESS_GROUP | win32.CREATE_NEW_CONSOLE)

taskcluster_proxy.go

@@ -53,15 +53,27 @@ func (l *TaskclusterProxyTask) RequiredScopes() scopes.Required {
 }
 
 func (l *TaskclusterProxyTask) Start() *CommandExecutionError {
+	// Set TASKCLUSTER_PROXY_URL in the task environment
+	err := l.task.setVariable("TASKCLUSTER_PROXY_URL",
+		fmt.Sprintf("http://localhost:%d", config.TaskclusterProxyPort))
+	if err != nil {
+		return MalformedPayloadError(err)
+	}
+
+	// include all scopes from task.scopes, as well as the scope to create artifacts on
+	// this task (which cannot be represented in task.scopes)
+	scopes := append(l.task.Definition.Scopes,
+		fmt.Sprintf("queue:create-artifact:%s/%d", l.task.TaskID, l.task.RunID))
 	taskclusterProxy, err := tcproxy.New(
 		config.TaskclusterProxyExecutable,
 		config.TaskclusterProxyPort,
+		config.RootURL,
 		&tcclient.Credentials{
-			AccessToken: l.task.TaskClaimResponse.Credentials.AccessToken,
-			Certificate: l.task.TaskClaimResponse.Credentials.Certificate,
-			ClientID:    l.task.TaskClaimResponse.Credentials.ClientID,
+			AccessToken:      l.task.TaskClaimResponse.Credentials.AccessToken,
+			Certificate:      l.task.TaskClaimResponse.Credentials.Certificate,
+			ClientID:         l.task.TaskClaimResponse.Credentials.ClientID,
+			AuthorizedScopes: scopes,
 		},
-		l.task.TaskID,
 	)
 	if err != nil {
 		return executionError(internalError, errored, fmt.Errorf("Could not start taskcluster proxy: %s", err))

taskcluster_proxy_test.go

@@ -7,16 +7,24 @@ import (
 )
 
 func TestTaskclusterProxy(t *testing.T) {
+	if os.Getenv("TASKCLUSTER_CLIENT_ID") == "" ||
+		os.Getenv("TASKCLUSTER_ACCESS_TOKEN") == "" ||
+		os.Getenv("TASKCLUSTER_ROOT_URL") == "" {
+		t.Skip("Skipping test since TASKCLUSTER_{CLIENT_ID,ACCESS_TOKEN,ROOT_URL} env vars not set")
+	}
+
 	defer setup(t)()
 	payload := GenericWorkerPayload{
 		Command: append(
 			append(
 				goEnv(),
+				// long enough to reclaim and get new credentials
 				sleep(12)...,
 			),
 			goRun(
 				"curlget.go",
-				fmt.Sprintf("http://localhost:%v/queue/v1/task/KTBKfEgxR5GdfIIREQIvFQ/runs/0/artifacts/SampleArtifacts/_/X.txt", config.TaskclusterProxyPort),
+				// note that curlget.go supports substituting the proxy URL from its runtime environment
+				fmt.Sprintf("TASKCLUSTER_PROXY_URL/queue/v1/task/KTBKfEgxR5GdfIIREQIvFQ/runs/0/artifacts/SampleArtifacts/_/X.txt"),
 			)...,
 		),
 		MaxRunTime: 60,

tcproxy/tcproxy.go

@@ -25,19 +25,17 @@ type TaskclusterProxy struct {
 
 // New starts a tcproxy OS process using the executable specified, and returns
 // a *TaskclusterProxy.
-func New(taskclusterProxyExecutable string, httpPort uint16, creds *tcclient.Credentials, taskID string) (*TaskclusterProxy, error) {
+func New(taskclusterProxyExecutable string, httpPort uint16, rootURL string, creds *tcclient.Credentials) (*TaskclusterProxy, error) {
 	args := []string{
 		"--port", strconv.Itoa(int(httpPort)),
+		"--root-url", rootURL,
 		"--client-id", creds.ClientID,
 		"--access-token", creds.AccessToken,
 		"--ip-address", "127.0.0.1",
 	}
 	if creds.Certificate != "" {
 		args = append(args, "--certificate", creds.Certificate)
 	}
-	if taskID != "" {
-		args = append(args, "--task-id", taskID)
-	}
 	for _, scope := range creds.AuthorizedScopes {
 		args = append(args, scope)
 	}