Bug 1460015 - add rootUrl support

.travis.yml

@@ -29,11 +29,11 @@ install:
 
 script:
   # need to test without gotestcover since error code is not reliable for gotestcover
-  - 'test $GIMME_OS.$GIMME_ARCH != linux.amd64 || CGO_ENABLED=1 GORACE=history_size=7 go test -v -ldflags "-X github.com/taskcluster/taskcluster-proxy.revision=$(git rev-parse HEAD)" ./...'
+  - 'if [ $GIMME_OS.$GIMME_ARCH = linux.amd64 ]; then CGO_ENABLED=1 GORACE=history_size=7 go test -v -ldflags "-X github.com/taskcluster/taskcluster-proxy.revision=$(git rev-parse HEAD)" ./...; fi'
 
 after_script:
   # using gotestcover means tests get run again, but need to do this since exit code of gotestcover is unreliable
-  - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || CGO_ENABLED=1 GORACE=history_size=7 ./gotestcover.sh coverage.report"
+  - "if [ $GIMME_OS.$GIMME_ARCH = linux.amd64 ]; then CGO_ENABLED=1 GORACE=history_size=7 ./gotestcover.sh coverage.report; fi"
 
 notifications:
   irc:

README.md

@@ -227,17 +227,26 @@ longer to complete.
 
 ### Proxy Request (`/`)
 
-All other requests will be treated like proxy requests, and the following
-naming translation will be applied to determine the desired endpoint of the
-target request:
+All other requests will be treated like proxy requests, with the proxy adding
+the credentials for the task to the outgoing request.
 
-```
-http://<proxyhost>:<proxyport>/<service>/<path> -> https://<service>.taskcluster.net/<path>
-```
+There are three URL formats supported:
+
+* *Root URL*: `https://<proxy-host>/api/<service-name>/<api-version>/<service-path>`. This format is
+  preferred, and is easily generated by giving Taskcluster clients a rootUrl
+  pointing to the proxy.
+
+* *Hostname*: `https://<proxy-host>/<service-hostname>/<service-path>`.  This format
+  proxies to the given hostname, and is useful for connecting to external
+  services that use Taskcluster Hawk authentication.
+
+* *Shortcut*: `https://<proxy-host>/<service-name>/<api-version>/<service-path>`.  This format
+  is supported only for backward compatibility with hard-coded URLs.
 
 For example, a PUT request to
-http://localhost:8080/auth/v1/clients/project/nss-nspr/rpi-64 would be proxied
-to https://auth.taskcluster.net/v1/clients/project/nss-nspr/rpi-64.
+`http://localhost:8080/api/auth/v1/clients/project/nss-nspr/rpi-64`, given a
+rootUrl of `https://tc.example.com`, would be proxied to
+`https://tc.example.com/api/auth/v1/clients/project/nss-nspr/rpi-64`.
 
 ## Making a release
 

authorization_test.go

@@ -20,6 +20,7 @@ import (
 )
 
 var (
+	rootURL         = os.Getenv("TASKCLUSTER_ROOT_URL")
 	permCredentials = &tcclient.Credentials{
 		ClientID:    os.Getenv("TASKCLUSTER_CLIENT_ID"),
 		AccessToken: os.Getenv("TASKCLUSTER_ACCESS_TOKEN"),
@@ -42,6 +43,9 @@ func newTestClient() *httpbackoff.Client {
 type IntegrationTest func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder
 
 func skipIfNoPermCreds(t *testing.T) {
+	if rootURL == "" {
+		t.Skip("TASKCLUSTER_ROOT_URL not set - skipping test")
+	}
 	if permCredentials.ClientID == "" {
 		t.Skip("TASKCLUSTER_CLIENT_ID not set - skipping test")
 	}
@@ -153,11 +157,12 @@ func TestBewit(t *testing.T) {
 	test := func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder {
 
 		// Test setup
-		routes := Routes{
-			Client: tcclient.Client{
+		routes := NewRoutes(
+			rootURL,
+			tcclient.Client{
 				Credentials: creds,
 			},
-		}
+		)
 		req, err := http.NewRequest(
 			"POST",
 			"http://localhost:60024/bewit",
@@ -203,8 +208,9 @@ func TestAuthorizationDelegate(t *testing.T) {
 	test := func(name string, scopes []string) IntegrationTest {
 		return func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder {
 			// Test setup
-			routes := Routes{
-				Client: tcclient.Client{
+			routes := NewRoutes(
+				rootURL,
+				tcclient.Client{
 					Authenticate: true,
 					Credentials: &tcclient.Credentials{
 						ClientID:         creds.ClientID,
@@ -213,13 +219,13 @@ func TestAuthorizationDelegate(t *testing.T) {
 						AuthorizedScopes: scopes,
 					},
 				},
-			}
+			)
 
 			// Requires scope "auth:azure-table:read-write:fakeaccount/DuMmYtAbLe"
 			req, err := http.NewRequest(
 				"GET",
 				fmt.Sprintf(
-					"http://localhost:60024/auth/v1/azure/%s/table/%s/read-write",
+					"http://localhost:60024/api/auth/v1/azure/%s/table/%s/read-write",
 					"fakeaccount",
 					"DuMmYtAbLe",
 				),
@@ -234,7 +240,7 @@ func TestAuthorizationDelegate(t *testing.T) {
 			res := httptest.NewRecorder()
 
 			// Function to test
-			routes.RootHandler(res, req)
+			routes.APIHandler(res, req)
 			return res
 		}
 	}
@@ -248,12 +254,13 @@ func TestAPICallWithPayload(t *testing.T) {
 	test := func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder {
 
 		// Test setup
-		routes := Routes{
-			Client: tcclient.Client{
+		routes := NewRoutes(
+			rootURL,
+			tcclient.Client{
 				Authenticate: true,
 				Credentials:  creds,
 			},
-		}
+		)
 		taskID := slugid.Nice()
 		taskGroupID := slugid.Nice()
 		created := time.Now()
@@ -323,12 +330,13 @@ func TestNon200HasErrorBody(t *testing.T) {
 	test := func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder {
 
 		// Test setup
-		routes := Routes{
-			Client: tcclient.Client{
+		routes := NewRoutes(
+			rootURL,
+			tcclient.Client{
 				Authenticate: true,
 				Credentials:  creds,
 			},
-		}
+		)
 		taskID := slugid.Nice()
 
 		req, err := http.NewRequest(
@@ -358,12 +366,13 @@ func TestOversteppedScopes(t *testing.T) {
 	test := func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder {
 
 		// Test setup
-		routes := Routes{
-			Client: tcclient.Client{
+		routes := NewRoutes(
+			rootURL,
+			tcclient.Client{
 				Authenticate: true,
 				Credentials:  creds,
 			},
-		}
+		)
 
 		// This scope is not in the scopes of the temp credentials, which would
 		// happen if a task declares a scope that the provisioner does not
@@ -398,16 +407,17 @@ func TestOversteppedScopes(t *testing.T) {
 }
 
 func TestBadCredsReturns500(t *testing.T) {
-	routes := Routes{
-		Client: tcclient.Client{
+	routes := NewRoutes(
+		rootURL,
+		tcclient.Client{
 			Authenticate: true,
 			Credentials: &tcclient.Credentials{
 				ClientID:    "abc",
 				AccessToken: "def",
 				Certificate: "ghi", // baaaad certificate
 			},
 		},
-	}
+	)
 	req, err := http.NewRequest(
 		"GET",
 		"http://localhost:60024/secrets/v1/secret/garbage/pmoore/foo",
@@ -428,12 +438,13 @@ func TestInvalidEndpoint(t *testing.T) {
 	test := func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder {
 
 		// Test setup
-		routes := Routes{
-			Client: tcclient.Client{
+		routes := NewRoutes(
+			rootURL,
+			tcclient.Client{
 				Authenticate: true,
 				Credentials:  creds,
 			},
-		}
+		)
 
 		req, err := http.NewRequest(
 			"GET",
@@ -466,12 +477,13 @@ func TestRetrievePrivateArtifact(t *testing.T) {
 	test := func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder {
 
 		// Test setup
-		routes := Routes{
-			Client: tcclient.Client{
+		routes := NewRoutes(
+			rootURL,
+			tcclient.Client{
 				Authenticate: true,
 				Credentials:  creds,
 			},
-		}
+		)
 
 		// See https://github.com/taskcluster/generic-worker/blob/c91adbc9fc65c28b3c9e76da1fb0f7f84a69eebf/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json
 		req, err := http.NewRequest(

main.go

@@ -34,6 +34,7 @@ task id.
                                     If not provided, will bind listener to all available network
                                     interfaces [default: ].
     -t --task-id <taskId>           Restrict given scopes to those defined in taskId.
+    --root-url <rootUrl>            The rootUrl for the TC deployment to access
     --client-id <clientId>          Use a specific auth.taskcluster hawk client id [default: ].
     --access-token <accessToken>    Use a specific auth.taskcluster hawk access token [default: ].
     --certificate <certificate>     Use a specific auth.taskcluster hawk certificate [default: ].
@@ -48,6 +49,7 @@ func main() {
 
 	http.HandleFunc("/bewit", routes.BewitHandler)
 	http.HandleFunc("/credentials", routes.CredentialsHandler)
+	http.HandleFunc("/api", routes.APIHandler)
 	http.HandleFunc("/", routes.RootHandler)
 
 	// Only listen on loopback interface to reduce attack surface. If we later
@@ -97,6 +99,15 @@ func ParseCommandArgs(argv []string, exit bool) (routes Routes, address string,
 	address = ipAddress + ":" + portStr
 	log.Printf("Listening on: %v", address)
 
+	rootURL := arguments["--root-url"]
+	if rootURL == nil || rootURL == "" {
+		rootURL = os.Getenv("TASKCLUSTER_ROOT_URL")
+	}
+	if rootURL == "" {
+		log.Fatal("Root URL must be passed via environment variable TASKCLUSTER_ROOT_URL or command line option --root-url")
+	}
+	log.Printf("Root URL: '%v'", rootURL)
+
 	clientID := arguments["--client-id"]
 	if clientID == nil || clientID == "" {
 		clientID = os.Getenv("TASKCLUSTER_CLIENT_ID")
@@ -154,6 +165,12 @@ func ParseCommandArgs(argv []string, exit bool) (routes Routes, address string,
 		authorizedScopes = nil
 	}
 
+	// This will include rootURL in creds, once the client supports it; until then it had better be https://taskcluster.net
+	if rootURL != "https://taskcluster.net" {
+		err = fmt.Errorf("Only the legacy rootUrl is currently supported, not %s", rootURL)
+		return
+	}
+
 	creds := &tcclient.Credentials{
 		ClientID:         clientID.(string),
 		AccessToken:      accessToken.(string),
@@ -167,11 +184,12 @@ func ParseCommandArgs(argv []string, exit bool) (routes Routes, address string,
 		log.Println("Proxy with scopes: ", authorizedScopes)
 	}
 
-	routes = Routes{
-		Client: tcclient.Client{
+	routes = NewRoutes(
+		rootURL.(string),
+		tcclient.Client{
 			Authenticate: true,
 			Credentials:  creds,
 		},
-	}
+	)
 	return
 }

routes.go

@@ -6,19 +6,24 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
+	"net/url"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/taskcluster/httpbackoff"
 	tcclient "github.com/taskcluster/taskcluster-client-go"
+	tcUrls "github.com/taskcluster/taskcluster-lib-urls"
 	tc "github.com/taskcluster/taskcluster-proxy/taskcluster"
 )
 
 // Routes represents the context of the running service
 type Routes struct {
+	RootURL string
 	tcclient.Client
-	lock sync.RWMutex
+	services tc.Services
+	lock     sync.RWMutex
 }
 
 // CredentialsUpdate is the internal representation of the json body which is
@@ -29,9 +34,17 @@ type CredentialsUpdate struct {
 	Certificate string `json:"certificate"`
 }
 
-var tcServices = tc.NewServices()
 var httpClient = &http.Client{}
 
+// NewRoutes creates a new Routes instance.
+func NewRoutes(rootURL string, client tcclient.Client) Routes {
+	return Routes{
+		RootURL:  rootURL,
+		Client:   client,
+		services: tc.NewServices(rootURL),
+	}
+}
+
 func (routes *Routes) setHeaders(res http.ResponseWriter) {
 	headersToSend := res.Header()
 	headersToSend.Set("X-Taskcluster-Proxy-Version", version)
@@ -127,7 +140,8 @@ func (routes *Routes) RootHandler(res http.ResponseWriter, req *http.Request) {
 	routes.lock.RLock()
 	defer routes.lock.RUnlock()
 
-	targetPath, err := tcServices.ConvertPath(req.URL)
+	fmt.Printf("root handler %s\n", req.URL)
+	targetPath, err := routes.services.ConvertPath(req.URL)
 
 	// Unkown service which we are trying to hit...
 	if err != nil {
@@ -136,6 +150,48 @@ func (routes *Routes) RootHandler(res http.ResponseWriter, req *http.Request) {
 		fmt.Fprintf(res, "Unkown taskcluster service: %s", err)
 		return
 	}
+	routes.commonHandler(res, req, targetPath)
+}
+
+var apiPath = regexp.MustCompile("^/api/(?P<service>[^/]*)/(?P<apiVersion>[^/]*)/(?P<path>.*)$")
+
+// APIHandler is the HTTP Handler for /api endpoint
+func (routes *Routes) APIHandler(res http.ResponseWriter, req *http.Request) {
+	routes.setHeaders(res)
+	routes.lock.RLock()
+	defer routes.lock.RUnlock()
+
+	rawPath := req.URL.EscapedPath()
+	fmt.Printf("api handler %s\n", rawPath)
+
+	query := req.URL.RawQuery
+	if query != "" {
+		query = "?" + query
+	}
+
+	var targetPath *url.URL
+	var err error
+	match := apiPath.FindStringSubmatch(rawPath)
+	if match != nil {
+		// reconstruct the target path from the matched parameters (service,
+		// apiVersion, path, query) based on the configured RootURL.
+		targetPath, err = url.Parse(tcUrls.API(routes.RootURL, match[1], match[2], match[3]+query))
+	} else {
+		err = fmt.Errorf("Invalid /api path")
+	}
+
+	if err != nil {
+		res.WriteHeader(404)
+		log.Printf("%s parsing %s", err, req.URL.String())
+		fmt.Fprintf(res, "%s", err)
+		return
+	}
+
+	routes.commonHandler(res, req, targetPath)
+}
+
+// Common code for RootHandler and APIHandler
+func (routes *Routes) commonHandler(res http.ResponseWriter, req *http.Request, targetPath *url.URL) {
 	res.Header().Set("X-Taskcluster-Endpoint", targetPath.String())
 	log.Printf("Proxying %s | %s | %s", req.URL, req.Method, targetPath)
 
@@ -152,6 +208,7 @@ func (routes *Routes) RootHandler(res http.ResponseWriter, req *http.Request) {
 	// this reason, and to avoid confusion around this, let's keep the nil
 	// check in here.
 	body := []byte{}
+	var err error
 	if req.Body != nil {
 		body, err = ioutil.ReadAll(req.Body)
 		// If we fail to create a request notify the client.