diff --git a/.travis.yml b/.travis.yml index d775151..5482176 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,11 +29,11 @@ install: script: # need to test without gotestcover since error code is not reliable for gotestcover - - 'test $GIMME_OS.$GIMME_ARCH != linux.amd64 || CGO_ENABLED=1 GORACE=history_size=7 go test -v -ldflags "-X github.com/taskcluster/taskcluster-proxy.revision=$(git rev-parse HEAD)" ./...' + - 'if [ $GIMME_OS.$GIMME_ARCH = linux.amd64 ]; then CGO_ENABLED=1 GORACE=history_size=7 go test -v -ldflags "-X github.com/taskcluster/taskcluster-proxy.revision=$(git rev-parse HEAD)" ./...; fi' after_script: # using gotestcover means tests get run again, but need to do this since exit code of gotestcover is unreliable - - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || CGO_ENABLED=1 GORACE=history_size=7 ./gotestcover.sh coverage.report" + - "if [ $GIMME_OS.$GIMME_ARCH = linux.amd64 ]; then CGO_ENABLED=1 GORACE=history_size=7 ./gotestcover.sh coverage.report; fi" notifications: irc: diff --git a/README.md b/README.md index a065e45..18f8948 100644 --- a/README.md +++ b/README.md @@ -227,17 +227,26 @@ longer to complete. ### Proxy Request (`/`) -All other requests will be treated like proxy requests, and the following -naming translation will be applied to determine the desired endpoint of the -target request: +All other requests will be treated like proxy requests, with the proxy adding +the credentials for the task to the outgoing request. -``` -http://:// -> https://.taskcluster.net/ -``` +There are three URL formats supported: + +* *Root URL*: `https:///api///`. This format is + preferred, and is easily generated by giving Taskcluster clients a rootUrl + pointing to the proxy. + +* *Hostname*: `https:////`. This format + proxies to the given hostname, and is useful for connecting to external + services that use Taskcluster Hawk authentication. + +* *Shortcut*: `https://///`. This format + is supported only for backward compatibility with hard-coded URLs. For example, a PUT request to -http://localhost:8080/auth/v1/clients/project/nss-nspr/rpi-64 would be proxied -to https://auth.taskcluster.net/v1/clients/project/nss-nspr/rpi-64. +`http://localhost:8080/api/auth/v1/clients/project/nss-nspr/rpi-64`, given a +rootUrl of `https://tc.example.com`, would be proxied to +`https://tc.example.com/api/auth/v1/clients/project/nss-nspr/rpi-64`. ## Making a release diff --git a/authorization_test.go b/authorization_test.go index f63c971..c25c54e 100644 --- a/authorization_test.go +++ b/authorization_test.go @@ -20,6 +20,7 @@ import ( ) var ( + rootURL = os.Getenv("TASKCLUSTER_ROOT_URL") permCredentials = &tcclient.Credentials{ ClientID: os.Getenv("TASKCLUSTER_CLIENT_ID"), AccessToken: os.Getenv("TASKCLUSTER_ACCESS_TOKEN"), @@ -42,6 +43,9 @@ func newTestClient() *httpbackoff.Client { type IntegrationTest func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder func skipIfNoPermCreds(t *testing.T) { + if rootURL == "" { + t.Skip("TASKCLUSTER_ROOT_URL not set - skipping test") + } if permCredentials.ClientID == "" { t.Skip("TASKCLUSTER_CLIENT_ID not set - skipping test") } @@ -153,11 +157,12 @@ func TestBewit(t *testing.T) { test := func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder { // Test setup - routes := Routes{ - Client: tcclient.Client{ + routes := NewRoutes( + rootURL, + tcclient.Client{ Credentials: creds, }, - } + ) req, err := http.NewRequest( "POST", "http://localhost:60024/bewit", @@ -203,8 +208,9 @@ func TestAuthorizationDelegate(t *testing.T) { test := func(name string, scopes []string) IntegrationTest { return func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder { // Test setup - routes := Routes{ - Client: tcclient.Client{ + routes := NewRoutes( + rootURL, + tcclient.Client{ Authenticate: true, Credentials: &tcclient.Credentials{ ClientID: creds.ClientID, @@ -213,13 +219,13 @@ func TestAuthorizationDelegate(t *testing.T) { AuthorizedScopes: scopes, }, }, - } + ) // Requires scope "auth:azure-table:read-write:fakeaccount/DuMmYtAbLe" req, err := http.NewRequest( "GET", fmt.Sprintf( - "http://localhost:60024/auth/v1/azure/%s/table/%s/read-write", + "http://localhost:60024/api/auth/v1/azure/%s/table/%s/read-write", "fakeaccount", "DuMmYtAbLe", ), @@ -234,7 +240,7 @@ func TestAuthorizationDelegate(t *testing.T) { res := httptest.NewRecorder() // Function to test - routes.RootHandler(res, req) + routes.APIHandler(res, req) return res } } @@ -248,12 +254,13 @@ func TestAPICallWithPayload(t *testing.T) { test := func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder { // Test setup - routes := Routes{ - Client: tcclient.Client{ + routes := NewRoutes( + rootURL, + tcclient.Client{ Authenticate: true, Credentials: creds, }, - } + ) taskID := slugid.Nice() taskGroupID := slugid.Nice() created := time.Now() @@ -323,12 +330,13 @@ func TestNon200HasErrorBody(t *testing.T) { test := func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder { // Test setup - routes := Routes{ - Client: tcclient.Client{ + routes := NewRoutes( + rootURL, + tcclient.Client{ Authenticate: true, Credentials: creds, }, - } + ) taskID := slugid.Nice() req, err := http.NewRequest( @@ -358,12 +366,13 @@ func TestOversteppedScopes(t *testing.T) { test := func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder { // Test setup - routes := Routes{ - Client: tcclient.Client{ + routes := NewRoutes( + rootURL, + tcclient.Client{ Authenticate: true, Credentials: creds, }, - } + ) // This scope is not in the scopes of the temp credentials, which would // happen if a task declares a scope that the provisioner does not @@ -398,8 +407,9 @@ func TestOversteppedScopes(t *testing.T) { } func TestBadCredsReturns500(t *testing.T) { - routes := Routes{ - Client: tcclient.Client{ + routes := NewRoutes( + rootURL, + tcclient.Client{ Authenticate: true, Credentials: &tcclient.Credentials{ ClientID: "abc", @@ -407,7 +417,7 @@ func TestBadCredsReturns500(t *testing.T) { Certificate: "ghi", // baaaad certificate }, }, - } + ) req, err := http.NewRequest( "GET", "http://localhost:60024/secrets/v1/secret/garbage/pmoore/foo", @@ -428,12 +438,13 @@ func TestInvalidEndpoint(t *testing.T) { test := func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder { // Test setup - routes := Routes{ - Client: tcclient.Client{ + routes := NewRoutes( + rootURL, + tcclient.Client{ Authenticate: true, Credentials: creds, }, - } + ) req, err := http.NewRequest( "GET", @@ -466,12 +477,13 @@ func TestRetrievePrivateArtifact(t *testing.T) { test := func(t *testing.T, creds *tcclient.Credentials) *httptest.ResponseRecorder { // Test setup - routes := Routes{ - Client: tcclient.Client{ + routes := NewRoutes( + rootURL, + tcclient.Client{ Authenticate: true, Credentials: creds, }, - } + ) // See https://github.com/taskcluster/generic-worker/blob/c91adbc9fc65c28b3c9e76da1fb0f7f84a69eebf/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json req, err := http.NewRequest( diff --git a/main.go b/main.go index 9c174cd..0023c95 100644 --- a/main.go +++ b/main.go @@ -34,6 +34,7 @@ task id. If not provided, will bind listener to all available network interfaces [default: ]. -t --task-id Restrict given scopes to those defined in taskId. + --root-url The rootUrl for the TC deployment to access --client-id Use a specific auth.taskcluster hawk client id [default: ]. --access-token Use a specific auth.taskcluster hawk access token [default: ]. --certificate Use a specific auth.taskcluster hawk certificate [default: ]. @@ -48,6 +49,7 @@ func main() { http.HandleFunc("/bewit", routes.BewitHandler) http.HandleFunc("/credentials", routes.CredentialsHandler) + http.HandleFunc("/api", routes.APIHandler) http.HandleFunc("/", routes.RootHandler) // Only listen on loopback interface to reduce attack surface. If we later @@ -97,6 +99,15 @@ func ParseCommandArgs(argv []string, exit bool) (routes Routes, address string, address = ipAddress + ":" + portStr log.Printf("Listening on: %v", address) + rootURL := arguments["--root-url"] + if rootURL == nil || rootURL == "" { + rootURL = os.Getenv("TASKCLUSTER_ROOT_URL") + } + if rootURL == "" { + log.Fatal("Root URL must be passed via environment variable TASKCLUSTER_ROOT_URL or command line option --root-url") + } + log.Printf("Root URL: '%v'", rootURL) + clientID := arguments["--client-id"] if clientID == nil || clientID == "" { clientID = os.Getenv("TASKCLUSTER_CLIENT_ID") @@ -154,6 +165,12 @@ func ParseCommandArgs(argv []string, exit bool) (routes Routes, address string, authorizedScopes = nil } + // This will include rootURL in creds, once the client supports it; until then it had better be https://taskcluster.net + if rootURL != "https://taskcluster.net" { + err = fmt.Errorf("Only the legacy rootUrl is currently supported, not %s", rootURL) + return + } + creds := &tcclient.Credentials{ ClientID: clientID.(string), AccessToken: accessToken.(string), @@ -167,11 +184,12 @@ func ParseCommandArgs(argv []string, exit bool) (routes Routes, address string, log.Println("Proxy with scopes: ", authorizedScopes) } - routes = Routes{ - Client: tcclient.Client{ + routes = NewRoutes( + rootURL.(string), + tcclient.Client{ Authenticate: true, Credentials: creds, }, - } + ) return } diff --git a/routes.go b/routes.go index 00ab25a..a21e506 100644 --- a/routes.go +++ b/routes.go @@ -6,19 +6,24 @@ import ( "io/ioutil" "log" "net/http" + "net/url" + "regexp" "strings" "sync" "time" "github.com/taskcluster/httpbackoff" tcclient "github.com/taskcluster/taskcluster-client-go" + tcUrls "github.com/taskcluster/taskcluster-lib-urls" tc "github.com/taskcluster/taskcluster-proxy/taskcluster" ) // Routes represents the context of the running service type Routes struct { + RootURL string tcclient.Client - lock sync.RWMutex + services tc.Services + lock sync.RWMutex } // CredentialsUpdate is the internal representation of the json body which is @@ -29,9 +34,17 @@ type CredentialsUpdate struct { Certificate string `json:"certificate"` } -var tcServices = tc.NewServices() var httpClient = &http.Client{} +// NewRoutes creates a new Routes instance. +func NewRoutes(rootURL string, client tcclient.Client) Routes { + return Routes{ + RootURL: rootURL, + Client: client, + services: tc.NewServices(rootURL), + } +} + func (routes *Routes) setHeaders(res http.ResponseWriter) { headersToSend := res.Header() headersToSend.Set("X-Taskcluster-Proxy-Version", version) @@ -127,7 +140,8 @@ func (routes *Routes) RootHandler(res http.ResponseWriter, req *http.Request) { routes.lock.RLock() defer routes.lock.RUnlock() - targetPath, err := tcServices.ConvertPath(req.URL) + fmt.Printf("root handler %s\n", req.URL) + targetPath, err := routes.services.ConvertPath(req.URL) // Unkown service which we are trying to hit... if err != nil { @@ -136,6 +150,48 @@ func (routes *Routes) RootHandler(res http.ResponseWriter, req *http.Request) { fmt.Fprintf(res, "Unkown taskcluster service: %s", err) return } + routes.commonHandler(res, req, targetPath) +} + +var apiPath = regexp.MustCompile("^/api/(?P[^/]*)/(?P[^/]*)/(?P.*)$") + +// APIHandler is the HTTP Handler for /api endpoint +func (routes *Routes) APIHandler(res http.ResponseWriter, req *http.Request) { + routes.setHeaders(res) + routes.lock.RLock() + defer routes.lock.RUnlock() + + rawPath := req.URL.EscapedPath() + fmt.Printf("api handler %s\n", rawPath) + + query := req.URL.RawQuery + if query != "" { + query = "?" + query + } + + var targetPath *url.URL + var err error + match := apiPath.FindStringSubmatch(rawPath) + if match != nil { + // reconstruct the target path from the matched parameters (service, + // apiVersion, path, query) based on the configured RootURL. + targetPath, err = url.Parse(tcUrls.API(routes.RootURL, match[1], match[2], match[3]+query)) + } else { + err = fmt.Errorf("Invalid /api path") + } + + if err != nil { + res.WriteHeader(404) + log.Printf("%s parsing %s", err, req.URL.String()) + fmt.Fprintf(res, "%s", err) + return + } + + routes.commonHandler(res, req, targetPath) +} + +// Common code for RootHandler and APIHandler +func (routes *Routes) commonHandler(res http.ResponseWriter, req *http.Request, targetPath *url.URL) { res.Header().Set("X-Taskcluster-Endpoint", targetPath.String()) log.Printf("Proxying %s | %s | %s", req.URL, req.Method, targetPath) @@ -152,6 +208,7 @@ func (routes *Routes) RootHandler(res http.ResponseWriter, req *http.Request) { // this reason, and to avoid confusion around this, let's keep the nil // check in here. body := []byte{} + var err error if req.Body != nil { body, err = ioutil.ReadAll(req.Body) // If we fail to create a request notify the client. diff --git a/taskcluster/services.go b/taskcluster/services.go index 37dd3cd..3fef575 100644 --- a/taskcluster/services.go +++ b/taskcluster/services.go @@ -5,31 +5,41 @@ import ( url "net/url" "regexp" "strings" + + tcUrls "github.com/taskcluster/taskcluster-lib-urls" ) // Services can convert urls to proxied urls type Services struct { - Domain string + RootURL string } // NewServices create a Service with default domain -func NewServices() Services { - return Services{Domain: "taskcluster.net"} +func NewServices(rootURL string) Services { + return Services{RootURL: rootURL} } var hostnamePattern = regexp.MustCompile(`^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$`) // ConvertPath converts a url for the proxy server into a url for the proper taskcluster -// service. +// service on the given rootURL // // Examples: // -// "/queue/v1/stuff" -> "http://queue.taskcluster.net/v1/stuff" +// "/queue/v1/stuff" -> "http://tc.example.com/api/queue/v1/stuff" +// "/myserver.com/v1/stuff" -> "http://myserver.com/v1/stuff" +// "/api/queue/v1/stuff" => "http://tc.example.com/api/queue/v1/stuff" // func (s *Services) ConvertPath(u *url.URL) (*url.URL, error) { + rawPath := u.EscapedPath() + + query := u.RawQuery + if query != "" { + query = "?" + query + } + // Find raw path, removing the initial slash if present // afaik initial slash should always be there. - rawPath := u.EscapedPath() if len(rawPath) > 0 && rawPath[0] == '/' { rawPath = rawPath[1:] } @@ -52,26 +62,30 @@ func (s *Services) ConvertPath(u *url.URL) (*url.URL, error) { return u, fmt.Errorf("%s is not a valid taskcluster service", service) } - var serviceEndpoint string + var realEndpoint *url.URL + var err error // If service name doesn't contain a dot, we assume it's a shortcut - // like: auth, queue, ... and suffix with self.Domain + // like: auth, queue, ... and qualify it with self.RootURL if !strings.Contains(service, ".") { - // This is pretty much legacy - serviceEndpoint = "https://" + service + "." + s.Domain + // extract version and path + i = strings.IndexByte(path, '/') + if i == -1 { + i = len(path) + } + realEndpoint, err = url.Parse(tcUrls.API(s.RootURL, service, path[:i], path[i+1:]+query)) + if err != nil { + return u, err + } } else { // Otherwise we assume service is the hostname - serviceEndpoint = "https://" + service - } + serviceEndpoint := "https://" + service - // Attempt to construct a new endpoint - query := u.RawQuery - if query != "" { - query = "?" + query - } - realEndpoint, err := url.Parse(serviceEndpoint + "/" + path + query) + // Attempt to construct a new endpoint + realEndpoint, err = url.Parse(serviceEndpoint + "/" + path + query) - if err != nil { - return u, err + if err != nil { + return u, err + } } return realEndpoint, nil diff --git a/taskcluster/services_test.go b/taskcluster/services_test.go index 671997a..b94ca35 100644 --- a/taskcluster/services_test.go +++ b/taskcluster/services_test.go @@ -8,51 +8,71 @@ import ( ) var urlConversions = []struct { + rootURL string given string expected string }{ { - "https://xfoo.com/queue/x/y/z", - "https://queue.taskcluster.net/x/y/z", + "https://taskcluster.net", + "https://xfoo.com/queue/v1/y/z", + "https://queue.taskcluster.net/v1/y/z", }, { - "https://xfoo.com/scheduler/x/y/z", - "https://scheduler.taskcluster.net/x/y/z", + "https://tc.example.com", + "https://xfoo.com/queue/v1/y/z", + "https://tc.example.com/api/queue/v1/y/z", }, { - "https://xfoo.com/index/x/y/z", - "https://index.taskcluster.net/x/y/z", + "https://taskcluster.net", + "https://xfoo.com/scheduler/v1/y/z", + "https://scheduler.taskcluster.net/v1/y/z", }, { - "https://xfoo.com/aws-provisioner/x/y/z", - "https://aws-provisioner.taskcluster.net/x/y/z", + "https://taskcluster.net", + "https://xfoo.com/index/v1/y/z", + "https://index.taskcluster.net/v1/y/z", }, { - "https://xfoo.com/queue/x/y%2fz", - "https://queue.taskcluster.net/x/y%2fz", + "https://taskcluster.net", + "https://xfoo.com/aws-provisioner/v1/y/z", + "https://aws-provisioner.taskcluster.net/v1/y/z", }, { - "https://xfoo.com/queue/x/y%2fz/a", - "https://queue.taskcluster.net/x/y%2fz/a", + "https://taskcluster.net", + "https://xfoo.com/queue/v1/y%2fz", + "https://queue.taskcluster.net/v1/y%2fz", }, { - "https://xfoo.com/queue/x/y/z?key=value", - "https://queue.taskcluster.net/x/y/z?key=value", + "https://tc.example.com", + "https://xfoo.com/queue/v1/y%2fz", + "https://tc.example.com/api/queue/v1/y%2fz", }, { - "https://xfoo.com/queue/x/y%20/z?key=value&key2=value2", - "https://queue.taskcluster.net/x/y%20/z?key=value&key2=value2", + "https://taskcluster.net", + "https://xfoo.com/queue/v1/y%2fz/a", + "https://queue.taskcluster.net/v1/y%2fz/a", }, { + "https://taskcluster.net", + "https://xfoo.com/queue/v1/y/z?key=value", + "https://queue.taskcluster.net/v1/y/z?key=value", + }, + { + "https://taskcluster.net", + "https://xfoo.com/queue/v1/y%20/z?key=value&key2=value2", + "https://queue.taskcluster.net/v1/y%20/z?key=value&key2=value2", + }, + { + "https://taskcluster.net", "https://xfoo.com/myqueue.somewhere.com/v1/task/tsdtwe34tgs%2ff5yh?k=v%20m", "https://myqueue.somewhere.com/v1/task/tsdtwe34tgs%2ff5yh?k=v%20m", }, } func TestConvertPathForQueue(t *testing.T) { - services := tc.NewServices() for _, test := range urlConversions { + services := tc.NewServices(test.rootURL) expected, _ := url.Parse(test.expected) given, _ := url.Parse(test.given) @@ -63,7 +83,7 @@ func TestConvertPathForQueue(t *testing.T) { } if expected.String() != realEndpoint.String() { - t.Errorf("Failed conversion %s expected to be %s", expected, realEndpoint) + t.Errorf("Failed conversion %s: got %s, expected %s", given, realEndpoint, expected) } } }