Add S3 v4 signature code for reading and writing.

Writing needs POST requests and the ability to read 301 (moved
permamently) response data.  As these features are not available
from hfile_libcurl, a new plug-in called hfile_s3_write is
added which uses libcurl directly.  The main hfile_s3 plug-in
passes write requests on to hfile_s3_write.

Author: whitwham

Makefile

@@ -1,6 +1,6 @@
 # Makefile for htslib, a C library for high-throughput sequencing data formats.
 #
-#    Copyright (C) 2013-2018 Genome Research Ltd.
+#    Copyright (C) 2013-2019 Genome Research Ltd.
 #
 #    Author: John Marshall <jm18@sanger.ac.uk>
 #
@@ -307,6 +307,7 @@ hfile.o hfile.pico: hfile.c config.h $(htslib_hfile_h) $(hfile_internal_h) $(hts
 hfile_gcs.o hfile_gcs.pico: hfile_gcs.c config.h $(htslib_hts_h) $(htslib_kstring_h) $(hfile_internal_h)
 hfile_libcurl.o hfile_libcurl.pico: hfile_libcurl.c config.h $(hfile_internal_h) $(htslib_hts_h) $(htslib_kstring_h) $(htslib_khash_h)
 hfile_net.o hfile_net.pico: hfile_net.c config.h $(hfile_internal_h) $(htslib_knetfile_h)
+hfile_s3_write.o hfile_s3_write.pico: hfile_s3_write.c config.h $(hfile_internal_h) $(htslib_hts_h) $(htslib_kstring_h)
 hfile_s3.o hfile_s3.pico: hfile_s3.c config.h $(hfile_internal_h) $(htslib_hts_h) $(htslib_kstring_h)
 hts.o hts.pico: hts.c config.h $(htslib_hts_h) $(htslib_bgzf_h) $(cram_h) $(htslib_hfile_h) $(htslib_hts_endian_h) version.h $(hts_internal_h) $(hfile_internal_h) $(htslib_hts_os_h) $(htslib_khash_h) $(htslib_kseq_h) $(htslib_ksort_h) $(htslib_tbx_h)
 hts_os.o hts_os.pico: hts_os.c config.h os/rand.c

NEWS

@@ -48,6 +48,28 @@ Noteworthy changes in release a.b
   example, "/path1/my_data.bam##idx##/path2/my_index.csi" will open bam file
   "/path1/my_data.bam" and index file "/path2/my_index.csi".
 
+* Changes to hfile_s3, which provides support for the AWS S3 API.
+
+  - hfile_s3 now uses version 4 signatures by default.  Attempting to write to
+    an S3 bucket will also now work correctly.  It is possible to force
+    version 2 signatures by creating environment variable HTS_S3_V2 (the exact
+    value does not matter, it just has to exist).  Note that writing depends
+    on features that need version 4 signatures, so forcing version 2 will
+    disable writes.
+
+  - hfile_s3 will automatically retry requests where the region endpoint
+    was not specified correctly, either by following the 301 redirect (when
+    using path-style requests) or reading the 400 response (when using
+    virtual-hosted style requests and version 4 signatures).  The first
+    region to try can be set by using the AWS_DEFAULT_REGION environment
+    variable, by setting "region" in ".aws/credentials" or by setting
+    "bucket_location" in ".s3cfg".
+
+  - hfile_s3 now percent-escapes the path component of s3:// URLs.  For
+    backwards-compatibility it will ignore any paths that have already
+    been escaped (detected by looking for '%' followed by two hexadecimal
+    digits.)
+
 * Fixed bug where some 8 or 16-bit negative integers were stored using values
   reserved by the BCF specification.  These numbers are now promoted to the
   next size up, so -121 to -128 are stored using at least 16 bits, and -32761

config.mk.in

@@ -1,6 +1,6 @@
 #  Optional configure Makefile overrides for htslib.
 #
-#    Copyright (C) 2015-2017 Genome Research Ltd.
+#    Copyright (C) 2015-2017, 2019 Genome Research Ltd.
 #
 #    Author: John Marshall <jm18@sanger.ac.uk>
 #
@@ -74,10 +74,12 @@ endif
 
 ifeq "s3-@s3@" "s3-enabled"
 plugin_OBJS += hfile_s3.o
+plugin_OBJS += hfile_s3_write.o
 
 CRYPTO_LIBS = @CRYPTO_LIBS@
 noplugin_LIBS += $(CRYPTO_LIBS)
 hfile_s3$(PLUGIN_EXT): LIBS += $(CRYPTO_LIBS)
+hfile_s3_write$(PLUGIN_EXT): LIBS += $(CRYPTO_LIBS) $(LIBCURL_LIBS)
 endif
 
 ifeq "plugins-@enable_plugins@" "plugins-yes"
@@ -94,6 +96,7 @@ plugin.o plugin.pico: CPPFLAGS += -DPLUGINPATH=\"$(pluginpath)\"
 hfile_gcs.o hfile_gcs.pico: version.h
 hfile_libcurl.o hfile_libcurl.pico: version.h
 hfile_s3.o hfile_s3.pico: version.h
+hfile_s3_write.o hfile_s3_write.pico: version.h
 
 # Windows DLL plugins depend on the import library, built as a byproduct.
 $(plugin_OBJS:.o=.cygdll): cyghts-$(LIBHTS_SOVERSION).dll

hfile.c

@@ -972,6 +972,7 @@ static void load_hfile_plugins()
 #endif
 #ifdef ENABLE_S3
     init_add_plugin(NULL, hfile_plugin_init_s3, "s3");
+    init_add_plugin(NULL, hfile_plugin_init_s3_write, "s3w");
 #endif
 
 #endif

hfile_internal.h

@@ -169,6 +169,7 @@ extern int hfile_plugin_init(struct hFILE_plugin *self);
 extern int hfile_plugin_init_gcs(struct hFILE_plugin *self);
 extern int hfile_plugin_init_libcurl(struct hFILE_plugin *self);
 extern int hfile_plugin_init_s3(struct hFILE_plugin *self);
+extern int hfile_plugin_init_s3_write(struct hFILE_plugin *self);
 #endif
 
 /* This one is never built as a separate plugin.  */
@@ -179,6 +180,18 @@ extern int hfile_plugin_init_net(struct hFILE_plugin *self);
 // although we may consider exposing it in the API later.
 typedef int (* hts_httphdr_callback) (void *cb_data, char ***hdrs);
 
+/** Callback for handling 3xx redirect responses from http connections.
+
+    @param data       is passed to the callback
+    @param response   http response code (e.g. 301)
+    @param headers    http response headers
+    @param new_url    the callback should write the url to switch to in here
+
+    Currently used by s3 to handle switching region endpoints.
+*/
+typedef int (*redirect_callback) (void *data, long response,
+                                  kstring_t *headers, kstring_t *new_url);
+
 #ifdef __cplusplus
 }
 #endif

hfile_libcurl.c

@@ -1,6 +1,6 @@
 /*  hfile_libcurl.c -- libcurl backend for low-level file streams.
 
-    Copyright (C) 2015-2017 Genome Research Ltd.
+    Copyright (C) 2015-2017, 2019 Genome Research Ltd.
 
     Author: John Marshall <jm18@sanger.ac.uk>
 
@@ -77,13 +77,18 @@ typedef struct {
     hdrlist fixed;                   // List of headers supplied at hopen()
     hdrlist extra;                   // List of headers from callback
     hts_httphdr_callback callback;   // Callback to get more headers
-    void *callback_data;             // Data to pass to callback
+    void *callback_data;             // Data to pass to httphdr callback
     auth_token *auth;                // Authentication token
     int auth_hdr_num;                // Location of auth_token in hdrlist extra
                                      // If -1, Authorization header is in fixed
                                      //    -2, it came from the callback
                                      //    -3, "auth_token_enabled", "false"
                                      //        passed to hopen()
+    redirect_callback redirect;      // Callback to handle 3xx redirects
+    void *redirect_data;             // Data to pass to redirect_callback
+    long *http_response_ptr;         // Location to store http response code.
+    int fail_on_error;               // Open fails on >400 response code
+                                     //    (default true)
 } http_headers;
 
 typedef struct {
@@ -107,6 +112,7 @@ typedef struct {
     unsigned tried_seek : 1; // At least one seek has been attempted
     int nrunning;
     http_headers headers;
+
     off_t delayed_seek;      // Location to seek to before reading
     off_t last_offset;       // Location we're seeking from
 } hFILE_libcurl;
@@ -720,7 +726,10 @@ static size_t recv_callback(char *ptr, size_t size, size_t nmemb, void *fpv)
     hFILE_libcurl *fp = (hFILE_libcurl *) fpv;
     size_t n = size * nmemb;
 
-    if (n > fp->buffer.len) { fp->paused = 1; return CURL_WRITEFUNC_PAUSE; }
+    if (n > fp->buffer.len) {
+        fp->paused = 1;
+        return CURL_WRITEFUNC_PAUSE;
+    }
     else if (n == 0) return 0;
 
     memcpy(fp->buffer.ptr.rd, ptr, n);
@@ -729,6 +738,19 @@ static size_t recv_callback(char *ptr, size_t size, size_t nmemb, void *fpv)
     return n;
 }
 
+
+size_t header_callback(void *contents, size_t size, size_t nmemb, void *userp) {
+    size_t realsize = size * nmemb;
+    kstring_t *resp = (kstring_t *)userp;
+
+    if (kputsn((const char *)contents, realsize, resp) == EOF) {
+        return 0;
+    }
+
+    return realsize;
+}
+
+
 static ssize_t libcurl_read(hFILE *fpv, void *bufferv, size_t nbytes)
 {
     hFILE_libcurl *fp = (hFILE_libcurl *) fpv;
@@ -764,8 +786,9 @@ static ssize_t libcurl_read(hFILE *fpv, void *bufferv, size_t nbytes)
         err = curl_easy_pause(fp->easy, CURLPAUSE_CONT);
         if (err != CURLE_OK) { errno = easy_errno(fp->easy, err); return -1; }
 
-        while (! fp->paused && ! fp->finished)
+        while (! fp->paused && ! fp->finished) {
             if (wait_perform(fp) < 0) return -1;
+        }
 
         got = fp->buffer.ptr.rd - buffer;
 
@@ -1087,6 +1110,8 @@ libcurl_open(const char *url, const char *modes, http_headers *headers)
     CURLcode err;
     CURLMcode errm;
     int save, is_recursive;
+    kstring_t in_header = {0, 0, NULL};
+    long response;
 
     is_recursive = strchr(modes, 'R') != NULL;
 
@@ -1163,28 +1188,83 @@ libcurl_open(const char *url, const char *modes, http_headers *headers)
         goto error;
     if ((list = get_header_list(fp)) != NULL)
         err |= curl_easy_setopt(fp->easy, CURLOPT_HTTPHEADER, list);
-    err |= curl_easy_setopt(fp->easy, CURLOPT_FOLLOWLOCATION, 1L);
-    if (hts_verbose <= 8)
+
+    if (hts_verbose <= 8 && fp->headers.fail_on_error)
         err |= curl_easy_setopt(fp->easy, CURLOPT_FAILONERROR, 1L);
     if (hts_verbose >= 8)
         err |= curl_easy_setopt(fp->easy, CURLOPT_VERBOSE, 1L);
 
+    if (fp->headers.redirect) {
+        err |= curl_easy_setopt(fp->easy, CURLOPT_HEADERFUNCTION, header_callback);
+        err |= curl_easy_setopt(fp->easy, CURLOPT_HEADERDATA, (void *)&in_header);
+    } else {
+        err |= curl_easy_setopt(fp->easy, CURLOPT_FOLLOWLOCATION, 1L);
+    }
+
     if (err != 0) { errno = ENOSYS; goto error; }
 
     errm = curl_multi_add_handle(fp->multi, fp->easy);
     if (errm != CURLM_OK) { errno = multi_errno(errm); goto error; }
     fp->nrunning++;
 
-    while (! fp->paused && ! fp->finished)
+    while (! fp->paused && ! fp->finished) {
         if (wait_perform(fp) < 0) goto error_remove;
+    }
+
+    curl_easy_getinfo(fp->easy, CURLINFO_RESPONSE_CODE, &response);
+    if (fp->headers.http_response_ptr) {
+        *fp->headers.http_response_ptr = response;
+    }
 
     if (fp->finished && fp->final_result != CURLE_OK) {
         errno = easy_errno(fp->easy, fp->final_result);
         goto error_remove;
     }
 
+    if (fp->headers.redirect) {
+        if (response >= 300 && response < 400) { // redirection
+            kstring_t new_url = {0, 0, NULL};
+
+            if (fp->headers.redirect(fp->headers.redirect_data, response,
+                                     &in_header, &new_url)) {
+                errno = ENOSYS;
+                goto error;
+            }
+
+            err |= curl_easy_setopt(fp->easy, CURLOPT_URL, new_url.s);
+            err |= curl_easy_setopt(fp->easy, CURLOPT_HEADERFUNCTION, NULL);
+            err |= curl_easy_setopt(fp->easy, CURLOPT_HEADERDATA, NULL);
+            free(ks_release(&in_header));
+
+            if (err != 0) { errno = ENOSYS; goto error; }
+            free(ks_release(&new_url));
+
+            if (restart_from_position(fp, 0) < 0) {
+                goto error_remove;
+            }
+
+            if (fp->headers.http_response_ptr) {
+                curl_easy_getinfo(fp->easy, CURLINFO_RESPONSE_CODE,
+                                  fp->headers.http_response_ptr);
+            }
+
+            if (fp->finished && fp->final_result != CURLE_OK) {
+                errno = easy_errno(fp->easy, fp->final_result);
+                goto error_remove;
+            }
+        } else {
+            // we no longer need to look at the headers
+            err |= curl_easy_setopt(fp->easy, CURLOPT_HEADERFUNCTION, NULL);
+            err |= curl_easy_setopt(fp->easy, CURLOPT_HEADERDATA, NULL);
+            free(ks_release(&in_header));
+
+            if (err != 0) { errno = ENOSYS; goto error; }
+        }
+    }
+
     if (mode == 'r') {
         double dval;
+
         if (curl_easy_getinfo(fp->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD,
                               &dval) == CURLE_OK && dval >= 0.0)
             fp->file_size = (off_t) (dval + 0.1);
@@ -1200,6 +1280,7 @@ libcurl_open(const char *url, const char *modes, http_headers *headers)
     errno = save;
 
 error:
+    if (fp->headers.redirect) free(in_header.s);
     save = errno;
     if (fp->easy) curl_easy_cleanup(fp->easy);
     if (fp->multi) curl_multi_cleanup(fp->multi);
@@ -1266,6 +1347,18 @@ static int parse_va_list(http_headers *headers, va_list args)
             if (strcmp(flag, "false") == 0)
                 headers->auth_hdr_num = -3;
         }
+        else if (strcmp(argtype, "redirect_callback") == 0) {
+            headers->redirect = va_arg(args, const redirect_callback);
+        }
+        else if (strcmp(argtype, "redirect_callback_data") == 0) {
+            headers->redirect_data = va_arg(args, void *);
+        }
+        else if (strcmp(argtype, "http_response_ptr") == 0) {
+            headers->http_response_ptr = va_arg(args, long *);
+        }
+        else if (strcmp(argtype, "fail_on_error") == 0) {
+            headers->fail_on_error = va_arg(args, int);
+        }
         else { errno = EINVAL; return -1; }
 
     return 0;
@@ -1318,7 +1411,8 @@ static int parse_va_list(http_headers *headers, va_list args)
 static hFILE *vhopen_libcurl(const char *url, const char *modes, va_list args)
 {
     hFILE *fp = NULL;
-    http_headers headers = { { NULL, 0, 0 }, { NULL, 0, 0 }, NULL, NULL };
+    http_headers headers = { .fail_on_error = 1 };
+
     if (parse_va_list(&headers, args) == 0) {
         fp = libcurl_open(url, modes, &headers);
     }