From 205b9f50b40bd89a5de1e0350acb0c976c05f01b Mon Sep 17 00:00:00 2001 From: Patrick Carlson Date: Fri, 6 May 2022 08:00:03 -0600 Subject: [PATCH 1/3] add instructions for seaweedfs and s3 connectivity --- .../source/installation/config.rst | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 55d96335a68..601b46e0ce1 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -617,6 +617,44 @@ For Amazon AWS, see comments in the edu.harvard.iq.dataverse.dataaccess.S3Access `open an issue at Github `_ and describe your setup. We will be glad to add it here. +_`Seaweedfs `_ + Seaweedfs is a distributed storage system that has s3 compatibility. Set the s3 storage options as explained above. Make sure to set `dataverse.files..path-style-access`` to `true`. You will need to create the bucket beforehand. You can do this with the filer API using curl commands. For example to create an empty bucket called `dataverse`: + + ``` + curl -X POST "http://localhost:8888/buckets/" + curl -X POST "http://localhost:8888/buckets/dataverse/" + ``` + + You will also need to set an access and secret key. One way to do this is via a [static file](https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API#static-configuration). As an example your `config.json` might look like this if you're using a bucket called `dataverse`: + + ``` + { + "identities": [ + { + "name": "anonymouse", + "credentials": [ + { + "accessKey": "secret", + "secretKey": "secret" + } + ], + "actions": [ + "Read:dataverse", + "List:dataverse", + "Tagging:dataverse", + "Write:dataverse" + ] + } + ] + } + ``` + + And lastly to start up the Seaweedfs server and various components you could use a command like this: + + ``` + weed server -s3 -metricsPort=9327 -dir=/data -s3.config=/config.json + ``` + Migrating from Local Storage to S3 ################################## From 40c73e9e0e6f21de5cb0b60a2c0c3d3d38058d0a Mon Sep 17 00:00:00 2001 From: Patrick Carlson Date: Fri, 6 May 2022 08:11:06 -0600 Subject: [PATCH 2/3] spelling --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 601b46e0ce1..b63e2db7037 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -631,7 +631,7 @@ _`Seaweedfs `_ { "identities": [ { - "name": "anonymouse", + "name": "anonymous", "credentials": [ { "accessKey": "secret", From b87eed6224a82651bdd91e60cd5b9892b00e17d6 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 6 May 2022 14:49:06 -0400 Subject: [PATCH 3/3] format SeaweedFS docs, small tweaks #8678 Mostly to get `make html` working. --- .../source/installation/config.rst | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index b63e2db7037..cd40221d7fc 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -613,21 +613,18 @@ For direct uploads and downloads, Dataverse redirects to the proxy-url but presi Additional configuration (appropriate CORS settings, proxy caching/timeout configuration, and proxy settings to pass headers to/from S3 and to avoid adding additional headers) will also be needed to enable use of a proxy with direct upload and download. For Amazon AWS, see comments in the edu.harvard.iq.dataverse.dataaccess.S3AccessIO class about support for AWS's bucket-specific DNS names. -**HINT:** If you are successfully using an S3 storage implementation not yet listed above, please feel free to -`open an issue at Github `_ and describe your setup. -We will be glad to add it here. - -_`Seaweedfs `_ - Seaweedfs is a distributed storage system that has s3 compatibility. Set the s3 storage options as explained above. Make sure to set `dataverse.files..path-style-access`` to `true`. You will need to create the bucket beforehand. You can do this with the filer API using curl commands. For example to create an empty bucket called `dataverse`: +`SeaweedFS `_ + SeaweedFS is a distributed storage system that has S3 compatibility. Set the S3 storage options as explained above. Make sure to set ``dataverse.files..path-style-access`` to ``true``. You will need to create the bucket beforehand. You can do this with the filer API using curl commands. For example, to create an empty bucket called ``dataverse``: - ``` +.. code-block:: bash + curl -X POST "http://localhost:8888/buckets/" curl -X POST "http://localhost:8888/buckets/dataverse/" - ``` - You will also need to set an access and secret key. One way to do this is via a [static file](https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API#static-configuration). As an example your `config.json` might look like this if you're using a bucket called `dataverse`: +You will also need to set an access and secret key. One way to do this is via a `static file `_. As an example, your ``config.json`` might look like this if you're using a bucket called ``dataverse``: - ``` +.. code-block:: json + { "identities": [ { @@ -647,13 +644,19 @@ _`Seaweedfs `_ } ] } - ``` - - And lastly to start up the Seaweedfs server and various components you could use a command like this: + +And lastly, to start up the SeaweedFS server and various components you could use a command like this: - ``` +.. code-block:: bash + weed server -s3 -metricsPort=9327 -dir=/data -s3.config=/config.json - ``` + +**Additional Reported Working S3-Compatible Storage** + +If you are successfully using an S3 storage implementation not yet listed above, please feel free to +`open an issue at Github `_ and describe your setup. +We will be glad to add it. + Migrating from Local Storage to S3 ##################################