Update Readme and add password protection (#13)

* add setup guide for clients

* add projekt link to header and load proxy password from env

* Implent password checking

* fix docker compose

---------

Signed-off-by: lowikian <38097264+bugmaschine@users.noreply.github.com>
Co-authored-by: Lowikian <lowikian@einfachzocken.eu>

Author: bugmaschine

README.md

@@ -19,6 +19,15 @@ Because:
 * Stores it in a local PostgreSQL database
 * Saves media files in your own S3-compatible bucket
 
+## Features
+
+* **Transparent Proxy**: Redirects all requests through e6-cache, then to your chosen instance.
+* **Passive Caching**: Automatically caches every post you view.
+* **Local Storage**: Stores metadata in a local PostgreSQL database and media files in your own S3-compatible bucket.
+* **Customizable**: Easily configurable to work with any e621-compatible instance.
+* **Self-Hosted**: Runs on your own server, giving you full control over your data.
+* **Authentication**: Supports authentication for secure access, even when exposed to the world.
+
 ## Dev Setup
 
 ### Start DB and S3 Storage
@@ -67,6 +76,7 @@ For The Wolf's Stash, it just reports the host not being supported.
 ### Other Clients
 Feel free to open a PR to add documentation for other clients.
 
+
 ## Speed Comparison (Speed depends on your internet, and database speed)
 
 ### Image 1:

dev/docker-compose.yml

@@ -29,13 +29,18 @@ services:
     image: minio/mc
     depends_on:
       - minio
-    entrypoint: >
-      /bin/sh -c "
-      sleep 5; # Waits for startup
-      /usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin;
-      /usr/bin/mc mb --ignore-existing minio/e6cache-media;
-      exit 0;
-      "   
+    entrypoint: 
+      - /bin/sh
+      - -c
+      - |
+        echo "Waiting for MinIO to be ready..."
+        until /usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin; do
+          echo "MinIO not ready yet, retrying in 2s..."
+          sleep 2
+        done
+        /usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin;
+        /usr/bin/mc mb minio/e6cache-media --ignore-existing;
+        exit 0;
 
 volumes:
   dev_db_data:

docker-compose.yml

@@ -41,6 +41,7 @@ services:
       # Proxy settings
       PROXY_URL: http://localhost:8080 # Set this to the Server IP / URL, as otherwise the proxy will not work.
       E6_BASE: https://e621.net
+      PROXY_AUTH: "" # Leave empty to disable proxy auth append like this to your username "Username:YourPassword"
     ports:
       - "8080:8080" # Point this to an Reverse Proxy and set the Proxy Url acordingly.
 
@@ -49,13 +50,18 @@ services:
     image: minio/mc
     depends_on:
       - minio
-    entrypoint: >
-      /bin/sh -c "
-      sleep 5; # Waits for startup
-      /usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin;
-      /usr/bin/mc mb --ignore-existing minio/e6cache-media;
-      exit 0;
-      "   
+    entrypoint: 
+      - /bin/sh
+      - -c
+      - |
+        echo "Waiting for MinIO to be ready..."
+        until /usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin; do
+          echo "MinIO not ready yet, retrying in 2s..."
+          sleep 2
+        done
+        /usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin;
+        /usr/bin/mc mb minio/e6cache-media --ignore-existing;
+        exit 0; 
 volumes:
   db_data:
   minio_data:

src/.env.example

@@ -16,4 +16,5 @@ S3_REGION=us-east-1
 
 # Proxy settings
 PROXY_URL=http://localhost:8080
-E6_BASE=https://e621.net
+E6_BASE=https://e621.net
+PROXY_AUTH="" # Leave empty to disable proxy auth append like this to your username "Username:YourPassword"

src/http_calls.go

@@ -53,6 +53,51 @@ func makeProxyLink(original string) string {
 }
 
 func proxyAndTransform(c *gin.Context) {
+
+	logging.Debug("Headers: ", c.Request.Header)
+
+	auth := c.Request.Header.Get("Authorization")
+
+	requestUsername := ""
+	// if proxy auth is enabled, check for auth header
+	if PROXY_AUTH != "" {
+		if auth == "" {
+			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"})
+			return
+		}
+
+		// an auth header should normaly look like this: base64 hashed username:password
+		// ours should look like this username:proxy_auth:password
+		// the idea is that the user enters their username and then a colon and then the proxy auth. Because most clients dont support colons in the api key, but will accept it in user names
+		auth = strings.TrimPrefix(auth, "Basic ")
+		decodedAuth, err := base64.StdEncoding.DecodeString(auth)
+		if err != nil {
+			logging.Error("Error decoding auth header: ", err)
+			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"})
+			return
+		}
+		// parse it into username, password and proxy auth
+		authParts := strings.Split(string(decodedAuth), ":")
+		suppliedProxyAuth := authParts[1] // if you want to check the password, set it to 2
+		if len(authParts) != 3 || suppliedProxyAuth != PROXY_AUTH {
+			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"})
+			return
+		}
+
+		c.Request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(authParts[0]+":"+authParts[2])))
+	} else {
+
+		if auth == "" {
+			requestUsername = "anonymous"
+		}
+
+		// parse the Authorization header
+		auth = strings.ReplaceAll(auth, "Basic ", "")
+		decodedAuth, _ := base64.URLEncoding.DecodeString(auth)
+		splitAuth := strings.Split(string(decodedAuth), ":")
+		requestUsername = splitAuth[0] // 0 is username, 1 is api key
+	}
+
 	// Construct full target URL
 	originalURL := baseURL + c.Request.URL.Path
 	if c.Request.URL.RawQuery != "" {
@@ -77,10 +122,9 @@ func proxyAndTransform(c *gin.Context) {
 	copyHeaders(c.Request.Header, req.Header)
 
 	// useragent stuff
-	setUseragent(c, req)
+	setUseragent(requestUsername, req)
 
 	logging.Debug("Host: ", c.Request.Host)
-	logging.Debug("Headers: ", c.Request.Header)
 	logging.Debug("Proxied Headers: ", req.Header)
 
 	// Perform request
@@ -322,7 +366,9 @@ func proxyFile(c *gin.Context) {
 
 	// download the image from the api
 	req, _ := http.NewRequest("GET", string(url), nil)
-	setUseragent(c, req)
+
+	// i dont think the username is required for downloading files
+	setUseragent("", req)
 	client := &http.Client{}
 	resp, err := client.Do(req)
 	if err != nil {
@@ -380,30 +426,15 @@ func ProcessPost(c *gin.Context, post *Post) {
 	post.Sample.URL = makeProxyLink(post.Sample.URL)
 }
 
-func setUseragent(c *gin.Context, req *http.Request) {
-	auth := c.Request.Header.Get("Authorization")
-
+func setUseragent(username string, req *http.Request) {
 	var useragent string
-
-	if auth == "" {
-		useragent = useragentBase
-		req.Header.Set("User-Agent", useragent)
-		return
-	}
-
-	// parse the Authorization header
-	auth = strings.ReplaceAll(auth, "Basic ", "")
-	decodedAuth, _ := base64.URLEncoding.DecodeString(auth)
-	splitAuth := strings.Split(string(decodedAuth), ":") // 0 is username, 1 is api key
-
-	if splitAuth[0] == "" {
+	if len(username) < 1 { // if it's too small, then forget about it
 		useragent = useragentBase
 		req.Header.Set("User-Agent", useragent)
 		return
 	}
 
-	//finally assemble the useragent
-	useragent = useragentBase + " (Request made on behalf of " + splitAuth[0] + ")"
+	useragent = useragentBase + " (Request made on behalf of " + username + ")"
 
 	req.Header.Set("User-Agent", useragent)
 }

src/main.go

@@ -21,7 +21,7 @@ func isDebug() bool {
 var (
 	debugMode     string = "false"
 	Database      DB
-	useragentBase = "e6-cache"
+	useragentBase = "e6-cache (https://github.com/bugmaschine/e6-cache)"
 	port          = ":8080"
 	Key           []byte          // gets randomly generated every launch, and used for signing the urls.
 	maxCacheAge   = 1 * time.Hour // idk what's a good value, but 1 hours seems enough
@@ -45,8 +45,9 @@ var (
 	DB_PASS string
 
 	// Proxy settings
-	PROXY_URL string
-	baseURL   string
+	PROXY_URL  string
+	baseURL    string
+	PROXY_AUTH string
 )
 
 func loadEnv() {
@@ -74,6 +75,13 @@ func loadEnv() {
 	// Proxy Settings
 	PROXY_URL = os.Getenv("PROXY_URL")
 	baseURL = os.Getenv("E6_BASE")
+	PROXY_AUTH = os.Getenv("PROXY_AUTH")
+
+	if PROXY_AUTH != "" {
+		logging.Debug("Proxy auth is enabled with key: ", PROXY_AUTH)
+	} else {
+		logging.Debug("Proxy auth is disabled")
+	}
 
 }