add IP_FILTERLIST_BYPASS_HTTP_AUTH (#538)

* add IP_FILTERLIST_BYPASS_HTTP_AUTH

* refactor to separated ip whitelist

README.md

@@ -86,51 +86,52 @@ https://transfer.sh/1lDau/test.txt --> https://transfer.sh/inline/1lDau/test.txt
 
 ## Usage
 
-Parameter | Description                                                                                 | Value | Env                         
---- |---------------------------------------------------------------------------------------------| --- |-----------------------------
-listener | port to use for http (:80)                                                                  | | LISTENER                    |
-profile-listener | port to use for profiler (:6060)                                                            | | PROFILE_LISTENER            |
-force-https | redirect to https                                                                           | false | FORCE_HTTPS                 
-tls-listener | port to use for https (:443)                                                                | | TLS_LISTENER                |
-tls-listener-only | flag to enable tls listener only                                                            | | TLS_LISTENER_ONLY           |
-tls-cert-file | path to tls certificate                                                                     | | TLS_CERT_FILE               |
-tls-private-key | path to tls private key                                                                     | | TLS_PRIVATE_KEY             |
-http-auth-user | user for basic http auth on upload                                                          | | HTTP_AUTH_USER              |
-http-auth-pass | pass for basic http auth on upload                                                          | | HTTP_AUTH_PASS              |
-http-auth-htpasswd | htpasswd file path for basic http auth on upload                                            | | HTTP_AUTH_HTPASSWD          |
-ip-whitelist | comma separated list of ips allowed to connect to the service                               | | IP_WHITELIST                |
-ip-blacklist | comma separated list of ips not allowed to connect to the service                           | | IP_BLACKLIST                |
-temp-path | path to temp folder                                                                         | system temp | TEMP_PATH                   |
-web-path | path to static web files (for development or custom front end)                              | | WEB_PATH                    |
-proxy-path | path prefix when service is run behind a proxy                                              | | PROXY_PATH                  |
-proxy-port | port of the proxy when the service is run behind a proxy                                    | | PROXY_PORT                  |
-email-contact | email contact for the front end                                                             | | EMAIL_CONTACT               |
-ga-key | google analytics key for the front end                                                      | | GA_KEY                      |
+Parameter | Description                                                                                 | Value                        | Env                         
+--- |---------------------------------------------------------------------------------------------|------------------------------|-----------------------------
+listener | port to use for http (:80)                                                                  |                              | LISTENER                    |
+profile-listener | port to use for profiler (:6060)                                                            |                              | PROFILE_LISTENER            |
+force-https | redirect to https                                                                           | false                        | FORCE_HTTPS                 
+tls-listener | port to use for https (:443)                                                                |                              | TLS_LISTENER                |
+tls-listener-only | flag to enable tls listener only                                                            |                              | TLS_LISTENER_ONLY           |
+tls-cert-file | path to tls certificate                                                                     |                              | TLS_CERT_FILE               |
+tls-private-key | path to tls private key                                                                     |                              | TLS_PRIVATE_KEY             |
+http-auth-user | user for basic http auth on upload                                                          |                              | HTTP_AUTH_USER              |
+http-auth-pass | pass for basic http auth on upload                                                          |                              | HTTP_AUTH_PASS              |
+http-auth-htpasswd | htpasswd file path for basic http auth on upload                                            |                              | HTTP_AUTH_HTPASSWD          |
+http-auth-ip-whitelist | comma separated list of ips allowed to upload without being challenged an http auth        |                              | HTTP_AUTH_IP_WHITELIST      |
+ip-whitelist | comma separated list of ips allowed to connect to the service                               |                              | IP_WHITELIST                |
+ip-blacklist | comma separated list of ips not allowed to connect to the service                           |                              | IP_BLACKLIST                |
+temp-path | path to temp folder                                                                         | system temp                  | TEMP_PATH                   |
+web-path | path to static web files (for development or custom front end)                              |                              | WEB_PATH                    |
+proxy-path | path prefix when service is run behind a proxy                                              |                              | PROXY_PATH                  |
+proxy-port | port of the proxy when the service is run behind a proxy                                    |                              | PROXY_PORT                  |
+email-contact | email contact for the front end                                                             |                              | EMAIL_CONTACT               |
+ga-key | google analytics key for the front end                                                      |                              | GA_KEY                      |
 provider | which storage provider to use                                                               | (s3, storj, gdrive or local) |
-uservoice-key | user voice key for the front end                                                            | | USERVOICE_KEY               |
-aws-access-key | aws access key                                                                              | | AWS_ACCESS_KEY              |
-aws-secret-key | aws access key                                                                              | | AWS_SECRET_KEY              |
-bucket | aws bucket                                                                                  | | BUCKET                      |
-s3-endpoint | Custom S3 endpoint.                                                                         | | S3_ENDPOINT                 |
-s3-region | region of the s3 bucket                                                                     | eu-west-1 | S3_REGION                   |
-s3-no-multipart | disables s3 multipart upload                                                                | false | S3_NO_MULTIPART             |
-s3-path-style | Forces path style URLs, required for Minio.                                                 | false | S3_PATH_STYLE               |
-storj-access | Access for the project                                                                      | | STORJ_ACCESS                |
-storj-bucket | Bucket to use within the project                                                            | | STORJ_BUCKET                |
-basedir | path storage for local/gdrive provider                                                      | | BASEDIR                     |
-gdrive-client-json-filepath | path to oauth client json config for gdrive provider                                        | | GDRIVE_CLIENT_JSON_FILEPATH |
-gdrive-local-config-path | path to store local transfer.sh config cache for gdrive provider                            | | GDRIVE_LOCAL_CONFIG_PATH    |
-gdrive-chunk-size | chunk size for gdrive upload in megabytes, must be lower than available memory (8 MB)       | | GDRIVE_CHUNK_SIZE           |
-lets-encrypt-hosts | hosts to use for lets encrypt certificates (comma seperated)                                | | HOSTS                       |
-log | path to log file                                                                            | | LOG                         |
-cors-domains | comma separated list of domains for CORS, setting it enable CORS                            | | CORS_DOMAINS                |
-clamav-host | host for clamav feature                                                                     | | CLAMAV_HOST                 |
-perform-clamav-prescan | prescan every upload through clamav feature (clamav-host must be a local clamd unix socket) | | PERFORM_CLAMAV_PRESCAN      |
-rate-limit | request per minute                                                                          | | RATE_LIMIT                  |
-max-upload-size | max upload size in kilobytes                                                                | | MAX_UPLOAD_SIZE             |
-purge-days | number of days after the uploads are purged automatically                                   | | PURGE_DAYS                  |   
-purge-interval | interval in hours to run the automatic purge for (not applicable to S3 and Storj)           | | PURGE_INTERVAL              |   
-random-token-length | length of the random token for the upload path (double the size for delete path)            | 6 | RANDOM_TOKEN_LENGTH         |   
+uservoice-key | user voice key for the front end                                                            |                              | USERVOICE_KEY               |
+aws-access-key | aws access key                                                                              |                              | AWS_ACCESS_KEY              |
+aws-secret-key | aws access key                                                                              |                              | AWS_SECRET_KEY              |
+bucket | aws bucket                                                                                  |                              | BUCKET                      |
+s3-endpoint | Custom S3 endpoint.                                                                         |                              | S3_ENDPOINT                 |
+s3-region | region of the s3 bucket                                                                     | eu-west-1                    | S3_REGION                   |
+s3-no-multipart | disables s3 multipart upload                                                                | false                        | S3_NO_MULTIPART             |
+s3-path-style | Forces path style URLs, required for Minio.                                                 | false                        | S3_PATH_STYLE               |
+storj-access | Access for the project                                                                      |                              | STORJ_ACCESS                |
+storj-bucket | Bucket to use within the project                                                            |                              | STORJ_BUCKET                |
+basedir | path storage for local/gdrive provider                                                      |                              | BASEDIR                     |
+gdrive-client-json-filepath | path to oauth client json config for gdrive provider                                        |                              | GDRIVE_CLIENT_JSON_FILEPATH |
+gdrive-local-config-path | path to store local transfer.sh config cache for gdrive provider                            |                              | GDRIVE_LOCAL_CONFIG_PATH    |
+gdrive-chunk-size | chunk size for gdrive upload in megabytes, must be lower than available memory (8 MB)       |                              | GDRIVE_CHUNK_SIZE           |
+lets-encrypt-hosts | hosts to use for lets encrypt certificates (comma seperated)                                |                              | HOSTS                       |
+log | path to log file                                                                            |                              | LOG                         |
+cors-domains | comma separated list of domains for CORS, setting it enable CORS                            |                              | CORS_DOMAINS                |
+clamav-host | host for clamav feature                                                                     |                              | CLAMAV_HOST                 |
+perform-clamav-prescan | prescan every upload through clamav feature (clamav-host must be a local clamd unix socket) |                              | PERFORM_CLAMAV_PRESCAN      |
+rate-limit | request per minute                                                                          |                              | RATE_LIMIT                  |
+max-upload-size | max upload size in kilobytes                                                                |                              | MAX_UPLOAD_SIZE             |
+purge-days | number of days after the uploads are purged automatically                                   |                              | PURGE_DAYS                  |   
+purge-interval | interval in hours to run the automatic purge for (not applicable to S3 and Storj)           |                              | PURGE_INTERVAL              |   
+random-token-length | length of the random token for the upload path (double the size for delete path)            | 6                            | RANDOM_TOKEN_LENGTH         |   
 
 If you want to use TLS using lets encrypt certificates, set lets-encrypt-hosts to your domain, set tls-listener to :443 and enable force-https.
 

cmd/cmd.go

@@ -276,6 +276,12 @@ var globalFlags = []cli.Flag{
 		Value:  "",
 		EnvVar: "HTTP_AUTH_HTPASSWD",
 	},
+	cli.StringFlag{
+		Name:   "http-auth-ip-whitelist",
+		Usage:  "comma separated list of ips allowed to upload without being challenged an http auth",
+		Value:  "",
+		EnvVar: "HTTP_AUTH_IP_WHITELIST",
+	},
 	cli.StringFlag{
 		Name:   "ip-whitelist",
 		Usage:  "comma separated list of ips allowed to connect to the service",
@@ -450,6 +456,13 @@ func New() *Cmd {
 			options = append(options, server.HTTPAuthHtpasswd(httpAuthHtpasswd))
 		}
 
+		if httpAuthIPWhitelist := c.String("http-auth-ip-whitelist"); httpAuthIPWhitelist != "" {
+			ipFilterOptions := server.IPFilterOptions{}
+			ipFilterOptions.AllowedIPs = strings.Split(httpAuthIPWhitelist, ",")
+			ipFilterOptions.BlockByDefault = false
+			options = append(options, server.HTTPAUTHFilterOptions(ipFilterOptions))
+		}
+
 		applyIPFilter := false
 		ipFilterOptions := server.IPFilterOptions{}
 		if ipWhitelist := c.String("ip-whitelist"); ipWhitelist != "" {

flake.nix

@@ -45,6 +45,7 @@
               http-auth-user = mkOption { type = types.nullOr types.str; description = "user for basic http auth on upload"; };
               http-auth-pass = mkOption { type = types.nullOr types.str; description = "pass for basic http auth on upload"; };
               http-auth-htpasswd = mkOption { type = types.nullOr types.str; description = "htpasswd file path for basic http auth on upload"; };
+              http-auth-ip-whitelist = mkOption { type = types.nullOr types.str; description = "comma separated list of ips allowed to upload without being challenged an http auth"; };
               ip-whitelist = mkOption { type = types.nullOr types.str; description = "comma separated list of ips allowed to connect to the service"; };
               ip-blacklist = mkOption { type = types.nullOr types.str; description = "comma separated list of ips not allowed to connect to the service"; };
               temp-path = mkOption { type = types.nullOr types.str; description = "path to temp folder"; };

server/handlers.go

@@ -58,6 +58,7 @@ import (
 	"github.com/ProtonMail/gopenpgp/v2/constants"
 	"github.com/dutchcoders/transfer.sh/server/storage"
 	"github.com/tg123/go-htpasswd"
+	"github.com/tomasen/realip"
 
 	web "github.com/dutchcoders/transfer.sh-web"
 	"github.com/gorilla/mux"
@@ -1313,20 +1314,20 @@ func ipFilterHandler(h http.Handler, ipFilterOptions *IPFilterOptions) http.Hand
 		if ipFilterOptions == nil {
 			h.ServeHTTP(w, r)
 		} else {
-			WrapIPFilter(h, *ipFilterOptions).ServeHTTP(w, r)
+			WrapIPFilter(h, ipFilterOptions).ServeHTTP(w, r)
 		}
 	}
 }
 
 func (s *Server) basicAuthHandler(h http.Handler) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		if s.AuthUser == "" || s.AuthPass == "" || s.AuthHtpasswd == "" {
+		if s.authUser == "" || s.authPass == "" || s.authHtpasswd == "" {
 			h.ServeHTTP(w, r)
 			return
 		}
 
-		if s.htpasswdFile == nil && s.AuthHtpasswd != "" {
-			htpasswdFile, err := htpasswd.New(s.AuthHtpasswd, htpasswd.DefaultSystems, nil)
+		if s.htpasswdFile == nil && s.authHtpasswd != "" {
+			htpasswdFile, err := htpasswd.New(s.authHtpasswd, htpasswd.DefaultSystems, nil)
 			if err != nil {
 				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
@@ -1335,20 +1336,29 @@ func (s *Server) basicAuthHandler(h http.Handler) http.HandlerFunc {
 			s.htpasswdFile = htpasswdFile
 		}
 
+		if s.authIPFilter == nil && s.authIPFilterOptions != nil {
+			s.authIPFilter = newIPFilter(s.authIPFilterOptions)
+		}
+
 		w.Header().Set("WWW-Authenticate", "Basic realm=\"Restricted\"")
 
+		var authorized bool
+		if s.authIPFilter != nil {
+			remoteIP := realip.FromRequest(r)
+			authorized = s.authIPFilter.Allowed(remoteIP)
+		}
+
 		username, password, authOK := r.BasicAuth()
-		if !authOK {
+		if !authOK && !authorized {
 			http.Error(w, "Not authorized", http.StatusUnauthorized)
 			return
 		}
 
-		var authorized bool
-		if username == s.AuthUser && password == s.AuthPass {
+		if !authorized && username == s.authUser && password == s.authPass {
 			authorized = true
 		}
 
-		if s.htpasswdFile != nil && !authorized {
+		if !authorized && s.htpasswdFile != nil {
 			authorized = s.htpasswdFile.Match(username, password)
 		}
 

server/ip_filter.go

@@ -45,7 +45,6 @@ type IPFilterOptions struct {
 
 // ipFilter
 type ipFilter struct {
-	opts IPFilterOptions
 	//mut protects the below
 	//rw since writes are rare
 	mut            sync.RWMutex
@@ -60,13 +59,12 @@ type subnet struct {
 	allowed bool
 }
 
-func newIPFilter(opts IPFilterOptions) *ipFilter {
+func newIPFilter(opts *IPFilterOptions) *ipFilter {
 	if opts.Logger == nil {
 		flags := log.LstdFlags
 		opts.Logger = log.New(os.Stdout, "", flags)
 	}
 	f := &ipFilter{
-		opts:           opts,
 		ips:            map[string]bool{},
 		defaultAllowed: !opts.BlockByDefault,
 	}
@@ -189,7 +187,7 @@ func (f *ipFilter) Wrap(next http.Handler) http.Handler {
 }
 
 // WrapIPFilter is equivalent to newIPFilter(opts) then Wrap(next)
-func WrapIPFilter(next http.Handler, opts IPFilterOptions) http.Handler {
+func WrapIPFilter(next http.Handler, opts *IPFilterOptions) http.Handler {
 	return newIPFilter(opts).Wrap(next)
 }
 

server/server.go

@@ -295,15 +295,26 @@ func TLSConfig(cert, pk string) OptionFn {
 // HTTPAuthCredentials sets basic http auth credentials
 func HTTPAuthCredentials(user string, pass string) OptionFn {
 	return func(srvr *Server) {
-		srvr.AuthUser = user
-		srvr.AuthPass = pass
+		srvr.authUser = user
+		srvr.authPass = pass
 	}
 }
 
 // HTTPAuthHtpasswd sets basic http auth htpasswd file
 func HTTPAuthHtpasswd(htpasswdPath string) OptionFn {
 	return func(srvr *Server) {
-		srvr.AuthHtpasswd = htpasswdPath
+		srvr.authHtpasswd = htpasswdPath
+	}
+}
+
+// HTTPAUTHFilterOptions sets basic http auth ips whitelist
+func HTTPAUTHFilterOptions(options IPFilterOptions) OptionFn {
+	for i, allowedIP := range options.AllowedIPs {
+		options.AllowedIPs[i] = strings.TrimSpace(allowedIP)
+	}
+
+	return func(srvr *Server) {
+		srvr.authIPFilterOptions = &options
 	}
 }
 
@@ -324,11 +335,13 @@ func FilterOptions(options IPFilterOptions) OptionFn {
 
 // Server is the main application
 type Server struct {
-	AuthUser     string
-	AuthPass     string
-	AuthHtpasswd string
+	authUser            string
+	authPass            string
+	authHtpasswd        string
+	authIPFilterOptions *IPFilterOptions
 
 	htpasswdFile *htpasswd.File
+	authIPFilter *ipFilter
 
 	logger *log.Logger