move SFTPGo package to the internal folder

SFTPGo is a daemon and command line tool, not a library. The public API are provided by the SDK Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
2025-12-07 23:00:55 +03:00 · 2022-07-24 16:18:54 +02:00
parent e96ae5ca51
commit c8158e14e0
216 changed files with 691 additions and 697 deletions
--- a/internal/vfs/httpfs.go
+++ b/internal/vfs/httpfs.go
@@ -0,0 +1,773 @@
+// Copyright (C) 2019-2022  Nicola Murino
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, version 3.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package vfs
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"mime"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/eikenb/pipeat"
+	"github.com/pkg/sftp"
+	"github.com/sftpgo/sdk"
+
+	"github.com/drakkan/sftpgo/v2/internal/kms"
+	"github.com/drakkan/sftpgo/v2/internal/logger"
+	"github.com/drakkan/sftpgo/v2/internal/metric"
+	"github.com/drakkan/sftpgo/v2/internal/util"
+)
+
+const (
+	// httpFsName is the name for the HTTP Fs implementation
+	httpFsName = "httpfs"
+)
+
+var (
+	supportedEndpointSchema = []string{"http://", "https://"}
+)
+
+// HTTPFsConfig defines the configuration for HTTP based filesystem
+type HTTPFsConfig struct {
+	sdk.BaseHTTPFsConfig
+	Password *kms.Secret `json:"password,omitempty"`
+	APIKey   *kms.Secret `json:"api_key,omitempty"`
+}
+
+func (c *HTTPFsConfig) isUnixDomainSocket() bool {
+	return strings.HasPrefix(c.Endpoint, "http://unix") || strings.HasPrefix(c.Endpoint, "https://unix")
+}
+
+// HideConfidentialData hides confidential data
+func (c *HTTPFsConfig) HideConfidentialData() {
+	if c.Password != nil {
+		c.Password.Hide()
+	}
+	if c.APIKey != nil {
+		c.APIKey.Hide()
+	}
+}
+
+func (c *HTTPFsConfig) setNilSecretsIfEmpty() {
+	if c.Password != nil && c.Password.IsEmpty() {
+		c.Password = nil
+	}
+	if c.APIKey != nil && c.APIKey.IsEmpty() {
+		c.APIKey = nil
+	}
+}
+
+func (c *HTTPFsConfig) setEmptyCredentialsIfNil() {
+	if c.Password == nil {
+		c.Password = kms.NewEmptySecret()
+	}
+	if c.APIKey == nil {
+		c.APIKey = kms.NewEmptySecret()
+	}
+}
+
+func (c *HTTPFsConfig) isEqual(other *HTTPFsConfig) bool {
+	if c.Endpoint != other.Endpoint {
+		return false
+	}
+	if c.Username != other.Username {
+		return false
+	}
+	if c.SkipTLSVerify != other.SkipTLSVerify {
+		return false
+	}
+	c.setEmptyCredentialsIfNil()
+	other.setEmptyCredentialsIfNil()
+	if !c.Password.IsEqual(other.Password) {
+		return false
+	}
+	return c.APIKey.IsEqual(other.APIKey)
+}
+
+// validate returns an error if the configuration is not valid
+func (c *HTTPFsConfig) validate() error {
+	c.setEmptyCredentialsIfNil()
+	if c.Endpoint == "" {
+		return errors.New("httpfs: endpoint cannot be empty")
+	}
+	c.Endpoint = strings.TrimRight(c.Endpoint, "/")
+	endpointURL, err := url.Parse(c.Endpoint)
+	if err != nil {
+		return fmt.Errorf("httpfs: invalid endpoint: %w", err)
+	}
+	if !util.IsStringPrefixInSlice(c.Endpoint, supportedEndpointSchema) {
+		return errors.New("httpfs: invalid endpoint schema: http and https are supported")
+	}
+	if endpointURL.Host == "unix" {
+		socketPath := endpointURL.Query().Get("socket_path")
+		if !filepath.IsAbs(socketPath) {
+			return fmt.Errorf("httpfs: invalid unix domain socket path: %q", socketPath)
+		}
+	}
+	if c.Password.IsEncrypted() && !c.Password.IsValid() {
+		return errors.New("httpfs: invalid encrypted password")
+	}
+	if !c.Password.IsEmpty() && !c.Password.IsValidInput() {
+		return errors.New("httpfs: invalid password")
+	}
+	if c.APIKey.IsEncrypted() && !c.APIKey.IsValid() {
+		return errors.New("httpfs: invalid encrypted API key")
+	}
+	if !c.APIKey.IsEmpty() && !c.APIKey.IsValidInput() {
+		return errors.New("httpfs: invalid API key")
+	}
+	return nil
+}
+
+// ValidateAndEncryptCredentials validates the config and encrypts credentials if they are in plain text
+func (c *HTTPFsConfig) ValidateAndEncryptCredentials(additionalData string) error {
+	if err := c.validate(); err != nil {
+		return util.NewValidationError(fmt.Sprintf("could not validate HTTP fs config: %v", err))
+	}
+	if c.Password.IsPlain() {
+		c.Password.SetAdditionalData(additionalData)
+		if err := c.Password.Encrypt(); err != nil {
+			return util.NewValidationError(fmt.Sprintf("could not encrypt HTTP fs password: %v", err))
+		}
+	}
+	if c.APIKey.IsPlain() {
+		c.APIKey.SetAdditionalData(additionalData)
+		if err := c.APIKey.Encrypt(); err != nil {
+			return util.NewValidationError(fmt.Sprintf("could not encrypt HTTP fs API key: %v", err))
+		}
+	}
+	return nil
+}
+
+// HTTPFs is a Fs implementation for the SFTPGo HTTP filesystem backend
+type HTTPFs struct {
+	connectionID string
+	localTempDir string
+	// if not empty this fs is mouted as virtual folder in the specified path
+	mountPath  string
+	config     *HTTPFsConfig
+	client     *http.Client
+	ctxTimeout time.Duration
+}
+
+// NewHTTPFs returns an HTTPFs object that allows to interact with SFTPGo HTTP filesystem backends
+func NewHTTPFs(connectionID, localTempDir, mountPath string, config HTTPFsConfig) (Fs, error) {
+	if localTempDir == "" {
+		if tempPath != "" {
+			localTempDir = tempPath
+		} else {
+			localTempDir = filepath.Clean(os.TempDir())
+		}
+	}
+	config.setEmptyCredentialsIfNil()
+	if !config.Password.IsEmpty() {
+		if err := config.Password.TryDecrypt(); err != nil {
+			return nil, err
+		}
+	}
+	if !config.APIKey.IsEmpty() {
+		if err := config.APIKey.TryDecrypt(); err != nil {
+			return nil, err
+		}
+	}
+	fs := &HTTPFs{
+		connectionID: connectionID,
+		localTempDir: localTempDir,
+		mountPath:    mountPath,
+		config:       &config,
+		ctxTimeout:   30 * time.Second,
+	}
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.MaxResponseHeaderBytes = 1 << 16
+	transport.WriteBufferSize = 1 << 16
+	transport.ReadBufferSize = 1 << 16
+	if fs.config.isUnixDomainSocket() {
+		endpointURL, err := url.Parse(fs.config.Endpoint)
+		if err != nil {
+			return nil, err
+		}
+		if endpointURL.Host == "unix" {
+			socketPath := endpointURL.Query().Get("socket_path")
+			if !filepath.IsAbs(socketPath) {
+				return nil, fmt.Errorf("httpfs: invalid unix domain socket path: %q", socketPath)
+			}
+			if endpointURL.Scheme == "https" {
+				transport.DialTLSContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+					var tlsConfig *tls.Config
+					var d tls.Dialer
+					if config.SkipTLSVerify {
+						tlsConfig = getInsecureTLSConfig()
+					}
+					d.Config = tlsConfig
+					return d.DialContext(ctx, "unix", socketPath)
+				}
+			} else {
+				transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+					var d net.Dialer
+					return d.DialContext(ctx, "unix", socketPath)
+				}
+			}
+			endpointURL.Path = path.Join(endpointURL.Path, endpointURL.Query().Get("api_prefix"))
+			endpointURL.RawQuery = ""
+			endpointURL.RawFragment = ""
+			fs.config.Endpoint = endpointURL.String()
+		}
+	}
+	if config.SkipTLSVerify {
+		if transport.TLSClientConfig != nil {
+			transport.TLSClientConfig.InsecureSkipVerify = true
+		} else {
+			transport.TLSClientConfig = getInsecureTLSConfig()
+		}
+	}
+	fs.client = &http.Client{
+		Transport: transport,
+	}
+	return fs, nil
+}
+
+// Name returns the name for the Fs implementation
+func (fs *HTTPFs) Name() string {
+	return fmt.Sprintf("%v %#v", httpFsName, fs.config.Endpoint)
+}
+
+// ConnectionID returns the connection ID associated to this Fs implementation
+func (fs *HTTPFs) ConnectionID() string {
+	return fs.connectionID
+}
+
+// Stat returns a FileInfo describing the named file
+func (fs *HTTPFs) Stat(name string) (os.FileInfo, error) {
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+
+	resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "stat", name, "", "", nil)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var response statResponse
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	if err != nil {
+		return nil, err
+	}
+	return response.getFileInfo(), nil
+}
+
+// Lstat returns a FileInfo describing the named file
+func (fs *HTTPFs) Lstat(name string) (os.FileInfo, error) {
+	return fs.Stat(name)
+}
+
+// Open opens the named file for reading
+func (fs *HTTPFs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, func(), error) {
+	r, w, err := pipeat.PipeInDir(fs.localTempDir)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	ctx, cancelFn := context.WithCancel(context.Background())
+
+	var queryString string
+	if offset > 0 {
+		queryString = fmt.Sprintf("?offset=%d", offset)
+	}
+
+	go func() {
+		defer cancelFn()
+
+		resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "open", name, queryString, "", nil)
+		if err != nil {
+			fsLog(fs, logger.LevelError, "download error, path %q, err: %v", name, err)
+			w.CloseWithError(err) //nolint:errcheck
+			metric.HTTPFsTransferCompleted(0, 1, err)
+			return
+		}
+		defer resp.Body.Close()
+		n, err := io.Copy(w, resp.Body)
+		w.CloseWithError(err) //nolint:errcheck
+		fsLog(fs, logger.LevelDebug, "download completed, path %q size: %v, err: %+v", name, n, err)
+		metric.HTTPFsTransferCompleted(n, 1, err)
+	}()
+
+	return nil, r, cancelFn, nil
+}
+
+// Create creates or opens the named file for writing
+func (fs *HTTPFs) Create(name string, flag int) (File, *PipeWriter, func(), error) {
+	r, w, err := pipeat.PipeInDir(fs.localTempDir)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	p := NewPipeWriter(w)
+	ctx, cancelFn := context.WithCancel(context.Background())
+
+	var queryString string
+	if flag > 0 {
+		queryString = fmt.Sprintf("?flags=%d", flag)
+	}
+
+	go func() {
+		defer cancelFn()
+
+		contentType := mime.TypeByExtension(path.Ext(name))
+		resp, err := fs.sendHTTPRequest(ctx, http.MethodPost, "create", name, queryString, contentType,
+			&wrapReader{reader: r})
+		if err != nil {
+			fsLog(fs, logger.LevelError, "upload error, path %q, err: %v", name, err)
+			r.CloseWithError(err) //nolint:errcheck
+			p.Done(err)
+			metric.HTTPFsTransferCompleted(0, 0, err)
+			return
+		}
+		defer resp.Body.Close()
+
+		r.CloseWithError(err) //nolint:errcheck
+		p.Done(err)
+		fsLog(fs, logger.LevelDebug, "upload completed, path: %q, readed bytes: %d", name, r.GetReadedBytes())
+		metric.HTTPFsTransferCompleted(r.GetReadedBytes(), 0, err)
+	}()
+
+	return nil, p, cancelFn, nil
+}
+
+// Rename renames (moves) source to target.
+func (fs *HTTPFs) Rename(source, target string) error {
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+
+	queryString := fmt.Sprintf("?target=%s", url.QueryEscape(target))
+	resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "rename", source, queryString, "", nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return nil
+}
+
+// Remove removes the named file or (empty) directory.
+func (fs *HTTPFs) Remove(name string, isDir bool) error {
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+
+	resp, err := fs.sendHTTPRequest(ctx, http.MethodDelete, "remove", name, "", "", nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return nil
+}
+
+// Mkdir creates a new directory with the specified name and default permissions
+func (fs *HTTPFs) Mkdir(name string) error {
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+
+	resp, err := fs.sendHTTPRequest(ctx, http.MethodPost, "mkdir", name, "", "", nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return nil
+}
+
+// Symlink creates source as a symbolic link to target.
+func (*HTTPFs) Symlink(source, target string) error {
+	return ErrVfsUnsupported
+}
+
+// Readlink returns the destination of the named symbolic link
+func (*HTTPFs) Readlink(name string) (string, error) {
+	return "", ErrVfsUnsupported
+}
+
+// Chown changes the numeric uid and gid of the named file.
+func (fs *HTTPFs) Chown(name string, uid int, gid int) error {
+	return ErrVfsUnsupported
+}
+
+// Chmod changes the mode of the named file to mode.
+func (fs *HTTPFs) Chmod(name string, mode os.FileMode) error {
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+
+	queryString := fmt.Sprintf("?mode=%d", mode)
+	resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "chmod", name, queryString, "", nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return nil
+}
+
+// Chtimes changes the access and modification times of the named file.
+func (fs *HTTPFs) Chtimes(name string, atime, mtime time.Time, isUploading bool) error {
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+
+	queryString := fmt.Sprintf("?access_time=%s&modification_time=%s", atime.UTC().Format(time.RFC3339),
+		mtime.UTC().Format(time.RFC3339))
+	resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "chtimes", name, queryString, "", nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return nil
+}
+
+// Truncate changes the size of the named file.
+// Truncate by path is not supported, while truncating an opened
+// file is handled inside base transfer
+func (fs *HTTPFs) Truncate(name string, size int64) error {
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+
+	queryString := fmt.Sprintf("?size=%d", size)
+	resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "truncate", name, queryString, "", nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return nil
+}
+
+// ReadDir reads the directory named by dirname and returns
+// a list of directory entries.
+func (fs *HTTPFs) ReadDir(dirname string) ([]os.FileInfo, error) {
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+
+	resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "readdir", dirname, "", "", nil)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var response []statResponse
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	if err != nil {
+		return nil, err
+	}
+	result := make([]os.FileInfo, 0, len(response))
+	for _, stat := range response {
+		result = append(result, stat.getFileInfo())
+	}
+	return result, nil
+}
+
+// IsUploadResumeSupported returns true if resuming uploads is supported.
+func (*HTTPFs) IsUploadResumeSupported() bool {
+	return false
+}
+
+// IsAtomicUploadSupported returns true if atomic upload is supported.
+func (*HTTPFs) IsAtomicUploadSupported() bool {
+	return false
+}
+
+// IsNotExist returns a boolean indicating whether the error is known to
+// report that a file or directory does not exist
+func (*HTTPFs) IsNotExist(err error) bool {
+	return errors.Is(err, fs.ErrNotExist)
+}
+
+// IsPermission returns a boolean indicating whether the error is known to
+// report that permission is denied.
+func (*HTTPFs) IsPermission(err error) bool {
+	return errors.Is(err, fs.ErrPermission)
+}
+
+// IsNotSupported returns true if the error indicate an unsupported operation
+func (*HTTPFs) IsNotSupported(err error) bool {
+	if err == nil {
+		return false
+	}
+	return err == ErrVfsUnsupported
+}
+
+// CheckRootPath creates the specified local root directory if it does not exists
+func (fs *HTTPFs) CheckRootPath(username string, uid int, gid int) bool {
+	// we need a local directory for temporary files
+	osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "")
+	return osFs.CheckRootPath(username, uid, gid)
+}
+
+// ScanRootDirContents returns the number of files and their size
+func (fs *HTTPFs) ScanRootDirContents() (int, int64, error) {
+	return fs.GetDirSize("/")
+}
+
+// CheckMetadata checks the metadata consistency
+func (*HTTPFs) CheckMetadata() error {
+	return nil
+}
+
+// GetDirSize returns the number of files and the size for a folder
+// including any subfolders
+func (fs *HTTPFs) GetDirSize(dirname string) (int, int64, error) {
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+
+	resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "dirsize", dirname, "", "", nil)
+	if err != nil {
+		return 0, 0, err
+	}
+	defer resp.Body.Close()
+
+	var response dirSizeResponse
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	if err != nil {
+		return 0, 0, err
+	}
+	return response.Files, response.Size, nil
+}
+
+// GetAtomicUploadPath returns the path to use for an atomic upload.
+func (*HTTPFs) GetAtomicUploadPath(name string) string {
+	return ""
+}
+
+// GetRelativePath returns the path for a file relative to the user's home dir.
+// This is the path as seen by SFTPGo users
+func (fs *HTTPFs) GetRelativePath(name string) string {
+	rel := path.Clean(name)
+	if rel == "." {
+		rel = ""
+	}
+	if !path.IsAbs(rel) {
+		rel = "/" + rel
+	}
+	if fs.mountPath != "" {
+		rel = path.Join(fs.mountPath, rel)
+	}
+	return rel
+}
+
+// Walk walks the file tree rooted at root, calling walkFn for each file or
+// directory in the tree, including root. The result are unordered
+func (fs *HTTPFs) Walk(root string, walkFn filepath.WalkFunc) error {
+	info, err := fs.Lstat(root)
+	if err != nil {
+		return walkFn(root, nil, err)
+	}
+	return fs.walk(root, info, walkFn)
+}
+
+// Join joins any number of path elements into a single path
+func (*HTTPFs) Join(elem ...string) string {
+	return strings.TrimPrefix(path.Join(elem...), "/")
+}
+
+// HasVirtualFolders returns true if folders are emulated
+func (*HTTPFs) HasVirtualFolders() bool {
+	return false
+}
+
+// ResolvePath returns the matching filesystem path for the specified virtual path
+func (fs *HTTPFs) ResolvePath(virtualPath string) (string, error) {
+	if fs.mountPath != "" {
+		virtualPath = strings.TrimPrefix(virtualPath, fs.mountPath)
+	}
+	if !path.IsAbs(virtualPath) {
+		virtualPath = path.Clean("/" + virtualPath)
+	}
+	return virtualPath, nil
+}
+
+// GetMimeType returns the content type
+func (fs *HTTPFs) GetMimeType(name string) (string, error) {
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+
+	resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "stat", name, "", "", nil)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	var response mimeTypeResponse
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	if err != nil {
+		return "", err
+	}
+	return response.Mime, nil
+}
+
+// Close closes the fs
+func (fs *HTTPFs) Close() error {
+	fs.client.CloseIdleConnections()
+	return nil
+}
+
+// GetAvailableDiskSize returns the available size for the specified path
+func (fs *HTTPFs) GetAvailableDiskSize(dirName string) (*sftp.StatVFS, error) {
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+
+	resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "statvfs", dirName, "", "", nil)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var response statVFSResponse
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	if err != nil {
+		return nil, err
+	}
+	return response.toSFTPStatVFS(), nil
+}
+
+func (fs *HTTPFs) sendHTTPRequest(ctx context.Context, method, base, name, queryString, contentType string,
+	body io.Reader,
+) (*http.Response, error) {
+	url := fmt.Sprintf("%s/%s/%s%s", fs.config.Endpoint, base, url.PathEscape(name), queryString)
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return nil, err
+	}
+	if contentType != "" {
+		req.Header.Set("Content-Type", contentType)
+	}
+	if fs.config.APIKey.GetPayload() != "" {
+		req.Header.Set("X-API-KEY", fs.config.APIKey.GetPayload())
+	}
+	if fs.config.Username != "" || fs.config.Password.GetPayload() != "" {
+		req.SetBasicAuth(fs.config.Username, fs.config.Password.GetPayload())
+	}
+	resp, err := fs.client.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, fmt.Errorf("unable to send HTTP request to URL %v: %w", url, err)
+	}
+	if err = getErrorFromResponseCode(resp.StatusCode); err != nil {
+		resp.Body.Close()
+		return nil, err
+	}
+	return resp, nil
+}
+
+// walk recursively descends path, calling walkFn.
+func (fs *HTTPFs) walk(filePath string, info fs.FileInfo, walkFn filepath.WalkFunc) error {
+	if !info.IsDir() {
+		return walkFn(filePath, info, nil)
+	}
+	files, err := fs.ReadDir(filePath)
+	err1 := walkFn(filePath, info, err)
+	if err != nil || err1 != nil {
+		return err1
+	}
+	for _, fi := range files {
+		objName := path.Join(filePath, fi.Name())
+		err = fs.walk(objName, fi, walkFn)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func getErrorFromResponseCode(code int) error {
+	switch code {
+	case 401, 403:
+		return os.ErrPermission
+	case 404:
+		return os.ErrNotExist
+	case 501:
+		return ErrVfsUnsupported
+	case 200, 201:
+		return nil
+	default:
+		return fmt.Errorf("unexpected response code: %v", code)
+	}
+}
+
+func getInsecureTLSConfig() *tls.Config {
+	return &tls.Config{
+		NextProtos:         []string{"h2", "http/1.1"},
+		InsecureSkipVerify: true,
+	}
+}
+
+type wrapReader struct {
+	reader io.Reader
+}
+
+func (r *wrapReader) Read(p []byte) (n int, err error) {
+	return r.reader.Read(p)
+}
+
+type statResponse struct {
+	Name         string    `json:"name"`
+	Size         int64     `json:"size"`
+	Mode         uint32    `json:"mode"`
+	LastModified time.Time `json:"last_modified"`
+}
+
+func (s *statResponse) getFileInfo() os.FileInfo {
+	info := NewFileInfo(s.Name, false, s.Size, s.LastModified, false)
+	info.SetMode(fs.FileMode(s.Mode))
+	return info
+}
+
+type dirSizeResponse struct {
+	Files int   `json:"files"`
+	Size  int64 `json:"size"`
+}
+
+type mimeTypeResponse struct {
+	Mime string `json:"mime"`
+}
+
+type statVFSResponse struct {
+	ID      uint32 `json:"-"`
+	Bsize   uint64 `json:"bsize"`
+	Frsize  uint64 `json:"frsize"`
+	Blocks  uint64 `json:"blocks"`
+	Bfree   uint64 `json:"bfree"`
+	Bavail  uint64 `json:"bavail"`
+	Files   uint64 `json:"files"`
+	Ffree   uint64 `json:"ffree"`
+	Favail  uint64 `json:"favail"`
+	Fsid    uint64 `json:"fsid"`
+	Flag    uint64 `json:"flag"`
+	Namemax uint64 `json:"namemax"`
+}
+
+func (s *statVFSResponse) toSFTPStatVFS() *sftp.StatVFS {
+	return &sftp.StatVFS{
+		Bsize:   s.Bsize,
+		Frsize:  s.Frsize,
+		Blocks:  s.Blocks,
+		Bfree:   s.Bfree,
+		Bavail:  s.Bavail,
+		Files:   s.Files,
+		Ffree:   s.Ffree,
+		Favail:  s.Ffree,
+		Flag:    s.Flag,
+		Namemax: s.Namemax,
+	}
+}