set version to 2.4.0

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>

.github/workflows/development.yml

@@ -2,7 +2,7 @@ name: CI
 
 on:
   push:
-    branches: [2.3.x]
+    branches: [main]
   pull_request:
 
 jobs:
@@ -11,11 +11,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go: [1.18]
+        go: [1.19]
         os: [ubuntu-latest, macos-latest]
         upload-coverage: [true]
         include:
-          - go: 1.18
+          - go: 1.19
             os: windows-latest
             upload-coverage: false
 
@@ -32,22 +32,24 @@ jobs:
       - name: Build for Linux/macOS x86_64
         if: startsWith(matrix.os, 'windows-') != true
         run: |
-          go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -o sftpgo
+          go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo
           cd tests/eventsearcher
           go build -trimpath -ldflags "-s -w" -o eventsearcher
           cd -
           cd tests/ipfilter
           go build -trimpath -ldflags "-s -w" -o ipfilter
           cd -
+          ./sftpgo initprovider
+          ./sftpgo resetprovider --force
 
       - name: Build for macOS arm64
         if: startsWith(matrix.os, 'macos-') == true
-        run: CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 SDKROOT=$(xcrun --sdk macosx --show-sdk-path) go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -o sftpgo_arm64
+        run: CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 SDKROOT=$(xcrun --sdk macosx --show-sdk-path) go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo_arm64
 
       - name: Build for Windows
         if: startsWith(matrix.os, 'windows-')
         run: |
-          $GIT_COMMIT = (git describe --always --dirty) | Out-String
+          $GIT_COMMIT = (git describe --always --abbrev=8 --dirty) | Out-String
           $DATE_TIME = ([datetime]::Now.ToUniversalTime().toString("yyyy-MM-ddTHH:mm:ssZ")) | Out-String
           $LATEST_TAG = ((git describe --tags $(git rev-list --tags --max-count=1)) | Out-String).Trim()
           $REV_LIST=$LATEST_TAG+"..HEAD"
@@ -55,7 +57,7 @@ jobs:
           $FILE_VERSION = $LATEST_TAG.substring(1)  + "." + $COMMITS_FROM_TAG
           go install github.com/tc-hib/go-winres@latest
           go-winres simply --arch amd64 --product-version $LATEST_TAG-dev-$GIT_COMMIT --file-version $FILE_VERSION --file-description "SFTPGo server" --product-name SFTPGo --copyright "AGPL-3.0" --original-filename sftpgo.exe --icon  .\windows-installer\icon.ico
-          go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/version.date=$DATE_TIME" -o sftpgo.exe
+          go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o sftpgo.exe
           cd tests/eventsearcher
           go build -trimpath -ldflags "-s -w" -o eventsearcher.exe
           cd ../..
@@ -67,17 +69,17 @@ jobs:
           $Env:GOOS='windows'
           $Env:GOARCH='arm64'
           go-winres simply --arch arm64 --product-version $LATEST_TAG-dev-$GIT_COMMIT --file-version $FILE_VERSION --file-description "SFTPGo server" --product-name SFTPGo --copyright "AGPL-3.0" --original-filename sftpgo.exe --icon  .\windows-installer\icon.ico
-          go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/version.date=$DATE_TIME" -o .\arm64\sftpgo.exe
+          go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o .\arm64\sftpgo.exe
           mkdir x86
           $Env:GOARCH='386'
           go-winres simply --arch 386 --product-version $LATEST_TAG-dev-$GIT_COMMIT --file-version $FILE_VERSION --file-description "SFTPGo server" --product-name SFTPGo --copyright "AGPL-3.0" --original-filename sftpgo.exe --icon  .\windows-installer\icon.ico
-          go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/version.date=$DATE_TIME" -o .\x86\sftpgo.exe
+          go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o .\x86\sftpgo.exe
           Remove-Item Env:\CGO_ENABLED
           Remove-Item Env:\GOOS
           Remove-Item Env:\GOARCH
 
       - name: Run test cases using SQLite provider
-        run: go test -v -p 1 -timeout 15m ./... -coverprofile=coverage.txt -covermode=atomic
+        run: go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -coverprofile=coverage.txt -covermode=atomic
 
       - name: Upload coverage to Codecov
         if: ${{ matrix.upload-coverage }}
@@ -88,21 +90,21 @@ jobs:
 
       - name: Run test cases using bolt provider
         run: |
-          go test -v -p 1 -timeout 2m ./config -covermode=atomic
-          go test -v -p 1 -timeout 5m ./common -covermode=atomic
-          go test -v -p 1 -timeout 5m ./httpd -covermode=atomic
-          go test -v -p 1 -timeout 8m ./sftpd -covermode=atomic
-          go test -v -p 1 -timeout 5m ./ftpd -covermode=atomic
-          go test -v -p 1 -timeout 5m ./webdavd -covermode=atomic
-          go test -v -p 1 -timeout 2m ./telemetry -covermode=atomic
-          go test -v -p 1 -timeout 2m ./mfa -covermode=atomic
-          go test -v -p 1 -timeout 2m ./command -covermode=atomic
+          go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 2m ./internal/config -covermode=atomic
+          go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 5m ./internal/common -covermode=atomic
+          go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 5m ./internal/httpd -covermode=atomic
+          go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 8m ./internal/sftpd -covermode=atomic
+          go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 5m ./internal/ftpd -covermode=atomic
+          go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 5m ./internal/webdavd -covermode=atomic
+          go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 2m ./internal/telemetry -covermode=atomic
+          go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 2m ./internal/mfa -covermode=atomic
+          go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 2m ./internal/command -covermode=atomic
         env:
           SFTPGO_DATA_PROVIDER__DRIVER: bolt
           SFTPGO_DATA_PROVIDER__NAME: 'sftpgo_bolt.db'
 
       - name: Run test cases using memory provider
-        run: go test -v -p 1 -timeout 15m ./... -covermode=atomic
+        run: go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic
         env:
           SFTPGO_DATA_PROVIDER__DRIVER: memory
           SFTPGO_DATA_PROVIDER__NAME: ''
@@ -220,6 +222,24 @@ jobs:
           name: sftpgo-${{ matrix.os }}-go-${{ matrix.go }}
           path: output
 
+  test-bundle:
+    name: Build in bundle mode
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+
+      - name: Build
+        run: |
+          cp -r openapi static templates internal/bundle/
+          go build -trimpath -tags nopgxregisterdefaulttypes,bundle -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo
+          ./sftpgo -v
+
   test-goarch-386:
     name: Run test cases on 32-bit arch
     runs-on: ubuntu-latest
@@ -230,7 +250,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.19
 
       - name: Build
         run: |
@@ -244,7 +264,7 @@ jobs:
           GOARCH: 386
 
       - name: Run test cases
-        run: go test -v -p 1 -timeout 15m ./... -covermode=atomic
+        run: go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic
         env:
           SFTPGO_DATA_PROVIDER__DRIVER: memory
           SFTPGO_DATA_PROVIDER__NAME: ''
@@ -304,10 +324,11 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.19
 
       - name: Build
         run: |
+          go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo
           cd tests/eventsearcher
           go build -trimpath -ldflags "-s -w" -o eventsearcher
           cd -
@@ -317,7 +338,9 @@ jobs:
 
       - name: Run tests using PostgreSQL provider
         run: |
-          go test -v -p 1 -timeout 15m ./... -covermode=atomic
+          ./sftpgo initprovider
+          ./sftpgo resetprovider --force
+          go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic
         env:
           SFTPGO_DATA_PROVIDER__DRIVER: postgresql
           SFTPGO_DATA_PROVIDER__NAME: sftpgo
@@ -328,7 +351,9 @@ jobs:
 
       - name: Run tests using MySQL provider
         run: |
-          go test -v -p 1 -timeout 15m ./... -covermode=atomic
+          ./sftpgo initprovider
+          ./sftpgo resetprovider --force
+          go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic
         env:
           SFTPGO_DATA_PROVIDER__DRIVER: mysql
           SFTPGO_DATA_PROVIDER__NAME: sftpgo
@@ -339,7 +364,9 @@ jobs:
 
       - name: Run tests using MariaDB provider
         run: |
-          go test -v -p 1 -timeout 15m ./... -covermode=atomic
+          ./sftpgo initprovider
+          ./sftpgo resetprovider --force
+          go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic
         env:
           SFTPGO_DATA_PROVIDER__DRIVER: mysql
           SFTPGO_DATA_PROVIDER__NAME: sftpgo
@@ -347,13 +374,16 @@ jobs:
           SFTPGO_DATA_PROVIDER__PORT: 3307
           SFTPGO_DATA_PROVIDER__USERNAME: sftpgo
           SFTPGO_DATA_PROVIDER__PASSWORD: sftpgo
+          SFTPGO_DATA_PROVIDER__SQL_TABLES_PREFIX: prefix_
 
       - name: Run tests using CockroachDB provider
         run: |
           docker run --rm --name crdb --health-cmd "curl -I http://127.0.0.1:8080" --health-interval 10s --health-timeout 5s --health-retries 6 -p 26257:26257 -d cockroachdb/cockroach:latest start-single-node --insecure --listen-addr :26257
           sleep 10
           docker exec crdb cockroach sql --insecure -e 'create database "sftpgo"'
-          go test -v -p 1 -timeout 15m ./... -covermode=atomic
+          ./sftpgo initprovider
+          ./sftpgo resetprovider --force
+          go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic
           docker stop crdb
         env:
           SFTPGO_DATA_PROVIDER__DRIVER: cockroachdb
@@ -362,15 +392,17 @@ jobs:
           SFTPGO_DATA_PROVIDER__PORT: 26257
           SFTPGO_DATA_PROVIDER__USERNAME: root
           SFTPGO_DATA_PROVIDER__PASSWORD:
+          SFTPGO_DATA_PROVIDER__SQL_TABLES_PREFIX: prefix_
 
   build-linux-packages:
     name: Build Linux packages
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         include:
           - arch: amd64
-            go: 1.18
+            distro: ubuntu:18.04
+            go: latest
             go-arch: amd64
           - arch: aarch64
             distro: ubuntu18.04
@@ -388,16 +420,36 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Set up Go
-        if: ${{ matrix.arch == 'amd64' }}
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ matrix.go }}
+
+      - name: Get commit SHA
+        id: get_commit
+        run: echo "COMMIT=${GITHUB_SHA::8}" >> $GITHUB_OUTPUT
+        shell: bash
 
       - name: Build on amd64
         if: ${{ matrix.arch == 'amd64' }}
         run: |
-          go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -o sftpgo
+          echo '#!/bin/bash' > build.sh
+          echo '' >> build.sh
+          echo 'set -e' >> build.sh
+          echo 'apt-get update -q -y' >> build.sh
+          echo 'apt-get install -q -y curl gcc' >> build.sh
+          if [ ${{ matrix.go }} == 'latest' ]
+          then
+            echo 'GO_VERSION=$(curl -L https://go.dev/VERSION?m=text)' >> build.sh
+          else
+            echo 'GO_VERSION=${{ matrix.go }}' >> build.sh
+          fi
+          echo 'GO_DOWNLOAD_ARCH=${{ matrix.go-arch }}' >> build.sh
+          echo 'curl --retry 5 --retry-delay 2 --connect-timeout 10 -o go.tar.gz -L https://go.dev/dl/${GO_VERSION}.linux-${GO_DOWNLOAD_ARCH}.tar.gz' >> build.sh
+          echo 'tar -C /usr/local -xzf go.tar.gz' >> build.sh
+          echo 'export PATH=$PATH:/usr/local/go/bin' >> build.sh
+          echo 'go version' >> build.sh
+          echo 'cd /usr/local/src' >> build.sh
+          echo 'go build -buildvcs=false -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${{ steps.get_commit.outputs.COMMIT }} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo' >> build.sh
+
+          chmod 755 build.sh
+          docker run --rm --name ubuntu-build --mount type=bind,source=`pwd`,target=/usr/local/src ${{ matrix.distro }} /usr/local/src/build.sh
           mkdir -p output/{init,bash_completion,zsh_completion}
           cp sftpgo.json output/
           cp -r templates output/
@@ -424,7 +476,7 @@ jobs:
           shell: /bin/bash
           install: |
             apt-get update -q -y
-            apt-get install -q -y curl gcc git
+            apt-get install -q -y curl gcc
             if [ ${{ matrix.go }} == 'latest' ]
             then
               GO_VERSION=$(curl -L https://go.dev/VERSION?m=text)
@@ -440,11 +492,12 @@ jobs:
             tar -C /usr/local -xzf go.tar.gz
           run: |
             export PATH=$PATH:/usr/local/go/bin
+            go version
             if [ ${{ matrix.arch}} == 'armv7' ]
             then
               export GOARM=7
             fi
-            go build -buildvcs=false -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -o sftpgo
+            go build -buildvcs=false -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${{ steps.get_commit.outputs.COMMIT }} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo
             mkdir -p output/{init,bash_completion,zsh_completion}
             cp sftpgo.json output/
             cp -r templates output/
@@ -470,7 +523,7 @@ jobs:
           cd pkgs
           ./build.sh
           PKG_VERSION=$(cat dist/version)
-          echo "::set-output name=pkg-version::${PKG_VERSION}"
+          echo "pkg-version=${PKG_VERSION}" >> $GITHUB_OUTPUT
 
       - name: Upload Debian Package
         uses: actions/upload-artifact@v3
@@ -491,7 +544,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/checkout@v3
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v3

.github/workflows/docker.yml

@@ -5,7 +5,7 @@ on:
   #  - cron: '0 4 * * *' # everyday at 4:00 AM UTC
   push:
     branches:
-      - 2.3.x
+      - main
     tags:
       - v*
   pull_request:
@@ -28,6 +28,9 @@ jobs:
           - os: ubuntu-latest
             docker_pkg: distroless
             optional_deps: false
+          - os: ubuntu-latest
+            docker_pkg: debian-plugins
+            optional_deps: true
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -64,6 +67,9 @@ jobs:
             VERSION="${VERSION}-distroless"
             VERSION_SLIM="${VERSION}-slim"
             DOCKERFILE=Dockerfile.distroless
+          elif [[ $DOCKER_PKG == debian-plugins ]]; then
+            VERSION="${VERSION}-plugins"
+            VERSION_SLIM="${VERSION}-slim"
           fi
           DOCKER_IMAGES=("drakkan/sftpgo" "ghcr.io/drakkan/sftpgo")
           TAGS="${DOCKER_IMAGES[0]}:${VERSION}"
@@ -89,6 +95,13 @@ jobs:
                 fi
                 TAGS="${TAGS},${DOCKER_IMAGE}:distroless"
                 TAGS_SLIM="${TAGS_SLIM},${DOCKER_IMAGE}:distroless-slim"
+              elif [[ $DOCKER_PKG == debian-plugins ]]; then
+                if [[ -n $MAJOR && -n $MINOR ]]; then
+                  TAGS="${TAGS},${DOCKER_IMAGE}:${MINOR}-plugins,${DOCKER_IMAGE}:${MAJOR}-plugins"
+                  TAGS_SLIM="${TAGS_SLIM},${DOCKER_IMAGE}:${MINOR}-plugins-slim,${DOCKER_IMAGE}:${MAJOR}-plugins-slim"
+                fi
+                TAGS="${TAGS},${DOCKER_IMAGE}:plugins"
+                TAGS_SLIM="${TAGS_SLIM},${DOCKER_IMAGE}:plugins-slim"
               else
                 if [[ -n $MAJOR && -n $MINOR ]]; then
                   TAGS="${TAGS},${DOCKER_IMAGE}:${MINOR}-alpine,${DOCKER_IMAGE}:${MAJOR}-alpine"
@@ -101,17 +114,22 @@ jobs:
           done
 
           if [[ $OPTIONAL_DEPS == true ]]; then
-            echo ::set-output name=version::${VERSION}
-            echo ::set-output name=tags::${TAGS}
-            echo ::set-output name=full::true
+            echo "version=${VERSION}" >> $GITHUB_OUTPUT
+            echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+            echo "full=true" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=version::${VERSION_SLIM}
-            echo ::set-output name=tags::${TAGS_SLIM}
-            echo ::set-output name=full::false
+            echo "version=${VERSION_SLIM}" >> $GITHUB_OUTPUT
+            echo "tags=${TAGS_SLIM}" >> $GITHUB_OUTPUT
+            echo "full=false" >> $GITHUB_OUTPUT
           fi
-          echo ::set-output name=dockerfile::${DOCKERFILE}
-          echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-          echo ::set-output name=sha::${GITHUB_SHA::8}
+          if [[ $DOCKER_PKG == debian-plugins ]]; then
+            echo "plugins=true" >> $GITHUB_OUTPUT
+          else
+            echo "plugins=false" >> $GITHUB_OUTPUT
+          fi
+          echo "dockerfile=${DOCKERFILE}" >> $GITHUB_OUTPUT
+          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
+          echo "sha=${GITHUB_SHA::8}" >> $GITHUB_OUTPUT
         env:
           DOCKER_PKG: ${{ matrix.docker_pkg }}
           OPTIONAL_DEPS: ${{ matrix.optional_deps }}
@@ -150,6 +168,8 @@ jobs:
           build-args: |
             COMMIT_SHA=${{ steps.info.outputs.sha }}
             INSTALL_OPTIONAL_PACKAGES=${{ steps.info.outputs.full }}
+            DOWNLOAD_PLUGINS=${{ steps.info.outputs.plugins }}
+            FEATURES=nopgxregisterdefaulttypes
           labels: |
             org.opencontainers.image.title=SFTPGo
             org.opencontainers.image.description=Fully featured and highly configurable SFTP server with optional HTTP, FTP/S and WebDAV support
@@ -159,4 +179,4 @@ jobs:
             org.opencontainers.image.version=${{ steps.info.outputs.version }}
             org.opencontainers.image.created=${{ steps.info.outputs.created }}
             org.opencontainers.image.revision=${{ github.sha }}
-            org.opencontainers.image.licenses=AGPL-3.0
+            org.opencontainers.image.licenses=AGPL-3.0-only

.github/workflows/release.yml

@@ -5,7 +5,7 @@ on:
     tags: 'v*'
 
 env:
-  GO_VERSION: 1.18.3
+  GO_VERSION: 1.19.2
 
 jobs:
   prepare-sources-with-deps:
@@ -20,12 +20,13 @@ jobs:
 
       - name: Get SFTPGo version
         id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+        run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
 
       - name: Prepare release
         run: |
           go mod vendor
           echo "${SFTPGO_VERSION}" > VERSION.txt
+          echo "${GITHUB_SHA::8}" >> VERSION.txt
           tar cJvf sftpgo_${SFTPGO_VERSION}_src_with_deps.tar.xz *
         env:
           SFTPGO_VERSION: ${{ steps.get_version.outputs.VERSION }}
@@ -53,7 +54,7 @@ jobs:
 
       - name: Get SFTPGo version
         id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+        run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
         shell: bash
 
       - name: Get OS name
@@ -61,9 +62,9 @@ jobs:
         run: |
           if [[ $MATRIX_OS =~ ^macos.* ]]
           then
-            echo ::set-output name=OS::macOS
+            echo "OS=macOS" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=OS::windows
+            echo "OS=windows" >> $GITHUB_OUTPUT
           fi
         shell: bash
         env:
@@ -71,31 +72,31 @@ jobs:
 
       - name: Build for macOS x86_64
         if: startsWith(matrix.os, 'windows-') != true
-        run: go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -o sftpgo
+        run: go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo
 
       - name: Build for macOS arm64
         if: startsWith(matrix.os, 'macos-') == true
-        run: CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 SDKROOT=$(xcrun --sdk macosx --show-sdk-path) go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -o sftpgo_arm64
+        run: CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 SDKROOT=$(xcrun --sdk macosx --show-sdk-path) go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo_arm64
 
       - name: Build for Windows
         if: startsWith(matrix.os, 'windows-')
         run: |
-          $GIT_COMMIT = (git describe --always --dirty) | Out-String
+          $GIT_COMMIT = (git describe --always --abbrev=8 --dirty) | Out-String
           $DATE_TIME = ([datetime]::Now.ToUniversalTime().toString("yyyy-MM-ddTHH:mm:ssZ")) | Out-String
           $FILE_VERSION = $Env:SFTPGO_VERSION.substring(1)  + ".0"
           go install github.com/tc-hib/go-winres@latest
           go-winres simply --arch amd64 --product-version $Env:SFTPGO_VERSION-$GIT_COMMIT --file-version $FILE_VERSION --file-description "SFTPGo server" --product-name SFTPGo --copyright "AGPL-3.0" --original-filename sftpgo.exe --icon  .\windows-installer\icon.ico
-          go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/version.date=$DATE_TIME" -o sftpgo.exe
+          go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o sftpgo.exe
           mkdir arm64
           $Env:CGO_ENABLED='0'
           $Env:GOOS='windows'
           $Env:GOARCH='arm64'
           go-winres simply --arch arm64 --product-version $Env:SFTPGO_VERSION-$GIT_COMMIT --file-version $FILE_VERSION --file-description "SFTPGo server" --product-name SFTPGo --copyright "AGPL-3.0" --original-filename sftpgo.exe --icon  .\windows-installer\icon.ico
-          go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/version.date=$DATE_TIME" -o .\arm64\sftpgo.exe
+          go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o .\arm64\sftpgo.exe
           mkdir x86
           $Env:GOARCH='386'
           go-winres simply --arch 386 --product-version $Env:SFTPGO_VERSION-$GIT_COMMIT --file-version $FILE_VERSION --file-description "SFTPGo server" --product-name SFTPGo --copyright "AGPL-3.0" --original-filename sftpgo.exe --icon  .\windows-installer\icon.ico
-          go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/version.date=$DATE_TIME" -o .\x86\sftpgo.exe
+          go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o .\x86\sftpgo.exe
           Remove-Item Env:\CGO_ENABLED
           Remove-Item Env:\GOOS
           Remove-Item Env:\GOARCH
@@ -254,11 +255,12 @@ jobs:
 
   prepare-linux:
     name: Prepare Linux binaries
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         include:
           - arch: amd64
+            distro: ubuntu:18.04
             go-arch: amd64
             deb-arch: amd64
             rpm-arch: x86_64
@@ -284,17 +286,13 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Go
-        if: ${{ matrix.arch == 'amd64' }}
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ env.GO_VERSION }}
 
       - name: Get versions
         id: get_version
         run: |
-          echo ::set-output name=SFTPGO_VERSION::${GITHUB_REF/refs\/tags\//}
-          echo ::set-output name=GO_VERSION::${GO_VERSION}
+          echo "SFTPGO_VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
+          echo "GO_VERSION=${GO_VERSION}" >> $GITHUB_OUTPUT
+          echo "COMMIT=${GITHUB_SHA::8}" >> $GITHUB_OUTPUT
         shell: bash
         env:
           GO_VERSION: ${{ env.GO_VERSION }}
@@ -302,7 +300,20 @@ jobs:
       - name: Build on amd64
         if: ${{ matrix.arch == 'amd64' }}
         run: |
-          go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -o sftpgo
+          echo '#!/bin/bash' > build.sh
+          echo '' >> build.sh
+          echo 'set -e' >> build.sh
+          echo 'apt-get update -q -y' >> build.sh
+          echo 'apt-get install -q -y curl gcc' >> build.sh
+          echo 'curl --retry 5 --retry-delay 2 --connect-timeout 10 -o go.tar.gz -L https://go.dev/dl/go${{ steps.get_version.outputs.GO_VERSION }}.linux-${{ matrix.go-arch }}.tar.gz' >> build.sh
+          echo 'tar -C /usr/local -xzf go.tar.gz' >> build.sh
+          echo 'export PATH=$PATH:/usr/local/go/bin' >> build.sh
+          echo 'go version' >> build.sh
+          echo 'cd /usr/local/src' >> build.sh
+          echo 'go build -buildvcs=false -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${{ steps.get_version.outputs.COMMIT }} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo' >> build.sh
+
+          chmod 755 build.sh
+          docker run --rm --name ubuntu-build --mount type=bind,source=`pwd`,target=/usr/local/src ${{ matrix.distro }} /usr/local/src/build.sh
           mkdir -p output/{init,sqlite,bash_completion,zsh_completion}
           echo "For documentation please take a look here:" > output/README.txt
           echo "" >> output/README.txt
@@ -340,7 +351,7 @@ jobs:
           shell: /bin/bash
           install: |
             apt-get update -q -y
-            apt-get install -q -y curl gcc git xz-utils
+            apt-get install -q -y curl gcc xz-utils
             GO_DOWNLOAD_ARCH=${{ matrix.go-arch }}
             if [ ${{ matrix.arch}} == 'armv7' ]
             then
@@ -350,7 +361,8 @@ jobs:
             tar -C /usr/local -xzf go.tar.gz
           run: |
             export PATH=$PATH:/usr/local/go/bin
-            go build -buildvcs=false -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -o sftpgo
+            go version
+            go build -buildvcs=false -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${{ steps.get_version.outputs.COMMIT }} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo
             mkdir -p output/{init,sqlite,bash_completion,zsh_completion}
             echo "For documentation please take a look here:" > output/README.txt
             echo "" >> output/README.txt
@@ -386,7 +398,7 @@ jobs:
           cd pkgs
           ./build.sh
           PKG_VERSION=${SFTPGO_VERSION:1}
-          echo "::set-output name=pkg-version::${PKG_VERSION}"
+          echo "pkg-version=${PKG_VERSION}" >> $GITHUB_OUTPUT
         env:
           SFTPGO_VERSION: ${{ steps.get_version.outputs.SFTPGO_VERSION }}
 
@@ -413,7 +425,7 @@ jobs:
       - name: Get versions
         id: get_version
         run: |
-          echo ::set-output name=SFTPGO_VERSION::${GITHUB_REF/refs\/tags\//}
+          echo "SFTPGO_VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
         shell: bash
 
       - name: Download amd64 artifact
@@ -473,8 +485,8 @@ jobs:
         run: |
           SFTPGO_VERSION=${GITHUB_REF/refs\/tags\//}
           PKG_VERSION=${SFTPGO_VERSION:1}
-          echo ::set-output name=SFTPGO_VERSION::${SFTPGO_VERSION}
-          echo "::set-output name=PKG_VERSION::${PKG_VERSION}"
+          echo "SFTPGO_VERSION=${SFTPGO_VERSION}" >> $GITHUB_OUTPUT
+          echo "PKG_VERSION=${PKG_VERSION}" >> $GITHUB_OUTPUT
         shell: bash
 
       - name: Download amd64 artifact

.golangci.yml

@@ -1,5 +1,5 @@
 run:
-  timeout: 5m
+  timeout: 10m
   issues-exit-code: 1
   tests: true
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18-bullseye as builder
+FROM golang:1.19-bullseye as builder
 
 ENV GOFLAGS="-mod=readonly"
 
@@ -20,8 +20,13 @@ ARG FEATURES
 COPY . .
 
 RUN set -xe && \
-    export COMMIT_SHA=${COMMIT_SHA:-$(git describe --always --dirty)} && \
-    go build $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=${COMMIT_SHA} -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -v -o sftpgo
+    export COMMIT_SHA=${COMMIT_SHA:-$(git describe --always --abbrev=8 --dirty)} && \
+    go build $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${COMMIT_SHA} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -v -o sftpgo
+
+# Set to "true" to download the "official" plugins in /usr/local/bin
+ARG DOWNLOAD_PLUGINS=false
+
+RUN if [ "${DOWNLOAD_PLUGINS}" = "true" ]; then apt-get update && apt-get install --no-install-recommends -y curl && ./docker/scripts/download-plugins.sh; fi
 
 FROM debian:bullseye-slim
 
@@ -43,7 +48,7 @@ COPY --from=builder /workspace/sftpgo.json /etc/sftpgo/sftpgo.json
 COPY --from=builder /workspace/templates /usr/share/sftpgo/templates
 COPY --from=builder /workspace/static /usr/share/sftpgo/static
 COPY --from=builder /workspace/openapi /usr/share/sftpgo/openapi
-COPY --from=builder /workspace/sftpgo /usr/local/bin/
+COPY --from=builder /workspace/sftpgo /usr/local/bin/sftpgo-plugin-* /usr/local/bin/
 
 # Log to the stdout so the logs will be available using docker logs
 ENV SFTPGO_LOG_FILE_PATH=""

Dockerfile.alpine

@@ -1,4 +1,4 @@
-FROM golang:1.18-alpine3.16 AS builder
+FROM golang:1.19-alpine3.16 AS builder
 
 ENV GOFLAGS="-mod=readonly"
 
@@ -22,8 +22,8 @@ ARG FEATURES
 COPY . .
 
 RUN set -xe && \
-    export COMMIT_SHA=${COMMIT_SHA:-$(git describe --always --dirty)} && \
-    go build $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=${COMMIT_SHA} -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -v -o sftpgo
+    export COMMIT_SHA=${COMMIT_SHA:-$(git describe --always --abbrev=8 --dirty)} && \
+    go build $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${COMMIT_SHA} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -v -o sftpgo
 
 
 FROM alpine:3.16

Dockerfile.distroless

@@ -1,4 +1,4 @@
-FROM golang:1.18-bullseye as builder
+FROM golang:1.19-bullseye as builder
 
 ENV CGO_ENABLED=0 GOFLAGS="-mod=readonly"
 
@@ -20,8 +20,8 @@ ARG FEATURES=nosqlite
 COPY . .
 
 RUN set -xe && \
-    export COMMIT_SHA=${COMMIT_SHA:-$(git describe --always --dirty)} && \
-    go build $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=${COMMIT_SHA} -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -v -o sftpgo
+    export COMMIT_SHA=${COMMIT_SHA:-$(git describe --always --abbrev=8 --dirty)} && \
+    go build $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${COMMIT_SHA} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -v -o sftpgo
 
 # Modify the default configuration file
 RUN sed -i 's|"users_base_dir": "",|"users_base_dir": "/srv/sftpgo/data",|' sftpgo.json && \

README.md

@@ -1,8 +1,8 @@
 # SFTPGo
 
-![CI Status](https://github.com/drakkan/sftpgo/workflows/CI/badge.svg?branch=main&event=push)
+[![CI Status](https://github.com/drakkan/sftpgo/workflows/CI/badge.svg?branch=main&event=push)](https://github.com/drakkan/sftpgo/workflows/CI/badge.svg?branch=main&event=push)
 [![Code Coverage](https://codecov.io/gh/drakkan/sftpgo/branch/main/graph/badge.svg)](https://codecov.io/gh/drakkan/sftpgo/branch/main)
-[![License: AGPL v3](https://img.shields.io/badge/License-AGPLv3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
+[![License: AGPL-3.0-only](https://img.shields.io/badge/License-AGPLv3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
 [![Docker Pulls](https://img.shields.io/docker/pulls/drakkan/sftpgo)](https://hub.docker.com/r/drakkan/sftpgo)
 [![Mentioned in Awesome Go](https://awesome.re/mentioned-badge.svg)](https://github.com/avelino/awesome-go)
 
@@ -11,6 +11,39 @@
 Fully featured and highly configurable SFTP server with optional HTTP/S, FTP/S and WebDAV support.
 Several storage backends are supported: local filesystem, encrypted local filesystem, S3 (compatible) Object Storage, Google Cloud Storage, Azure Blob Storage, SFTP.
 
+## Sponsors
+
+If you find SFTPGo useful please consider supporting this Open Source project.
+
+Maintaining and evolving SFTPGo is a lot of work - easily the equivalent of a full time job - for me.
+
+I'd like to make SFTPGo into a sustainable long term project and would not like to introduce a dual licensing option and limit some features to the proprietary version only.
+
+If you use SFTPGo, it is in your best interest to ensure that the project you rely on stays healthy and well maintained.
+This can only happen with your donations and [sponsorships](https://github.com/sponsors/drakkan) :heart:
+
+If you just take and don't return anything back, the project will die in the long run and you will be forced to pay for a similar proprietary solution.
+
+More [info](https://github.com/drakkan/sftpgo/issues/452).
+
+### Thank you to our sponsors
+
+#### Platinum sponsors
+
+[<img src="./img/Aledade_logo.png" alt="Aledade logo" width="202" height="70">](https://www.aledade.com/)
+
+#### Bronze sponsors
+
+[<img src="https://www.7digital.com/wp-content/themes/sevendigital/images/top_logo.png" alt="7digital logo">](https://www.7digital.com/)
+
+## Support policy
+
+SFTPGo is an Open Source project and you can of course use it for free but please don't ask for free support as well.
+
+We will check the reported issues to see if you are experiencing a bug and if so we'll will fix it, but will only provide support to project [sponsors/donors](#sponsors).
+
+If you report an invalid issue or ask for step-by-step support, your issue will remain open with no answer or will be closed as invalid without further explanation. Thanks for understanding.
+
 ## Features
 
 - Support for serving local filesystem, encrypted local filesystem, S3 Compatible Object Storage, Google Cloud Storage, Azure Blob Storage or other SFTP accounts over SFTP/SCP/FTP/WebDAV.
@@ -21,6 +54,7 @@ Several storage backends are supported: local filesystem, encrypted local filesy
 - Chroot isolation for local accounts. Cloud-based accounts can be restricted to a certain base path.
 - Per-user and per-directory virtual permissions, for each exposed path you can allow or deny: directory listing, upload, overwrite, download, delete, rename, create directories, create symlinks, change owner/group/file mode and modification time.
 - [REST API](./docs/rest-api.md) for users and folders management, data retention, backup, restore and real time reports of the active connections with possibility of forcibly closing a connection.
+- The [Event Manager](./docs/eventmanager.md) allows to define custom workflows based on server events or schedules.
 - [Web based administration interface](./docs/web-admin.md) to easily manage users, folders and connections.
 - [Web client interface](./docs/web-client.md) so that end users can change their credentials, manage and share their files in the browser.
 - Public key and password authentication. Multiple public keys per-user are supported.
@@ -30,10 +64,10 @@ Several storage backends are supported: local filesystem, encrypted local filesy
 - Per-user authentication methods.
 - [Two-factor authentication](./docs/howto/two-factor-authentication.md) based on time-based one time passwords (RFC 6238) which works with Authy, Google Authenticator and other compatible apps.
 - Simplified user administrations using [groups](./docs/groups.md).
-- Custom authentication via external programs/HTTP API.
+- Custom authentication via [external programs/HTTP API](./docs/external-auth.md).
 - Web Client and Web Admin user interfaces support [OpenID Connect](https://openid.net/connect/) authentication and so they can be integrated with identity providers such as [Keycloak](https://www.keycloak.org/). You can find more details [here](./docs/oidc.md).
 - [Data At Rest Encryption](./docs/dare.md).
-- Dynamic user modification before login via external programs/HTTP API.
+- Dynamic user modification before login via [external programs/HTTP API](./docs/dynamic-user-mod.md).
 - Quota support: accounts can have individual disk quota expressed as max total size and/or max number of files.
 - Bandwidth throttling, with separate settings for upload and download and overrides based on the client's IP address.
 - Data transfer bandwidth limits, with total limit or separate settings for uploads and downloads and overrides based on the client's IP address. Limits can be reset using the REST API.
@@ -70,8 +104,10 @@ SFTPGo is developed and tested on Linux. After each commit, the code is automati
 ## Requirements
 
 - Go as build only dependency. We support the Go version(s) used in [continuous integration workflows](./.github/workflows).
-- A suitable SQL server to use as data provider: PostgreSQL 9.4+, MySQL 5.6+, SQLite 3.x, CockroachDB stable.
-- The SQL server is optional: you can choose to use an embedded bolt database as key/value store or an in memory data provider.
+- A suitable SQL server to use as data provider:
+  - upstream supported versions of PostgreSQL, MySQL and MariaDB.
+  - CockroachDB stable.
+- The SQL server is optional: you can choose to use an embedded SQLite, bolt or in memory data provider.
 
 ## Installation
 
@@ -95,7 +131,13 @@ An official Docker image is available. Documentation is [here](./docker/README.m
 
 APT and YUM repositories are [available](./docs/repo.md).
 
-SFTPGo is also available on [AWS Marketplace](https://aws.amazon.com/marketplace/seller-profile?id=6e849ab8-70a6-47de-9a43-13c3fa849335) and [Azure Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/prasselsrl1645470739547.sftpgo_linux), purchasing from there will help keep SFTPGo a long-term sustainable project.
+SFTPGo is also available on some marketplaces:
+
+- [AWS Marketplace](https://aws.amazon.com/marketplace/seller-profile?id=6e849ab8-70a6-47de-9a43-13c3fa849335)
+- [Azure Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/prasselsrl1645470739547.sftpgo_linux)
+- [Elest.io](https://elest.io/open-source/sftpgo)
+
+Purchasing from there will help keep SFTPGo a long-term sustainable project.
 
 <details><summary>Windows packages</summary>
 
@@ -206,16 +248,18 @@ The `revertprovider` command is not supported for the memory provider.
 
 Please note that we only support the current release branch and the current main branch, if you find a bug it is better to report it rather than downgrading to an older unsupported version.
 
-## Users and folders management
+## Users, groups and folders management
 
-After starting SFTPGo you can manage users and folders using:
+After starting SFTPGo you can manage users, groups, folders and other resources using:
 
 - the [web based administration interface](./docs/web-admin.md)
 - the [REST API](./docs/rest-api.md)
 
-To support embedded data providers like `bolt` and `SQLite` we can't have a CLI that directly write users and folders to the data provider, we always have to use the REST API.
+To support embedded data providers like `bolt` and `SQLite`, which do not support concurrent connections, we can't have a CLI that directly write users and other resources to the data provider, we always have to use the REST API.
 
-Full details for users, folders, admins and other resources are documented in the [OpenAPI](./openapi/openapi.yaml) schema. If you want to render the schema without importing it manually, you can explore it on [Stoplight](https://sftpgo.stoplight.io/docs/sftpgo/openapi.yaml).
+Full details for users, groups, folders, admins and other resources are documented in the [OpenAPI](./openapi/openapi.yaml) schema. If you want to render the schema without importing it manually, you can explore it on [Stoplight](https://sftpgo.stoplight.io/docs/sftpgo/openapi.yaml).
+
+:warning: SFTPGo users, groups and folders are virtual and therefore unrelated to the system ones. There is no need to create system-wide users and groups.
 
 ## Tutorials
 
@@ -271,6 +315,10 @@ Each user can be mapped to another SFTP server account or a subfolder of it. Mor
 
 Data at-rest encryption is supported via the [cryptfs backend](./docs/dare.md).
 
+### HTTP/S backend
+
+HTTP/S backend allows you to write your own custom storage backend by implementing a REST API. More information can be found [here](./docs/httpfs.md).
+
 ### Other Storage backends
 
 Adding new storage backends is quite easy:
@@ -310,14 +358,6 @@ We are very grateful to all the people who contributed with ideas and/or pull re
 
 Thank you [ysura](https://www.ysura.com/) for granting me stable access to a test AWS S3 account.
 
-## Sponsors
-
-I'd like to make SFTPGo into a sustainable long term project and your [sponsorship](https://github.com/sponsors/drakkan) will really help :heart:
-
-Thank you to our sponsors!
-
-[<img src="https://www.7digital.com/wp-content/themes/sevendigital/images/top_logo.png" alt="7digital logo">](https://www.7digital.com/)
-
 ## License
 
-GNU AGPLv3
+GNU AGPL-3.0-only

README.zh_CN.md

@@ -1,8 +1,8 @@
 # SFTPGo
 
-![CI Status](https://github.com/drakkan/sftpgo/workflows/CI/badge.svg?branch=main&event=push)
+[![CI Status](https://github.com/drakkan/sftpgo/workflows/CI/badge.svg?branch=main&event=push)](https://github.com/drakkan/sftpgo/workflows/CI/badge.svg?branch=main&event=push)
 [![Code Coverage](https://codecov.io/gh/drakkan/sftpgo/branch/main/graph/badge.svg)](https://codecov.io/gh/drakkan/sftpgo/branch/main)
-[![License: AGPL v3](https://img.shields.io/badge/License-AGPLv3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
+[![License: AGPL-3.0-only](https://img.shields.io/badge/License-AGPLv3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
 [![Docker Pulls](https://img.shields.io/docker/pulls/drakkan/sftpgo)](https://hub.docker.com/r/drakkan/sftpgo)
 [![Mentioned in Awesome Go](https://awesome.re/mentioned-badge.svg)](https://github.com/avelino/awesome-go)
 
@@ -11,6 +11,39 @@
 功能齐全、高度可配置化、支持自定义 HTTP/S，FTP/S 和 WebDAV 的 SFTP 服务。
 一些存储后端支持：本地文件系统、加密本地文件系统、S3（兼容）对象存储，Google Cloud 存储，Azure Blob 存储，SFTP。
 
+## 赞助商
+
+如果你觉得 SFTPGo 有用，请考虑支持这个开源项目。
+
+维护和发展 SFTPGo 对我来说是很多工作——很容易相当于一份全职工作。
+
+我想让 SFTPGo 成为一个可持续的长期项目，并且不想引入双重许可选项并将某些功能仅限于专有版本。
+
+如果您使用 SFTPGo，确保您所依赖的项目保持健康和维护良好符合您的最大利益。
+这只能通过您的捐款和[赞助](https://github.com/sponsors/drakkan) 发生：heart：
+
+如果您只是拿走任何东西而不返回任何东西，从长远来看，该项目将失败，您将被迫为类似的专有解决方案付费。
+
+[更多信息](https://github.com/drakkan/sftpgo/issues/452)。
+
+### 感谢我们的赞助商
+
+#### 白金赞助商
+
+[<img src="./img/Aledade_logo.png" alt="Aledade logo" width="202" height="70">](https://www.aledade.com/)
+
+#### 铜牌赞助商
+
+[<img src="https://www.7digital.com/wp-content/themes/sevendigital/images/top_logo.png" alt="7digital logo">](https://www.7digital.com/)
+
+## 支持政策
+
+SFTPGo 是一个开源项目，您当然可以免费使用它，但也请不要要求免费支持。
+
+我们将检查报告的问题以查看您是否遇到错误，如果是，我们将修复它，但只会为项目赞助商/捐助者提供支持。
+
+如果您报告无效问题或要求逐步支持，您的问题将保持打开状态而没有答案，或者将被关闭为无效而无需进一步解释。 感谢您的理解。
+
 ## 特性
 
 - 支持服务本地文件系统、加密本地文件系统、S3 兼容对象存储、Google Cloud 存储、Azure Blob 存储或其它基于 SFTP/SCP/FTP/WebDAV 协议的 SFTP 账户。
@@ -115,7 +148,7 @@ SFTPGo 在 [AWS Marketplace](https://aws.amazon.com/marketplace/seller-profile?i
 
 可以完整的配置项方法说明可以参考 [配置项](./docs/full-configuration.md)。
 
-请确保按需运行之前，[初始化数据提供程序](#data-provider-initialization-and-management)。
+请确保按需运行之前，[初始化数据提供程序](#数据提供程序初始化和管理)。
 
 默认配置启动 STFPGo，运行：
 
@@ -315,4 +348,4 @@ SFTPGo 使用了 [go.mod](./go.mod) 中列出的第三方库。
 
 ## 许可证
 
-GNU AGPLv3
+GNU AGPL-3.0-only

cmd/awscontainer_disabled.go

@@ -1,10 +0,0 @@
-//go:build !awscontainer
-// +build !awscontainer
-
-package cmd
-
-import (
-	"github.com/spf13/cobra"
-)
-
-func addAWSContainerFlags(cmd *cobra.Command) {}

cmd/gen.go

@@ -1,12 +0,0 @@
-package cmd
-
-import "github.com/spf13/cobra"
-
-var genCmd = &cobra.Command{
-	Use:   "gen",
-	Short: "A collection of useful generators",
-}
-
-func init() {
-	rootCmd.AddCommand(genCmd)
-}

cmd/portable_disabled.go

@@ -1,10 +0,0 @@
-//go:build noportable
-// +build noportable
-
-package cmd
-
-import "github.com/drakkan/sftpgo/v2/version"
-
-func init() {
-	version.AddFeature("-portable")
-}

cmd/reload_windows.go

@@ -1,35 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/spf13/cobra"
-
-	"github.com/drakkan/sftpgo/v2/service"
-)
-
-var (
-	reloadCmd = &cobra.Command{
-		Use:   "reload",
-		Short: "Reload the SFTPGo Windows Service sending a \"paramchange\" request",
-		Run: func(cmd *cobra.Command, args []string) {
-			s := service.WindowsService{
-				Service: service.Service{
-					Shutdown: make(chan bool),
-				},
-			}
-			err := s.Reload()
-			if err != nil {
-				fmt.Printf("Error sending reload signal: %v\r\n", err)
-				os.Exit(1)
-			} else {
-				fmt.Printf("Reload signal sent!\r\n")
-			}
-		},
-	}
-)
-
-func init() {
-	serviceCmd.AddCommand(reloadCmd)
-}

cmd/rotatelogs_windows.go

@@ -1,35 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/spf13/cobra"
-
-	"github.com/drakkan/sftpgo/v2/service"
-)
-
-var (
-	rotateLogCmd = &cobra.Command{
-		Use:   "rotatelogs",
-		Short: "Signal to the running service to rotate the logs",
-		Run: func(cmd *cobra.Command, args []string) {
-			s := service.WindowsService{
-				Service: service.Service{
-					Shutdown: make(chan bool),
-				},
-			}
-			err := s.RotateLogFile()
-			if err != nil {
-				fmt.Printf("Error sending rotate log file signal to the service: %v\r\n", err)
-				os.Exit(1)
-			} else {
-				fmt.Printf("Rotate log file signal sent!\r\n")
-			}
-		},
-	}
-)
-
-func init() {
-	serviceCmd.AddCommand(rotateLogCmd)
-}

cmd/service_windows.go

@@ -1,16 +0,0 @@
-package cmd
-
-import (
-	"github.com/spf13/cobra"
-)
-
-var (
-	serviceCmd = &cobra.Command{
-		Use:   "service",
-		Short: "Manage the SFTPGo Windows Service",
-	}
-)
-
-func init() {
-	rootCmd.AddCommand(serviceCmd)
-}

cmd/status_windows.go

@@ -1,35 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/spf13/cobra"
-
-	"github.com/drakkan/sftpgo/v2/service"
-)
-
-var (
-	statusCmd = &cobra.Command{
-		Use:   "status",
-		Short: "Retrieve the status for the SFTPGo Windows Service",
-		Run: func(cmd *cobra.Command, args []string) {
-			s := service.WindowsService{
-				Service: service.Service{
-					Shutdown: make(chan bool),
-				},
-			}
-			status, err := s.Status()
-			if err != nil {
-				fmt.Printf("Error querying service status: %v\r\n", err)
-				os.Exit(1)
-			} else {
-				fmt.Printf("Service status: %#v\r\n", status.String())
-			}
-		},
-	}
-)
-
-func init() {
-	serviceCmd.AddCommand(statusCmd)
-}

cmd/stop_windows.go

@@ -1,35 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/spf13/cobra"
-
-	"github.com/drakkan/sftpgo/v2/service"
-)
-
-var (
-	stopCmd = &cobra.Command{
-		Use:   "stop",
-		Short: "Stop the SFTPGo Windows Service",
-		Run: func(cmd *cobra.Command, args []string) {
-			s := service.WindowsService{
-				Service: service.Service{
-					Shutdown: make(chan bool),
-				},
-			}
-			err := s.Stop()
-			if err != nil {
-				fmt.Printf("Error stopping service: %v\r\n", err)
-				os.Exit(1)
-			} else {
-				fmt.Printf("Service stopped!\r\n")
-			}
-		},
-	}
-)
-
-func init() {
-	serviceCmd.AddCommand(stopCmd)
-}

cmd/uninstall_windows.go

@@ -1,35 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/spf13/cobra"
-
-	"github.com/drakkan/sftpgo/v2/service"
-)
-
-var (
-	uninstallCmd = &cobra.Command{
-		Use:   "uninstall",
-		Short: "Uninstall the SFTPGo Windows Service",
-		Run: func(cmd *cobra.Command, args []string) {
-			s := service.WindowsService{
-				Service: service.Service{
-					Shutdown: make(chan bool),
-				},
-			}
-			err := s.Uninstall()
-			if err != nil {
-				fmt.Printf("Error removing service: %v\r\n", err)
-				os.Exit(1)
-			} else {
-				fmt.Printf("Service uninstalled\r\n")
-			}
-		},
-	}
-)
-
-func init() {
-	serviceCmd.AddCommand(uninstallCmd)
-}

command/command.go

@@ -1,100 +0,0 @@
-package command
-
-import (
-	"fmt"
-	"os"
-	"strings"
-	"time"
-)
-
-const (
-	minTimeout     = 1
-	maxTimeout     = 300
-	defaultTimeout = 30
-)
-
-var (
-	config Config
-)
-
-// Command define the configuration for a specific commands
-type Command struct {
-	// Path is the command path as defined in the hook configuration
-	Path string `json:"path" mapstructure:"path"`
-	// Timeout specifies a time limit, in seconds, for the command execution.
-	// This value overrides the global timeout if set.
-	// Do not use variables with the SFTPGO_ prefix to avoid conflicts with env
-	// vars that SFTPGo sets
-	Timeout int `json:"timeout" mapstructure:"timeout"`
-	// Env defines additional environment variable for the commands.
-	// Each entry is of the form "key=value".
-	// These values are added to the global environment variables if any
-	Env []string `json:"env" mapstructure:"env"`
-}
-
-// Config defines the configuration for external commands such as
-// program based hooks
-type Config struct {
-	// Timeout specifies a global time limit, in seconds, for the external commands execution
-	Timeout int `json:"timeout" mapstructure:"timeout"`
-	// Env defines additional environment variable for the commands.
-	// Each entry is of the form "key=value".
-	// Do not use variables with the SFTPGO_ prefix to avoid conflicts with env
-	// vars that SFTPGo sets
-	Env []string `json:"env" mapstructure:"env"`
-	// Commands defines configuration for specific commands
-	Commands []Command `json:"commands" mapstructure:"commands"`
-}
-
-func init() {
-	config = Config{
-		Timeout: defaultTimeout,
-	}
-}
-
-// Initialize configures commands
-func (c Config) Initialize() error {
-	if c.Timeout < minTimeout || c.Timeout > maxTimeout {
-		return fmt.Errorf("invalid timeout %v", c.Timeout)
-	}
-	for _, env := range c.Env {
-		if len(strings.Split(env, "=")) != 2 {
-			return fmt.Errorf("invalid env var %#v", env)
-		}
-	}
-	for idx, cmd := range c.Commands {
-		if cmd.Path == "" {
-			return fmt.Errorf("invalid path %#v", cmd.Path)
-		}
-		if cmd.Timeout == 0 {
-			c.Commands[idx].Timeout = c.Timeout
-		} else {
-			if cmd.Timeout < minTimeout || cmd.Timeout > maxTimeout {
-				return fmt.Errorf("invalid timeout %v for command %#v", cmd.Timeout, cmd.Path)
-			}
-		}
-		for _, env := range cmd.Env {
-			if len(strings.Split(env, "=")) != 2 {
-				return fmt.Errorf("invalid env var %#v for command %#v", env, cmd.Path)
-			}
-		}
-	}
-	config = c
-	return nil
-}
-
-// GetConfig returns the configuration for the specified command
-func GetConfig(command string) (time.Duration, []string) {
-	env := os.Environ()
-	timeout := time.Duration(config.Timeout) * time.Second
-	env = append(env, config.Env...)
-	for _, cmd := range config.Commands {
-		if cmd.Path == command {
-			timeout = time.Duration(cmd.Timeout) * time.Second
-			env = append(env, cmd.Env...)
-			break
-		}
-	}
-
-	return timeout, env
-}

command/command_test.go

@@ -1,105 +0,0 @@
-package command
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestCommandConfig(t *testing.T) {
-	require.Equal(t, defaultTimeout, config.Timeout)
-	cfg := Config{
-		Timeout: 10,
-		Env:     []string{"a=b"},
-	}
-	err := cfg.Initialize()
-	require.NoError(t, err)
-	assert.Equal(t, cfg.Timeout, config.Timeout)
-	assert.Equal(t, cfg.Env, config.Env)
-	assert.Len(t, cfg.Commands, 0)
-	timeout, env := GetConfig("cmd")
-	assert.Equal(t, time.Duration(config.Timeout)*time.Second, timeout)
-	assert.Contains(t, env, "a=b")
-
-	cfg.Commands = []Command{
-		{
-			Path:    "cmd1",
-			Timeout: 30,
-			Env:     []string{"c=d"},
-		},
-		{
-			Path:    "cmd2",
-			Timeout: 0,
-			Env:     []string{"e=f"},
-		},
-	}
-	err = cfg.Initialize()
-	require.NoError(t, err)
-	assert.Equal(t, cfg.Timeout, config.Timeout)
-	assert.Equal(t, cfg.Env, config.Env)
-	if assert.Len(t, config.Commands, 2) {
-		assert.Equal(t, cfg.Commands[0].Path, config.Commands[0].Path)
-		assert.Equal(t, cfg.Commands[0].Timeout, config.Commands[0].Timeout)
-		assert.Equal(t, cfg.Commands[0].Env, config.Commands[0].Env)
-		assert.Equal(t, cfg.Commands[1].Path, config.Commands[1].Path)
-		assert.Equal(t, cfg.Timeout, config.Commands[1].Timeout)
-		assert.Equal(t, cfg.Commands[1].Env, config.Commands[1].Env)
-	}
-	timeout, env = GetConfig("cmd1")
-	assert.Equal(t, time.Duration(config.Commands[0].Timeout)*time.Second, timeout)
-	assert.Contains(t, env, "a=b")
-	assert.Contains(t, env, "c=d")
-	assert.NotContains(t, env, "e=f")
-	timeout, env = GetConfig("cmd2")
-	assert.Equal(t, time.Duration(config.Timeout)*time.Second, timeout)
-	assert.Contains(t, env, "a=b")
-	assert.NotContains(t, env, "c=d")
-	assert.Contains(t, env, "e=f")
-}
-
-func TestConfigErrors(t *testing.T) {
-	c := Config{}
-	err := c.Initialize()
-	if assert.Error(t, err) {
-		assert.Contains(t, err.Error(), "invalid timeout")
-	}
-	c.Timeout = 10
-	c.Env = []string{"a"}
-	err = c.Initialize()
-	if assert.Error(t, err) {
-		assert.Contains(t, err.Error(), "invalid env var")
-	}
-	c.Env = nil
-	c.Commands = []Command{
-		{
-			Path: "",
-		},
-	}
-	err = c.Initialize()
-	if assert.Error(t, err) {
-		assert.Contains(t, err.Error(), "invalid path")
-	}
-	c.Commands = []Command{
-		{
-			Path:    "path",
-			Timeout: 10000,
-		},
-	}
-	err = c.Initialize()
-	if assert.Error(t, err) {
-		assert.Contains(t, err.Error(), "invalid timeout")
-	}
-	c.Commands = []Command{
-		{
-			Path:    "path",
-			Timeout: 30,
-			Env:     []string{"b"},
-		},
-	}
-	err = c.Initialize()
-	if assert.Error(t, err) {
-		assert.Contains(t, err.Error(), "invalid env var")
-	}
-}

common/clientsmap.go

@@ -1,51 +0,0 @@
-package common
-
-import (
-	"sync"
-	"sync/atomic"
-
-	"github.com/drakkan/sftpgo/v2/logger"
-)
-
-// clienstMap is a struct containing the map of the connected clients
-type clientsMap struct {
-	totalConnections int32
-	mu               sync.RWMutex
-	clients          map[string]int
-}
-
-func (c *clientsMap) add(source string) {
-	atomic.AddInt32(&c.totalConnections, 1)
-
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	c.clients[source]++
-}
-
-func (c *clientsMap) remove(source string) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	if val, ok := c.clients[source]; ok {
-		atomic.AddInt32(&c.totalConnections, -1)
-		c.clients[source]--
-		if val > 1 {
-			return
-		}
-		delete(c.clients, source)
-	} else {
-		logger.Warn(logSender, "", "cannot remove client %v it is not mapped", source)
-	}
-}
-
-func (c *clientsMap) getTotal() int32 {
-	return atomic.LoadInt32(&c.totalConnections)
-}
-
-func (c *clientsMap) getTotalFrom(source string) int {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	return c.clients[source]
-}

config/config_linux.go

@@ -1,12 +0,0 @@
-//go:build linux
-// +build linux
-
-package config
-
-import "github.com/spf13/viper"
-
-// linux specific config search path
-func setViperAdditionalConfigPaths() {
-	viper.AddConfigPath("$HOME/.config/sftpgo")
-	viper.AddConfigPath("/etc/sftpgo")
-}

config/config_nolinux.go

@@ -1,6 +0,0 @@
-//go:build !linux
-// +build !linux
-
-package config
-
-func setViperAdditionalConfigPaths() {}

dataprovider/bolt_disabled.go

@@ -1,18 +0,0 @@
-//go:build nobolt
-// +build nobolt
-
-package dataprovider
-
-import (
-	"errors"
-
-	"github.com/drakkan/sftpgo/v2/version"
-)
-
-func init() {
-	version.AddFeature("-bolt")
-}
-
-func initializeBoltProvider(basePath string) error {
-	return errors.New("bolt disabled at build time")
-}

dataprovider/mysql_disabled.go

@@ -1,18 +0,0 @@
-//go:build nomysql
-// +build nomysql
-
-package dataprovider
-
-import (
-	"errors"
-
-	"github.com/drakkan/sftpgo/v2/version"
-)
-
-func init() {
-	version.AddFeature("-mysql")
-}
-
-func initializeMySQLProvider() error {
-	return errors.New("MySQL disabled at build time")
-}

dataprovider/pgsql_disabled.go

@@ -1,18 +0,0 @@
-//go:build nopgsql
-// +build nopgsql
-
-package dataprovider
-
-import (
-	"errors"
-
-	"github.com/drakkan/sftpgo/v2/version"
-)
-
-func init() {
-	version.AddFeature("-pgsql")
-}
-
-func initializePGSQLProvider() error {
-	return errors.New("PostgreSQL disabled at build time")
-}

dataprovider/scheduler.go

@@ -1,96 +0,0 @@
-package dataprovider
-
-import (
-	"fmt"
-	"sync/atomic"
-	"time"
-
-	"github.com/robfig/cron/v3"
-
-	"github.com/drakkan/sftpgo/v2/logger"
-	"github.com/drakkan/sftpgo/v2/metric"
-	"github.com/drakkan/sftpgo/v2/util"
-)
-
-var (
-	scheduler        *cron.Cron
-	lastCachesUpdate int64
-	// used for bolt and memory providers, so we avoid iterating all users
-	// to find recently modified ones
-	lastUserUpdate int64
-)
-
-func stopScheduler() {
-	if scheduler != nil {
-		scheduler.Stop()
-		scheduler = nil
-	}
-}
-
-func startScheduler() error {
-	stopScheduler()
-
-	scheduler = cron.New()
-	_, err := scheduler.AddFunc("@every 30s", checkDataprovider)
-	if err != nil {
-		return fmt.Errorf("unable to schedule dataprovider availability check: %w", err)
-	}
-
-	if config.AutoBackup.Enabled {
-		spec := fmt.Sprintf("0 %v * * %v", config.AutoBackup.Hour, config.AutoBackup.DayOfWeek)
-		_, err = scheduler.AddFunc(spec, config.doBackup)
-		if err != nil {
-			return fmt.Errorf("unable to schedule auto backup: %w", err)
-		}
-	}
-
-	err = addScheduledCacheUpdates()
-	if err != nil {
-		return err
-	}
-	scheduler.Start()
-	return nil
-}
-
-func addScheduledCacheUpdates() error {
-	lastCachesUpdate = util.GetTimeAsMsSinceEpoch(time.Now())
-	_, err := scheduler.AddFunc("@every 10m", checkCacheUpdates)
-	if err != nil {
-		return fmt.Errorf("unable to schedule cache updates: %w", err)
-	}
-	return nil
-}
-
-func checkDataprovider() {
-	err := provider.checkAvailability()
-	if err != nil {
-		providerLog(logger.LevelError, "check availability error: %v", err)
-	}
-	metric.UpdateDataProviderAvailability(err)
-}
-
-func checkCacheUpdates() {
-	providerLog(logger.LevelDebug, "start caches check, update time %v", util.GetTimeFromMsecSinceEpoch(lastCachesUpdate))
-	checkTime := util.GetTimeAsMsSinceEpoch(time.Now())
-	users, err := provider.getRecentlyUpdatedUsers(lastCachesUpdate)
-	if err != nil {
-		providerLog(logger.LevelError, "unable to get recently updated users: %v", err)
-		return
-	}
-	for _, user := range users {
-		providerLog(logger.LevelDebug, "invalidate caches for user %#v", user.Username)
-		webDAVUsersCache.swap(&user)
-		cachedPasswords.Remove(user.Username)
-	}
-
-	lastCachesUpdate = checkTime
-	providerLog(logger.LevelDebug, "end caches check, new update time %v", util.GetTimeFromMsecSinceEpoch(lastCachesUpdate))
-}
-
-func setLastUserUpdate() {
-	atomic.StoreInt64(&lastUserUpdate, util.GetTimeAsMsSinceEpoch(time.Now()))
-}
-
-func getLastUserUpdate() int64 {
-	return atomic.LoadInt64(&lastUserUpdate)
-}

dataprovider/sqlite_disabled.go

@@ -1,18 +0,0 @@
-//go:build nosqlite
-// +build nosqlite
-
-package dataprovider
-
-import (
-	"errors"
-
-	"github.com/drakkan/sftpgo/v2/version"
-)
-
-func init() {
-	version.AddFeature("-sqlite")
-}
-
-func initializeSQLiteProvider(basePath string) error {
-	return errors.New("SQLite disabled at build time")
-}

dataprovider/sqlqueries.go

@@ -1,720 +0,0 @@
-package dataprovider
-
-import (
-	"fmt"
-	"strconv"
-	"strings"
-
-	"github.com/drakkan/sftpgo/v2/vfs"
-)
-
-const (
-	selectUserFields = "id,username,password,public_keys,home_dir,uid,gid,max_sessions,quota_size,quota_files,permissions,used_quota_size," +
-		"used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,expiration_date,last_login,status,filters,filesystem," +
-		"additional_info,description,email,created_at,updated_at,upload_data_transfer,download_data_transfer,total_data_transfer," +
-		"used_upload_data_transfer,used_download_data_transfer"
-	selectFolderFields = "id,path,used_quota_size,used_quota_files,last_quota_update,name,description,filesystem"
-	selectAdminFields  = "id,username,password,status,email,permissions,filters,additional_info,description,created_at,updated_at,last_login"
-	selectAPIKeyFields = "key_id,name,api_key,scope,created_at,updated_at,last_use_at,expires_at,description,user_id,admin_id"
-	selectShareFields  = "s.share_id,s.name,s.description,s.scope,s.paths,u.username,s.created_at,s.updated_at,s.last_use_at," +
-		"s.expires_at,s.password,s.max_tokens,s.used_tokens,s.allow_from"
-	selectGroupFields = "id,name,description,created_at,updated_at,user_settings"
-)
-
-func getSQLPlaceholders() []string {
-	var placeholders []string
-	for i := 1; i <= 50; i++ {
-		if config.Driver == PGSQLDataProviderName || config.Driver == CockroachDataProviderName {
-			placeholders = append(placeholders, fmt.Sprintf("$%v", i))
-		} else {
-			placeholders = append(placeholders, "?")
-		}
-	}
-	return placeholders
-}
-
-func getSQLTableGroups() string {
-	if config.Driver == MySQLDataProviderName {
-		return fmt.Sprintf("`%s`", sqlTableGroups)
-	}
-
-	return sqlTableGroups
-}
-
-func getAddSessionQuery() string {
-	if config.Driver == MySQLDataProviderName {
-		return fmt.Sprintf("INSERT INTO %s (`key`,`data`,`type`,`timestamp`) VALUES (%s,%s,%s,%s) "+
-			"ON DUPLICATE KEY UPDATE `data`=VALUES(`data`), `timestamp`=VALUES(`timestamp`)",
-			sqlTableSharedSessions, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
-	}
-	return fmt.Sprintf(`INSERT INTO %s (key,data,type,timestamp) VALUES (%s,%s,%s,%s) ON CONFLICT(key) DO UPDATE SET data=
-		EXCLUDED.data, timestamp=EXCLUDED.timestamp`,
-		sqlTableSharedSessions, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
-}
-
-func getDeleteSessionQuery() string {
-	if config.Driver == MySQLDataProviderName {
-		return fmt.Sprintf("DELETE FROM %s WHERE `key` = %s", sqlTableSharedSessions, sqlPlaceholders[0])
-	}
-	return fmt.Sprintf(`DELETE FROM %s WHERE key = %s`, sqlTableSharedSessions, sqlPlaceholders[0])
-}
-
-func getSessionQuery() string {
-	if config.Driver == MySQLDataProviderName {
-		return fmt.Sprintf("SELECT `key`,`data`,`type`,`timestamp` FROM %s WHERE `key` = %s", sqlTableSharedSessions,
-			sqlPlaceholders[0])
-	}
-	return fmt.Sprintf(`SELECT key,data,type,timestamp FROM %s WHERE key = %s`, sqlTableSharedSessions,
-		sqlPlaceholders[0])
-}
-
-func getCleanupSessionsQuery() string {
-	return fmt.Sprintf(`DELETE from %s WHERE type = %s AND timestamp < %s`,
-		sqlTableSharedSessions, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getAddDefenderHostQuery() string {
-	if config.Driver == MySQLDataProviderName {
-		return fmt.Sprintf("INSERT INTO %v (`ip`,`updated_at`,`ban_time`) VALUES (%v,%v,0) ON DUPLICATE KEY UPDATE `updated_at`=VALUES(`updated_at`)",
-			sqlTableDefenderHosts, sqlPlaceholders[0], sqlPlaceholders[1])
-	}
-	return fmt.Sprintf(`INSERT INTO %v (ip,updated_at,ban_time) VALUES (%v,%v,0) ON CONFLICT (ip) DO UPDATE SET updated_at = EXCLUDED.updated_at RETURNING id`,
-		sqlTableDefenderHosts, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getAddDefenderEventQuery() string {
-	return fmt.Sprintf(`INSERT INTO %v (date_time,score,host_id) VALUES (%v,%v,(SELECT id from %v WHERE ip = %v))`,
-		sqlTableDefenderEvents, sqlPlaceholders[0], sqlPlaceholders[1], sqlTableDefenderHosts, sqlPlaceholders[2])
-}
-
-func getDefenderHostsQuery() string {
-	return fmt.Sprintf(`SELECT id,ip,ban_time FROM %v WHERE updated_at >= %v OR ban_time > 0 ORDER BY updated_at DESC LIMIT %v`,
-		sqlTableDefenderHosts, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getDefenderHostQuery() string {
-	return fmt.Sprintf(`SELECT id,ip,ban_time FROM %v WHERE ip = %v AND (updated_at >= %v OR ban_time > 0)`,
-		sqlTableDefenderHosts, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getDefenderEventsQuery(hostIDS []int64) string {
-	var sb strings.Builder
-	for _, hID := range hostIDS {
-		if sb.Len() == 0 {
-			sb.WriteString("(")
-		} else {
-			sb.WriteString(",")
-		}
-		sb.WriteString(strconv.FormatInt(hID, 10))
-	}
-	if sb.Len() > 0 {
-		sb.WriteString(")")
-	} else {
-		sb.WriteString("(0)")
-	}
-	return fmt.Sprintf(`SELECT host_id,SUM(score) FROM %v WHERE date_time >= %v AND host_id IN %v GROUP BY host_id`,
-		sqlTableDefenderEvents, sqlPlaceholders[0], sb.String())
-}
-
-func getDefenderIsHostBannedQuery() string {
-	return fmt.Sprintf(`SELECT id FROM %v WHERE ip = %v AND ban_time >= %v`,
-		sqlTableDefenderHosts, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getDefenderIncrementBanTimeQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET ban_time = ban_time + %v WHERE ip = %v`,
-		sqlTableDefenderHosts, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getDefenderSetBanTimeQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET ban_time = %v WHERE ip = %v`,
-		sqlTableDefenderHosts, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getDeleteDefenderHostQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE ip = %v`, sqlTableDefenderHosts, sqlPlaceholders[0])
-}
-
-func getDefenderHostsCleanupQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE ban_time < %v AND NOT EXISTS (
-		SELECT id FROM %v WHERE %v.host_id = %v.id AND %v.date_time > %v)`,
-		sqlTableDefenderHosts, sqlPlaceholders[0], sqlTableDefenderEvents, sqlTableDefenderEvents, sqlTableDefenderHosts,
-		sqlTableDefenderEvents, sqlPlaceholders[1])
-}
-
-func getDefenderEventsCleanupQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE date_time < %v`, sqlTableDefenderEvents, sqlPlaceholders[0])
-}
-
-func getGroupByNameQuery() string {
-	return fmt.Sprintf(`SELECT %s FROM %s WHERE name = %s`, selectGroupFields, getSQLTableGroups(), sqlPlaceholders[0])
-}
-
-func getGroupsQuery(order string, minimal bool) string {
-	var fieldSelection string
-	if minimal {
-		fieldSelection = "id,name"
-	} else {
-		fieldSelection = selectGroupFields
-	}
-	return fmt.Sprintf(`SELECT %s FROM %s ORDER BY name %s LIMIT %v OFFSET %v`, fieldSelection, getSQLTableGroups(),
-		order, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getGroupsWithNamesQuery(numArgs int) string {
-	var sb strings.Builder
-	for idx := 0; idx < numArgs; idx++ {
-		if sb.Len() == 0 {
-			sb.WriteString("(")
-		} else {
-			sb.WriteString(",")
-		}
-		sb.WriteString(sqlPlaceholders[idx])
-	}
-	if sb.Len() > 0 {
-		sb.WriteString(")")
-	} else {
-		sb.WriteString("('')")
-	}
-	return fmt.Sprintf(`SELECT %s FROM %s WHERE name in %s`, selectGroupFields, getSQLTableGroups(), sb.String())
-}
-
-func getUsersInGroupsQuery(numArgs int) string {
-	var sb strings.Builder
-	for idx := 0; idx < numArgs; idx++ {
-		if sb.Len() == 0 {
-			sb.WriteString("(")
-		} else {
-			sb.WriteString(",")
-		}
-		sb.WriteString(sqlPlaceholders[idx])
-	}
-	if sb.Len() > 0 {
-		sb.WriteString(")")
-	} else {
-		sb.WriteString("('')")
-	}
-	return fmt.Sprintf(`SELECT username FROM %s WHERE id IN (SELECT user_id from %s WHERE group_id IN (SELECT id FROM %s WHERE name IN (%s)))`,
-		sqlTableUsers, sqlTableUsersGroupsMapping, getSQLTableGroups(), sb.String())
-}
-
-func getDumpGroupsQuery() string {
-	return fmt.Sprintf(`SELECT %s FROM %s`, selectGroupFields, getSQLTableGroups())
-}
-
-func getAddGroupQuery() string {
-	return fmt.Sprintf(`INSERT INTO %s (name,description,created_at,updated_at,user_settings)
-		VALUES (%v,%v,%v,%v,%v)`, getSQLTableGroups(), sqlPlaceholders[0], sqlPlaceholders[1],
-		sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4])
-}
-
-func getUpdateGroupQuery() string {
-	return fmt.Sprintf(`UPDATE %s SET description=%v,user_settings=%v,updated_at=%v
-		WHERE name = %s`, getSQLTableGroups(), sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2],
-		sqlPlaceholders[3])
-}
-
-func getDeleteGroupQuery() string {
-	return fmt.Sprintf(`DELETE FROM %s WHERE name = %s`, getSQLTableGroups(), sqlPlaceholders[0])
-}
-
-func getAdminByUsernameQuery() string {
-	return fmt.Sprintf(`SELECT %v FROM %v WHERE username = %v`, selectAdminFields, sqlTableAdmins, sqlPlaceholders[0])
-}
-
-func getAdminsQuery(order string) string {
-	return fmt.Sprintf(`SELECT %v FROM %v ORDER BY username %v LIMIT %v OFFSET %v`, selectAdminFields, sqlTableAdmins,
-		order, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getDumpAdminsQuery() string {
-	return fmt.Sprintf(`SELECT %v FROM %v`, selectAdminFields, sqlTableAdmins)
-}
-
-func getAddAdminQuery() string {
-	return fmt.Sprintf(`INSERT INTO %v (username,password,status,email,permissions,filters,additional_info,description,created_at,updated_at,last_login)
-		VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0)`, sqlTableAdmins, sqlPlaceholders[0], sqlPlaceholders[1],
-		sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7],
-		sqlPlaceholders[8], sqlPlaceholders[9])
-}
-
-func getUpdateAdminQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET password=%v,status=%v,email=%v,permissions=%v,filters=%v,additional_info=%v,description=%v,updated_at=%v
-		WHERE username = %v`, sqlTableAdmins, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2],
-		sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8])
-}
-
-func getDeleteAdminQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE username = %v`, sqlTableAdmins, sqlPlaceholders[0])
-}
-
-func getShareByIDQuery(filterUser bool) string {
-	if filterUser {
-		return fmt.Sprintf(`SELECT %v FROM %v s INNER JOIN %v u ON s.user_id = u.id WHERE s.share_id = %v AND u.username = %v`,
-			selectShareFields, sqlTableShares, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1])
-	}
-	return fmt.Sprintf(`SELECT %v FROM %v s INNER JOIN %v u ON s.user_id = u.id WHERE s.share_id = %v`,
-		selectShareFields, sqlTableShares, sqlTableUsers, sqlPlaceholders[0])
-}
-
-func getSharesQuery(order string) string {
-	return fmt.Sprintf(`SELECT %v FROM %v s INNER JOIN %v u ON s.user_id = u.id WHERE u.username = %v ORDER BY s.share_id %v LIMIT %v OFFSET %v`,
-		selectShareFields, sqlTableShares, sqlTableUsers, sqlPlaceholders[0], order, sqlPlaceholders[1], sqlPlaceholders[2])
-}
-
-func getDumpSharesQuery() string {
-	return fmt.Sprintf(`SELECT %v FROM %v s INNER JOIN %v u ON s.user_id = u.id`,
-		selectShareFields, sqlTableShares, sqlTableUsers)
-}
-
-func getAddShareQuery() string {
-	return fmt.Sprintf(`INSERT INTO %v (share_id,name,description,scope,paths,created_at,updated_at,last_use_at,
-		expires_at,password,max_tokens,used_tokens,allow_from,user_id) VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v)`,
-		sqlTableShares, sqlPlaceholders[0], sqlPlaceholders[1],
-		sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6],
-		sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10], sqlPlaceholders[11],
-		sqlPlaceholders[12], sqlPlaceholders[13])
-}
-
-func getUpdateShareRestoreQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET name=%v,description=%v,scope=%v,paths=%v,created_at=%v,updated_at=%v,
-		last_use_at=%v,expires_at=%v,password=%v,max_tokens=%v,used_tokens=%v,allow_from=%v,user_id=%v WHERE share_id = %v`, sqlTableShares,
-		sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4],
-		sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9],
-		sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13])
-}
-
-func getUpdateShareQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET name=%v,description=%v,scope=%v,paths=%v,updated_at=%v,expires_at=%v,
-		password=%v,max_tokens=%v,allow_from=%v,user_id=%v WHERE share_id = %v`, sqlTableShares,
-		sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4],
-		sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9],
-		sqlPlaceholders[10])
-}
-
-func getDeleteShareQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE share_id = %v`, sqlTableShares, sqlPlaceholders[0])
-}
-
-func getAPIKeyByIDQuery() string {
-	return fmt.Sprintf(`SELECT %v FROM %v WHERE key_id = %v`, selectAPIKeyFields, sqlTableAPIKeys, sqlPlaceholders[0])
-}
-
-func getAPIKeysQuery(order string) string {
-	return fmt.Sprintf(`SELECT %v FROM %v ORDER BY key_id %v LIMIT %v OFFSET %v`, selectAPIKeyFields, sqlTableAPIKeys,
-		order, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getDumpAPIKeysQuery() string {
-	return fmt.Sprintf(`SELECT %v FROM %v`, selectAPIKeyFields, sqlTableAPIKeys)
-}
-
-func getAddAPIKeyQuery() string {
-	return fmt.Sprintf(`INSERT INTO %v (key_id,name,api_key,scope,created_at,updated_at,last_use_at,expires_at,description,user_id,admin_id)
-		VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v)`, sqlTableAPIKeys, sqlPlaceholders[0], sqlPlaceholders[1],
-		sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6],
-		sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10])
-}
-
-func getUpdateAPIKeyQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET name=%v,scope=%v,expires_at=%v,user_id=%v,admin_id=%v,description=%v,updated_at=%v
-		WHERE key_id = %v`, sqlTableAPIKeys, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2],
-		sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7])
-}
-
-func getDeleteAPIKeyQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE key_id = %v`, sqlTableAPIKeys, sqlPlaceholders[0])
-}
-
-func getRelatedUsersForAPIKeysQuery(apiKeys []APIKey) string {
-	var sb strings.Builder
-	for _, k := range apiKeys {
-		if k.userID == 0 {
-			continue
-		}
-		if sb.Len() == 0 {
-			sb.WriteString("(")
-		} else {
-			sb.WriteString(",")
-		}
-		sb.WriteString(strconv.FormatInt(k.userID, 10))
-	}
-	if sb.Len() > 0 {
-		sb.WriteString(")")
-	} else {
-		sb.WriteString("(0)")
-	}
-	return fmt.Sprintf(`SELECT id,username FROM %v WHERE id IN %v`, sqlTableUsers, sb.String())
-}
-
-func getRelatedAdminsForAPIKeysQuery(apiKeys []APIKey) string {
-	var sb strings.Builder
-	for _, k := range apiKeys {
-		if k.adminID == 0 {
-			continue
-		}
-		if sb.Len() == 0 {
-			sb.WriteString("(")
-		} else {
-			sb.WriteString(",")
-		}
-		sb.WriteString(strconv.FormatInt(k.adminID, 10))
-	}
-	if sb.Len() > 0 {
-		sb.WriteString(")")
-	} else {
-		sb.WriteString("(0)")
-	}
-	return fmt.Sprintf(`SELECT id,username FROM %v WHERE id IN %v`, sqlTableAdmins, sb.String())
-}
-
-func getUserByUsernameQuery() string {
-	return fmt.Sprintf(`SELECT %v FROM %v WHERE username = %v`, selectUserFields, sqlTableUsers, sqlPlaceholders[0])
-}
-
-func getUsersQuery(order string) string {
-	return fmt.Sprintf(`SELECT %v FROM %v ORDER BY username %v LIMIT %v OFFSET %v`, selectUserFields, sqlTableUsers,
-		order, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getUsersForQuotaCheckQuery(numArgs int) string {
-	var sb strings.Builder
-	for idx := 0; idx < numArgs; idx++ {
-		if sb.Len() == 0 {
-			sb.WriteString("(")
-		} else {
-			sb.WriteString(",")
-		}
-		sb.WriteString(sqlPlaceholders[idx])
-	}
-	if sb.Len() > 0 {
-		sb.WriteString(")")
-	}
-	return fmt.Sprintf(`SELECT id,username,quota_size,used_quota_size,total_data_transfer,upload_data_transfer,
-		download_data_transfer,used_upload_data_transfer,used_download_data_transfer,filters FROM %v WHERE username IN %v`,
-		sqlTableUsers, sb.String())
-}
-
-func getRecentlyUpdatedUsersQuery() string {
-	return fmt.Sprintf(`SELECT %v FROM %v WHERE updated_at >= %v`, selectUserFields, sqlTableUsers, sqlPlaceholders[0])
-}
-
-func getDumpUsersQuery() string {
-	return fmt.Sprintf(`SELECT %v FROM %v`, selectUserFields, sqlTableUsers)
-}
-
-func getDumpFoldersQuery() string {
-	return fmt.Sprintf(`SELECT %v FROM %v`, selectFolderFields, sqlTableFolders)
-}
-
-func getUpdateTransferQuotaQuery(reset bool) string {
-	if reset {
-		return fmt.Sprintf(`UPDATE %v SET used_upload_data_transfer = %v,used_download_data_transfer = %v,last_quota_update = %v
-			WHERE username = %v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
-	}
-	return fmt.Sprintf(`UPDATE %v SET used_upload_data_transfer = used_upload_data_transfer + %v,
-		used_download_data_transfer = used_download_data_transfer + %v,last_quota_update = %v
-		WHERE username = %v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
-}
-
-func getUpdateQuotaQuery(reset bool) string {
-	if reset {
-		return fmt.Sprintf(`UPDATE %v SET used_quota_size = %v,used_quota_files = %v,last_quota_update = %v
-			WHERE username = %v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
-	}
-	return fmt.Sprintf(`UPDATE %v SET used_quota_size = used_quota_size + %v,used_quota_files = used_quota_files + %v,last_quota_update = %v
-		WHERE username = %v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
-}
-
-func getSetUpdateAtQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET updated_at = %v WHERE username = %v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getUpdateLastLoginQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET last_login = %v WHERE username = %v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getUpdateAdminLastLoginQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET last_login = %v WHERE username = %v`, sqlTableAdmins, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getUpdateAPIKeyLastUseQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET last_use_at = %v WHERE key_id = %v`, sqlTableAPIKeys, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getUpdateShareLastUseQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET last_use_at = %v, used_tokens = used_tokens +%v WHERE share_id = %v`,
-		sqlTableShares, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2])
-}
-
-func getQuotaQuery() string {
-	return fmt.Sprintf(`SELECT used_quota_size,used_quota_files,used_upload_data_transfer,
-		used_download_data_transfer FROM %v WHERE username = %v`,
-		sqlTableUsers, sqlPlaceholders[0])
-}
-
-func getAddUserQuery() string {
-	return fmt.Sprintf(`INSERT INTO %v (username,password,public_keys,home_dir,uid,gid,max_sessions,quota_size,quota_files,permissions,
-		used_quota_size,used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,status,last_login,expiration_date,filters,
-		filesystem,additional_info,description,email,created_at,updated_at,upload_data_transfer,download_data_transfer,total_data_transfer,
-		used_upload_data_transfer,used_download_data_transfer)
-		VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0,0,0,%v,%v,%v,0,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0,0)`,
-		sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4],
-		sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9],
-		sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13], sqlPlaceholders[14],
-		sqlPlaceholders[15], sqlPlaceholders[16], sqlPlaceholders[17], sqlPlaceholders[18], sqlPlaceholders[19],
-		sqlPlaceholders[20], sqlPlaceholders[21], sqlPlaceholders[22], sqlPlaceholders[23])
-}
-
-func getUpdateUserQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET password=%v,public_keys=%v,home_dir=%v,uid=%v,gid=%v,max_sessions=%v,quota_size=%v,
-		quota_files=%v,permissions=%v,upload_bandwidth=%v,download_bandwidth=%v,status=%v,expiration_date=%v,filters=%v,filesystem=%v,
-		additional_info=%v,description=%v,email=%v,updated_at=%v,upload_data_transfer=%v,download_data_transfer=%v,
-		total_data_transfer=%v WHERE id = %v`,
-		sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4],
-		sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9],
-		sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13], sqlPlaceholders[14],
-		sqlPlaceholders[15], sqlPlaceholders[16], sqlPlaceholders[17], sqlPlaceholders[18], sqlPlaceholders[19],
-		sqlPlaceholders[20], sqlPlaceholders[21], sqlPlaceholders[22])
-}
-
-func getUpdateUserPasswordQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET password=%v WHERE username = %v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getDeleteUserQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE id = %v`, sqlTableUsers, sqlPlaceholders[0])
-}
-
-func getFolderByNameQuery() string {
-	return fmt.Sprintf(`SELECT %v FROM %v WHERE name = %v`, selectFolderFields, sqlTableFolders, sqlPlaceholders[0])
-}
-
-func getAddFolderQuery() string {
-	return fmt.Sprintf(`INSERT INTO %v (path,used_quota_size,used_quota_files,last_quota_update,name,description,filesystem)
-		VALUES (%v,%v,%v,%v,%v,%v,%v)`, sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2],
-		sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6])
-}
-
-func getUpdateFolderQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET path=%v,description=%v,filesystem=%v WHERE name = %v`, sqlTableFolders, sqlPlaceholders[0],
-		sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
-}
-
-func getDeleteFolderQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE id = %v`, sqlTableFolders, sqlPlaceholders[0])
-}
-
-func getUpsertFolderQuery() string {
-	if config.Driver == MySQLDataProviderName {
-		return fmt.Sprintf("INSERT INTO %v (`path`,`used_quota_size`,`used_quota_files`,`last_quota_update`,`name`,"+
-			"`description`,`filesystem`) VALUES (%v,%v,%v,%v,%v,%v,%v) ON DUPLICATE KEY UPDATE "+
-			"`path`=VALUES(`path`),`description`=VALUES(`description`),`filesystem`=VALUES(`filesystem`)",
-			sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4],
-			sqlPlaceholders[5], sqlPlaceholders[6])
-	}
-	return fmt.Sprintf(`INSERT INTO %v (path,used_quota_size,used_quota_files,last_quota_update,name,description,filesystem)
-		VALUES (%v,%v,%v,%v,%v,%v,%v) ON CONFLICT (name) DO UPDATE SET path = EXCLUDED.path,description=EXCLUDED.description,
-		filesystem=EXCLUDED.filesystem`, sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2],
-		sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6])
-}
-
-func getClearUserGroupMappingQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE user_id = (SELECT id FROM %v WHERE username = %v)`, sqlTableUsersGroupsMapping,
-		sqlTableUsers, sqlPlaceholders[0])
-}
-
-func getAddUserGroupMappingQuery() string {
-	return fmt.Sprintf(`INSERT INTO %v (user_id,group_id,group_type) VALUES ((SELECT id FROM %v WHERE username = %v),
-		(SELECT id FROM %v WHERE name = %v),%v)`,
-		sqlTableUsersGroupsMapping, sqlTableUsers, sqlPlaceholders[0], getSQLTableGroups(), sqlPlaceholders[1], sqlPlaceholders[2])
-}
-
-func getClearGroupFolderMappingQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE group_id = (SELECT id FROM %v WHERE name = %v)`, sqlTableGroupsFoldersMapping,
-		getSQLTableGroups(), sqlPlaceholders[0])
-}
-
-func getAddGroupFolderMappingQuery() string {
-	return fmt.Sprintf(`INSERT INTO %v (virtual_path,quota_size,quota_files,folder_id,group_id)
-		VALUES (%v,%v,%v,(SELECT id FROM %v WHERE name = %v),(SELECT id FROM %v WHERE name = %v))`,
-		sqlTableGroupsFoldersMapping, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlTableFolders,
-		sqlPlaceholders[3], getSQLTableGroups(), sqlPlaceholders[4])
-}
-
-func getClearUserFolderMappingQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE user_id = (SELECT id FROM %v WHERE username = %v)`, sqlTableUsersFoldersMapping,
-		sqlTableUsers, sqlPlaceholders[0])
-}
-
-func getAddUserFolderMappingQuery() string {
-	return fmt.Sprintf(`INSERT INTO %v (virtual_path,quota_size,quota_files,folder_id,user_id)
-		VALUES (%v,%v,%v,(SELECT id FROM %v WHERE name = %v),(SELECT id FROM %v WHERE username = %v))`,
-		sqlTableUsersFoldersMapping, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlTableFolders,
-		sqlPlaceholders[3], sqlTableUsers, sqlPlaceholders[4])
-}
-
-func getFoldersQuery(order string, minimal bool) string {
-	var fieldSelection string
-	if minimal {
-		fieldSelection = "id,name"
-	} else {
-		fieldSelection = selectFolderFields
-	}
-	return fmt.Sprintf(`SELECT %v FROM %v ORDER BY name %v LIMIT %v OFFSET %v`, fieldSelection, sqlTableFolders,
-		order, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getUpdateFolderQuotaQuery(reset bool) string {
-	if reset {
-		return fmt.Sprintf(`UPDATE %v SET used_quota_size = %v,used_quota_files = %v,last_quota_update = %v
-			WHERE name = %v`, sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
-	}
-	return fmt.Sprintf(`UPDATE %v SET used_quota_size = used_quota_size + %v,used_quota_files = used_quota_files + %v,last_quota_update = %v
-		WHERE name = %v`, sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
-}
-
-func getQuotaFolderQuery() string {
-	return fmt.Sprintf(`SELECT used_quota_size,used_quota_files FROM %v WHERE name = %v`, sqlTableFolders,
-		sqlPlaceholders[0])
-}
-
-func getRelatedGroupsForUsersQuery(users []User) string {
-	var sb strings.Builder
-	for _, u := range users {
-		if sb.Len() == 0 {
-			sb.WriteString("(")
-		} else {
-			sb.WriteString(",")
-		}
-		sb.WriteString(strconv.FormatInt(u.ID, 10))
-	}
-	if sb.Len() > 0 {
-		sb.WriteString(")")
-	}
-	return fmt.Sprintf(`SELECT g.name,ug.group_type,ug.user_id FROM %v g INNER JOIN %v ug ON g.id = ug.group_id WHERE
-		ug.user_id IN %v ORDER BY ug.user_id`, getSQLTableGroups(), sqlTableUsersGroupsMapping, sb.String())
-}
-
-func getRelatedFoldersForUsersQuery(users []User) string {
-	var sb strings.Builder
-	for _, u := range users {
-		if sb.Len() == 0 {
-			sb.WriteString("(")
-		} else {
-			sb.WriteString(",")
-		}
-		sb.WriteString(strconv.FormatInt(u.ID, 10))
-	}
-	if sb.Len() > 0 {
-		sb.WriteString(")")
-	}
-	return fmt.Sprintf(`SELECT f.id,f.name,f.path,f.used_quota_size,f.used_quota_files,f.last_quota_update,fm.virtual_path,
-		fm.quota_size,fm.quota_files,fm.user_id,f.filesystem,f.description FROM %v f INNER JOIN %v fm ON f.id = fm.folder_id WHERE
-		fm.user_id IN %v ORDER BY fm.user_id`, sqlTableFolders, sqlTableUsersFoldersMapping, sb.String())
-}
-
-func getRelatedUsersForFoldersQuery(folders []vfs.BaseVirtualFolder) string {
-	var sb strings.Builder
-	for _, f := range folders {
-		if sb.Len() == 0 {
-			sb.WriteString("(")
-		} else {
-			sb.WriteString(",")
-		}
-		sb.WriteString(strconv.FormatInt(f.ID, 10))
-	}
-	if sb.Len() > 0 {
-		sb.WriteString(")")
-	}
-	return fmt.Sprintf(`SELECT fm.folder_id,u.username FROM %v fm INNER JOIN %v u ON fm.user_id = u.id
-		WHERE fm.folder_id IN %v ORDER BY fm.folder_id`, sqlTableUsersFoldersMapping, sqlTableUsers, sb.String())
-}
-
-func getRelatedGroupsForFoldersQuery(folders []vfs.BaseVirtualFolder) string {
-	var sb strings.Builder
-	for _, f := range folders {
-		if sb.Len() == 0 {
-			sb.WriteString("(")
-		} else {
-			sb.WriteString(",")
-		}
-		sb.WriteString(strconv.FormatInt(f.ID, 10))
-	}
-	if sb.Len() > 0 {
-		sb.WriteString(")")
-	}
-	return fmt.Sprintf(`SELECT fm.folder_id,g.name FROM %v fm INNER JOIN %v g ON fm.group_id = g.id
-		WHERE fm.folder_id IN %v ORDER BY fm.folder_id`, sqlTableGroupsFoldersMapping, getSQLTableGroups(), sb.String())
-}
-
-func getRelatedUsersForGroupsQuery(groups []Group) string {
-	var sb strings.Builder
-	for _, g := range groups {
-		if sb.Len() == 0 {
-			sb.WriteString("(")
-		} else {
-			sb.WriteString(",")
-		}
-		sb.WriteString(strconv.FormatInt(g.ID, 10))
-	}
-	if sb.Len() > 0 {
-		sb.WriteString(")")
-	}
-	return fmt.Sprintf(`SELECT um.group_id,u.username FROM %v um INNER JOIN %v u ON um.user_id = u.id
-		WHERE um.group_id IN %v ORDER BY um.group_id`, sqlTableUsersGroupsMapping, sqlTableUsers, sb.String())
-}
-
-func getRelatedFoldersForGroupsQuery(groups []Group) string {
-	var sb strings.Builder
-	for _, g := range groups {
-		if sb.Len() == 0 {
-			sb.WriteString("(")
-		} else {
-			sb.WriteString(",")
-		}
-		sb.WriteString(strconv.FormatInt(g.ID, 10))
-	}
-	if sb.Len() > 0 {
-		sb.WriteString(")")
-	}
-	return fmt.Sprintf(`SELECT f.id,f.name,f.path,f.used_quota_size,f.used_quota_files,f.last_quota_update,fm.virtual_path,
-		fm.quota_size,fm.quota_files,fm.group_id,f.filesystem,f.description FROM %s f INNER JOIN %s fm ON f.id = fm.folder_id WHERE
-		fm.group_id IN %v ORDER BY fm.group_id`, sqlTableFolders, sqlTableGroupsFoldersMapping, sb.String())
-}
-
-func getActiveTransfersQuery() string {
-	return fmt.Sprintf(`SELECT transfer_id,connection_id,transfer_type,username,folder_name,ip,truncated_size,
-		current_ul_size,current_dl_size,created_at,updated_at FROM %v WHERE updated_at > %v`,
-		sqlTableActiveTransfers, sqlPlaceholders[0])
-}
-
-func getAddActiveTransferQuery() string {
-	return fmt.Sprintf(`INSERT INTO %v (transfer_id,connection_id,transfer_type,username,folder_name,ip,truncated_size,
-		current_ul_size,current_dl_size,created_at,updated_at) VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v)`,
-		sqlTableActiveTransfers, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3],
-		sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8],
-		sqlPlaceholders[9], sqlPlaceholders[10])
-}
-
-func getUpdateActiveTransferSizesQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET current_ul_size=%v,current_dl_size=%v,updated_at=%v WHERE connection_id = %v AND transfer_id = %v`,
-		sqlTableActiveTransfers, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4])
-}
-
-func getRemoveActiveTransferQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE connection_id = %v AND transfer_id = %v`,
-		sqlTableActiveTransfers, sqlPlaceholders[0], sqlPlaceholders[1])
-}
-
-func getCleanupActiveTransfersQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE updated_at < %v`, sqlTableActiveTransfers, sqlPlaceholders[0])
-}
-
-func getDatabaseVersionQuery() string {
-	return fmt.Sprintf("SELECT version from %v LIMIT 1", sqlTableSchemaVersion)
-}
-
-func getUpdateDBVersionQuery() string {
-	return fmt.Sprintf(`UPDATE %v SET version=%v`, sqlTableSchemaVersion, sqlPlaceholders[0])
-}

docker/README.md

@@ -4,12 +4,14 @@ SFTPGo provides an official Docker image, it is available on both [Docker Hub](h
 
 ## Supported tags and respective Dockerfile links
 
-- [v2.3.1, v2.3, v2, latest](https://github.com/drakkan/sftpgo/blob/v2.3.1/Dockerfile)
-- [v2.3.1-alpine, v2.3-alpine, v2-alpine, alpine](https://github.com/drakkan/sftpgo/blob/v2.3.1/Dockerfile.alpine)
-- [v2.3.1-slim, v2.3-slim, v2-slim, slim](https://github.com/drakkan/sftpgo/blob/v2.3.1/Dockerfile)
-- [v2.3.1-alpine-slim, v2.3-alpine-slim, v2-alpine-slim, alpine-slim](https://github.com/drakkan/sftpgo/blob/v2.3.1/Dockerfile.alpine)
-- [v2.3.1-distroless-slim, v2.3-distroless-slim, v2-distroless-slim, distroless-slim](https://github.com/drakkan/sftpgo/blob/v2.3.1/Dockerfile.distroless)
+- [v2.4.0, v2.4, v2, latest](https://github.com/drakkan/sftpgo/blob/v2.4.0/Dockerfile)
+- [v2.4.0-plugins, v2.4-plugins, v2-plugins, plugins](https://github.com/drakkan/sftpgo/blob/v2.4.0/Dockerfile)
+- [v2.4.0-alpine, v2.4-alpine, v2-alpine, alpine](https://github.com/drakkan/sftpgo/blob/v2.4.0/Dockerfile.alpine)
+- [v2.4.0-slim, v2.4-slim, v2-slim, slim](https://github.com/drakkan/sftpgo/blob/v2.4.0/Dockerfile)
+- [v2.4.0-alpine-slim, v2.4-alpine-slim, v2-alpine-slim, alpine-slim](https://github.com/drakkan/sftpgo/blob/v2.4.0/Dockerfile.alpine)
+- [v2.4.0-distroless-slim, v2.4-distroless-slim, v2-distroless-slim, distroless-slim](https://github.com/drakkan/sftpgo/blob/v2.4.0/Dockerfile.distroless)
 - [edge](../Dockerfile)
+- [edge-plugins](../Dockerfile)
 - [edge-alpine](../Dockerfile.alpine)
 - [edge-slim](../Dockerfile)
 - [edge-alpine-slim](../Dockerfile.alpine)
@@ -197,8 +199,17 @@ We only provide the slim variant and so the optional `git` dependency is not ava
 
 ### `sftpgo:<suite>-slim`
 
-These tags provide a slimmer image that does not include the optional `git` dependency.
+These tags provide a slimmer image that does not include `jq` and the optional `git` and `rsync` dependencies.
+
+### `sftpgo:<suite>-plugins`
+
+These tags provide the standard image with the addition of all "official" plugins installed in `/usr/local/bin`.
 
 ## Helm Chart
 
-An helm chart is [available](https://artifacthub.io/packages/helm/sagikazarmark/sftpgo). You can find the source code [here](https://github.com/sagikazarmark/helm-charts/tree/master/charts/sftpgo).
+Some helm charts are available:
+
+- [sagikazarmark/sftpgo](https://artifacthub.io/packages/helm/sagikazarmark/sftpgo)
+- [truecharts/sftpgo](https://artifacthub.io/packages/helm/truecharts/sftpgo)
+
+These charts are not maintained by the SFTPGo project and any issues with the charts should be raised to the upstream repo.

docker/scripts/download-plugins.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -e
+
+ARCH=`uname -m`
+
+case ${ARCH} in
+    "x86_64")
+        SUFFIX=amd64
+        ;;
+    "aarch64")
+        SUFFIX=arm64
+        ;;
+    *)
+        SUFFIX=ppc64le
+        ;;
+esac
+
+echo "download plugins for arch ${SUFFIX}"
+
+for PLUGIN in geoipfilter kms pubsub eventstore eventsearch metadata
+do
+    echo "download plugin from https://github.com/sftpgo/sftpgo-plugin-${PLUGIN}/releases/latest/download/sftpgo-plugin-${PLUGIN}-linux-${SUFFIX}"
+    curl -L "https://github.com/sftpgo/sftpgo-plugin-${PLUGIN}/releases/latest/download/sftpgo-plugin-${PLUGIN}-linux-${SUFFIX}" --output "/usr/local/bin/sftpgo-plugin-${PLUGIN}"
+    chmod 755 "/usr/local/bin/sftpgo-plugin-${PLUGIN}"
+done

docker/scripts/entrypoint-alpine.sh

@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-
-SFTPGO_PUID=${SFTPGO_PUID:-1000}
-SFTPGO_PGID=${SFTPGO_PGID:-1000}
-
-if [ "$1" = 'sftpgo' ]; then
-    if [ "$(id -u)" = '0' ]; then
-        for DIR in "/etc/sftpgo" "/var/lib/sftpgo" "/srv/sftpgo"
-        do
-            DIR_UID=$(stat -c %u ${DIR})
-            DIR_GID=$(stat -c %g ${DIR})
-            if [ ${DIR_UID} != ${SFTPGO_PUID} ] || [ ${DIR_GID} != ${SFTPGO_PGID} ]; then
-                echo '{"level":"info","time":"'`date +%Y-%m-%dT%H:%M:%S.000`'","sender":"entrypoint","message":"change owner for \"'${DIR}'\" UID: '${SFTPGO_PUID}' GID: '${SFTPGO_PGID}'"}'
-                if [ ${DIR} = "/etc/sftpgo" ]; then
-                    chown -R ${SFTPGO_PUID}:${SFTPGO_PGID} ${DIR}
-                else
-                    chown ${SFTPGO_PUID}:${SFTPGO_PGID} ${DIR}
-                fi
-            fi
-        done
-        echo '{"level":"info","time":"'`date +%Y-%m-%dT%H:%M:%S.000`'","sender":"entrypoint","message":"run as UID: '${SFTPGO_PUID}' GID: '${SFTPGO_PGID}'"}'
-        exec su-exec ${SFTPGO_PUID}:${SFTPGO_PGID} "$@"
-    fi
-
-    exec "$@"
-fi
-
-exec "$@"

docker/scripts/entrypoint.sh

@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-
-SFTPGO_PUID=${SFTPGO_PUID:-1000}
-SFTPGO_PGID=${SFTPGO_PGID:-1000}
-
-if [ "$1" = 'sftpgo' ]; then
-    if [ "$(id -u)" = '0' ]; then
-        getent passwd ${SFTPGO_PUID} > /dev/null
-	    HAS_PUID=$?
-	    getent group ${SFTPGO_PGID} > /dev/null
-	    HAS_PGID=$?
-        if [ ${HAS_PUID} -ne 0 ] || [ ${HAS_PGID} -ne 0 ]; then
-            echo '{"level":"info","time":"'`date +%Y-%m-%dT%H:%M:%S.%3N`'","sender":"entrypoint","message":"prepare to run as UID: '${SFTPGO_PUID}' GID: '${SFTPGO_PGID}'"}'
-            if [ ${HAS_PGID} -ne 0 ];  then
-                echo '{"level":"info","time":"'`date +%Y-%m-%dT%H:%M:%S.%3N`'","sender":"entrypoint","message":"set GID to: '${SFTPGO_PGID}'"}'
-                groupmod -g ${SFTPGO_PGID} sftpgo
-            fi
-            if [ ${HAS_PUID} -ne 0 ]; then
-                echo '{"level":"info","time":"'`date +%Y-%m-%dT%H:%M:%S.%3N`'","sender":"entrypoint","message":"set UID to: '${SFTPGO_PUID}'"}'
-                usermod -u ${SFTPGO_PUID} sftpgo
-            fi
-            chown -R ${SFTPGO_PUID}:${SFTPGO_PGID} /etc/sftpgo
-            chown ${SFTPGO_PUID}:${SFTPGO_PGID} /var/lib/sftpgo /srv/sftpgo
-        fi
-        echo '{"level":"info","time":"'`date +%Y-%m-%dT%H:%M:%S.%3N`'","sender":"entrypoint","message":"run as UID: '${SFTPGO_PUID}' GID: '${SFTPGO_PGID}'"}'
-        exec gosu ${SFTPGO_PUID}:${SFTPGO_PGID} "$@"
-    fi
-
-    exec "$@"
-fi
-
-exec "$@"

docker/sftpgo/alpine/Dockerfile

@@ -1,50 +0,0 @@
-FROM golang:alpine as builder
-
-RUN apk add --no-cache git gcc g++ ca-certificates \
-  && go get -v -d github.com/drakkan/sftpgo
-WORKDIR /go/src/github.com/drakkan/sftpgo
-ARG TAG
-ARG FEATURES
-# Use --build-arg TAG=LATEST for latest tag. Use e.g. --build-arg TAG=v1.0.0 for a specific tag/commit. Otherwise HEAD (master) is built.
-RUN git checkout $(if [ "${TAG}" = LATEST ]; then echo `git rev-list --tags --max-count=1`; elif [ -n "${TAG}" ]; then echo "${TAG}"; else echo HEAD; fi)
-RUN go build $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -v -o /go/bin/sftpgo
-
-FROM alpine:latest
-
-RUN apk add --no-cache ca-certificates su-exec \
-  && mkdir -p /data /etc/sftpgo /srv/sftpgo/config /srv/sftpgo/web /srv/sftpgo/backups
-
-# git and rsync are optional, uncomment the next line to add support for them if needed.
-#RUN apk add --no-cache git rsync
-
-COPY --from=builder /go/bin/sftpgo /bin/
-COPY --from=builder /go/src/github.com/drakkan/sftpgo/sftpgo.json /etc/sftpgo/sftpgo.json
-COPY --from=builder /go/src/github.com/drakkan/sftpgo/templates /srv/sftpgo/web/templates
-COPY --from=builder /go/src/github.com/drakkan/sftpgo/static /srv/sftpgo/web/static
-COPY docker-entrypoint.sh /bin/entrypoint.sh
-RUN chmod +x /bin/entrypoint.sh
-
-VOLUME [ "/data", "/srv/sftpgo/config", "/srv/sftpgo/backups" ]
-EXPOSE 2022 8080
-
-# uncomment the following settings to enable FTP support
-#ENV SFTPGO_FTPD__BIND_PORT=2121
-#ENV SFTPGO_FTPD__FORCE_PASSIVE_IP=<your FTP visibile IP here>
-#EXPOSE 2121
-
-# we need to expose the passive ports range too
-#EXPOSE 50000-50100
-
-# it is a good idea to provide certificates to enable FTPS too
-#ENV SFTPGO_FTPD__CERTIFICATE_FILE=/srv/sftpgo/config/mycert.crt
-#ENV SFTPGO_FTPD__CERTIFICATE_KEY_FILE=/srv/sftpgo/config/mycert.key
-
-# uncomment the following setting to enable WebDAV support
-#ENV SFTPGO_WEBDAVD__BIND_PORT=8090
-
-# it is a good idea to provide certificates to enable WebDAV over HTTPS
-#ENV SFTPGO_WEBDAVD__CERTIFICATE_FILE=${CONFIG_DIR}/mycert.crt
-#ENV SFTPGO_WEBDAVD__CERTIFICATE_KEY_FILE=${CONFIG_DIR}/mycert.key
-
-ENTRYPOINT ["/bin/entrypoint.sh"]
-CMD ["serve"]

docker/sftpgo/alpine/README.md

@@ -1,61 +0,0 @@
-# SFTPGo with Docker and Alpine
-
-:warning: The recommended way to run SFTPGo on Docker is to use the official [images](https://hub.docker.com/r/drakkan/sftpgo). The documentation here is now obsolete.
-
-This DockerFile is made to build image to host multiple instances of SFTPGo started with different users.
-
-## Example
-
-> 1003 is a custom uid:gid for this instance of SFTPGo
-
-```bash
-# Prereq on docker host
-sudo groupadd -g 1003 sftpgrp && \
-  sudo useradd -u 1003 -g 1003 sftpuser -d /home/sftpuser/ && \
-  sudo -u sftpuser mkdir /home/sftpuser/{conf,data} && \
-  curl https://raw.githubusercontent.com/drakkan/sftpgo/master/sftpgo.json -o /home/sftpuser/conf/sftpgo.json
-
-# Edit sftpgo.json as you need
-
-# Get and build SFTPGo image.
-# Add --build-arg TAG=LATEST to build the latest tag or e.g. TAG=v1.0.0 for a specific tag/commit.
-# Add --build-arg FEATURES=<build features comma separated> to specify the features to build.
-git clone https://github.com/drakkan/sftpgo.git && \
-  cd sftpgo && \
-  sudo docker build -t sftpgo docker/sftpgo/alpine/
-
-# Initialize the configured provider. For PostgreSQL and MySQL providers you need to create the configured database and the "initprovider" command will create the required tables.
-sudo docker run --name sftpgo \
-  -e PUID=1003 \
-  -e GUID=1003 \
-  -v /home/sftpuser/conf/:/srv/sftpgo/config \
-  sftpgo initprovider -c /srv/sftpgo/config
-
-# Start the image
-sudo docker rm sftpgo && sudo docker run --name sftpgo \
-  -e SFTPGO_LOG_FILE_PATH= \
-  -e SFTPGO_CONFIG_DIR=/srv/sftpgo/config \
-  -e SFTPGO_HTTPD__TEMPLATES_PATH=/srv/sftpgo/web/templates \
-  -e SFTPGO_HTTPD__STATIC_FILES_PATH=/srv/sftpgo/web/static \
-  -e SFTPGO_HTTPD__BACKUPS_PATH=/srv/sftpgo/backups \
-  -p 8080:8080 \
-  -p 2022:2022 \
-  -e PUID=1003 \
-  -e GUID=1003 \
-  -v /home/sftpuser/conf/:/srv/sftpgo/config \
-  -v /home/sftpuser/data:/data \
-  -v /home/sftpuser/backups:/srv/sftpgo/backups \
-  sftpgo
-```
-
-If you want to enable FTP/S you also need the publish the FTP port and the FTP passive port range, defined in your `Dockerfile`, by adding, for example, the following options to the `docker run` command `-p 2121:2121 -p 50000-50100:50000-50100`. The same goes for WebDAV, you need to publish the configured port.
-
-The script `entrypoint.sh` makes sure to correct the permissions of directories and start the process with the right user.
-
-Several images can be run with different parameters.
-
-## Custom systemd script
-
-An example of systemd script is present [here](sftpgo.service), with `Environment` parameter to set `PUID` and `GUID`
-
-`WorkingDirectory` parameter must be exist with one file in this directory like `sftpgo-${PUID}.env` corresponding to the variable file for SFTPGo instance.

docker/sftpgo/alpine/docker-entrypoint.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-chown -R "${PUID}:${GUID}" /data /etc/sftpgo /srv/sftpgo/config /srv/sftpgo/backups \
-	&& exec su-exec "${PUID}:${GUID}" \
-  /bin/sftpgo "$@"

docker/sftpgo/alpine/sftpgo.service

@@ -1,35 +0,0 @@
-[Unit]
-Description=SFTPGo server
-After=docker.service
-
-[Service]
-User=root
-Group=root
-WorkingDirectory=/etc/sftpgo
-Environment=PUID=1003
-Environment=GUID=1003
-EnvironmentFile=-/etc/sysconfig/sftpgo.env
-ExecStartPre=-docker kill sftpgo
-ExecStartPre=-docker rm sftpgo
-ExecStart=docker run --name sftpgo \
-  --env-file sftpgo-${PUID}.env \
-  -e PUID=${PUID} \
-  -e GUID=${GUID} \
-  -e SFTPGO_LOG_FILE_PATH= \
-  -e SFTPGO_CONFIG_DIR=/srv/sftpgo/config \
-  -e SFTPGO_HTTPD__TEMPLATES_PATH=/srv/sftpgo/web/templates \
-  -e SFTPGO_HTTPD__STATIC_FILES_PATH=/srv/sftpgo/web/static \
-  -e SFTPGO_HTTPD__BACKUPS_PATH=/srv/sftpgo/backups \
-  -p 8080:8080 \
-  -p 2022:2022 \
-  -v /home/sftpuser/conf/:/srv/sftpgo/config \
-  -v /home/sftpuser/data:/data \
-  -v /home/sftpuser/backups:/srv/sftpgo/backups \
-  sftpgo
-ExecStop=docker stop sftpgo
-SyslogIdentifier=sftpgo
-Restart=always
-RestartSec=10s
-
-[Install]
-WantedBy=multi-user.target

docker/sftpgo/debian/Dockerfile

@@ -1,93 +0,0 @@
-# we use a multi stage build to have a separate build and run env
-FROM golang:latest as buildenv
-LABEL maintainer="nicola.murino@gmail.com"
-RUN go get -v -d github.com/drakkan/sftpgo
-WORKDIR /go/src/github.com/drakkan/sftpgo
-ARG TAG
-ARG FEATURES
-# Use --build-arg TAG=LATEST for latest tag. Use e.g. --build-arg TAG=v1.0.0 for a specific tag/commit. Otherwise HEAD (master) is built.
-RUN git checkout $(if [ "${TAG}" = LATEST ]; then echo `git rev-list --tags --max-count=1`; elif [ -n "${TAG}" ]; then echo "${TAG}"; else echo HEAD; fi)
-RUN go build $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -v -o sftpgo
-
-# now define the run environment
-FROM debian:latest
-
-# ca-certificates is needed for Cloud Storage Support and for HTTPS/FTPS.
-RUN apt-get update && apt-get install -y ca-certificates && apt-get clean
-
-# git and rsync are optional, uncomment the next line to add support for them if needed.
-#RUN apt-get update && apt-get install -y git rsync && apt-get clean
-
-ARG BASE_DIR=/app
-ARG DATA_REL_DIR=data
-ARG CONFIG_REL_DIR=config
-ARG BACKUP_REL_DIR=backups
-ARG USERNAME=sftpgo
-ARG GROUPNAME=sftpgo
-ARG UID=515
-ARG GID=515
-ARG WEB_REL_PATH=web
-
-# HOME_DIR for sftpgo itself
-ENV HOME_DIR=${BASE_DIR}/${USERNAME}
-# DATA_DIR, this is a volume that you can use hold user's home dirs
-ENV DATA_DIR=${BASE_DIR}/${DATA_REL_DIR}
-# CONFIG_DIR, this is a volume to persist the daemon private keys, configuration file ecc..
-ENV CONFIG_DIR=${BASE_DIR}/${CONFIG_REL_DIR}
-# BACKUPS_DIR, this is a volume to store backups done using "dumpdata" REST API
-ENV BACKUPS_DIR=${BASE_DIR}/${BACKUP_REL_DIR}
-ENV WEB_DIR=${BASE_DIR}/${WEB_REL_PATH}
-
-RUN mkdir -p ${DATA_DIR} ${CONFIG_DIR} ${WEB_DIR} ${BACKUPS_DIR}
-RUN groupadd --system -g ${GID} ${GROUPNAME}
-RUN useradd --system --create-home --no-log-init --home-dir ${HOME_DIR} --comment "SFTPGo user" --shell /usr/sbin/nologin --gid ${GID} --uid ${UID} ${USERNAME}
-
-WORKDIR ${HOME_DIR}
-RUN mkdir -p bin .config/sftpgo
-ENV PATH ${HOME_DIR}/bin:$PATH
-COPY --from=buildenv /go/src/github.com/drakkan/sftpgo/sftpgo bin/sftpgo
-# default config file to use if no config file is found inside the CONFIG_DIR volume.
-# You can override each configuration options via env vars too
-COPY --from=buildenv /go/src/github.com/drakkan/sftpgo/sftpgo.json .config/sftpgo/
-COPY --from=buildenv /go/src/github.com/drakkan/sftpgo/templates ${WEB_DIR}/templates
-COPY --from=buildenv /go/src/github.com/drakkan/sftpgo/static ${WEB_DIR}/static
-RUN chown -R ${UID}:${GID} ${DATA_DIR} ${BACKUPS_DIR}
-
-# run as non root user
-USER ${USERNAME}
-
-EXPOSE 2022 8080
-
-# the defined volumes must have write access for the UID and GID defined above
-VOLUME [ "$DATA_DIR", "$CONFIG_DIR", "$BACKUPS_DIR" ]
-
-# override some default configuration options using env vars
-ENV SFTPGO_CONFIG_DIR=${CONFIG_DIR}
-# setting SFTPGO_LOG_FILE_PATH to an empty string will log to stdout
-ENV SFTPGO_LOG_FILE_PATH=""
-ENV SFTPGO_HTTPD__BIND_ADDRESS=""
-ENV SFTPGO_HTTPD__TEMPLATES_PATH=${WEB_DIR}/templates
-ENV SFTPGO_HTTPD__STATIC_FILES_PATH=${WEB_DIR}/static
-ENV SFTPGO_DATA_PROVIDER__USERS_BASE_DIR=${DATA_DIR}
-ENV SFTPGO_HTTPD__BACKUPS_PATH=${BACKUPS_DIR}
-
-# uncomment the following settings to enable FTP support
-#ENV SFTPGO_FTPD__BIND_PORT=2121
-#ENV SFTPGO_FTPD__FORCE_PASSIVE_IP=<your FTP visibile IP here>
-#EXPOSE 2121
-# we need to expose the passive ports range too
-#EXPOSE 50000-50100
-
-# it is a good idea to provide certificates to enable FTPS too
-#ENV SFTPGO_FTPD__CERTIFICATE_FILE=${CONFIG_DIR}/mycert.crt
-#ENV SFTPGO_FTPD__CERTIFICATE_KEY_FILE=${CONFIG_DIR}/mycert.key
-
-# uncomment the following setting to enable WebDAV support
-#ENV SFTPGO_WEBDAVD__BIND_PORT=8090
-
-# it is a good idea to provide certificates to enable WebDAV over HTTPS
-#ENV SFTPGO_WEBDAVD__CERTIFICATE_FILE=${CONFIG_DIR}/mycert.crt
-#ENV SFTPGO_WEBDAVD__CERTIFICATE_KEY_FILE=${CONFIG_DIR}/mycert.key
-
-ENTRYPOINT ["sftpgo"]
-CMD ["serve"]

docker/sftpgo/debian/README.md

@@ -1,59 +0,0 @@
-# Dockerfile based on Debian stable
-
-:warning: The recommended way to run SFTPGo on Docker is to use the official [images](https://hub.docker.com/r/drakkan/sftpgo). The documentation here is now obsolete.
-
-Please read the comments inside the `Dockerfile` to learn how to customize things for your setup.
-
-You can build the container image using `docker build`, for example:
-
-```bash
-docker build -t="drakkan/sftpgo" .
-```
-
-This will build master of github.com/drakkan/sftpgo.
-
-To build the latest tag you can add `--build-arg TAG=LATEST` and to build a specific tag/commit you can use for example `TAG=v1.0.0`, like this:
-
-```bash
-docker build -t="drakkan/sftpgo" --build-arg TAG=v1.0.0 .
-```
-
-To specify the features to build you can add `--build-arg FEATURES=<build features comma separated>`. For example you can disable SQLite and S3 support like this:
-
-```bash
-docker build -t="drakkan/sftpgo" --build-arg FEATURES=nosqlite,nos3 .
-```
-
-Please take a look at the [build from source](./../../../docs/build-from-source.md) documentation for the complete list of the features that can be disabled.
-
-Now create the required folders on the host system, for example:
-
-```bash
-sudo mkdir -p /srv/sftpgo/data /srv/sftpgo/config /srv/sftpgo/backups
-```
-
-and give write access to them to the UID/GID defined inside the `Dockerfile`. You can choose to create a new user, on the host system, with a matching UID/GID pair, or simply do something like this:
-
-```bash
-sudo chown -R <UID>:<GID> /srv/sftpgo/data /srv/sftpgo/config /srv/sftpgo/backups
-```
-
-Download the default configuration file and edit it as you need:
-
-```bash
-sudo curl https://raw.githubusercontent.com/drakkan/sftpgo/master/sftpgo.json -o /srv/sftpgo/config/sftpgo.json
-```
-
-Initialize the configured provider. For PostgreSQL and MySQL providers you need to create the configured database and the `initprovider` command will create the required tables:
-
-```bash
-docker run --name sftpgo --mount type=bind,source=/srv/sftpgo/config,target=/app/config drakkan/sftpgo initprovider -c /app/config
-```
-
-and finally you can run the image using something like this:
-
-```bash
-docker rm sftpgo && docker run --name sftpgo -p 8080:8080 -p 2022:2022 --mount type=bind,source=/srv/sftpgo/data,target=/app/data --mount type=bind,source=/srv/sftpgo/config,target=/app/config --mount type=bind,source=/srv/sftpgo/backups,target=/app/backups drakkan/sftpgo
-```
-
-If you want to enable FTP/S you also need the publish the FTP port and the FTP passive port range, defined in your `Dockerfile`, by adding, for example, the following options to the `docker run` command `-p 2121:2121 -p 50000-50100:50000-50100`. The same goes for WebDAV, you need to publish the configured port.

docs/account.md

@@ -14,7 +14,7 @@ the dump is a JSON with all SFTPGo data including users, folders, admins.
 
 These properties are stored inside the configured data provider.
 
-SFTPGo supports checking passwords stored with bcrypt, pbkdf2, md5crypt and sha512crypt too. For pbkdf2 the supported format is `$<algo>$<iterations>$<salt>$<hashed pwd base64 encoded>`, where algo is `pbkdf2-sha1` or `pbkdf2-sha256` or `pbkdf2-sha512` or `$pbkdf2-b64salt-sha256$`. For example the pbkdf2-sha256 of the word password using 150000 iterations and E86a9YMX3zC7 as salt must be stored as `$pbkdf2-sha256$150000$E86a9YMX3zC7$R5J62hsSq+pYw00hLLPKBbcGXmq7fj5+/M0IFoYtZbo=`. In pbkdf2 variant with b64salt the salt is base64 encoded. For bcrypt the format must be the one supported by golang's crypto/bcrypt package, for example the password secret with cost 14 must be stored as `$2a$14$ajq8Q7fbtFRQvXpdCq7Jcuy.Rx1h/L4J60Otx.gyNLbAYctGMJ9tK`. For md5crypt and sha512crypt we support the format used in `/etc/shadow` with the `$1$` and `$6$` prefix, this is useful if you are migrating from Unix system user accounts. We support Apache md5crypt (`$apr1$` prefix) too. Using the REST API you can send a password hashed as bcrypt, pbkdf2, md5crypt or sha512crypt and it will be stored as is.
+SFTPGo supports checking passwords stored with argon2id, bcrypt, pbkdf2, md5cryptm sha256crypt and sha512crypt too. For pbkdf2 the supported format is `$<algo>$<iterations>$<salt>$<hashed pwd base64 encoded>`, where algo is `pbkdf2-sha1` or `pbkdf2-sha256` or `pbkdf2-sha512` or `$pbkdf2-b64salt-sha256$`. For example the pbkdf2-sha256 of the word password using 150000 iterations and E86a9YMX3zC7 as salt must be stored as `$pbkdf2-sha256$150000$E86a9YMX3zC7$R5J62hsSq+pYw00hLLPKBbcGXmq7fj5+/M0IFoYtZbo=`. In pbkdf2 variant with b64salt the salt is base64 encoded. For bcrypt the format must be the one supported by golang's crypto/bcrypt package, for example the password secret with cost 14 must be stored as `$2a$14$ajq8Q7fbtFRQvXpdCq7Jcuy.Rx1h/L4J60Otx.gyNLbAYctGMJ9tK`. For md5crypt, sha256crypt and sha512crypt we support the format used in `/etc/shadow` with the `$1$`, `$5$` and `$6$` prefix, this is useful if you are migrating from Unix system user accounts. We support Apache md5crypt (`$apr1$` prefix) too. Using the REST API you can send a password hashed as argon2id, bcrypt, pbkdf2, md5crypt, sha256crypt  or sha512crypt and it will be stored as is.
 
 If you want to use your existing accounts, you have these options:
 

docs/build-from-source.md

@@ -13,6 +13,7 @@ The following build tags are available:
 - `nosqlite`, disable SQLite data provider, default enabled
 - `noportable`, disable portable mode, default enabled
 - `nometrics`, disable Prometheus metrics, default enabled
+- `bundle`, embed static files and templates. Before building with this tag enabled you have to copy `openapi`, `static` and `templates` dirs to `internal/bundle` directory. Default disabled
 
 If no build tag is specified the build will include the default features.
 
@@ -23,18 +24,18 @@ The compiler is a build time only dependency. It is not required at runtime.
 
 Version info, such as git commit and build date, can be embedded setting the following string variables at build time:
 
-- `github.com/drakkan/sftpgo/v2/version.commit`
-- `github.com/drakkan/sftpgo/v2/version.date`
+- `github.com/drakkan/sftpgo/v2/internal/version.commit`
+- `github.com/drakkan/sftpgo/v2/internal/version.date`
 
 For example, you can build using the following command:
 
 ```bash
-go build -tags nogcs,nos3,nosqlite -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -o sftpgo
+go build -tags nogcs,nos3,nosqlite -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo
 ```
 
 You should get a version that includes git commit, build date and available features like this one:
 
 ```bash
 $ ./sftpgo -v
-SFTPGo 0.9.6-dev-b30614e-dirty-2020-06-19T11:04:56Z +metrics -gcs -s3 +bolt +mysql +pgsql -sqlite +portable
+SFTPGo 2.3.1-dev-c8158e1-2022-07-24T17:25:45Z +metrics +azblob +gcs +s3 +bolt +mysql +pgsql +sqlite +portable
 ```

docs/check-password-hook.md

@@ -18,7 +18,7 @@ If the hook defines an external program it can read the following environment va
 - `SFTPGO_AUTHD_IP`
 - `SFTPGO_AUTHD_PROTOCOL`, possible values are `SSH`, `FTP`, `DAV`, `HTTP`
 
-Previous global environment variables aren't cleared when the script is called. The content of these variables is _not_ quoted. They may contain special characters. They are under the control of a possibly malicious remote user.
+Global environment variables are cleared, for security reasons, when the script is called. You can set additional environment variables in the "command" configuration section.
 
 The program must write, on its standard output, the expected JSON serialized response described above.
 

docs/custom-actions.md

@@ -10,8 +10,10 @@ The `hook` can be defined as the absolute path of your program or an HTTP URL.
 The following `actions` are supported:
 
 - `download`
+- `first-download`
 - `pre-download`
 - `upload`
+- `first-upload`
 - `pre-upload`
 - `delete`
 - `pre-delete`
@@ -20,7 +22,7 @@ The following `actions` are supported:
 - `rmdir`
 - `ssh_cmd`
 
-The `upload` condition includes both uploads to new files and overwrite of existing ones. If an upload is aborted for quota limits SFTPGo tries to remove the partial file, so if the notification reports a zero size file and a quota exceeded error the file has been deleted. The `ssh_cmd` condition will be triggered after a command is successfully executed via SSH. `scp` will trigger the `download` and `upload` conditions and not `ssh_cmd`.
+The `upload` condition includes both uploads to new files and overwrite of existing ones. If an upload is aborted for quota limits SFTPGo tries to remove the partial file, so if the notification reports a zero size file and a quota exceeded error the file has been deleted. The `ssh_cmd` condition will be triggered after a command is successfully executed via SSH. `scp` will trigger the `download` and `upload` conditions and not `ssh_cmd`. The `first-download` and `first-upload` action are executed only if no error occour and they don't exclude the `download` and `upload` notifications, so you will get both the `first-upload` and `upload` notification after the first successful upload and the same for the first successful download.
 For cloud backends directories are virtual, they are created implicitly when you upload a file and are implicitly removed when the last file within a directory is removed. The `mkdir` and `rmdir` notifications are sent only when a directory is explicitly created or removed.
 
 The notification will indicate if an error is detected and so, for example, a partial file is uploaded.
@@ -43,13 +45,13 @@ If the `hook` defines a path to an external program, then this program can read
 - `SFTPGO_ACTION_BUCKET`, non-empty for S3, GCS and Azure backends
 - `SFTPGO_ACTION_ENDPOINT`, non-empty for S3, SFTP and Azure backend if configured
 - `SFTPGO_ACTION_STATUS`, integer. Status for `upload`, `download` and `ssh_cmd` actions. 1 means no error, 2 means a generic error occurred, 3 means quota exceeded error
-- `SFTPGO_ACTION_PROTOCOL`, string. Possible values are `SSH`, `SFTP`, `SCP`, `FTP`, `DAV`, `HTTP`, `HTTPShare`, `OIDC`, `DataRetention`
+- `SFTPGO_ACTION_PROTOCOL`, string. Possible values are `SSH`, `SFTP`, `SCP`, `FTP`, `DAV`, `HTTP`, `HTTPShare`, `OIDC`, `DataRetention`, `EventAction`
 - `SFTPGO_ACTION_IP`, the action was executed from this IP address
 - `SFTPGO_ACTION_SESSION_ID`, string. Unique protocol session identifier. For stateless protocols such as HTTP the session id will change for each request
 - `SFTPGO_ACTION_OPEN_FLAGS`, integer. File open flags, can be non-zero for `pre-upload` action. If `SFTPGO_ACTION_FILE_SIZE` is greater than zero and `SFTPGO_ACTION_OPEN_FLAGS&512 == 0` the target file will not be truncated
 - `SFTPGO_ACTION_TIMESTAMP`, int64. Event timestamp as nanoseconds since epoch
 
-Previous global environment variables aren't cleared when the script is called.
+Global environment variables are cleared, for security reasons, when the script is called. You can set additional environment variables in the "command" configuration section.
 The program must finish within 30 seconds.
 
 If the `hook` defines an HTTP URL then this URL will be invoked as HTTP POST. The request body will contain a JSON serialized struct with the following fields:
@@ -62,11 +64,11 @@ If the `hook` defines an HTTP URL then this URL will be invoked as HTTP POST. Th
 - `virtual_target_path`, string, virtual target path, seen by SFTPGo users
 - `ssh_cmd`, string, included for `ssh_cmd` action
 - `file_size`, int64, included for `pre-upload`, `upload`, `download`, `delete` actions if the file size is greater than `0`
-- `fs_provider`, integer, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend, `3` for Azure Blob Storage backend, `4` for local encrypted backend, `5` for SFTP backend
+- `fs_provider`, integer, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend, `3` for Azure Blob Storage backend, `4` for local encrypted backend, `5` for SFTP backend, `6` for HTTPFs backend
 - `bucket`, string, included for S3, GCS and Azure backends
 - `endpoint`, string, included for S3, SFTP and Azure backend if configured
 - `status`, integer. Status for `upload`, `download` and `ssh_cmd` actions. 1 means no error, 2 means a generic error occurred, 3 means quota exceeded error
-- `protocol`, string. Possible values are `SSH`, `SFTP`, `SCP`, `FTP`, `DAV`, `HTTP`, `HTTPShare`, `OIDC`, `DataRetention`
+- `protocol`, string. Possible values are `SSH`, `SFTP`, `SCP`, `FTP`, `DAV`, `HTTP`, `HTTPShare`, `OIDC`, `DataRetention`, `EventAction`
 - `ip`, string. The action was executed from this IP address
 - `session_id`, string. Unique protocol session identifier. For stateless protocols such as HTTP the session id will change for each request
 - `open_flags`, integer. File open flags, can be non-zero for `pre-upload` action. If `file_size` is greater than zero and `file_size&512 == 0` the target file will not be truncated
@@ -75,6 +77,8 @@ If the `hook` defines an HTTP URL then this URL will be invoked as HTTP POST. Th
 The HTTP hook will use the global configuration for HTTP clients and will respect the retry configurations.
 
 The `pre-*` actions are always executed synchronously while the other ones are asynchronous. You can specify the actions to run synchronously via the `execute_sync` configuration key. Executing an action synchronously means that SFTPGo will not return a result code to the client (which is waiting for it) until your hook have completed its execution. If your hook takes a long time to complete this could cause a timeout on the client side, which wouldn't receive the server response in a timely manner and eventually drop the connection.
+If you add the `upload` action to the `execute_sync` configuration key, SFTPGo will try to delete the uploaded file and return an error to the client if the hook fails. A hook is considered failed if the external command completes with a non-zero exit status or the HTTP notification response code is other than `200` (or the HTTP endpoint cannot be reached or times out).
+After a hook failure, the uploaded size is removed from the quota if SFTPGo is able to remove the file.
 
 ## Provider events
 
@@ -83,8 +87,13 @@ The `actions` struct inside the `data_provider` configuration section allows you
 The supported object types are:
 
 - `user`
+- `folder`
+- `group`
 - `admin`
 - `api_key`
+- `share`
+- `event_action`
+- `event_rule`
 
 Actions will not be fired for internal updates, such as the last login or the user quota fields, or after external authentication.
 
@@ -98,7 +107,7 @@ If the `hook` defines a path to an external program, then this program can read
 - `SFTPGO_PROVIDER_TIMESTAMP`, event timestamp as nanoseconds since epoch
 - `SFTPGO_PROVIDER_OBJECT`, object serialized as JSON with sensitive fields removed
 
-Previous global environment variables aren't cleared when the script is called.
+Global environment variables are cleared, for security reasons, when the script is called. You can set additional environment variables in the "command" configuration section.
 The program must finish within 15 seconds.
 
 If the `hook` defines an HTTP URL then this URL will be invoked as HTTP POST. The action, username, ip, object_type and object_name and timestamp are added to the query string, for example `<hook>?action=update&username=admin&ip=127.0.0.1&object_type=user&object_name=user1&timestamp=1633860803249`, and the full object is sent serialized as JSON inside the POST body with sensitive fields removed.

docs/data-retention-hook.md

@@ -8,7 +8,7 @@ If the hook defines an external program it can read the following environment va
 
 - `SFTPGO_DATA_RETENTION_RESULT`, it contains the data retention check result JSON serialized.
 
-Previous global environment variables aren't cleared when the script is called.
+Global environment variables are cleared, for security reasons, when the script is called. You can set additional environment variables in the "command" configuration section.
 The program must finish within 20 seconds.
 
 If the hook defines an HTTP URL then this URL will be invoked as HTTP POST and the POST body contains the data retention check result JSON serialized.

docs/dynamic-user-mod.md

@@ -6,7 +6,7 @@ To enable dynamic user modification, you must set the absolute path of your prog
 The external program can read the following environment variables to get info about the user trying to login:
 
 - `SFTPGO_LOGIND_USER`, it contains the user trying to login serialized as JSON. A JSON serialized user id equal to zero means the user does not exist inside SFTPGo
-- `SFTPGO_LOGIND_METHOD`, possible values are: `password`, `publickey`, `keyboard-interactive`, `TLSCertificate`, `IDP` (external identity provider)
+- `SFTPGO_LOGIND_METHOD`, possible values are: `password`, `publickey`, `keyboard-interactive`, `TLSCertificate`, `IDP` (external identity provider) or empty if the hook is executed after receiving the FTP `USER` command
 - `SFTPGO_LOGIND_IP`, ip address of the user trying to login
 - `SFTPGO_LOGIND_PROTOCOL`, possible values are `SSH`, `FTP`, `DAV`, `HTTP`, `OIDC` (OpenID Connect)
 
@@ -35,7 +35,9 @@ If an error happens while executing the hook then login will be denied.
 "Dynamic user creation or modification" and "External Authentication" are mutually exclusive, they are quite similar, the difference is that "External Authentication" returns an already authenticated user while using "Dynamic users modification" you simply create or update a user. The authentication will be checked inside SFTPGo.
 In other words while using "External Authentication" the external program receives the credentials of the user trying to login (for example the cleartext password) and it needs to validate them. While using "Dynamic users modification" the pre-login program receives the user stored inside the dataprovider (it includes the hashed password if any) and it can modify it, after the modification SFTPGo will check the credentials of the user trying to login.
 
-For SFTPGo users (not admins) authenticating using an external identity provider such as OpenID Connect, the pre-login hook will be executed after a successful authentication against the external IDP so that you can create/update the SFTPGo user matching the one authenticated against the identity provider. This is the only case where the pre-login hook is executed even if an external authentication hook is defined.
+For SFTPGo users (not admins) authenticating using an external identity provider such as OpenID Connect, the pre-login hook will be executed after a successful authentication against the external IDP so that you can create/update the SFTPGo user matching the one authenticated against the identity provider. In this case where the pre-login hook is executed even if an external authentication hook is defined.
+
+If you enable FTP and allow both encrypted and plain text sessions, the pre-login hook is executed after receiving the FTP `USER` command. If you return an SFTPGo user with `ftp_security` set to `1` and the FTP session is not encrypted, it will be terminated. In this case where the pre-login hook is executed even if an external authentication hook is defined.
 
 You can disable the hook on a per-user basis.
 

docs/eventmanager.md

@@ -0,0 +1,71 @@
+# Event Manager
+
+The Event Manager allows an administrator to configure HTTP notifications, commands execution, email notifications and carry out certain server operations based on server events or schedules.
+
+The following actions are supported:
+
+- `HTTP notification`. You can notify an HTTP/S endpoing via GET, POST, PUT methods. You can define custom headers, query parameters and a body for POST and PUT request. Placeholders are supported for username, body, header and query parameter values.
+- `Command execution`. You can launch custom commands passing parameters via environment variables. Placeholders are supported for environment variable values.
+- `Email notification`. Placeholders are supported in subject and body. The email will be sent as plain text. For this action to work you have to configure an SMTP server in the SFTPGo configuration file.
+- `Backup`. A backup will be saved in the configured backup directory. The backup will contain the week day and the hour in the file name.
+- `User quota reset`. The quota used by users will be updated based on current usage.
+- `Folder quota reset`. The quota used by virtual folders will be updated based on current usage.
+- `Transfer quota reset`. The transfer quota values will be reset to `0`.
+- `Data retention check`. You can define per-folder retention policies.
+- `Metadata check`. A metadata check requires a metadata plugin such as [this one](https://github.com/sftpgo/sftpgo-plugin-metadata) and removes the metadata associated to missing items (for example objects deleted outside SFTPGo). A metadata check does nothing is no metadata plugin is installed or external metadata are not supported for a filesystem.
+- `Filesystem`. For these actions, the required permissions are automatically granted. This is the same as executing the actions from an SFTP client and the same restrictions applies. Supported actions:
+  - `Rename`. You can rename one or more files or directories.
+  - `Delete`. You can delete one or more files and directories.
+  - `Create directories`. You can create one or more directories including sub-directories.
+  - `Path exists`. Check if the specified path exists.
+  - `Compress paths`. You can compress (currently as zip) ore or more files and directories.
+
+The following placeholders are supported:
+
+- `{{Name}}`. Username, folder name or admin username for provider events.
+- `{{Event}}`. Event name, for example `upload`, `download` for filesystem events or `add`, `update` for provider events.
+- `{{Status}}`. Status for `upload`, `download` and `ssh_cmd` events. 1 means no error, 2 means a generic error occurred, 3 means quota exceeded error.
+- `{{StatusString}}`. Status as string. Possible values "OK", "KO".
+- `{{ErrorString}}`. Error details. Replaced with an empty string if no errors occur.
+- `{{VirtualPath}}`. Path seen by SFTPGo users, for example `/adir/afile.txt`.
+- `{{FsPath}}`. Full filesystem path, for example `/user/homedir/adir/afile.txt` or `C:/data/user/homedir/adir/afile.txt` on Windows.
+- `{{ObjectName}}`. File/directory name, for example `afile.txt` or provider object name.
+- `{{ObjectType}}`. Object type for provider events: `user`, `group`, `admin`, etc.
+- `{{VirtualTargetPath}}`. Virtual target path for renames.
+- `{{FsTargetPath}}`. Full filesystem target path for renames.
+- `{{FileSize}}`. File size.
+- `{{Protocol}}`. Used protocol, for example `SFTP`, `FTP`.
+- `{{IP}}`. Client IP address.
+- `{{Timestamp}}`. Event timestamp as nanoseconds since epoch.
+- `{{ObjectData}}`. Provider object data serialized as JSON with sensitive fields removed.
+- `{{RetentionReports}}`. Data retention reports as zip compressed CSV files. Supported as email attachment, file path for multipart HTTP request and as single parameter for HTTP requests body. Data retention reports contain details on the number of files deleted and the total size deleted for each folder.
+
+Event rules are based on the premise that an event occours. To each rule you can associate one or more actions.
+The following trigger events are supported:
+
+- `Filesystem events`, for example `upload`, `download` etc.
+- `Provider events`, for example `add`, `update`, `delete` user or other resources.
+- `Schedules`. The scheduler uses UTC time.
+- `IP Blocked`, this event can be generated if you enable the [defender](./defender.md).
+- `Certificate`, this event is generated when a certificate is renewed using the built-in ACME protocol. Both successful and failed renewals are notified.
+
+You can further restrict a rule by specifying additional conditions that must be met before the rule’s actions are taken. For example you can react to uploads only if they are performed by a particular user or using a specified protocol.
+
+Actions such as user quota reset, transfer quota reset, data retention check, folder quota reset and filesystem events are executed for all matching users if the trigger is a schedule or for the affected user if the trigger is a provider event or a filesystem action.
+
+Actions are executed in a sequential order except for sync actions that are executed before the others. For each action associated to a rule you can define the following settings:
+
+- `Stop on failure`, the next action will not be executed if the current one fails.
+- `Failure action`, this action will be executed only if at least another one fails. :warning: Please note that a failure action isn't executed if the event fails, for example if a download fails the main action is executed. The failure action is executed only if one of the non-failure actions associated to a rule fails.
+- `Execute sync`, for upload events, you can execute the action synchronously. Executing an action synchronously means that SFTPGo will not return a result code to the client (which is waiting for it) until your action have completed its execution. If your acion takes a long time to complete this could cause a timeout on the client side, which wouldn't receive the server response in a timely manner and eventually drop the connection.
+
+If you are running multiple SFTPGo instances connected to the same data provider, you can choose whether to allow simultaneous execution for scheduled actions.
+
+Some actions are not supported for some triggers, rules containing incompatible actions are skipped at runtime:
+
+- `Filesystem events`, folder quota reset cannot be executed, we don't have a direct way to get the affected folder.
+- `Provider events`, user quota reset, transfer quota reset, data retention check and filesystem actions can be executed only if  a user is updated. They will be executed for the affected user. Folder quota reset can be executed only for folders. Filesystem actions are not executed for `delete` user events because the actions is executed after the user deletion.
+- `IP Blocked`, user quota reset, folder quota reset, transfer quota reset, data retention check and filesystem actions cannot be executed, we only have an IP.
+- `Certificate`, user quota reset, folder quota reset, transfer quota reset, data retention check and filesystem actions cannot be executed.
+- `Email with attachments` are supported for filesystem events and provider events if a user is added/updated. We need a user to get the files to attach.
+- `HTTP multipart requests with files as attachments` are supported for filesystem events and provider events if a user is added/updated. We need a user to get the files to attach.

docs/external-auth.md

@@ -13,12 +13,12 @@ The external program can read the following environment variables to get info ab
 - `SFTPGO_AUTHD_KEYBOARD_INTERACTIVE`, not empty for keyboard interactive authentication
 - `SFTPGO_AUTHD_TLS_CERT`, TLS client certificate PEM encoded. Not empty for TLS certificate authentication
 
-Previous global environment variables aren't cleared when the script is called. The content of these variables is _not_ quoted. They may contain special characters. They are under the control of a possibly malicious remote user.
+Global environment variables are cleared, for security reasons, when the script is called. You can set additional environment variables in the "command" configuration section.
 The program can inspect the SFTPGo user, if it exists, using the `SFTPGO_AUTHD_USER` environment variable.
 The program must write, on its standard output:
 
 - a valid SFTPGo user serialized as JSON if the authentication succeeds. The user will be added/updated within the defined data provider
-- an empty string, or no response at all, if authentication succeeds and the existing SFTPGo user does not need to be updated. Please note that in versions 2.0.x and earlier an empty response was interpreted as an authentication error
+- an empty string, or no response at all, if authentication succeeds and the existing SFTPGo user does not need to be updated. This means that the credentials already stored in SFTPGo must match those used for the current authentication.
 - a user with an empty username if the authentication fails
 
 If the hook is an HTTP URL then it will be invoked as HTTP POST. The request body will contain a JSON serialized struct with the following fields:

docs/full-configuration.md

@@ -31,6 +31,7 @@ The `serve` command supports the following flags:
 
 - `--config-dir` string. Location of the config dir. This directory is used as the base for files with a relative path, eg. the private keys for the SFTP server or the SQLite database if you use SQLite as data provider. The configuration file, if not explicitly set, is looked for in this dir. We support reading from JSON, TOML, YAML, HCL, envfile and Java properties config files. The default config file name is `sftpgo` and therefore `sftpgo.json`, `sftpgo.yaml` and so on are searched. The default value is the working directory (".") or the value of `SFTPGO_CONFIG_DIR` environment variable.
 - `--config-file` string. This flag explicitly defines the path, name and extension of the config file. If must be an absolute path or a path relative to the configuration directory. The specified file name must have a supported extension (JSON, YAML, TOML, HCL or Java properties). The default value is empty or the value of `SFTPGO_CONFIG_FILE` environment variable.
+- `--grace-time`, integer. Graceful shutdown is an option to initiate a shutdown without abrupt cancellation of the currently ongoing client-initiated transfer sessions. This grace time defines the number of seconds allowed for existing transfers to get completed before shutting down. 0 means disabled. The default value is `0` or the value of `SFTPGO_GRACE_TIME` environment variable. A graceful shutdown is triggered by an interrupt signal or by a service `stop` request on Windows, if a grace time is configured.
 - `--loaddata-from` string. Load users and folders from this file. The file must be specified as absolute path and it must contain a backup obtained using the `dumpdata` REST API or compatible content. The default value is empty or the value of `SFTPGO_LOADDATA_FROM` environment variable.
 - `--loaddata-clean` boolean. Determine if the loaddata-from file should be removed after a successful load. Default `false` or the value of `SFTPGO_LOADDATA_CLEAN` environment variable (1 or `true`, 0 or `false`).
 - `--loaddata-mode`, integer. Restore mode for data to load. 0 means new users are added, existing users are updated. 1 means new users are added, existing users are not modified. Default 1 or the value of `SFTPGO_LOADDATA_MODE` environment variable.
@@ -40,7 +41,7 @@ The `serve` command supports the following flags:
 - `--log-max-age` int. Maximum number of days to retain old log files. Default 28 or the value of `SFTPGO_LOG_MAX_AGE` environment variable. It is unused if `log-file-path` is empty.
 - `--log-max-backups` int. Maximum number of old log files to retain. Default 5 or the value of `SFTPGO_LOG_MAX_BACKUPS` environment variable. It is unused if `log-file-path` is empty.
 - `--log-max-size` int. Maximum size in megabytes of the log file before it gets rotated. Default 10 or the value of `SFTPGO_LOG_MAX_SIZE` environment variable. It is unused if `log-file-path` is empty.
-- `--log-verbose` boolean. Enable verbose logs. Default `true` or the value of `SFTPGO_LOG_VERBOSE` environment variable (1 or `true`, 0 or `false`).
+- `--log-level` string. Set the log level. Supported values: `debug`, `info`, `warn`, `error`. Default `debug` or the value of `SFTPGO_LOG_LEVEL` environment variable.
 - `--log-utc-time` boolean. Enable UTC time for logging. Default `false` or the value of `SFTPGO_LOG_UTC_TIME` environment variable (1 or `true`, 0 or `false`)
 
 Log file can be rotated on demand sending a `SIGUSR1` signal on Unix based systems and using the command `sftpgo service rotatelogs` on Windows.
@@ -57,7 +58,7 @@ The configuration file contains the following sections:
 
 - **"common"**, configuration parameters shared among all the supported protocols
   - `idle_timeout`, integer. Time in minutes after which an idle client will be disconnected. 0 means disabled. Default: 15
-  - `upload_mode` integer. 0 means standard: the files are uploaded directly to the requested path. 1 means atomic: files are uploaded to a temporary path and renamed to the requested path when the client ends the upload. Atomic mode avoids problems such as a web server that serves partial files when the files are being uploaded. In atomic mode, if there is an upload error, the temporary file is deleted and so the requested upload path will not contain a partial file. 2 means atomic with resume support: same as atomic but if there is an upload error, the temporary file is renamed to the requested path and not deleted. This way, a client can reconnect and resume the upload. Default: 0
+  - `upload_mode` integer. 0 means standard: the files are uploaded directly to the requested path. 1 means atomic: files are uploaded to a temporary path and renamed to the requested path when the client ends the upload. Atomic mode avoids problems such as a web server that serves partial files when the files are being uploaded. In atomic mode, if there is an upload error, the temporary file is deleted and so the requested upload path will not contain a partial file. 2 means atomic with resume support: same as atomic but if there is an upload error, the temporary file is renamed to the requested path and not deleted. This way, a client can reconnect and resume the upload. Ignored for cloud-based storage backends (uploads are always atomic and resume is not supported for these backends) and for SFTP backend if buffering is enabled. Default: 0
   - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions. See [Custom Actions](./custom-actions.md) for more details
     - `execute_on`, list of strings. Valid values are `pre-download`, `download`, `pre-upload`, `upload`, `pre-delete`, `delete`, `rename`, `mkdir`, `rmdir`, `ssh_cmd`. Leave empty to disable actions.
     - `execute_sync`, list of strings. Actions, defined in the `execute_on` list above, to be performed synchronously. The `pre-*` actions are always executed synchronously while the other ones are asynchronous. Executing an action synchronously means that SFTPGo will not return a result code to the client (which is waiting for it) until your hook have completed its execution. Leave empty to execute only the defined `pre-*` hook synchronously
@@ -78,6 +79,7 @@ The configuration file contains the following sections:
   - `max_total_connections`, integer. Maximum number of concurrent client connections. 0 means unlimited. Default: 0.
   - `max_per_host_connections`, integer.  Maximum number of concurrent client connections from the same host (IP). If the defender is enabled, exceeding this limit will generate `score_limit_exceeded` events and thus hosts that repeatedly exceed the max allowed connections can be automatically blocked. 0 means unlimited. Default: 20.
   - `whitelist_file`, string. Path to a file containing a list of IP addresses and/or networks to allow. Only the listed IPs/networks can access the configured services, all other client connections will be dropped before they even try to authenticate. The whitelist must be a JSON file with the same structure documented for the [defenders's list](./defender.md). The whitelist can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. Default: "".
+  - `allow_self_connections`, integer. Allow users on this instance to use other users/virtual folders on this instance as storage backend. Enable this setting if you know what you are doing. Set to `1` to enable. Default: `0`.
   - `defender`, struct containing the defender configuration. See [Defender](./defender.md) for more details.
     - `enabled`, boolean. Default `false`.
     - `driver`, string. Supported drivers are `memory` and `provider`. The `provider` driver will use the configured data provider to store defender events and it is supported for `MySQL`, `PostgreSQL` and `CockroachDB` data providers. Using the `provider` driver you can share the defender events among multiple SFTPGO instances. For a single instance the `memory` driver will be much faster. Default: `memory`.
@@ -182,6 +184,7 @@ The configuration file contains the following sections:
     - `proxy_allowed`, list of IP addresses and IP ranges allowed to set client IP proxy header such as `X-Forwarded-For`. Any client IP proxy headers, if set on requests from a connection address not in this list, will be silently ignored. Default: empty.
     - `client_ip_proxy_header`, string. Defines the allowed client IP proxy header such as `X-Forwarded-For`, `X-Real-IP` etc. Default: empty
     - `client_ip_header_depth`, integer. Some client IP headers such as `X-Forwarded-For` can contain multiple IP address, this setting define the position to trust starting from the right. For example if we have: `10.0.0.1,11.0.0.1,12.0.0.1,13.0.0.1` and the depth is `0`, SFTPGo will use `13.0.0.1` as client IP, if depth is `1`, `12.0.0.1` will be used and so on. Default: `0`.
+    - `disable_www_auth_header`, boolean. Set to `true` to not add the WWW-Authenticate header after an authentication failure, only the `401` status code will be sent. Default: `false`.
   - `certificate_file`, string. Certificate for WebDAV over HTTPS. This can be an absolute path or a path relative to the config dir.
   - `certificate_key_file`, string. Private key matching the above certificate. This can be an absolute path or a path relative to the config dir. A certificate and a private key are required to enable HTTPS connections. Certificate and key files can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows.
   - `ca_certificates`, list of strings. Set of root certificate authorities to be used to verify client certificates.
@@ -194,6 +197,9 @@ The configuration file contains the following sections:
     - `exposed_headers`, list of strings.
     - `allow_credentials` boolean.
     - `max_age`, integer.
+    - `options_passthrough`, boolean.
+    - `options_success_status`, integer.
+    - `allow_private_network`, boolean.
   - `cache` struct containing cache configuration for the authenticated users.
     - `enabled`, boolean, set to true to enable user caching. Default: true.
     - `expiration_time`, integer. Expiration time, in minutes, for the cached users. 0 means unlimited. Default: 0.
@@ -201,12 +207,14 @@ The configuration file contains the following sections:
 - **"data_provider"**, the configuration for the data provider
   - `driver`, string. Supported drivers are `sqlite`, `mysql`, `postgresql`, `cockroachdb`, `bolt`, `memory`
   - `name`, string. Database name. For driver `sqlite` this can be the database name relative to the config dir or the absolute path to the SQLite database. For driver `memory` this is the (optional) path relative to the config dir or the absolute path to the provider dump, obtained using the `dumpdata` REST API, to load. This dump will be loaded at startup and can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. The `memory` provider will not modify the provided file so quota usage and last login will not be persisted. If you plan to use a SQLite database over a `cifs` network share (this is not recommended in general) you must use the `nobrl` mount option otherwise you will get the `database is locked` error. Some users reported that the `bolt` provider works fine over `cifs` shares.
-  - `host`, string. Database host. Leave empty for drivers `sqlite`, `bolt` and `memory`
+  - `host`, string. Database host. For `postgresql` and `cockroachdb` drivers you can specify multiple hosts separated by commas. Leave empty for drivers `sqlite`, `bolt` and `memory`
   - `port`, integer. Database port. Leave empty for drivers `sqlite`, `bolt` and `memory`
   - `username`, string. Database user. Leave empty for drivers `sqlite`, `bolt` and `memory`
   - `password`, string. Database password. Leave empty for drivers `sqlite`, `bolt` and `memory`
   - `sslmode`, integer. Used for drivers `mysql` and `postgresql`. 0 disable TLS connections, 1 require TLS, 2 set TLS mode to `verify-ca` for driver `postgresql` and `skip-verify` for driver `mysql`, 3 set TLS mode to `verify-full` for driver `postgresql` and `preferred` for driver `mysql`
   - `root_cert`, string. Path to the root certificate authority used to verify that the server certificate was signed by a trusted CA
+  - `disable_sni`, boolean. Allows to opt out Server Name Indication (SNI) for TLS connections. Default: `false`
+  - `target_session_attrs`, string. This is a `postgresql` and `cockroachdb` specific option. It determines whether the session must have certain properties to be acceptable. It's typically used in combination with multiple host names to select the first acceptable alternative among several hosts. Supported values: `any`, `read-write`, `read-only`, `primary`, `standby`, `prefer-standby`. If empty, `any` is assumed.
   - `client_cert`, string. Path to the client certificate for two-way TLS authentication
   - `client_key`,string. Path to the client key for two-way TLS authentication
   - `connection_string`, string. Provide a custom database connection string. If not empty, this connection string will be used instead of building one using the previous parameters. Leave empty for drivers `bolt` and `memory`
@@ -220,7 +228,7 @@ The configuration file contains the following sections:
   - `users_base_dir`, string. Users default base directory. If no home dir is defined while adding a new user, and this value is a valid absolute path, then the user home dir will be automatically defined as the path obtained joining the base dir and the username
   - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions. See [Custom Actions](./custom-actions.md) for more details
     - `execute_on`, list of strings. Valid values are `add`, `update`, `delete`. `update` action will not be fired for internal updates such as the last login or the user quota fields.
-    - `execute_for`, list of strings. Defines the provider objects that trigger the action. Valid values are `user`, `admin`, `api_key`.
+    - `execute_for`, list of strings. Defines the provider objects that trigger the action. Valid values are `user`, `folder`, `group`, `admin`, `api_key`, `share`, `event_action`, `event_rule`.
     - `hook`, string. Absolute path to the command to execute or HTTP URL to notify.
   - `external_auth_hook`, string. Absolute path to an external program or an HTTP URL to invoke for users authentication. See [External Authentication](./external-auth.md) for more details. Leave empty to disable.
   - `external_auth_scope`, integer. 0 means all supported authentication scopes (passwords, public keys and keyboard interactive). 1 means passwords only. 2 means public keys only. 4 means key keyboard interactive only. 8 means TLS certificate. The flags can be combined, for example 6 means public keys and keyboard interactive
@@ -247,18 +255,20 @@ The configuration file contains the following sections:
   - `update_mode`, integer. Defines how the database will be initialized/updated. 0 means automatically. 1 means manually using the initprovider sub-command.
   - `create_default_admin`, boolean. Before you can use SFTPGo you need to create an admin account. If you open the admin web UI, a setup screen will guide you in creating the first admin account. You can automatically create the first admin account by enabling this setting and setting the environment variables `SFTPGO_DEFAULT_ADMIN_USERNAME` and `SFTPGO_DEFAULT_ADMIN_PASSWORD`. You can also create the first admin by loading initial data. This setting has no effect if an admin account is already found within the data provider. Default `false`.
   - `naming_rules`, integer. Naming rules for usernames, folder and group names. `0` means no rules. `1` means you can use any UTF-8 character. The names are used in URIs for REST API and Web admin. If not set only unreserved URI characters are allowed: ALPHA / DIGIT / "-" / "." / "_" / "~". `2` means names are converted to lowercase before saving/matching and so case insensitive matching is possible. `3` means trimming trailing and leading white spaces before saving/matching. Rules can be combined, for example `3` means both converting to lowercase and allowing any UTF-8 character. Enabling these options for existing installations could be backward incompatible, some users could be unable to login, for example existing users with mixed cases in their usernames. You have to ensure that all existing users respect the defined rules. Default: `1`.
-  - `is_shared`, integer. If the data provider is shared across multiple SFTPGo instances, set this parameter to `1`. `MySQL`, `PostgreSQL` and `CockroachDB` can be shared, this setting is ignored for other data providers. For shared data providers, active transfers are persisted in the database and thus quota checks between ongoing transfers will work cross multiple instances. Password reset requests and OIDC tokens/states are also persisted in the database if the provider is shared. The database table `shared_sessions` is used only to store temporary sessions. In performance critical installations, you might consider using a database-specific optimization, for example you might use an `UNLOGGED` table for PostgreSQL. This optimization in only required in very limited use cases. Default: `0`.
+  - `is_shared`, integer. If the data provider is shared across multiple SFTPGo instances, set this parameter to `1`. `MySQL`, `PostgreSQL` and `CockroachDB` can be shared, this setting is ignored for other data providers. For shared data providers, active transfers are persisted in the database and thus quota checks between ongoing transfers will work cross multiple instances. Password reset requests and OIDC tokens/states are also persisted in the database if the provider is shared. For shared data providers, scheduled event actions are only executed on a single SFTPGo instance by default, you can override this behavior on a per-action basis. The database table `shared_sessions` is used only to store temporary sessions. In performance critical installations, you might consider using a database-specific optimization, for example you might use an `UNLOGGED` table for PostgreSQL. This optimization in only required in very limited use cases. Default: `0`.
+  - `node`, struct. Node-specific configurations to allow inter-node communications. If your provider is shared across multiple nodes, the nodes can exchange information to present a uniform view for node-specific data. The current implementation allows to obtain active connections from all nodes. Nodes connect to each other using the REST API.
+    - `host`, string. IP address or hostname that other nodes can use to connect to this node via REST API. Empty means inter-node communications disabled. Default: empty.
+    - `port`, integer. The port that other nodes can use to connect to this node via REST API. Default: `0`
+    - `proto`, string. Supported values `http` or `https`. For `https` the configurations for http clients is used, so you can, for example, enable mutual TLS authentication. Default: `http`
   - `backups_path`, string. Path to the backup directory. This can be an absolute path or a path relative to the config dir. We don't allow backups in arbitrary paths for security reasons.
-  - `auto_backup`, struct. Defines the configuration for automatic data provider backups. Example: hour `0` and day_of_week `*` means a backup every day at midnight. The backup file name is in the format `backup_<day_of_week>_<hour>.json`, files with the same name will be overwritten. Note, this process will only backup provider data (users, folders, shares, admins, api keys) and will not backup the configuration file and users files.
-    - `enabled`, boolean. Set to `true` to enable automatic backups. Default: `true`.
-    - `hour`, string. Hour as standard cron expression. Allowed values: 0-23. Allowed special characters: asterisk (`*`), slash (`/`), comma (`,`), hyphen (`-`). More info about special characters [here](https://pkg.go.dev/github.com/robfig/cron#hdr-Special_Characters). Default: `0`.
-    - `day_of_week`, string. Day of week as standard cron expression. Allowed values: 0-6 (Sunday to Saturday). Allowed special characters: asterisk (`*`), slash (`/`), comma (`,`), hyphen (`-`),  question mark (`?`). More info about special characters [here](https://pkg.go.dev/github.com/robfig/cron#hdr-Special_Characters). Default: `*`.
 - **"httpd"**, the configuration for the HTTP server used to serve REST API and to expose the built-in web interface
   - `bindings`, list of structs. Each struct has the following fields:
     - `port`, integer. The port used for serving HTTP requests. Default: 8080.
     - `address`, string. Leave blank to listen on all available network interfaces. On *NIX you can specify an absolute path to listen on a Unix-domain socket Default: blank.
     - `enable_web_admin`, boolean. Set to `false` to disable the built-in web admin for this binding. You also need to define `templates_path` and `static_files_path` to use the built-in web admin interface. Default `true`.
     - `enable_web_client`, boolean. Set to `false` to disable the built-in web client for this binding. You also need to define `templates_path` and `static_files_path` to use the built-in web client interface. Default `true`.
+    - `enable_rest_api`, boolean. Set to `false` to disable REST API. Default `true`.
+    - `enabled_login_methods`, integer. Defines the login methods available for the WebAdmin and WebClient UIs. `0` means any configured method: username/password login form and OIDC, if enabled. `1` means OIDC for the WebAdmin UI. `2` means OIDC for the WebClient UI. `4` means login form for the WebAdmin UI. `8` means login form for the WebClient UI. You can combine the values. For example `3` means that you can only login using OIDC on both WebClient and WebAdmin UI. Default: `0`.
     - `enable_https`, boolean. Set to `true` and provide both a certificate and a key file to enable HTTPS connection for this binding. Default `false`.
     - `certificate_file`, string. Binding specific TLS certificate. This can be an absolute path or a path relative to the config dir.
     - `certificate_key_file`, string. Binding specific private key matching the above certificate. This can be an absolute path or a path relative to the config dir. If not set the global ones will be used, if any.
@@ -279,9 +289,12 @@ The configuration file contains the following sections:
       - `client_secret`, string. Defines the application's secret. Default: blank.
       - `redirect_base_url`, string. Defines the base URL to redirect to after OpenID authentication. The suffix `/web/oidc/redirect` will be added to this base URL, adding also the `web_root` if configured. Default: blank.
       - `username_field`, string. Defines the ID token claims field to map to the SFTPGo username. Default: blank.
-      - `role_field`, string. Defines the optional ID token claims field to map to a SFTPGo role. If the defined ID token claims field is set to `admin` the authenticated user is mapped to an SFTPGo admin. You don't need to specify this field if you want to use OpenID only for the Web Client UI. Default: blank.
+      - `scopes`, list of strings. Request the OAuth provider to provide the scope information from an authenticated users. The `openid` scope is mandatory. Default: `"openid", "profile", "email"`.
+      - `role_field`, string. Defines the optional ID token claims field to map to a SFTPGo role. If the defined ID token claims field is set to `admin` the authenticated user is mapped to an SFTPGo admin. You don't need to specify this field if you want to use OpenID only for the Web Client UI. If the field is inside a nested structure, you can use the dot notation to traverse the structures. Default: blank.
       - `implicit_roles`, boolean. If set, the `role_field` is ignored and the SFTPGo role is assumed based on the login link used. Default: `false`.
       - `custom_fields`, list of strings. Custom token claims fields to pass to the pre-login hook. Default: empty.
+      - `insecure_skip_signature_check`, boolean. This setting causes SFTPGo to skip JWT signature validation. It's intended for special cases where providers, such as Azure, use the `none` algorithm. Skipping the signature validation can cause security issues. Default: `false`.
+      - `debug`, boolean. If set, the received id tokens will be logged at debug level. Default: `false`.
     - `security`, struct. Defines security headers to add to HTTP responses and allows to restrict allowed hosts. The following parameters are supported:
       - `enabled`, boolean. Set to `true` to enable security configurations. Default: `false`.
       - `allowed_hosts`, list of strings. Fully qualified domain names that are allowed. An empty list allows any and all host names. Default: empty.
@@ -327,9 +340,13 @@ The configuration file contains the following sections:
     - `exposed_headers`, list of strings.
     - `allow_credentials` boolean.
     - `max_age`, integer.
+    - `options_passthrough`, boolean.
+    - `options_success_status`, integer.
+    - `allow_private_network`, boolean.
   - `setup` struct containing configurations for the initial setup screen
     - `installation_code`, string. If set, this installation code will be required when creating the first admin account. Please note that even if set using an environment variable this field is read at SFTPGo startup and not at runtime. This is not a license key or similar, the purpose here is to prevent anyone who can access to the initial setup screen from creating an admin user. Default: blank.
     - `installation_code_hint`, string. Description for the installation code input field. Default: `Installation code`.
+  - `hide_support_link`, boolean. If set, the link to the [sponsors section](../README.md#sponsors) will not appear on the setup screen page. Default: `false`.
 - **"telemetry"**, the configuration for the telemetry server, more details [below](#telemetry-server)
   - `bind_port`, integer. The port used for serving HTTP requests. Set to 0 to disable HTTP server. Default: 0
   - `bind_address`, string. Leave blank to listen on all available network interfaces. On \*NIX you can specify an absolute path to listen on a Unix-domain socket. Default: `127.0.0.1`
@@ -355,11 +372,13 @@ The configuration file contains the following sections:
     - `url`, string, optional. If not empty, the header will be added only if the request URL starts with the one specified here
 - **command**, configuration for external commands such as program based hooks
   - `timeout`, integer. Timeout specifies a time limit, in seconds, to execute external commands. Valid range: `1-300`. Default: `30`
-  - `env`, list of strings. Additional environment variable to pass to all the external commands. Each entry is of the form `key=value`. Do not use environment variables prefixed with `SFTPGO_` to avoid conflicts with environment variables that SFTPGo hooks can set. Default: empty
+  - `env`, list of strings. Environment variables to pass to all the external commands. Global environment variables are cleared, for security reasons, you have to explicitly set any environment variable such as `PATH` etc. if you need them. Each entry is of the form `key=value`. Do not use environment variables prefixed with `SFTPGO_` to avoid conflicts with environment variables that SFTPGo hooks can set. Default: empty
   - `commands`, list of structs. Allow to customize configuration per-command. Each struct has the following fields:
     - `path`, string. Define the command path as defined in the hook configuration
     - `timeout`, integer. This value overrides the global timeout if set
-    - `env`, list of strings. These values are added to the environment variables defined for all commands, if any
+    - `env`, list of strings. These values are added to the environment variables defined for all commands, if any. Default: empty
+    - `args`, list of strings. Arguments to pass to the command identified by `path`. Default: empty
+    - `hook`, string. If not empty this configuration only apply to the specified hook name. Supported hook names: `fs_actions`, `provider_actions`, `startup`, `post_connect`, `post_disconnect`, `data_retention`, `check_password`, `pre_login`, `post_login`, `external_auth`, `keyboard_interactive`. Default: empty
 - **kms**, configuration for the Key Management Service, more details can be found [here](./kms.md)
   - `secrets`
     - `url`, string. Defines the URI to the KMS service. Default: blank.
@@ -448,6 +467,13 @@ You can select `sha256-simd` setting the environment variable `SFTPGO_MINIO_SHA2
 
  `sha256-simd` is particularly useful if you have an Intel CPU with SHA extensions or an ARM CPU with Cryptography Extensions.
 
+The configuration file can change between different versions and merging your custom settings with the default configuration file, after updating SFTPGo, may be time-consuming. For this reason we suggest to set your custom settings using environment variables. This eliminates the need to merge your changes with the default configuration file after each update, you have to just check that your custom configuration keys still exists.
+
+Setting configuration options from environment variables is natural in Docker/Kubernetes.
+If you install SFTPGo on Linux using the official deb/rpm packages you can set your custom environment variables in the file `/etc/sftpgo/sftpgo.env` (create this file if it does not exist, it is defined as `EnvironmentFile` in the SFTPGo systemd unit).
+SFTPGo also reads files inside the `env.d` directory relative to config dir and then exports the valid variables into environment variables if they are not already set. With this method you can override any configuration options, set environment variables for SFTPGo plugins but you cannot set command flags because these files are read after that SFTPGo starts and the config dir must already be set.
+Of course you can also set environment variables with the method provided by the operating system of your choice.
+
 </details>
 
 <details><summary><font size=5>Binding to privileged ports</font></summary>
@@ -487,6 +513,7 @@ Supported hash algorithms:
 - PBKDF2 sha256 with base64 salt, prefix `$pbkdf2-b64salt-sha256$`
 - MD5 crypt, prefix `$1$`
 - MD5 crypt APR1, prefix `$apr1$`
+- SHA256 crypt, prefix `$5$`
 - SHA512 crypt, prefix `$6$`
 - LDAP MD5, prefix `{MD5}`
 

docs/groups.md

@@ -7,19 +7,21 @@ SFTPGo supports two types of groups:
 - primary groups
 - secondary groups
 
-A user can be a member of a primary group and many secondary groups. Depending on the group type, the settings are inherited differently.
+A user can be a member of a primary group and many secondary and membership groups. Depending on the group type, the settings are inherited differently.
+
+:warning: SFTPGo groups are completely unrelated to system groups. Therefore, it is not necessary to add Linux/Windows groups to use SFTPGo groups.
 
 The following settings are inherited from the primary group:
 
 - home dir, if set for the group will replace the one defined for the user. The `%username%` placeholder is replaced with the username
 - filesystem config, if the provider set for the group is different from the "local provider" will replace the one defined for the user. The `%username%` placeholder is replaced with the username within the defined "prefix", for any vfs, and the "username" for the SFTP filesystem config
-- max sessions, quota size/files, upload/download bandwidth, upload/download/total data transfer, max upload size, external auth cache time: if they are set to `0` for the user they are replaced with the value set for the group, if different from `0`
-- TLS username, check password hook disabled, pre-login hook disabled, external auth hook disabled, filesystem checks disabled, allow API key authentication: if they are not set for the user they are replaced with the value set for the group
+- max sessions, quota size/files, upload/download bandwidth, upload/download/total data transfer, max upload size, external auth cache time, ftp_security, default share expiration: if they are set to `0` for the user they are replaced with the value set for the group, if different from `0`
+- TLS username, check password hook disabled, pre-login hook disabled, external auth hook disabled, filesystem checks disabled, allow API key authentication, anonymous user: if they are not set for the user they are replaced with the value set for the group
 - starting directory, if the user does not have a starting directory set, the value set for the group is used, if any. The `%username%` placeholder is replaced with the username
 
 The following settings are inherited from the primary and secondary groups:
 
-- virtual folders, file patterns, permissions: they are added to the user configuration if the user does not already have a setting for the configured path. The `/` path is ignored for secondary groups. The `%username%` placeholder is replaced with the username within the virtual path, the defined "prefix", for any vfs, and the "username" for the SFTP filesystem config
+- virtual folders, file patterns, permissions: they are added to the user configuration if the user does not already have a setting for the configured path. The `/` path is ignored for secondary groups. The `%username%` placeholder is replaced with the username within the virtual path, the defined "prefix", for any vfs, and the "username" for the SFTP and HTTP filesystem config
 - per-source bandwidth limits
 - per-source data transfer limits
 - allowed/denied IPs
@@ -27,7 +29,7 @@ The following settings are inherited from the primary and secondary groups:
 - two factor auth protocols
 - web client/REST API permissions
 
-The settings from the primary group are always merged first.
+The settings from the primary group are always merged first. no setting is inherited from "membership" groups.
 
 The final settings are a combination of the user settings and the group ones.
 For example you can define the following groups:

docs/howto/README.md

@@ -5,6 +5,7 @@ Here we collect step-to-step tutorials. SFTPGo users are encouraged to contribut
 - [Getting Started](./getting-started.md)
 - [Securing SFTPGo with a free Let's Encrypt TLS Certificate](./lets-encrypt-certificate.md)
 - [Two-factor Authentication](./two-factor-authentication.md)
+- [Event Manager](./eventmanager.md)
 - [SFTPGo as OpenSSH's SFTP subsystem](./openssh-sftp-subsystem.md)
 - [SFTPGo with PostgreSQL data provider and S3 backend](./postgresql-s3.md)
 - [SFTPGo on Windows with Active Directory Integration + Caddy Static File Server](https://www.youtube.com/watch?v=M5UcJI8t4AI)

docs/howto/eventmanager.md

@@ -0,0 +1,96 @@
+# Event Manager
+
+The Event Manager allows an administrator to configure HTTP notifications, commands execution, email notifications and carry out certain server operations based on server events or schedules. More details [here](../eventmanager.md).
+
+Let's see some common use cases.
+
+- [Preliminary Note](#preliminary-note)
+- [Daily backups](#daily-backups)
+- [Automatically create a folder structure](#automatically-create-a-folder-structure)
+- [Upload notifications](#upload-notifications)
+
+## Preliminary Note
+
+We will use email actions in the following paragraphs, so let's assume you have a working SMTP configuration.
+You can adapt the following snippet to configure an SMTP server using environment variables.
+
+```shell
+SFTPGO_SMTP__HOST="your smtp server host"
+SFTPGO_SMTP__FROM="SFTPGo <sftpgo@example.com>"
+SFTPGO_SMTP__USER=sftpgo@example.com
+SFTPGO_SMTP__PASSWORD="your password"
+SFTPGO_SMTP__AUTH_TYPE=1 # change based on what your server supports
+SFTPGO_SMTP__ENCRYPTION=2 # change based on what your server supports
+```
+
+SFTPGo supports several placeholders for event actions. You can see all supported placeholders by clicking on the "info" icon at the top right of the add/update action page.
+
+## Daily backups
+
+You can schedule SFTPGo data backups (users, folders, groups, admins etc.) on a regular basis, such as daily.
+
+From the WebAdmin expand the `Event Manager` section, select `Event actions` and add a new action.
+Create an action named `backup` and set the type to `Backup`.
+
+![Backup action](./img/backup-action.png)
+
+Create another action named `backup notification`, set the type to `Email` and fill the recipient/s.
+As email subject set `Backup {{StatusString}}`. The `{{StatusString}}` placeholder will be expanded to `OK` or `KO`.
+As email body set `Backup done {{ErrorString}}`. The error string will be empty if no errors occur.
+
+![Backup notification action](./img/backup-notification-action.png)
+
+Now select `Event rules` and create a rule named `Daily backup`, select `Schedule` as trigger and schedule a backup at midnight UTC time.
+
+![Daily backup schedule](./img/daily-backup-schedule.png)
+
+As actions select `backup` and `backup notification`.
+
+![Daily backup actions](./img/daily-backup-actions.png)
+
+Done! SFTPGo will make a new backup every day and you will receive an email with the status of the backup. The backup will be saved on the server side in the configured backup directory. The backup files will have names like this `backup_<week day>_<hour>.json`.
+
+## Automatically create a folder structure
+
+Suppose you want to automatically create the folders `in` and `out` when you create new users.
+
+From the WebAdmin expand the `Event Manager` section, select `Event actions` and add a new action.
+Create an action named `create dirs`, with the settings you can see in the following screen.
+
+![Create dirs action](./img/create-dirs-action.png)
+
+Create another action named `create dirs failure notification`, set the type to `Email` and fill the recipient/s.
+As email subject set `Unable to create dirs for user {{ObjectName}}`.
+As email body set `Error: {{ErrorString}}`.
+
+![Create dirs notification](./img/create-dirs-failure-notification.png)
+
+Now select `Event rules` and create a rule named `Create dirs for users`, select `Provider event` as trigger, `add` as provider event and `user` as object filters.
+
+![Create dirs rule](./img/create-dirs-rule.png)
+
+As actions select `create dirs` and `create dirs failure notification`, check `Is failure action` for the notification action.
+This way you will only be notified by email if an error occurs.
+
+![Create dirs rule actions](./img/create-dirs-rule-actions.png)
+
+Done! Create a new user and check that the defined directories are automatically created.
+
+## Upload notifications
+
+Let's see how you can receive an email notification after each upload and, optionally, the uploaded file as well.
+
+From the WebAdmin expand the `Event Manager` section, select `Event actions` and add a new action.
+Create an action named `upload notification`, with the settings you can see in the following screen.
+
+![Upload notification action](./img/upload-notification.png)
+
+You can optionally add the uploaded file as an attachment but note that SFTPGo allows you to attach a maximum of 10MB. Then the action will fail for files bigger than 10MB.
+
+Now select `Event rules` and create a rule named `Upload rule`, select `Filesystem evens` as trigger and `upload` as filesystem event.
+You can also filters events based on protocol, user and group name, filepath shell-like patterns, file size. We omit these additional filters for simplicity.
+
+![Upload rule](./img/upload-rule.png)
+
+As actions, select `upload notification`.
+Done! Try uploading a new file and you will receive the configured email notification.

docs/howto/getting-started.md

@@ -7,19 +7,22 @@ Virtual folders can be private or shared among multiple users, for shared virtua
 
 In this tutorial we explore the main features and concepts using the built-in web admin interface. Advanced users can also use the SFTPGo [REST API](https://sftpgo.stoplight.io/docs/sftpgo/openapi.yaml)
 
-- [Installation](#Installation)
-- [Initial configuration](#Initial-configuration)
-- [Creating users](#Creating-users)
-  - [Creating users with a Cloud Storage backend](#Creating-users-with-a-Cloud-Storage-backend)
-  - [Creating users with a local encrypted backend (Data At Rest Encryption)](#Creating-users-with-a-local-encrypted-backend-Data-At-Rest-Encryption))
-- [Virtual permissions](#Virtual-permissions)
-- [Virtual folders](#Virtual-folders)
-- [Configuration parameters](#Configuration-parameters)
-  - [Use PostgreSQL data provider](#Use-PostgreSQL-data-provider)
-  - [Use MySQL/MariaDB data provider](#Use-MySQLMariaDB-data-provider)
-  - [Use CockroachDB data provider](#Use-CockroachDB-data-provider)
-  - [Enable FTP service](#Enable-FTP-service)
-  - [Enable WebDAV service](#Enable-WebDAV-service)
+- [Installation](#installation)
+- [Initial configuration](#initial-configuration)
+- [Creating users](#creating-users)
+  - [Creating users with a Cloud Storage backend](#creating-users-with-a-cloud-storage-backend)
+  - [Creating users with a local encrypted backend (Data At Rest Encryption)](#creating-users-with-a-local-encrypted-backend-data-at-rest-Encryption)
+- [Virtual permissions](#virtual-permissions)
+- [Virtual folders](#virtual-folders)
+- [Groups](#groups)
+  - [Usage example](#usage-example)
+  - [Simplify user page](#simplify-user-page)
+- [Configuration parameters](#configuration-parameters)
+  - [Use PostgreSQL data provider](#use-postgresql-data-provider)
+  - [Use MySQL/MariaDB data provider](#use-mysqlmariadb-data-provider)
+  - [Use CockroachDB data provider](#use-cockroachdb-data-provider)
+  - [Enable FTP service](#enable-ftp-service)
+  - [Enable WebDAV service](#enable-webdav-service)
 
 ## Installation
 
@@ -60,6 +63,15 @@ Let's create our first local user:
 
 ![Add user](./img/add-user.png)
 
+:warning: Please note that, on Linux, SFTPGo runs using a dedicated system user and group called `sftpgo`, for added security. If you want to be able to use directories outside the `/srv/sftpgo` path you need to set the appropriate system level permissions. For example if you define `/home/username/test` as home dir you have to create this directory yourself, if it doesn't exist, and set the appropriate system-level permissions:
+
+```shell
+sudo mkdir /home/username/test
+sudo chown sftpgo:sftpgo /home/username/test
+```
+
+You also need to make sure that the `sftpgo` system user has at least the read permission for any parent directory, so in the example above `/home/username` and `/home` must not have `0700` permissions.
+
 Now test the new user, we use the `sftp` CLI here, you can use any SFTP client.
 
 ```shell
@@ -172,6 +184,9 @@ as you can see it worked as expected.
 
 ## Virtual folders
 
+A virtual folder is a mapping between a SFTPGo virtual path and a filesystem path outside the user home directory or on a different storage provider.
+Therefore, there is no need to create virtual folders for the users home directory or for directories within the users home directory.
+
 From the web admin interface click `Folders` and then the `+` icon.
 
 ![Add folder](./img/add-folder.png)
@@ -227,6 +242,87 @@ sftp> quit
 
 The last upload failed since we exceeded the number of files quota limit.
 
+## Groups
+
+Using groups simplifies the administration of multiple SFTPGo users: you can assign settings once to a group, instead of multiple times to each individual user.
+
+SFTPGo supports the following types of groups:
+
+- primary groups
+- secondary groups
+- membership groups
+
+A user can be a member of a primary group and many secondary and membership groups. Depending on the group type, the settings are inherited differently, more details [here](../groups.md).
+
+:warning: SFTPGo groups are completely unrelated to system groups. Therefore, it is not necessary to add Linux/Windows groups to use SFTPGo groups.
+
+### Usage example
+
+Suppose you have the following requirements:
+
+- each user must be restricted to a local home directory containing the username as last element of the path, for example `/srv/sftpgo/data/<username>`
+- for each user, the maximum upload size for a single file must be limited to 1GB
+- each user must have an S3 virtual folder available in the path `/s3<username>` and each user can only access a specified "prefix" of the S3 bucket. It must not be able to access other users' files
+- each user must have an S3 virtual folder available in the path `/shared`. This is a folder shared with other users
+- a group of users can only download and list contents in the `/shared` path while another group of users have full access
+
+We can easily meet these requirements by defining two groups.
+
+From the SFTPGo WebAdmin UI, click on `Folders` and then on the `+` icon.
+
+Create a folder named `S3private`.
+Set the storage to `AWS S3 (Compatible)` and fill the required parameters:
+
+- bucket name
+- region
+- credentials: access key and access secret
+
+![S3Private folder](./img/s3-private-folder.png)
+
+The important part is the `Key Prefix`, set it to `users/%username%/`
+
+![S3Private Key Prefix](./img/s3-key-prefix.png)
+
+The placeholder `%username%` will be replaced with the associated username.
+
+Create another folder named `S3shared` with the same settings as `S3private` but this time set the `Key Prefix` to `shared/`.
+The `Key Prefix` has no placeholder, so the folder will operate on a static path that won't change based on the associated user.
+
+Now click on `Groups` and then on the `+` icon and add a group named `Primary`.
+
+Set the `Home Dir` to `/srv/sftpgo/data/%username%`.
+
+![Add group](./img/add-group.png)
+
+As before, the placeholder `%username%` will be replaced with the associated username.
+
+Add the two virtual folders to this group and set the `Max file upload size` to 1GB.
+
+![Primary group settings](./img/primary-group-settings.png)
+
+Add a new group and name it `SharedReadOnly`, in the ACLs section set the permission on the `/shared` path so that read only access is granted.
+
+![Read-only share](./img/read-only-share.png)
+
+The group setup is now complete. We can now create our users and set the primary group to `Primary`.
+For the users who need read-only access to the `/shared` path we also have to set `SharedReadOnly` as a secondary group.
+
+You can now login with any SFTP client like FileZilla, WinSCP etc. and verify that the requirements are met.
+
+### Simplify user page
+
+The add/update user page has many configuration options and can be intimidating for some administrators. We can hide most of the settings and automatically add groups to newly created users. This way the hidden settings are inherited from the automatically assigned groups and therefore administrators can add new users simply by setting the username and credentials.
+
+Click on `Admins` and then on the `+` icon and add an admin named `simply`.
+In the `Groups for users` section set `Primary` as primary group and `SharedReadOnly` as `seconday` group.
+In the `User page preferences` section hide all the sections.
+
+![Simplified admin](./img/simplified-admin.png)
+
+Log in using the newly created administrator and try to add a new user. The user page is simplified as you can see in the following screen.
+
+![Simplified user add](./img/add-user-simplified.png)
+
 ## Configuration parameters
 
 Until now we used the default configuration, to change the global service parameters you have to edit the configuration file, or set appropriate environment variables, and restart SFTPGo to apply the changes.
@@ -235,6 +331,11 @@ A full explanation of all configuration methods can be found [here](./../full-co
 
 The default configuration file is `sftpgo.json` and it can be found within the `/etc/sftpgo` directory if you installed from Linux distro packages. On Windows the configuration file can be found within the `{commonappdata}\SFTPGo` directory where `{commonappdata}` is typically `C:\ProgramData`. SFTPGo also supports reading from TOML and YAML configuration files.
 
+The configuration file can change between different versions and merging your custom settings with the default configuration file, after updating SFTPGo, may be time-consuming. For this reason we suggest to set your custom settings using environment variables.
+If you install SFTPGo on Linux using the official deb/rpm packages you can set your custom environment variables in the file `/etc/sftpgo/sftpgo.env`.
+SFTPGo also reads files inside the `env.d` directory relative to config dir (`/etc/sftpgo/env.d` on Linux and `{commonappdata}\SFTPGo\env.d` on Windows) and then exports the valid variables into environment variables if they are not already set.
+Of course you can also set environment variables with the method provided by the operating system of your choice.
+
 The following snippets assume your are running SFTPGo on Linux but they can be easily adapted for other operating systems.
 
 ### Use PostgreSQL data provider
@@ -249,7 +350,7 @@ grant all privileges on database "sftpgo" to "sftpgo";
 \q
 ```
 
-Open the SFTPGo configuration file, search for the `data_provider` section and change it as follow.
+You can open the SFTPGo configuration file, search for the `data_provider` section and change it as follow.
 
 ```json
   "data_provider": {
@@ -263,12 +364,23 @@ Open the SFTPGo configuration file, search for the `data_provider` section and c
 }
 ```
 
+Alternatively (recommended), you can use environment variables by creating the file `/etc/sftpgo/env.d/postgresql.env` with the following content.
+
+```shell
+SFTPGO_DATA_PROVIDER__DRIVER=postgresql
+SFTPGO_DATA_PROVIDER__NAME=sftpgo
+SFTPGO_DATA_PROVIDER__HOST=127.0.0.1
+SFTPGO_DATA_PROVIDER__PORT=5432
+SFTPGO_DATA_PROVIDER__USERNAME=sftpgo
+SFTPGO_DATA_PROVIDER__PASSWORD=your password here
+```
+
 Confirm that the database connection works by initializing the data provider.
 
 ```shell
 $ sudo su - sftpgo -s /bin/bash -c 'sftpgo initprovider -c /etc/sftpgo'
 2021-05-19T22:21:54.000 INF Initializing provider: "postgresql" config file: "/etc/sftpgo/sftpgo.json"
-2021-05-19T22:21:54.000 INF updating database version: 8 -> 9
+2021-05-19T22:21:54.000 INF updating database schema version: 8 -> 9
 2021-05-19T22:21:54.000 INF Data provider successfully initialized/updated
 ```
 
@@ -303,7 +415,7 @@ MariaDB [(none)]> quit
 Bye
 ```
 
-Open the SFTPGo configuration file, search for the `data_provider` section and change it as follow.
+You can open the SFTPGo configuration file, search for the `data_provider` section and change it as follow.
 
 ```json
   "data_provider": {
@@ -317,12 +429,23 @@ Open the SFTPGo configuration file, search for the `data_provider` section and c
 }
 ```
 
+Alternatively (recommended), you can use environment variables by creating the file `/etc/sftpgo/env.d/mysql.env` with the following content.
+
+```shell
+SFTPGO_DATA_PROVIDER__DRIVER=mysql
+SFTPGO_DATA_PROVIDER__NAME=sftpgo
+SFTPGO_DATA_PROVIDER__HOST=127.0.0.1
+SFTPGO_DATA_PROVIDER__PORT=3306
+SFTPGO_DATA_PROVIDER__USERNAME=sftpgo
+SFTPGO_DATA_PROVIDER__PASSWORD=your password here
+```
+
 Confirm that the database connection works by initializing the data provider.
 
 ```shell
 $ sudo su - sftpgo -s /bin/bash -c 'sftpgo initprovider -c /etc/sftpgo'
 2021-05-19T22:29:30.000 INF Initializing provider: "mysql" config file: "/etc/sftpgo/sftpgo.json"
-2021-05-19T22:29:30.000 INF updating database version: 8 -> 9
+2021-05-19T22:29:30.000 INF updating database schema version: 8 -> 9
 2021-05-19T22:29:30.000 INF Data provider successfully initialized/updated
 ```
 
@@ -347,7 +470,7 @@ We suppose you have installed CockroachDB this way:
 
 ```shell
 sudo su
-export CRDB_VERSION=22.1.0 # set the latest available version here
+export CRDB_VERSION=22.1.8 # set the latest available version here
 wget -qO- https://binaries.cockroachdb.com/cockroach-v${CRDB_VERSION}.linux-amd64.tgz | tar xvz
 cp -i cockroach-v${CRDB_VERSION}.linux-amd64/cockroach /usr/local/bin/
 mkdir -p /usr/local/lib/cockroach
@@ -377,9 +500,8 @@ ExecStart=/usr/local/bin/cockroach start-single-node --certs-dir=/etc/cockroach/
 TimeoutStopSec=60
 Restart=always
 RestartSec=10
-StandardOutput=syslog
-StandardError=syslog
-SyslogIdentifier=cockroach
+StandardOutput=journal
+StandardError=journal
 User=sftpgo
 [Install]
 WantedBy=default.target
@@ -394,7 +516,7 @@ CREATE DATABASE
 Time: 13ms
 ```
 
-Open the SFTPGo configuration file, search for the `data_provider` section and change it as follow.
+You can open the SFTPGo configuration file, search for the `data_provider` section and change it as follow.
 
 ```json
   "data_provider": {
@@ -412,16 +534,30 @@ Open the SFTPGo configuration file, search for the `data_provider` section and c
 }
 ```
 
+Alternatively (recommended), you can use environment variables by creating the file `/etc/sftpgo/env.d/cockroachdb.env` with the following content.
+
+```shell
+SFTPGO_DATA_PROVIDER__DRIVER=cockroachdb
+SFTPGO_DATA_PROVIDER__NAME=sftpgo
+SFTPGO_DATA_PROVIDER__HOST=localhost
+SFTPGO_DATA_PROVIDER__PORT=26257
+SFTPGO_DATA_PROVIDER__USERNAME=root
+SFTPGO_DATA_PROVIDER__SSLMODE=3
+SFTPGO_DATA_PROVIDER__ROOT_CERT="/etc/cockroach/certs/ca.crt"
+SFTPGO_DATA_PROVIDER__CLIENT_CERT="/etc/cockroach/certs/client.root.crt"
+SFTPGO_DATA_PROVIDER__CLIENT_KEY="/etc/cockroach/certs/client.root.key"
+```
+
 Confirm that the database connection works by initializing the data provider.
 
 ```shell
 $ sudo su - sftpgo -s /bin/bash -c 'sftpgo initprovider -c /etc/sftpgo'
 2022-06-02T14:54:04.510 INF Initializing provider: "cockroachdb" config file: "/etc/sftpgo/sftpgo.json"
 2022-06-02T14:54:04.554 INF creating initial database schema, version 15
-2022-06-02T14:54:04.698 INF updating database version: 15 -> 16
-2022-06-02T14:54:07.093 INF updating database version: 16 -> 17
-2022-06-02T14:54:07.672 INF updating database version: 17 -> 18
-2022-06-02T14:54:07.699 INF updating database version: 18 -> 19
+2022-06-02T14:54:04.698 INF updating database schema version: 15 -> 16
+2022-06-02T14:54:07.093 INF updating database schema version: 16 -> 17
+2022-06-02T14:54:07.672 INF updating database schema version: 17 -> 18
+2022-06-02T14:54:07.699 INF updating database schema version: 18 -> 19
 2022-06-02T14:54:07.721 INF Data provider successfully initialized/updated
 ```
 
@@ -442,7 +578,7 @@ Restart SFTPGo to apply the changes.
 
 ### Enable FTP service
 
-Open the SFTPGo configuration file, search for the `ftpd` section and change it as follow.
+You can set the configuration options to enable the FTP service by opening the SFTPGo configuration file, looking for the `ftpd` section and editing it as follows.
 
 ```json
   "ftpd": {
@@ -475,6 +611,12 @@ Open the SFTPGo configuration file, search for the `ftpd` section and change it
   }
 ```
 
+Alternatively (recommended), you can use environment variables by creating the file `/etc/sftpgo/env.d/ftpd.env` with the following content.
+
+```shell
+SFTPGO_FTPD__BINDINGS__0__PORT=2121
+```
+
 Restart SFTPGo to apply the changes. The FTP service is now available on port `2121`.
 
 You can also configure the passive ports range (`50000-50100` by default), these ports must be reachable for passive FTP to work. If your FTP server is on the private network side of a NAT configuration you have to set `force_passive_ip` to your external IP address. You may also need to open the passive port range on your firewall.
@@ -483,7 +625,7 @@ It is recommended that you provide a certificate and key file to expose FTP over
 
 ### Enable WebDAV service
 
-Open the SFTPGo configuration file, search for the `webdavd` section and change it as follow.
+You can set the configuration options to enable the FTP service by opening the SFTPGo configuration file, looking for the `webdavd` section and editing it as follows.
 
 ```json
   "webdavd": {
@@ -500,11 +642,18 @@ Open the SFTPGo configuration file, search for the `webdavd` section and change
         "prefix": "",
         "proxy_allowed": [],
         "client_ip_proxy_header": "",
-        "client_ip_header_depth": 0
+        "client_ip_header_depth": 0,
+        "disable_www_auth_header": false
       }
     ],
     ...
   }
 ```
 
+Alternatively (recommended), you can use environment variables by creating the file `/etc/sftpgo/env.d/webdavd.env` with the following content.
+
+```shell
+SFTPGO_WEBDAVD__BINDINGS__0__PORT=10080
+```
+
 Restart SFTPGo to apply the changes. The WebDAV service is now available on port `10080`. It is recommended that you provide a certificate and key file to expose WebDAV over https.

docs/howto/lets-encrypt-certificate.md

@@ -117,7 +117,7 @@ When the certificate is renewed you should see SFTPGo logs like the following to
 
 ## Obtaining a certificate using the ACME protocol built into SFTPGo
 
-Open the SFTPGo configuration file, search for the `acme` section and change it as follow.
+You can open the SFTPGo configuration file, search for the `acme` section and change it as follow.
 
 ```json
   "acme": {
@@ -138,6 +138,14 @@ Open the SFTPGo configuration file, search for the `acme` section and change it
   }
 ```
 
+Alternatively (recommended), you can use environment variables by creating the file `/etc/sftpgo/env.d/acme.env` with the following content.
+
+```shell
+SFTPGO_ACME__DOMAINS="sftpgo.com"
+SFTPGO_ACME__EMAIL="<you email address here>"
+SFTPGO_ACME__HTTP01_CHALLENGE__WEBROOT="/var/www/sftpgo.com"
+```
+
 Make sure that the `sftpgo` user can write to the `/var/www/sftpgo.com` directory or pre-create the `/var/www/sftpgo.com/.well-known/acme-challenge` directory with the appropriate permissions.
 This directory must be publicly served by your web server.
 
@@ -151,7 +159,7 @@ If this command completes successfully, you are done. The SFTPGo service will ta
 
 ## Enable HTTPS for SFTPGo Web UI and REST API
 
-Open the SFTPGo configuration file, search for the `httpd` section and change it as follow.
+You can open the SFTPGo configuration file, search for the `httpd` section and change it as follow.
 
 ```json
   "httpd": {
@@ -161,17 +169,27 @@ Open the SFTPGo configuration file, search for the `httpd` section and change it
         "address": "",
         "enable_web_admin": true,
         "enable_web_client": true,
+        "enable_rest_api": true,
         "enable_https": true,
         "certificate_file": "/var/lib/sftpgo/certs/sftpgo.com.crt",
         "certificate_key_file": "/var/lib/sftpgo/certs/sftpgo.com.key",
         .....
 ```
 
+Alternatively (recommended), you can use environment variables by creating the file `/etc/sftpgo/env.d/httpd.env` with the following content.
+
+```shell
+SFTPGO_HTTPD__BINDINGS__0__PORT=9443
+SFTPGO_HTTPD__BINDINGS__0__ENABLE_HTTPS=1
+SFTPGO_HTTPD__BINDINGS__0__CERTIFICATE_FILE="/var/lib/sftpgo/certs/sftpgo.com.crt"
+SFTPGO_HTTPD__BINDINGS__0__CERTIFICATE_KEY_FILE="/var/lib/sftpgo/certs/sftpgo.com.key"
+```
+
 Restart SFTPGo to apply the changes. The HTTPS service is now available on port `9443`.
 
 ## Enable HTTPS for WebDAV service
 
-Open the SFTPGo configuration file, search for the `webdavd` section and change it as follow.
+You can open the SFTPGo configuration file, search for the `webdavd` section and change it as follow.
 
 ```json
   "webdavd": {
@@ -185,11 +203,20 @@ Open the SFTPGo configuration file, search for the `webdavd` section and change
         ...
 ```
 
+Alternatively (recommended), you can use environment variables by creating the file `/etc/sftpgo/env.d/webdavd.env` with the following content.
+
+```shell
+SFTPGO_WEBDAVD__BINDINGS__0__PORT=10443
+SFTPGO_WEBDAVD__BINDINGS__0__ENABLE_HTTPS=1
+SFTPGO_WEBDAVD__CERTIFICATE_FILE="/var/lib/sftpgo/certs/sftpgo.com.crt"
+SFTPGO_WEBDAVD__CERTIFICATE_KEY_FILE="/var/lib/sftpgo/certs/sftpgo.com.key"
+```
+
 Restart SFTPGo to apply the changes. WebDAV is now availble over HTTPS on port `10443`.
 
 ## Enable explicit FTP over TLS
 
-Open the SFTPGo configuration file, search for the `ftpd` section and change it as follow.
+You can open the SFTPGo configuration file, search for the `ftpd` section and change it as follow.
 
 ```json
   "ftpd": {
@@ -204,4 +231,13 @@ Open the SFTPGo configuration file, search for the `ftpd` section and change it
         ...
 ```
 
+Alternatively (recommended), you can use environment variables by creating the file `/etc/sftpgo/env.d/ftpd.env` with the following content.
+
+```shell
+SFTPGO_FTPD__BINDINGS__0__PORT=2121
+SFTPGO_FTPD__BINDINGS__0__TLS_MODE=1
+SFTPGO_FTPD__BINDINGS__0__CERTIFICATE_FILE="/var/lib/sftpgo/certs/sftpgo.com.crt"
+SFTPGO_FTPD__BINDINGS__0__CERTIFICATE_KEY_FILE="/var/lib/sftpgo/certs/sftpgo.com.key"
+```
+
 Restart SFTPGo to apply the changes. FTPES service is now available on port `2121` and TLS is required for both control and data connection (`tls_mode` is 1).

docs/howto/openssh-sftp-subsystem.md

@@ -3,6 +3,7 @@
 This tutorial shows how to run SFTPGo as OpenSSH's SFTP subsystem and still use its advanced features.
 
 Please note that when running in SFTP subsystem mode some SFTPGo features are not available, for example SFTPGo cannot limit the concurrent connnections or user sessions, restrict available ciphers etc. In this mode OpenSSH accepts the network connection, handles the SSH handshake and user authentication and then executes a separate SFTPGo process for each SFTP connection.
+Hooks may or may not work because OpenSSH stops the SFTPGo process when the user logs out and some hooks may not have run or ended yet. If you need these features use SFTPGo in standalone mode.
 
 ## Preliminary Note
 

docs/howto/postgresql-s3.md

@@ -144,6 +144,17 @@ Search for the `data_provider` section and change it as follow.
 }
 ```
 
+Alternatively (recommended), you can use environment variables by creating the file `/etc/sftpgo/env.d/postgresql.env` with the following content.
+
+```shell
+SFTPGO_DATA_PROVIDER__DRIVER=postgresql
+SFTPGO_DATA_PROVIDER__NAME="sftpgo.db"
+SFTPGO_DATA_PROVIDER__HOST=127.0.0.1
+SFTPGO_DATA_PROVIDER__PORT=5432
+SFTPGO_DATA_PROVIDER__USERNAME=sftpgo
+SFTPGO_DATA_PROVIDER__PASSWORD=sftpgo_pg_pwd
+```
+
 This way we set the PostgreSQL connection parameters.
 
 If you want to connect to PostgreSQL over a Unix Domain socket you have to set the value `/var/run/postgresql` for the `host` configuration key instead of `127.0.0.1`.
@@ -155,9 +166,9 @@ Next, initialize the data provider with the following command.
 ```shell
 $ sudo su - sftpgo -s /bin/bash -c 'sftpgo initprovider -c /etc/sftpgo'
 2020-10-09T21:07:50.000 INF Initializing provider: "postgresql" config file: "/etc/sftpgo/sftpgo.json"
-2020-10-09T21:07:50.000 INF updating database version: 1 -> 2
-2020-10-09T21:07:50.000 INF updating database version: 2 -> 3
-2020-10-09T21:07:50.000 INF updating database version: 3 -> 4
+2020-10-09T21:07:50.000 INF updating database schema version: 1 -> 2
+2020-10-09T21:07:50.000 INF updating database schema version: 2 -> 3
+2020-10-09T21:07:50.000 INF updating database schema version: 3 -> 4
 2020-10-09T21:07:50.000 INF Data provider successfully initialized/updated
 ```
 

docs/howto/two-factor-authentication.md

@@ -42,6 +42,8 @@ SFTPGo can use 2FA for `HTTP`, `SSH` (SFTP, SCP) and `FTP` protocols. If you pla
     ...
 ```
 
+Or setting the environment variable `SFTPGO_SFTPD__KEYBOARD_INTERACTIVE_AUTHENTICATION=1`.
+
 ## Enable 2FA for admins
 
 Each admin can view/change his/her two-factor authentication by selecting the `Two-Factor Auth` link from the top-right web UI menu.

docs/httpfs.md

@@ -0,0 +1,27 @@
+# HTTP/S storage backend
+
+SFTPGo can use custom storage backend implementations compliant with the REST API documented [here](./../openapi/httpfs.yaml).
+
+:warning: HTTPFs is a work in progress and makes no API stability promises.
+
+The only required parameter is the HTTP/S endpoint that SFTPGo must use to make API calls.
+If you define `http://127.0.0.1:9999/api/v1` as endpoint, SFTPGo will add the API path, for example for the `stat` API it will invoke `http://127.0.0.1:9999/api/v1/stat/{name}`.
+
+You can set a `username` and/or a `password` to instruct SFTPGo to use the basic authentication, or you can set an API key to instruct SFTPGo to add it to each API call in the `X-API-KEY` HTTP header.
+
+Here is a mapping between HTTP response codes and protocol errors:
+
+- `401`, `403` mean permission denied error
+- `404`, means not found error
+- `501`, means not supported error
+- `200`, `201`, mean no error
+- any other response code means a generic error
+
+HTTPFs can also connect to UNIX domain sockets. To use UNIX domain sockets you need to set an endpoint with the following conventions:
+
+- the URL schema can be `http` or `https` as usual.
+- The URL host must be `unix`.
+- The socket path is mandatory and is set using the `socket_path` query parameter. The path must be query escaped.
+- The optional API prefix can be set using the `api_prefix` query parameter. The prefix must be query escaped.
+
+Here is an example endpoint for UNIX domain socket connections: `http://unix?socket_path=%2Ftmp%2Fsftpgofs.sock&api_prefix=%2Fapi%2Fv1`. In this case we are connecting using the `HTTP` protocol to the socket `/tmp/sftpgofs.sock` and we use the `/api/v1` prefix for API URLs.

docs/keyboard-interactive.md

@@ -12,7 +12,7 @@ The external program can read the following environment variables to get info ab
 - `SFTPGO_AUTHD_IP`
 - `SFTPGO_AUTHD_PASSWORD`, this is the hashed password as stored inside the data provider
 
-Previous global environment variables aren't cleared when the script is called. The content of these variables is _not_ quoted. They may contain special characters.
+Global environment variables are cleared, for security reasons, when the script is called. You can set additional environment variables in the "command" configuration section.
 
 The program must write the questions on its standard output, in a single line, using the following struct JSON serialized:
 

docs/oidc.md

@@ -1,6 +1,7 @@
 # OpenID Connect
 
-OpenID Connect integration allows you to map your identity provider users to SFTPGo admins/users and so you can login to SFTPGo Web Client and Web Admin user interfaces using your identity provider.
+OpenID Connect integration allows you to map your identity provider users to SFTPGo admins/users,
+so you can login to SFTPGo Web Client and Web Admin user interfaces, using your own identity provider.
 
 SFTPGo allows to configure per-binding OpenID Connect configurations. The supported configuration parameters are documented within the `oidc` section [here](./full-configuration.md).
 
@@ -41,6 +42,11 @@ Add the following configuration parameters to the SFTPGo configuration file (or
       "client_secret": "jRsmE0SWnuZjP7djBqNq0mrf8QN77j2c",
       "config_url": "http://192.168.1.12:8086/auth/realms/sftpgo",
       "redirect_base_url": "http://192.168.1.50:8080",
+      "scopes": [
+        "openid",
+        "profile",
+        "email"
+      ],
       "username_field": "preferred_username",
       "role_field": "sftpgo_role",
       "implicit_roles": false,
@@ -104,8 +110,12 @@ And the following is an example ID token which allows the SFTPGo user `user1` to
 ```
 
 SFTPGo users (not admins) can be created/updated after successful OpenID authentication by defining a [pre-login hook](./dynamic-user-mod.md).
-You can use the `custom_fields` configuration parameter to define the token claims field names to pass to the pre-login hook, these fields are useful for implementing custom logic when creating/updating the SFTPGo user within the hook.
-For example you can set the field `sftpgo_home_dir` in your identity provider and add it to the `custom_fields` in the SFTPGo configuration like this:
+You can use `scopes` configuration to request additional information (claims) about authenticated users (See your provider's own documentation for more information).
+By default the scopes `"openid", "profile", "email"` are retrieved.
+The `custom_fields` configuration parameter can be used to define claim field names to pass to the pre-login hook,
+these fields can be used e.g. for implementing custom logic when creating/updating the SFTPGo user within the hook.
+For example, if you have created a scope with name `sftpgo` in your identity provider to provide a claim for `sftpgo_home_dir` ,
+then you can add it to the `custom_fields` in the SFTPGo configuration like this:
 
 ```json
 ...
@@ -115,6 +125,7 @@ For example you can set the field `sftpgo_home_dir` in your identity provider an
       "config_url": "http://192.168.1.12:8086/auth/realms/sftpgo",
       "redirect_base_url": "http://192.168.1.50:8080",
       "username_field": "preferred_username",
+      "scopes": [ "openid", "profile", "email", "sftpgo" ],
       "role_field": "sftpgo_role",
       "custom_fields": ["sftpgo_home_dir"]
     }

docs/portable-mode.md

@@ -74,10 +74,20 @@ Flags:
                                         virtual folder identified by this
                                         prefix and its contents
       --gcs-storage-class string
+      --grace-time int                  This grace time defines the number of
+                                        seconds allowed for existing transfers
+                                        to get completed before shutting down.
+                                        A graceful shutdown is triggered by an
+                                        interrupt signal.
+
   -h, --help                            help for portable
   -l, --log-file-path string            Leave empty to disable logging
+      --log-level string                Set the log level.
+                                        Supported values:
+
+                                        debug, info, warn, error.
+                                         (default "debug")
       --log-utc-time                    Use UTC time for logging
-  -v, --log-verbose                     Enable verbose logs
   -p, --password string                 Leave empty to use an auto generated
                                         value
   -g, --permissions strings             User's permissions. "*" means any

docs/post-connect-hook.md

@@ -2,8 +2,6 @@
 
 This hook is executed as soon as a new connection is established. It notifies the connection's IP address and protocol. Based on the received response, the connection is accepted or rejected. Combining this hook with the [Post-login hook](./post-login-hook.md) you can implement your own (even for Protocol) blacklist/whitelist of IP addresses.
 
-Please keep in mind that you can easily configure specialized program such as [Fail2ban](http://www.fail2ban.org/) for brute force protection. Executing a hook for each connection can be heavy.
-
 The `post_connect_hook` can be defined as the absolute path of your program or an HTTP URL.
 
 If the hook defines an external program it can read the following environment variables:
@@ -13,7 +11,7 @@ If the hook defines an external program it can read the following environment va
 
 If the external command completes with a zero exit status the connection will be accepted otherwise rejected.
 
-Previous global environment variables aren't cleared when the script is called.
+Global environment variables are cleared, for security reasons, when the script is called. You can set additional environment variables in the "command" configuration section.
 The program must finish within 20 seconds.
 
 If the hook defines an HTTP URL then this URL will be invoked as HTTP GET with the following query parameters:

docs/post-disconnect-hook.md

@@ -13,7 +13,7 @@ If the hook defines an external program it can read the following environment va
 - `SFTPGO_CONNECTION_USERNAME`, can be empty if the channel is closed before user authentication
 - `SFTPGO_CONNECTION_DURATION`, connection duration in milliseconds
 
-Previous global environment variables aren't cleared when the script is called.
+Global environment variables are cleared, for security reasons, when the script is called. You can set additional environment variables in the "command" configuration section.
 The program must finish within 20 seconds.
 
 If the hook defines an HTTP URL then this URL will be invoked as HTTP GET with the following query parameters:

docs/post-login-hook.md

@@ -2,8 +2,6 @@
 
 This hook is executed after a login or after closing a connection for authentication timeout. Defining an appropriate `post_login_scope` you can get notifications for failed logins, successful logins or both.
 
-Please keep in mind that executing a hook after each login can be heavy.
-
 The `post-login-hook` can be defined as the absolute path of your program or an HTTP URL.
 
 If the hook defines an external program it can reads the following environment variables:
@@ -14,7 +12,7 @@ If the hook defines an external program it can reads the following environment v
 - `SFTPGO_LOGIND_STATUS`, 1 means login OK, 0 login KO
 - `SFTPGO_LOGIND_PROTOCOL`, possible values are `SSH`, `FTP`, `DAV`, `HTTP`, `OIDC` (OpenID Connect)
 
-Previous global environment variables aren't cleared when the script is called.
+Global environment variables are cleared, for security reasons, when the script is called. You can set additional environment variables in the "command" configuration section.
 The program must finish within 20 seconds.
 
 If the hook is an HTTP URL then it will be invoked as HTTP POST. The login method, the used protocol, the ip address and the status of the user are added to the query string, for example `<http_url>?login_method=password&ip=1.2.3.4&protocol=SSH&status=1`.

docs/repo.md

@@ -9,7 +9,7 @@ Supported distributions:
 - Debian 10 "buster"
 - Debian 11 "bullseye"
 
-Import the public key used by the package management system using the following command:
+Import the public key used by the package management system:
 
 ```shell
 curl -sS https://ftp.osuosl.org/pub/sftpgo/apt/gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/sftpgo-archive-keyring.gpg
@@ -39,7 +39,7 @@ sudo apt install sftpgo
 
 The YUM repository supports generic Red Hat based distributions.
 
-Create the SFTPGo repository using the following command:
+Create the SFTPGo repository:
 
 ```shell
 ARCH=`uname -m`

docs/rest-api.md

@@ -20,6 +20,8 @@ If you define multiple bindings, each binding will sign JWT tokens with a differ
 
 If, instead, you want to use a persistent signing key for JWT tokens, you can define a signing passphrase via configuration file or environment variable.
 
+REST API can be disabled within the `httpd` configuration via the `enable_rest_api` key.
+
 You can create other administrator and assign them the following permissions:
 
 - add users
@@ -35,8 +37,11 @@ You can create other administrator and assign them the following permissions:
 - manage API keys
 - manage system
 - manage admins
+- manage groups
 - manage data retention
+- manage metadata
 - view events
+- manage event rules
 
 You can also restrict administrator access based on the source IP address. If you are running SFTPGo behind a reverse proxy you need to allow both the proxy IP address and the real client IP.
 

docs/service.md

@@ -6,9 +6,9 @@ Run the following instructions from the directory that contains the sftpgo binar
 
 ## Linux
 
-The easiest way to run SFTPGo as a service is to download and install the pre-compiled deb/rpm package or use one of the Arch Linux PKGBUILDs we maintain.
+The easiest way to run SFTPGo as a service is to use the [deb/rpm repos](./repo.md) or download and install a pre-compiled deb/rpm package or use one of the Arch Linux PKGBUILDs we maintain.
 
-This section describes the procedure to use if you prefer to build SFTPGo yourself or if you want to download and configure a pre-built release as tar.
+This section describes the manual procedure.
 
 A `systemd` sample [service](../init/sftpgo.service "systemd service") can be found inside the source tree.
 
@@ -67,6 +67,10 @@ sudo /usr/bin/sftpgo gen man -d /usr/share/man/man1
 
 ## macOS
 
+The easiest way to run SFTPGo as a service is to install it from the Homebrew [Formula](https://formulae.brew.sh/formula/sftpgo).
+
+This section describes the procedure to use if you prefer to build SFTPGo yourself or if you want to download and configure a pre-built release as tar.
+
 For macOS, a `launchd` sample [service](../init/com.github.drakkan.sftpgo.plist "launchd plist") can be found inside the source tree. The `launchd` plist assumes that SFTPGo has `/usr/local/opt/sftpgo` as base directory.
 
 Here are some basic instructions to run SFTPGo as service, please run the following commands from the directory where you downloaded SFTPGo:

docs/webdav.md

@@ -29,6 +29,8 @@ Know issues:
 - if a file or a directory cannot be accessed, for example due to OS permissions issues or because a mapped path for a virtual folder is a missing, it will be omitted from the directory listing. If there is a different error then the whole directory listing will fail. This behavior is different from SFTP/FTP where you will be able to see the problematic file/directory in the directory listing, you will only get an error if you try to access it
 - if you use the native Windows client please check its usage and pay particular attention to the [registry settings](https://docs.microsoft.com/en-us/iis/publish/using-webdav/using-the-webdav-redirector#webdav-redirector-registry-settings). The default file size limit is 50MB and if you don't configure SFTPGo to use HTTPS you have to set `BasicAuthLevel` to `2`
 
-We plan to add [Dead Properties](https://tools.ietf.org/html/rfc4918#section-3) support in future releases. We need a design decision here, probably the best solution is to store dead properties inside the data provider but this could increase a lot its size. Alternately we could store them on disk for local filesystem and add as metadata for Cloud Storage, this means that we need to do a separate `HEAD` request to retrieve dead properties for an S3 file. For big folders will do a lot of requests to the Cloud Provider, I don't like this solution. Another option is to expose a hook and allow you to implement `dead properties` outside SFTPGo.
+SFTPGo has a minimal implementation for [Dead Properties](https://tools.ietf.org/html/rfc4918#section-3). We support setting the last modification time and we return the value in the "live" properties, so basically we don't store anything.
+
+To properly support dead properties we need a design decision, probably the best solution is to write a plugin and store them inside a supported data provider.
 
 If you find any other quirks or problems please let us know opening a GitHub issue, thank you!

go.mod

@@ -1,163 +1,168 @@
 module github.com/drakkan/sftpgo/v2
 
-go 1.18
+go 1.19
 
 require (
-	cloud.google.com/go/storage v1.22.1
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1
+	cloud.google.com/go/storage v1.27.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1
 	github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
 	github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387
-	github.com/aws/aws-sdk-go-v2 v1.16.5
-	github.com/aws/aws-sdk-go-v2/config v1.15.10
-	github.com/aws/aws-sdk-go-v2/credentials v1.12.5
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.15
-	github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.6
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.26.11
-	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.10
-	github.com/aws/aws-sdk-go-v2/service/sts v1.16.7
-	github.com/cockroachdb/cockroach-go/v2 v2.2.12
-	github.com/coreos/go-oidc/v3 v3.2.0
+	github.com/aws/aws-sdk-go-v2 v1.17.0
+	github.com/aws/aws-sdk-go-v2/config v1.17.9
+	github.com/aws/aws-sdk-go-v2/credentials v1.12.22
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.18
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.36
+	github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.20
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.29.0
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.3
+	github.com/aws/aws-sdk-go-v2/service/sts v1.17.0
+	github.com/cockroachdb/cockroach-go/v2 v2.2.16
+	github.com/coreos/go-oidc/v3 v3.4.0
 	github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001
-	github.com/fclairamb/ftpserverlib v0.18.1-0.20220515214847-f96d31ec626e
-	github.com/fclairamb/go-log v0.3.0
-	github.com/go-acme/lego/v4 v4.7.0
-	github.com/go-chi/chi/v5 v5.0.8-0.20220512131524-9e71a0d4b3d6
+	github.com/fclairamb/ftpserverlib v0.20.1-0.20221012093027-95be4ae0c9a6
+	github.com/fclairamb/go-log v0.4.1
+	github.com/go-acme/lego/v4 v4.9.0
+	github.com/go-chi/chi/v5 v5.0.8-0.20221018120124-e5529d9db4d3
 	github.com/go-chi/jwtauth/v5 v5.0.2
-	github.com/go-chi/render v1.0.1
+	github.com/go-chi/render v1.0.2
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/golang/mock v1.6.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/google/uuid v1.3.0
 	github.com/grandcat/zeroconf v1.0.0
-	github.com/hashicorp/go-hclog v1.2.1
-	github.com/hashicorp/go-plugin v1.4.4
+	github.com/hashicorp/go-hclog v1.3.1
+	github.com/hashicorp/go-plugin v1.4.5
 	github.com/hashicorp/go-retryablehttp v0.7.1
+	github.com/jackc/pgx/v5 v5.0.4-0.20221022150232-3e825ec8982f
 	github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126
-	github.com/klauspost/compress v1.15.6
+	github.com/klauspost/compress v1.15.11
 	github.com/lestrrat-go/jwx v1.2.25
-	github.com/lib/pq v1.10.6
 	github.com/lithammer/shortuuid/v3 v3.0.7
-	github.com/mattn/go-sqlite3 v1.14.13
+	github.com/mattn/go-sqlite3 v1.14.15
 	github.com/mhale/smtpd v0.8.0
 	github.com/minio/sio v0.3.0
 	github.com/otiai10/copy v1.7.0
 	github.com/pires/go-proxyproto v0.6.2
-	github.com/pkg/sftp v1.13.5-0.20220303113417-dcfc1d5e4162
+	github.com/pkg/sftp v1.13.6-0.20221020054726-e4133ab7e9bd
 	github.com/pquerna/otp v1.3.0
-	github.com/prometheus/client_golang v1.12.2
+	github.com/prometheus/client_golang v1.13.0
 	github.com/robfig/cron/v3 v3.0.1
-	github.com/rs/cors v1.8.2
+	github.com/rs/cors v1.8.3-0.20220619195839-da52b0701de5
 	github.com/rs/xid v1.4.0
-	github.com/rs/zerolog v1.27.0
-	github.com/sftpgo/sdk v0.1.1
-	github.com/shirou/gopsutil/v3 v3.22.5
-	github.com/spf13/afero v1.8.2
-	github.com/spf13/cobra v1.4.0
-	github.com/spf13/viper v1.12.0
-	github.com/stretchr/testify v1.7.2
-	github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
-	github.com/unrolled/secure v1.10.0
+	github.com/rs/zerolog v1.28.0
+	github.com/sftpgo/sdk v0.1.2
+	github.com/shirou/gopsutil/v3 v3.22.9
+	github.com/spf13/afero v1.9.2
+	github.com/spf13/cobra v1.6.0
+	github.com/spf13/viper v1.13.0
+	github.com/stretchr/testify v1.8.0
+	github.com/studio-b12/gowebdav v0.0.0-20221015232716-17255f2e7423
+	github.com/subosito/gotenv v1.4.1
+	github.com/unrolled/secure v1.13.0
 	github.com/wagslane/go-password-validator v0.3.0
-	github.com/xhit/go-simple-mail/v2 v2.11.0
+	github.com/xhit/go-simple-mail/v2 v2.12.0
 	github.com/yl2chen/cidranger v1.0.3-0.20210928021809-d1cb2c52f37a
 	go.etcd.io/bbolt v1.3.6
 	go.uber.org/automaxprocs v1.5.1
-	gocloud.dev v0.25.0
-	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e
-	golang.org/x/net v0.0.0-20220607020251-c690dde0001d
-	golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb
-	golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68
-	golang.org/x/time v0.0.0-20220609170525-579cf78fd858
-	google.golang.org/api v0.83.0
+	gocloud.dev v0.27.0
+	golang.org/x/crypto v0.1.0
+	golang.org/x/net v0.1.0
+	golang.org/x/oauth2 v0.1.0
+	golang.org/x/sys v0.1.0
+	golang.org/x/time v0.1.0
+	google.golang.org/api v0.100.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
 require (
-	cloud.google.com/go v0.102.0 // indirect
-	cloud.google.com/go/compute v1.6.1 // indirect
-	cloud.google.com/go/iam v0.3.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.11.8 // indirect
-	github.com/aws/smithy-go v1.11.3 // indirect
+	cloud.google.com/go v0.104.0 // indirect
+	cloud.google.com/go/compute v1.10.0 // indirect
+	cloud.google.com/go/iam v0.5.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect
+	github.com/ajg/form v1.5.1 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.24 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.18 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.25 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.11.24 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.7 // indirect
+	github.com/aws/smithy-go v1.13.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boombuler/barcode v1.0.1 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 // indirect
+	github.com/coreos/go-systemd/v22 v22.4.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
 	github.com/fatih/color v1.13.0 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-test/deep v1.0.8 // indirect
-	github.com/goccy/go-json v0.9.7 // indirect
+	github.com/goccy/go-json v0.9.11 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.8 // indirect
-	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
-	github.com/googleapis/go-type-adapters v1.0.0 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.6.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/hashicorp/yamux v0.1.1 // indirect
+	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.1.2 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.1 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
 	github.com/lestrrat-go/iter v1.0.2 // indirect
 	github.com/lestrrat-go/option v1.0.0 // indirect
-	github.com/lufia/plan9stats v0.0.0-20220517141722-cf486979b281 // indirect
+	github.com/lib/pq v1.10.7 // indirect
+	github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/miekg/dns v1.1.49 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
+	github.com/miekg/dns v1.1.50 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.34.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.4.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.5.0 // indirect
 	github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.11 // indirect
-	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
+	golang.org/x/mod v0.6.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/tools v0.2.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac // indirect
-	google.golang.org/grpc v1.47.0 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
-	gopkg.in/ini.v1 v1.66.6 // indirect
+	google.golang.org/genproto v0.0.0-20221018160656-63c7b68cfc55 // indirect
+	google.golang.org/grpc v1.50.1 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
@@ -165,6 +170,6 @@ require (
 
 replace (
 	github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9
-	golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20220527053356-5e1caf8ed0e1
-	golang.org/x/net => github.com/drakkan/net v0.0.0-20220610161024-1d26a897c1a1
+	golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20221020054403-a265c1cba3cb
+	golang.org/x/net => github.com/drakkan/net v0.0.0-20221020052826-79457e688bf9
 )

httpd/api_defender.go

@@ -1,162 +0,0 @@
-package httpd
-
-import (
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"net"
-	"net/http"
-	"time"
-
-	"github.com/go-chi/render"
-
-	"github.com/drakkan/sftpgo/v2/common"
-	"github.com/drakkan/sftpgo/v2/dataprovider"
-	"github.com/drakkan/sftpgo/v2/util"
-)
-
-func getDefenderHosts(w http.ResponseWriter, r *http.Request) {
-	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-	hosts, err := common.GetDefenderHosts()
-	if err != nil {
-		sendAPIResponse(w, r, err, "", getRespStatus(err))
-		return
-	}
-	if hosts == nil {
-		render.JSON(w, r, make([]dataprovider.DefenderEntry, 0))
-		return
-	}
-	render.JSON(w, r, hosts)
-}
-
-func getDefenderHostByID(w http.ResponseWriter, r *http.Request) {
-	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-	ip, err := getIPFromID(r)
-	if err != nil {
-		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
-		return
-	}
-	host, err := common.GetDefenderHost(ip)
-	if err != nil {
-		sendAPIResponse(w, r, err, "", getRespStatus(err))
-		return
-	}
-	render.JSON(w, r, host)
-}
-
-func deleteDefenderHostByID(w http.ResponseWriter, r *http.Request) {
-	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-	ip, err := getIPFromID(r)
-	if err != nil {
-		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
-		return
-	}
-	if !common.DeleteDefenderHost(ip) {
-		sendAPIResponse(w, r, nil, "Not found", http.StatusNotFound)
-		return
-	}
-
-	sendAPIResponse(w, r, nil, "OK", http.StatusOK)
-}
-
-func getBanTime(w http.ResponseWriter, r *http.Request) {
-	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-	ip := r.URL.Query().Get("ip")
-	err := validateIPAddress(ip)
-	if err != nil {
-		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
-		return
-	}
-
-	banStatus := make(map[string]*string)
-
-	banTime, err := common.GetDefenderBanTime(ip)
-	if err != nil {
-		if _, ok := err.(*util.RecordNotFoundError); ok {
-			banTime = nil
-		} else {
-			sendAPIResponse(w, r, err, "", getRespStatus(err))
-			return
-		}
-	}
-	var banTimeString *string
-	if banTime != nil {
-		rfc3339String := banTime.UTC().Format(time.RFC3339)
-		banTimeString = &rfc3339String
-	}
-
-	banStatus["date_time"] = banTimeString
-	render.JSON(w, r, banStatus)
-}
-
-func getScore(w http.ResponseWriter, r *http.Request) {
-	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-	ip := r.URL.Query().Get("ip")
-	err := validateIPAddress(ip)
-	if err != nil {
-		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
-		return
-	}
-
-	score, err := common.GetDefenderScore(ip)
-	if err != nil {
-		if _, ok := err.(*util.RecordNotFoundError); ok {
-			score = 0
-		} else {
-			sendAPIResponse(w, r, err, "", getRespStatus(err))
-			return
-		}
-	}
-
-	scoreStatus := make(map[string]int)
-	scoreStatus["score"] = score
-
-	render.JSON(w, r, scoreStatus)
-}
-
-func unban(w http.ResponseWriter, r *http.Request) {
-	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-
-	var postBody map[string]string
-	err := render.DecodeJSON(r.Body, &postBody)
-	if err != nil {
-		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
-		return
-	}
-
-	ip := postBody["ip"]
-	err = validateIPAddress(ip)
-	if err != nil {
-		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
-		return
-	}
-
-	if common.DeleteDefenderHost(ip) {
-		sendAPIResponse(w, r, nil, "OK", http.StatusOK)
-	} else {
-		sendAPIResponse(w, r, nil, "Not found", http.StatusNotFound)
-	}
-}
-
-func getIPFromID(r *http.Request) (string, error) {
-	decoded, err := hex.DecodeString(getURLParam(r, "id"))
-	if err != nil {
-		return "", errors.New("invalid host id")
-	}
-	ip := string(decoded)
-	err = validateIPAddress(ip)
-	if err != nil {
-		return "", err
-	}
-	return ip, nil
-}
-
-func validateIPAddress(ip string) error {
-	if ip == "" {
-		return errors.New("ip address is required")
-	}
-	if net.ParseIP(ip) == nil {
-		return fmt.Errorf("ip address %#v is not valid", ip)
-	}
-	return nil
-}

httpd/api_metadata.go

@@ -1,110 +0,0 @@
-package httpd
-
-import (
-	"fmt"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/go-chi/render"
-
-	"github.com/drakkan/sftpgo/v2/dataprovider"
-	"github.com/drakkan/sftpgo/v2/logger"
-	"github.com/drakkan/sftpgo/v2/util"
-)
-
-var (
-	activeMetadataChecks metadataChecks
-)
-
-type metadataCheck struct {
-	// Username to which the metadata check refers
-	Username string `json:"username"`
-	// check start time as unix timestamp in milliseconds
-	StartTime int64 `json:"start_time"`
-}
-
-// metadataChecks holds the active metadata checks
-type metadataChecks struct {
-	sync.RWMutex
-	checks []metadataCheck
-}
-
-func (c *metadataChecks) get() []metadataCheck {
-	c.RLock()
-	defer c.RUnlock()
-
-	checks := make([]metadataCheck, len(c.checks))
-	copy(checks, c.checks)
-
-	return checks
-}
-
-func (c *metadataChecks) add(username string) bool {
-	c.Lock()
-	defer c.Unlock()
-
-	for idx := range c.checks {
-		if c.checks[idx].Username == username {
-			return false
-		}
-	}
-
-	c.checks = append(c.checks, metadataCheck{
-		Username:  username,
-		StartTime: util.GetTimeAsMsSinceEpoch(time.Now()),
-	})
-
-	return true
-}
-
-func (c *metadataChecks) remove(username string) bool {
-	c.Lock()
-	defer c.Unlock()
-
-	for idx := range c.checks {
-		if c.checks[idx].Username == username {
-			lastIdx := len(c.checks) - 1
-			c.checks[idx] = c.checks[lastIdx]
-			c.checks = c.checks[:lastIdx]
-			return true
-		}
-	}
-
-	return false
-}
-
-func getMetadataChecks(w http.ResponseWriter, r *http.Request) {
-	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-	render.JSON(w, r, activeMetadataChecks.get())
-}
-
-func startMetadataCheck(w http.ResponseWriter, r *http.Request) {
-	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-
-	user, err := dataprovider.GetUserWithGroupSettings(getURLParam(r, "username"))
-	if err != nil {
-		sendAPIResponse(w, r, err, "", getRespStatus(err))
-		return
-	}
-	if !activeMetadataChecks.add(user.Username) {
-		sendAPIResponse(w, r, err, fmt.Sprintf("Another check is already in progress for user %#v", user.Username),
-			http.StatusConflict)
-		return
-	}
-	go doMetadataCheck(user) //nolint:errcheck
-
-	sendAPIResponse(w, r, err, "Check started", http.StatusAccepted)
-}
-
-func doMetadataCheck(user dataprovider.User) error {
-	defer activeMetadataChecks.remove(user.Username)
-
-	err := user.CheckMetadataConsistency()
-	if err != nil {
-		logger.Warn(logSender, "", "error checking metadata for user %#v: %v", user.Username, err)
-		return err
-	}
-	logger.Debug(logSender, "", "metadata check completed for user: %#v", user.Username)
-	return nil
-}

internal/acme/account.go

@@ -1,3 +1,17 @@
+// Copyright (C) 2019-2022  Nicola Murino
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, version 3.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 package acme
 
 import (

internal/acme/acme.go

@@ -1,3 +1,17 @@
+// Copyright (C) 2019-2022  Nicola Murino
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, version 3.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 // Package acme provides automatic access to certificates from Let's Encrypt and any other ACME-based CA
 // The code here is largely coiped from https://github.com/go-acme/lego/tree/master/cmd
 // This package is intended to provide basic functionality for obtaining and renewing certificates
@@ -31,13 +45,14 @@ import (
 	"github.com/go-acme/lego/v4/registration"
 	"github.com/robfig/cron/v3"
 
-	"github.com/drakkan/sftpgo/v2/ftpd"
-	"github.com/drakkan/sftpgo/v2/httpd"
-	"github.com/drakkan/sftpgo/v2/logger"
-	"github.com/drakkan/sftpgo/v2/telemetry"
-	"github.com/drakkan/sftpgo/v2/util"
-	"github.com/drakkan/sftpgo/v2/version"
-	"github.com/drakkan/sftpgo/v2/webdavd"
+	"github.com/drakkan/sftpgo/v2/internal/common"
+	"github.com/drakkan/sftpgo/v2/internal/ftpd"
+	"github.com/drakkan/sftpgo/v2/internal/httpd"
+	"github.com/drakkan/sftpgo/v2/internal/logger"
+	"github.com/drakkan/sftpgo/v2/internal/telemetry"
+	"github.com/drakkan/sftpgo/v2/internal/util"
+	"github.com/drakkan/sftpgo/v2/internal/version"
+	"github.com/drakkan/sftpgo/v2/internal/webdavd"
 )
 
 const (
@@ -547,6 +562,24 @@ func (c *Configuration) getCertificates() error {
 	return nil
 }
 
+func (c *Configuration) notifyCertificateRenewal(domain string, err error) {
+	if domain == "" {
+		domain = strings.Join(c.Domains, ",")
+	}
+	params := common.EventParams{
+		Name:      domain,
+		Event:     "Certificate renewal",
+		Timestamp: time.Now().UnixNano(),
+	}
+	if err != nil {
+		params.Status = 2
+		params.AddError(err)
+	} else {
+		params.Status = 1
+	}
+	common.HandleCertificateEvent(params)
+}
+
 func (c *Configuration) renewCertificates() error {
 	lockTime, err := c.getLockTime()
 	if err != nil {
@@ -559,22 +592,28 @@ func (c *Configuration) renewCertificates() error {
 	}
 	err = c.setLockTime()
 	if err != nil {
+		c.notifyCertificateRenewal("", err)
 		return err
 	}
 	account, client, err := c.setup()
 	if err != nil {
+		c.notifyCertificateRenewal("", err)
 		return err
 	}
 	if account.Registration == nil {
 		acmeLog(logger.LevelError, "cannot renew certificates, your account is not registered")
-		return fmt.Errorf("cannot renew certificates, your account is not registered")
+		err = errors.New("cannot renew certificates, your account is not registered")
+		c.notifyCertificateRenewal("", err)
+		return err
 	}
 	var errRenew error
 	needReload := false
 	for _, domain := range c.Domains {
 		certificates, err := c.loadCertificatesForDomain(domain)
 		if err != nil {
-			return err
+			c.notifyCertificateRenewal(domain, err)
+			errRenew = err
+			continue
 		}
 		cert := certificates[0]
 		if !c.needRenewal(cert, domain) {
@@ -582,8 +621,10 @@ func (c *Configuration) renewCertificates() error {
 		}
 		err = c.obtainAndSaveCertificate(client, domain)
 		if err != nil {
+			c.notifyCertificateRenewal(domain, err)
 			errRenew = err
 		} else {
+			c.notifyCertificateRenewal(domain, nil)
 			needReload = true
 		}
 	}

internal/bundle/bundle.go

@@ -0,0 +1,65 @@
+// Copyright (C) 2019-2022  Nicola Murino
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, version 3.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//go:build bundle
+// +build bundle
+
+package bundle
+
+import (
+	"embed"
+	"fmt"
+	"io/fs"
+	"net/http"
+
+	"github.com/drakkan/sftpgo/v2/internal/version"
+)
+
+func init() {
+	version.AddFeature("+bundle")
+}
+
+//go:embed templates/*
+var templatesFs embed.FS
+
+//go:embed static/*
+var staticFs embed.FS
+
+//go:embed openapi/*
+var openapiFs embed.FS
+
+// GetTemplatesFs returns the embedded filesystem with the SFTPGo templates
+func GetTemplatesFs() embed.FS {
+	return templatesFs
+}
+
+// GetStaticFs return the http Filesystem with the embedded static files
+func GetStaticFs() http.FileSystem {
+	fsys, err := fs.Sub(staticFs, "static")
+	if err != nil {
+		err = fmt.Errorf("unable to get embedded filesystem for static files: %w", err)
+		panic(err)
+	}
+	return http.FS(fsys)
+}
+
+// GetOpenAPIFs return the http Filesystem with the embedded static files
+func GetOpenAPIFs() http.FileSystem {
+	fsys, err := fs.Sub(openapiFs, "openapi")
+	if err != nil {
+		err = fmt.Errorf("unable to get embedded filesystem for OpenAPI files: %w", err)
+		panic(err)
+	}
+	return http.FS(fsys)
+}

internal/cmd/acme.go

@@ -1,3 +1,17 @@
+// Copyright (C) 2019-2022  Nicola Murino
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, version 3.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 package cmd
 
 import (
@@ -6,10 +20,10 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/spf13/cobra"
 
-	"github.com/drakkan/sftpgo/v2/acme"
-	"github.com/drakkan/sftpgo/v2/config"
-	"github.com/drakkan/sftpgo/v2/logger"
-	"github.com/drakkan/sftpgo/v2/util"
+	"github.com/drakkan/sftpgo/v2/internal/acme"
+	"github.com/drakkan/sftpgo/v2/internal/config"
+	"github.com/drakkan/sftpgo/v2/internal/logger"
+	"github.com/drakkan/sftpgo/v2/internal/util"
 )
 
 var (
@@ -26,13 +40,13 @@ Certificates are saved in the configured "certs_path".
 After this initial step, the certificates are automatically checked and
 renewed by the SFTPGo service
 `,
-		Run: func(cmd *cobra.Command, args []string) {
+		Run: func(_ *cobra.Command, _ []string) {
 			logger.DisableLogger()
 			logger.EnableConsoleLogger(zerolog.DebugLevel)
 			configDir = util.CleanDirInput(configDir)
 			err := config.LoadConfig(configDir, configFile)
 			if err != nil {
-				logger.ErrorToConsole("Unable to initialize data provider, config load error: %v", err)
+				logger.ErrorToConsole("Unable to initialize ACME, config load error: %v", err)
 				return
 			}
 			acmeConfig := config.GetACMEConfig()

internal/cmd/awscontainer.go

@@ -1,3 +1,17 @@
+// Copyright (C) 2019-2022  Nicola Murino
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, version 3.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 //go:build awscontainer
 // +build awscontainer
 

internal/cmd/awscontainer_disabled.go

@@ -0,0 +1,24 @@
+// Copyright (C) 2019-2022  Nicola Murino
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, version 3.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//go:build !awscontainer
+// +build !awscontainer
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+func addAWSContainerFlags(_ *cobra.Command) {}

internal/cmd/gen.go

@@ -0,0 +1,26 @@
+// Copyright (C) 2019-2022  Nicola Murino
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, version 3.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package cmd
+
+import "github.com/spf13/cobra"
+
+var genCmd = &cobra.Command{
+	Use:   "gen",
+	Short: "A collection of useful generators",
+}
+
+func init() {
+	rootCmd.AddCommand(genCmd)
+}

internal/cmd/gencompletion.go

@@ -1,3 +1,17 @@
+// Copyright (C) 2019-2022  Nicola Murino
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, version 3.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 package cmd
 
 import (

internal/cmd/genman.go

@@ -1,3 +1,17 @@
+// Copyright (C) 2019-2022  Nicola Murino
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, version 3.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 package cmd
 
 import (
@@ -10,8 +24,8 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/cobra/doc"
 
-	"github.com/drakkan/sftpgo/v2/logger"
-	"github.com/drakkan/sftpgo/v2/version"
+	"github.com/drakkan/sftpgo/v2/internal/logger"
+	"github.com/drakkan/sftpgo/v2/internal/version"
 )
 
 var (
@@ -24,7 +38,7 @@ command-line interface.
 By default, it creates the man page files in the "man" directory under the
 current directory.
 `,
-		Run: func(cmd *cobra.Command, args []string) {
+		Run: func(cmd *cobra.Command, _ []string) {
 			logger.DisableLogger()
 			logger.EnableConsoleLogger(zerolog.DebugLevel)
 			if _, err := os.Stat(manDir); errors.Is(err, fs.ErrNotExist) {

internal/cmd/initprovider.go

@@ -1,3 +1,17 @@
+// Copyright (C) 2019-2022  Nicola Murino
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, version 3.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 package cmd
 
 import (
@@ -7,11 +21,11 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/drakkan/sftpgo/v2/config"
-	"github.com/drakkan/sftpgo/v2/dataprovider"
-	"github.com/drakkan/sftpgo/v2/logger"
-	"github.com/drakkan/sftpgo/v2/service"
-	"github.com/drakkan/sftpgo/v2/util"
+	"github.com/drakkan/sftpgo/v2/internal/config"
+	"github.com/drakkan/sftpgo/v2/internal/dataprovider"
+	"github.com/drakkan/sftpgo/v2/internal/logger"
+	"github.com/drakkan/sftpgo/v2/internal/service"
+	"github.com/drakkan/sftpgo/v2/internal/util"
 )
 
 var (
@@ -34,8 +48,9 @@ To initialize/update the data provider from the configuration directory simply u
 
 $ sftpgo initprovider
 
+Any defined action is ignored.
 Please take a look at the usage below to customize the options.`,
-		Run: func(cmd *cobra.Command, args []string) {
+		Run: func(_ *cobra.Command, _ []string) {
 			logger.DisableLogger()
 			logger.EnableConsoleLogger(zerolog.DebugLevel)
 			configDir = util.CleanDirInput(configDir)
@@ -51,6 +66,10 @@ Please take a look at the usage below to customize the options.`,
 				os.Exit(1)
 			}
 			providerConf := config.GetProviderConf()
+			// ignore actions
+			providerConf.Actions.Hook = ""
+			providerConf.Actions.ExecuteFor = nil
+			providerConf.Actions.ExecuteOn = nil
 			logger.InfoToConsole("Initializing provider: %#v config file: %#v", providerConf.Driver, viper.ConfigFileUsed())
 			err = dataprovider.InitializeDatabase(providerConf, configDir)
 			if err == nil {