set version to 1.2.2

These patches are now merged upstream:

https://github.com/pkg/sftp/pull/392
https://github.com/pkg/sftp/pull/393

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: [drakkan] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/.editorconfig

@@ -0,0 +1,2 @@
+[*.yml]
+indent_size = 2

.github/workflows/development.yml

@@ -11,19 +11,21 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go: [1.14]
+        go: [1.15]
         os: [ubuntu-latest, macos-latest]
         upload-coverage: [true]
         include:
-          - go: 1.13
+          - go: 1.14
             os: ubuntu-latest
             upload-coverage: false
-          - go: 1.14
+          - go: 1.15
             os: windows-latest
             upload-coverage: false
 
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Set up Go
         uses: actions/setup-go@v2
@@ -32,21 +34,17 @@ jobs:
 
       - name: Build for Linux/macOS
         if: startsWith(matrix.os, 'windows-') != true
-        run: go build -i -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
+        run: go build -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
 
       - name: Build for Windows
         if: startsWith(matrix.os, 'windows-')
         run: |
           $GIT_COMMIT = (git describe --always --dirty) | Out-String
           $DATE_TIME = ([datetime]::Now.ToUniversalTime().toString("yyyy-MM-ddTHH:mm:ssZ")) | Out-String
-          go build -i -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/version.date=$DATE_TIME" -o sftpgo.exe
-
-      - name: Initialize data provider
-        run: ./sftpgo initprovider
-        shell: bash
+          go build -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/version.date=$DATE_TIME" -o sftpgo.exe
 
       - name: Run test cases using SQLite provider
-        run: go test -v ./... -coverprofile=coverage.txt -covermode=atomic
+        run: go test -v -p 1 -timeout 10m ./... -coverprofile=coverage.txt -covermode=atomic
 
       - name: Upload coverage to Codecov
         if: ${{ matrix.upload-coverage }}
@@ -57,27 +55,65 @@ jobs:
 
       - name: Run test cases using bolt provider
         run: |
-          go test -v ./config -covermode=atomic
-          go test -v ./httpd -covermode=atomic
-          go test -v ./sftpd -covermode=atomic
+          go test -v -p 1 -timeout 2m ./config -covermode=atomic
+          go test -v -p 1 -timeout 2m ./common -covermode=atomic
+          go test -v -p 1 -timeout 3m ./httpd -covermode=atomic
+          go test -v -p 1 -timeout 8m ./sftpd -covermode=atomic
+          go test -v -p 1 -timeout 2m ./ftpd -covermode=atomic
+          go test -v -p 1 -timeout 2m ./webdavd -covermode=atomic
         env:
           SFTPGO_DATA_PROVIDER__DRIVER: bolt
           SFTPGO_DATA_PROVIDER__NAME: 'sftpgo_bolt.db'
 
       - name: Run test cases using memory provider
-        run: go test -v ./... -covermode=atomic
+        run: go test -v -p 1 -timeout 10m ./... -covermode=atomic
         env:
           SFTPGO_DATA_PROVIDER__DRIVER: memory
           SFTPGO_DATA_PROVIDER__NAME: ''
 
+      - name: Gather cross build info
+        id: cross_info
+        if: ${{ matrix.upload-coverage && startsWith(matrix.os, 'ubuntu-') }}
+        run: |
+          GIT_COMMIT=$(git describe --always)
+          BUILD_DATE=$(date -u +%FT%TZ)
+          echo ::set-output name=sha::${GIT_COMMIT}
+          echo ::set-output name=created::${BUILD_DATE}
+
+      - name: Cross build with xgo
+        if: ${{ matrix.upload-coverage && startsWith(matrix.os, 'ubuntu-') }}
+        uses: crazy-max/ghaction-xgo@v1
+        with:
+          dest: cross
+          prefix: sftpgo
+          targets: linux/arm64,linux/ppc64le
+          v: true
+          x: false
+          race: false
+          ldflags: -s -w -X github.com/drakkan/sftpgo/version.commit=${{ steps.cross_info.outputs.sha }} -X github.com/drakkan/sftpgo/version.date=${{ steps.cross_info.outputs.created }}
+          buildmode: default
+
       - name: Prepare build artifact for Linux/macOS
         if: startsWith(matrix.os, 'windows-') != true
         run: |
-          mkdir output
+          mkdir -p output/{bash_completion,zsh_completion}
+          mkdir -p output/examples/rest-api-cli
           cp sftpgo output/
           cp sftpgo.json output/
           cp -r templates output/
           cp -r static output/
+          cp -r init output/
+          cp examples/rest-api-cli/sftpgo_api_cli output/examples/rest-api-cli/
+          ./sftpgo gen completion bash > output/bash_completion/sftpgo
+          ./sftpgo gen completion zsh > output/zsh_completion/_sftpgo
+          ./sftpgo gen man -d output/man/man1
+          gzip output/man/man1/*
+
+      - name: Copy cross compiled Linux binaries
+        if: ${{ matrix.upload-coverage && startsWith(matrix.os, 'ubuntu-') }}
+        run: |
+          cp cross/sftpgo-linux-arm64 output/
+          cp cross/sftpgo-linux-ppc64le output/
 
       - name: Prepare build artifact for Windows
         if: startsWith(matrix.os, 'windows-')
@@ -96,6 +132,71 @@ jobs:
           name: sftpgo-${{ matrix.os }}-go${{ matrix.go }}
           path: output
 
+      - name: Build Linux Packages
+        id: build_linux_pkgs
+        if: ${{ matrix.upload-coverage && startsWith(matrix.os, 'ubuntu-') }}
+        run: |
+          cp -r pkgs pkgs_arm64
+          cp -r pkgs pkgs_ppc64le
+          cd pkgs
+          ./build.sh
+          cd ..
+          export NFPM_ARCH=arm64
+          export BIN_SUFFIX=-linux-arm64
+          cp cross/sftpgo${BIN_SUFFIX} .
+          cd pkgs_arm64
+          ./build.sh
+          cd ..
+          export NFPM_ARCH=ppc64le
+          export BIN_SUFFIX=-linux-ppc64le
+          cp cross/sftpgo${BIN_SUFFIX} .
+          cd pkgs_ppc64le
+          ./build.sh
+          PKG_VERSION=$(cat dist/version)
+          echo "::set-output name=pkg-version::${PKG_VERSION}"
+
+      - name: Upload Debian Package
+        if: ${{ matrix.upload-coverage && startsWith(matrix.os, 'ubuntu-') }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: sftpgo-${{ steps.build_linux_pkgs.outputs.pkg-version }}-x86_64-deb
+          path: pkgs/dist/deb/*
+
+      - name: Upload RPM Package
+        if: ${{ matrix.upload-coverage && startsWith(matrix.os, 'ubuntu-') }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: sftpgo-${{ steps.build_linux_pkgs.outputs.pkg-version }}-x86_64-rpm
+          path: pkgs/dist/rpm/*
+
+      - name: Upload Debian Package arm64
+        if: ${{ matrix.upload-coverage && startsWith(matrix.os, 'ubuntu-') }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: sftpgo-${{ steps.build_linux_pkgs.outputs.pkg-version }}-arm64-deb
+          path: pkgs_arm64/dist/deb/*
+
+      - name: Upload RPM Package arm64
+        if: ${{ matrix.upload-coverage && startsWith(matrix.os, 'ubuntu-') }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: sftpgo-${{ steps.build_linux_pkgs.outputs.pkg-version }}-arm64-rpm
+          path: pkgs_arm64/dist/rpm/*
+
+      - name: Upload Debian Package ppc64le
+        if: ${{ matrix.upload-coverage && startsWith(matrix.os, 'ubuntu-') }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: sftpgo-${{ steps.build_linux_pkgs.outputs.pkg-version }}-ppc64le-deb
+          path: pkgs_ppc64le/dist/deb/*
+
+      - name: Upload RPM Package ppc64le
+        if: ${{ matrix.upload-coverage && startsWith(matrix.os, 'ubuntu-') }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: sftpgo-${{ steps.build_linux_pkgs.outputs.pkg-version }}-ppc64le-rpm
+          path: pkgs_ppc64le/dist/rpm/*
+
   test-postgresql-mysql:
     name: Test with PostgreSQL/MySQL
     runs-on: ubuntu-latest
@@ -135,15 +236,14 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.14
+          go-version: 1.15
 
       - name: Build
-        run: go build -i -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
+        run: go build -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
 
       - name: Run tests using PostgreSQL provider
         run: |
-          ./sftpgo initprovider
-          go test -v ./... -covermode=atomic
+          go test -v -p 1 -timeout 10m ./... -covermode=atomic
         env:
           SFTPGO_DATA_PROVIDER__DRIVER: postgresql
           SFTPGO_DATA_PROVIDER__NAME: sftpgo
@@ -154,8 +254,7 @@ jobs:
 
       - name: Run tests using MySQL provider
         run: |
-          ./sftpgo initprovider
-          go test -v ./... -covermode=atomic
+          go test -v -p 1 -timeout 10m ./... -covermode=atomic
         env:
           SFTPGO_DATA_PROVIDER__DRIVER: mysql
           SFTPGO_DATA_PROVIDER__NAME: sftpgo
@@ -170,6 +269,6 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v1
+        uses: golangci/golangci-lint-action@v2
         with:
-          version: v1.27
+          version: latest

.github/workflows/docker.yml

@@ -0,0 +1,176 @@
+name: Docker
+
+on:
+  #schedule:
+  #  - cron: '0 4 * * *' # everyday at 4:00 AM UTC
+  push:
+    branches:
+      - master
+    tags:
+      - v*
+  pull_request:
+
+jobs:
+  build:
+    name: Build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+        docker_pkg:
+          - debian
+          - alpine
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Repo metadata
+        id: repo
+        uses: actions/github-script@v3
+        with:
+          script: |
+            const repo = await github.repos.get(context.repo)
+            return repo.data
+
+      - name: Gather image information
+        id: info
+        run: |
+          VERSION=noop
+          DOCKERFILE_SLIM=Dockerfile
+          DOCKERFILE=Dockerfile.full
+          MINOR=""
+          MAJOR=""
+          if [ "${{ github.event_name }}" = "schedule" ]; then
+            VERSION=nightly
+          elif [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+            if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; then
+              VERSION=edge
+            fi
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            VERSION=pr-${{ github.event.number }}
+          fi
+          if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
+            MINOR=${VERSION%.*}
+            MAJOR=${MINOR%.*}
+          fi
+          VERSION_SLIM="${VERSION}-slim"
+          if [[ $DOCKER_PKG == alpine ]]; then
+            VERSION="${VERSION}-alpine"
+            VERSION_SLIM="${VERSION}-slim"
+            DOCKERFILE_SLIM=Dockerfile.alpine
+            DOCKERFILE=Dockerfile.full.alpine
+          fi
+
+          DOCKER_IMAGES=("drakkan/sftpgo" "ghcr.io/drakkan/sftpgo")
+          TAGS="${DOCKER_IMAGES[0]}:${VERSION}"
+          TAGS_SLIM="${DOCKER_IMAGES[0]}:${VERSION_SLIM}"
+          BASE_IMAGE="${TAGS_SLIM}"
+
+          for DOCKER_IMAGE in ${DOCKER_IMAGES[@]}; do
+            if [[ ${DOCKER_IMAGE} != ${DOCKER_IMAGES[0]} ]]; then
+              TAGS="${TAGS},${DOCKER_IMAGE}:${VERSION}"
+              TAGS_SLIM="${TAGS_SLIM},${DOCKER_IMAGE}:${VERSION_SLIM}"
+            fi
+            if [[ $GITHUB_REF == refs/tags/* ]]; then
+              if [[ $DOCKER_PKG == debian ]]; then
+                if [[ -n $MAJOR && -n $MINOR ]]; then
+                  TAGS="${TAGS},${DOCKER_IMAGE}:${MINOR},${DOCKER_IMAGE}:${MAJOR}"
+                  TAGS_SLIM="${TAGS_SLIM},${DOCKER_IMAGE}:${MINOR}-slim,${DOCKER_IMAGE}:${MAJOR}-slim"
+                fi
+                TAGS="${TAGS},${DOCKER_IMAGE}:latest"
+                TAGS_SLIM="${TAGS_SLIM},${DOCKER_IMAGE}:slim"
+              else
+                if [[ -n $MAJOR && -n $MINOR ]]; then
+                  TAGS="${TAGS},${DOCKER_IMAGE}:${MINOR}-alpine,${DOCKER_IMAGE}:${MAJOR}-alpine"
+                  TAGS_SLIM="${TAGS_SLIM},${DOCKER_IMAGE}:${MINOR}-alpine-slim,${DOCKER_IMAGE}:${MAJOR}-alpine-slim"
+                fi
+                TAGS="${TAGS},${DOCKER_IMAGE}:alpine"
+                TAGS_SLIM="${TAGS_SLIM},${DOCKER_IMAGE}:alpine-slim"
+              fi
+            fi
+          done
+
+          echo ::set-output name=dockerfile::${DOCKERFILE}
+          echo ::set-output name=dockerfile-slim::${DOCKERFILE_SLIM}
+          echo ::set-output name=version::${VERSION}
+          echo ::set-output name=version-slim::${VERSION_SLIM}
+          echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=tags-slim::${TAGS_SLIM}
+          echo ::set-output name=base-image::${BASE_IMAGE}
+          echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+          echo ::set-output name=sha::${GITHUB_SHA::8}
+        env:
+          DOCKER_PKG: ${{ matrix.docker_pkg }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up builder slim
+        uses: docker/setup-buildx-action@v1
+        id: builder-slim
+
+      - name: Set up builder full
+        uses: docker/setup-buildx-action@v1
+        id: builder-full
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+        if: ${{ github.event_name != 'pull_request' }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.CR_PAT }}
+        if: ${{ github.event_name != 'pull_request' }}
+
+      - name: Build and push slim
+        uses: docker/build-push-action@v2
+        with:
+          builder: ${{ steps.builder-slim.outputs.name }}
+          file: ./${{ steps.info.outputs.dockerfile-slim }}
+          platforms: linux/amd64,linux/arm64,linux/ppc64le
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.info.outputs.tags-slim }}
+          build-args: |
+            COMMIT_SHA=${{ steps.info.outputs.sha }}
+          labels: |
+            org.opencontainers.image.title=SFTPGo
+            org.opencontainers.image.description=Fully featured and highly configurable SFTP server with optional FTP/S and WebDAV support
+            org.opencontainers.image.url=${{ fromJson(steps.repo.outputs.result).html_url }}
+            org.opencontainers.image.documentation=${{ fromJson(steps.repo.outputs.result).html_url }}/blob/${{ github.sha }}/docker/README.md
+            org.opencontainers.image.source=${{ fromJson(steps.repo.outputs.result).html_url }}
+            org.opencontainers.image.version=${{ steps.info.outputs.version }}
+            org.opencontainers.image.created=${{ steps.info.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.licenses=${{ fromJson(steps.repo.outputs.result).license.spdx_id }}
+
+      - name: Build and push full
+        uses: docker/build-push-action@v2
+        with:
+          builder: ${{ steps.builder-full.outputs.name }}
+          file: ./${{ steps.info.outputs.dockerfile }}
+          platforms: linux/amd64,linux/arm64,linux/ppc64le
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.info.outputs.tags }}
+          build-args: |
+            COMMIT_SHA=${{ steps.info.outputs.sha }}
+            BASE_IMAGE=${{ steps.info.outputs.base-image }}
+          labels: |
+            org.opencontainers.image.title=SFTPGo
+            org.opencontainers.image.description=Fully featured and highly configurable SFTP server with optional FTP/S and WebDAV support
+            org.opencontainers.image.url=${{ fromJson(steps.repo.outputs.result).html_url }}
+            org.opencontainers.image.documentation=${{ fromJson(steps.repo.outputs.result).html_url }}/blob/${{ github.sha }}/docker/README.md
+            org.opencontainers.image.source=${{ fromJson(steps.repo.outputs.result).html_url }}
+            org.opencontainers.image.version=${{ steps.info.outputs.version }}
+            org.opencontainers.image.created=${{ steps.info.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.licenses=${{ fromJson(steps.repo.outputs.result).license.spdx_id }}

.github/workflows/release.yml

@@ -5,7 +5,7 @@ on:
     tags: 'v*'
 
 env:
-  GO_VERSION: 1.14
+  GO_VERSION: 1.15.5
 
 jobs:
   create-release:
@@ -19,7 +19,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           tag_name: ${{ github.ref }}
-          release_name: Release ${{ github.ref }}
+          release_name: ${{ github.ref }}
           draft: false
           prerelease: false
 
@@ -102,14 +102,14 @@ jobs:
 
       - name: Build for Linux/macOS
         if: startsWith(matrix.os, 'windows-') != true
-        run: go build -i -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
+        run: go build -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
 
       - name: Build for Windows
         if: startsWith(matrix.os, 'windows-')
         run: |
           $GIT_COMMIT = (git describe --always --dirty) | Out-String
           $DATE_TIME = ([datetime]::Now.ToUniversalTime().toString("yyyy-MM-ddTHH:mm:ssZ")) | Out-String
-          go build -i -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/version.date=$DATE_TIME" -o sftpgo.exe
+          go build -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/version.date=$DATE_TIME" -o sftpgo.exe
 
       - name: Initialize data provider
         run: ./sftpgo initprovider
@@ -143,12 +143,34 @@ jobs:
           pip install requests
           pip install pygments
           pip install pyinstaller
-          pyinstaller --hidden-import="pkg_resources.py2_warn" --noupx --onefile examples\rest-api-cli\sftpgo_api_cli.py
+          pyinstaller --hidden-import="pkg_resources.py2_warn" --noupx --onefile examples\rest-api-cli\sftpgo_api_cli
+
+      - name: Gather cross build info
+        id: cross_info
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          GIT_COMMIT=$(git describe --always)
+          BUILD_DATE=$(date -u +%FT%TZ)
+          echo ::set-output name=sha::${GIT_COMMIT}
+          echo ::set-output name=created::${BUILD_DATE}
+
+      - name: Cross build with xgo
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        uses: crazy-max/ghaction-xgo@v1
+        with:
+          dest: cross
+          prefix: sftpgo
+          targets: linux/arm64,linux/ppc64le
+          v: true
+          x: false
+          race: false
+          ldflags: -s -w -X github.com/drakkan/sftpgo/version.commit=${{ steps.cross_info.outputs.sha }} -X github.com/drakkan/sftpgo/version.date=${{ steps.cross_info.outputs.created }}
+          buildmode: default
 
       - name: Prepare Release for Linux/macOS
         if: startsWith(matrix.os, 'windows-') != true
         run: |
-          mkdir -p output/{init,examples/rest-api-cli,sqlite}
+          mkdir -p output/{init,examples/rest-api-cli,sqlite,bash_completion,zsh_completion}
           echo "For documentation please take a look here:" > output/README.txt
           echo "" >> output/README.txt
           echo "https://github.com/drakkan/sftpgo/blob/${SFTPGO_VERSION}/README.md" >> output/README.txt
@@ -160,18 +182,71 @@ jobs:
           cp -r templates output/
           if [ $OS == 'linux' ]
           then
-            cp -r init/sftpgo.service output/init/
+            cp init/sftpgo.service output/init/
           else
-            cp -r init/com.github.drakkan.sftpgo.plist output/init/
+            cp init/com.github.drakkan.sftpgo.plist output/init/
+          fi
+          ./sftpgo gen completion bash > output/bash_completion/sftpgo
+          ./sftpgo gen completion zsh > output/zsh_completion/_sftpgo
+          ./sftpgo gen man -d output/man/man1
+          gzip output/man/man1/*
+          cp examples/rest-api-cli/sftpgo_api_cli output/examples/rest-api-cli/
+          if [ $OS == 'linux' ]
+          then
+            cp -r output output_arm64
+            cp -r output output_ppc64le
+            cp -r output output_all
           fi
-          cp examples/rest-api-cli/sftpgo_api_cli.py output/examples/rest-api-cli/
           cd output
           tar cJvf sftpgo_${SFTPGO_VERSION}_${OS}_x86_64.tar.xz *
           cd ..
+          if [ $OS == 'linux' ]
+          then
+            cp cross/sftpgo-linux-arm64 output_arm64/sftpgo
+            cd output_arm64
+            tar cJvf sftpgo_${SFTPGO_VERSION}_${OS}_arm64.tar.xz *
+            cd ..
+            cp cross/sftpgo-linux-ppc64le output_ppc64le/sftpgo
+            cd output_ppc64le
+            tar cJvf sftpgo_${SFTPGO_VERSION}_${OS}_ppc64le.tar.xz *
+            cd ..
+            mkdir output_all/{arm64,ppc64le}
+            cp cross/sftpgo-linux-arm64 output_all/arm64/sftpgo
+            cp cross/sftpgo-linux-ppc64le output_all/ppc64le/sftpgo
+            cd output_all
+            tar cJvf sftpgo_${SFTPGO_VERSION}_${OS}_bundle.tar.xz *
+            cd ..
+          fi
         env:
           SFTPGO_VERSION: ${{ steps.get_version.outputs.VERSION }}
           OS: ${{ steps.get_os_name.outputs.OS }}
 
+      - name: Prepare Linux Packages
+        id: build_linux_pkgs
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          cp -r pkgs pkgs_arm64
+          cp -r pkgs pkgs_ppc64le
+          cd pkgs
+          ./build.sh
+          cd ..
+          export NFPM_ARCH=arm64
+          export BIN_SUFFIX=-linux-arm64
+          cp cross/sftpgo${BIN_SUFFIX} .
+          cd pkgs_arm64
+          ./build.sh
+          cd ..
+          export NFPM_ARCH=ppc64le
+          export BIN_SUFFIX=-linux-ppc64le
+          cp cross/sftpgo${BIN_SUFFIX} .
+          cd pkgs_ppc64le
+          ./build.sh
+          cd ..
+          PKG_VERSION=${SFTPGO_VERSION:1}
+          echo "::set-output name=pkg-version::${PKG_VERSION}"
+        env:
+          SFTPGO_VERSION: ${{ steps.get_version.outputs.VERSION }}
+
       - name: Prepare Release for Windows
         if: startsWith(matrix.os, 'windows-')
         run: |
@@ -190,6 +265,24 @@ jobs:
           SFTPGO_ISS_VERSION: ${{ steps.get_version.outputs.VERSION }}
           SFTPGO_ISS_DOC_URL: https://github.com/drakkan/sftpgo/blob/${{ steps.get_version.outputs.VERSION }}/README.md
 
+      - name: Prepare Portable Release for Windows
+        if: startsWith(matrix.os, 'windows-')
+        run: |
+          mkdir win-portable\examples\rest-api-cli
+          copy .\sftpgo.exe .\win-portable
+          copy .\sftpgo.json .\win-portable
+          copy .\sftpgo.db .\win-portable
+          copy .\dist\sftpgo_api_cli.exe .\win-portable\examples\rest-api-cli
+          copy .\LICENSE .\win-portable\LICENSE.txt
+          mkdir win-portable\templates
+          xcopy .\templates .\win-portable\templates\ /E
+          mkdir win-portable\static
+          xcopy .\static .\win-portable\static\ /E
+          Compress-Archive .\win-portable\* sftpgo_portable_x86_64.zip
+        env:
+          SFTPGO_VERSION: ${{ steps.get_version.outputs.VERSION }}
+          OS: ${{ steps.get_os_name.outputs.OS }}
+
       - name: Download release upload URL
         uses: actions/download-artifact@v2
         with:
@@ -213,6 +306,39 @@ jobs:
           asset_name: sftpgo_${{ steps.get_version.outputs.VERSION }}_${{ steps.get_os_name.outputs.OS }}_x86_64.tar.xz
           asset_content_type: application/x-xz
 
+      - name: Upload Linux/arm64 Release
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.upload_url.outputs.url }}
+          asset_path: ./output_arm64/sftpgo_${{ steps.get_version.outputs.VERSION }}_${{ steps.get_os_name.outputs.OS }}_arm64.tar.xz
+          asset_name: sftpgo_${{ steps.get_version.outputs.VERSION }}_${{ steps.get_os_name.outputs.OS }}_arm64.tar.xz
+          asset_content_type: application/x-xz
+
+      - name: Upload Linux/ppc64le Release
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.upload_url.outputs.url }}
+          asset_path: ./output_ppc64le/sftpgo_${{ steps.get_version.outputs.VERSION }}_${{ steps.get_os_name.outputs.OS }}_ppc64le.tar.xz
+          asset_name: sftpgo_${{ steps.get_version.outputs.VERSION }}_${{ steps.get_os_name.outputs.OS }}_ppc64le.tar.xz
+          asset_content_type: application/x-xz
+
+      - name: Upload Linux Bundle Release
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.upload_url.outputs.url }}
+          asset_path: ./output_all/sftpgo_${{ steps.get_version.outputs.VERSION }}_${{ steps.get_os_name.outputs.OS }}_bundle.tar.xz
+          asset_name: sftpgo_${{ steps.get_version.outputs.VERSION }}_${{ steps.get_os_name.outputs.OS }}_bundle.tar.xz
+          asset_content_type: application/x-xz
+
       - name: Upload Windows Release
         if: startsWith(matrix.os, 'windows-')
         uses: actions/upload-release-asset@v1
@@ -223,3 +349,80 @@ jobs:
           asset_path: ./sftpgo_windows_x86_64.exe
           asset_name: sftpgo_${{ steps.get_version.outputs.VERSION }}_${{ steps.get_os_name.outputs.OS }}_x86_64.exe
           asset_content_type: application/x-dosexec
+
+      - name: Upload Portable Windows Release
+        if: startsWith(matrix.os, 'windows-')
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.upload_url.outputs.url }}
+          asset_path: ./sftpgo_portable_x86_64.zip
+          asset_name: sftpgo_${{ steps.get_version.outputs.VERSION }}_${{ steps.get_os_name.outputs.OS }}_portable_x86_64.zip
+          asset_content_type: application/zip
+
+      - name: Upload Debian Package
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.upload_url.outputs.url }}
+          asset_path: ./pkgs/dist/deb/sftpgo_${{ steps.build_linux_pkgs.outputs.pkg-version }}-1_amd64.deb
+          asset_name: sftpgo_${{ steps.build_linux_pkgs.outputs.pkg-version }}-1_amd64.deb
+          asset_content_type: application/vnd.debian.binary-package
+
+      - name: Upload RPM Package
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.upload_url.outputs.url }}
+          asset_path: ./pkgs/dist/rpm/sftpgo-${{ steps.build_linux_pkgs.outputs.pkg-version }}-1.x86_64.rpm
+          asset_name: sftpgo-${{ steps.build_linux_pkgs.outputs.pkg-version }}-1.x86_64.rpm
+          asset_content_type: application/x-rpm
+
+      - name: Upload Debian Package arm64
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.upload_url.outputs.url }}
+          asset_path: ./pkgs_arm64/dist/deb/sftpgo_${{ steps.build_linux_pkgs.outputs.pkg-version }}-1_arm64.deb
+          asset_name: sftpgo_${{ steps.build_linux_pkgs.outputs.pkg-version }}-1_arm64.deb
+          asset_content_type: application/vnd.debian.binary-package
+
+      - name: Upload RPM Package arm64
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.upload_url.outputs.url }}
+          asset_path: ./pkgs_arm64/dist/rpm/sftpgo-${{ steps.build_linux_pkgs.outputs.pkg-version }}-1.aarch64.rpm
+          asset_name: sftpgo-${{ steps.build_linux_pkgs.outputs.pkg-version }}-1.aarch64.rpm
+          asset_content_type: application/x-rpm
+
+      - name: Upload Debian Package ppc64le
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.upload_url.outputs.url }}
+          asset_path: ./pkgs_ppc64le/dist/deb/sftpgo_${{ steps.build_linux_pkgs.outputs.pkg-version }}-1_ppc64el.deb
+          asset_name: sftpgo_${{ steps.build_linux_pkgs.outputs.pkg-version }}-1_ppc64el.deb
+          asset_content_type: application/vnd.debian.binary-package
+
+      - name: Upload RPM Package ppc64le
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.upload_url.outputs.url }}
+          asset_path: ./pkgs_ppc64le/dist/rpm/sftpgo-${{ steps.build_linux_pkgs.outputs.pkg-version }}-1.ppc64le.rpm
+          asset_name: sftpgo-${{ steps.build_linux_pkgs.outputs.pkg-version }}-1.ppc64le.rpm
+          asset_content_type: application/x-rpm

Dockerfile

@@ -0,0 +1,60 @@
+FROM golang:1.15 as builder
+
+ENV GOFLAGS="-mod=readonly"
+
+RUN mkdir -p /workspace
+WORKDIR /workspace
+
+ARG GOPROXY
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+ARG COMMIT_SHA
+
+# This ARG allows to disable some optional features and it might be useful if you build the image yourself.
+# For example you can disable S3 and GCS support like this:
+# --build-arg FEATURES=nos3,nogcs
+ARG FEATURES
+
+COPY . .
+
+RUN set -xe && \
+    export COMMIT_SHA=${COMMIT_SHA:-$(git describe --always --dirty)} && \
+    go build $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=${COMMIT_SHA} -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -v -o sftpgo
+
+FROM debian:buster-slim
+
+RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates mime-support && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /etc/sftpgo /var/lib/sftpgo /usr/share/sftpgo /srv/sftpgo
+
+RUN groupadd --system -g 1000 sftpgo && \
+    useradd --system --gid sftpgo --no-create-home \
+    --home-dir /var/lib/sftpgo --shell /usr/sbin/nologin \
+    --comment "SFTPGo user" --uid 1000 sftpgo
+
+COPY --from=builder /workspace/sftpgo.json /etc/sftpgo/sftpgo.json
+COPY --from=builder /workspace/templates /usr/share/sftpgo/templates
+COPY --from=builder /workspace/static /usr/share/sftpgo/static
+COPY --from=builder /workspace/sftpgo /usr/local/bin/
+
+# Log to the stdout so the logs will be available using docker logs
+ENV SFTPGO_LOG_FILE_PATH=""
+# templates and static paths are inside the container
+ENV SFTPGO_HTTPD__TEMPLATES_PATH=/usr/share/sftpgo/templates
+ENV SFTPGO_HTTPD__STATIC_FILES_PATH=/usr/share/sftpgo/static
+
+# Modify the default configuration file
+RUN sed -i "s|\"users_base_dir\": \"\",|\"users_base_dir\": \"/srv/sftpgo/data\",|" /etc/sftpgo/sftpgo.json && \
+    sed -i "s|\"backups\"|\"/srv/sftpgo/backups\"|" /etc/sftpgo/sftpgo.json && \
+    sed -i "s|\"bind_address\": \"127.0.0.1\",|\"bind_address\": \"\",|" /etc/sftpgo/sftpgo.json
+
+COPY ./docker/scripts/entrypoint.sh /docker-entrypoint.sh
+
+RUN chown -R sftpgo:sftpgo /etc/sftpgo && chown sftpgo:sftpgo /var/lib/sftpgo /srv/sftpgo
+
+WORKDIR /var/lib/sftpgo
+USER 1000:1000
+
+CMD ["sftpgo", "serve"]

Dockerfile.alpine

@@ -0,0 +1,63 @@
+FROM golang:1.15-alpine AS builder
+
+ENV GOFLAGS="-mod=readonly"
+
+RUN apk add --update --no-cache bash ca-certificates curl git gcc g++
+
+RUN mkdir -p /workspace
+WORKDIR /workspace
+
+ARG GOPROXY
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+ARG COMMIT_SHA
+
+# This ARG allows to disable some optional features and it might be useful if you build the image yourself.
+# For example you can disable S3 and GCS support like this:
+# --build-arg FEATURES=nos3,nogcs
+ARG FEATURES
+
+COPY . .
+
+RUN set -xe && \
+    export COMMIT_SHA=${COMMIT_SHA:-$(git describe --always --dirty)} && \
+    go build $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=${COMMIT_SHA} -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -v -o sftpgo
+
+
+FROM alpine:3.12
+
+RUN apk add --update --no-cache ca-certificates tzdata mailcap
+
+# set up nsswitch.conf for Go's "netgo" implementation
+# https://github.com/gliderlabs/docker-alpine/issues/367#issuecomment-424546457
+RUN test ! -e /etc/nsswitch.conf && echo 'hosts: files dns' > /etc/nsswitch.conf
+
+RUN mkdir -p /etc/sftpgo /var/lib/sftpgo /usr/share/sftpgo /srv/sftpgo
+
+RUN addgroup -g 1000 -S sftpgo && \
+    adduser -u 1000 -h /var/lib/sftpgo -s /sbin/nologin -G sftpgo -S -D -H -g "SFTPGo user" sftpgo
+
+COPY --from=builder /workspace/sftpgo.json /etc/sftpgo/sftpgo.json
+COPY --from=builder /workspace/templates /usr/share/sftpgo/templates
+COPY --from=builder /workspace/static /usr/share/sftpgo/static
+COPY --from=builder /workspace/sftpgo /usr/local/bin/
+
+# Log to the stdout so the logs will be available using docker logs
+ENV SFTPGO_LOG_FILE_PATH=""
+# templates and static paths are inside the container
+ENV SFTPGO_HTTPD__TEMPLATES_PATH=/usr/share/sftpgo/templates
+ENV SFTPGO_HTTPD__STATIC_FILES_PATH=/usr/share/sftpgo/static
+
+# Modify the default configuration file
+RUN sed -i "s|\"users_base_dir\": \"\",|\"users_base_dir\": \"/srv/sftpgo/data\",|" /etc/sftpgo/sftpgo.json && \
+    sed -i "s|\"backups\"|\"/srv/sftpgo/backups\"|" /etc/sftpgo/sftpgo.json && \
+    sed -i "s|\"bind_address\": \"127.0.0.1\",|\"bind_address\": \"\",|" /etc/sftpgo/sftpgo.json
+
+RUN chown -R sftpgo:sftpgo /etc/sftpgo && chown sftpgo:sftpgo /var/lib/sftpgo /srv/sftpgo
+
+WORKDIR /var/lib/sftpgo
+USER 1000:1000
+
+CMD ["sftpgo", "serve"]

Dockerfile.full

@@ -0,0 +1,10 @@
+ARG BASE_IMAGE
+
+FROM ${BASE_IMAGE}
+
+USER root
+
+# Install some optional packages used by SFTPGo features
+RUN apt-get update && apt-get install --no-install-recommends -y git rsync && rm -rf /var/lib/apt/lists/*
+
+USER 1000:1000

Dockerfile.full.alpine

@@ -0,0 +1,10 @@
+ARG BASE_IMAGE
+
+FROM ${BASE_IMAGE}
+
+USER root
+
+# Install some optional packages used by SFTPGo features
+RUN apk add --update --no-cache rsync git
+
+USER 1000:1000

README.md

@@ -4,15 +4,17 @@
 [![Code Coverage](https://codecov.io/gh/drakkan/sftpgo/branch/master/graph/badge.svg)](https://codecov.io/gh/drakkan/sftpgo/branch/master)
 [![Go Report Card](https://goreportcard.com/badge/github.com/drakkan/sftpgo)](https://goreportcard.com/report/github.com/drakkan/sftpgo)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
+[![Docker Pulls](https://img.shields.io/docker/pulls/drakkan/sftpgo)](https://hub.docker.com/r/drakkan/sftpgo)
 [![Mentioned in Awesome Go](https://awesome.re/mentioned-badge.svg)](https://github.com/avelino/awesome-go)
 
-Fully featured and highly configurable SFTP server, written in Go
+Fully featured and highly configurable SFTP server with optional FTP/S and WebDAV support, written in Go.
+It can serve local filesystem, S3 (compatible) Object Storage, Google Cloud Storage and Azure Blob Storage.
 
 ## Features
 
-- Each account is chrooted to its home directory.
-- SFTP accounts are virtual accounts stored in a "data provider".
+- SFTPGo uses virtual accounts stored inside a "data provider".
 - SQLite, MySQL, PostgreSQL, bbolt (key/value store in pure Go) and in-memory data providers are supported.
+- Each account is chrooted to its home directory.
 - Public key and password authentication. Multiple public keys per user are supported.
 - SSH user [certificate authentication](https://cvsweb.openbsd.org/src/usr.bin/ssh/PROTOCOL.certkeys?rev=1.8).
 - Keyboard interactive authentication. You can easily setup a customizable multi-factor authentication.
@@ -26,38 +28,43 @@ Fully featured and highly configurable SFTP server, written in Go
 - Per user and per directory permission management: list directory contents, upload, overwrite, download, delete, rename, create directories, create symlinks, change owner/group and mode, change access and modification times.
 - Per user files/folders ownership mapping: you can map all the users to the system account that runs SFTPGo (all platforms are supported) or you can run SFTPGo as root user and map each user or group of users to a different system account (\*NIX only).
 - Per user IP filters are supported: login can be restricted to specific ranges of IP addresses or to a specific IP address.
-- Per user and per directory file extensions filters are supported: files can be allowed or denied based on their extensions.
+- Per user and per directory shell like patterns filters are supported: files can be allowed or denied based on shell like patterns.
 - Virtual folders are supported: directories outside the user home directory can be exposed as virtual folders.
 - Configurable custom commands and/or HTTP notifications on file upload, download, pre-delete, delete, rename, on SSH commands and on user add, update and delete.
 - Automatically terminating idle connections.
 - Atomic uploads are configurable.
 - Support for Git repositories over SSH.
 - SCP and rsync are supported.
-- Support for serving local filesystem, S3 Compatible Object Storage and Google Cloud Storage over SFTP/SCP.
+- FTP/S is supported. You can configure the FTP service to require TLS for both control and data connections.
+- [WebDAV](./docs/webdav.md) is supported.
+- Support for serving local filesystem, S3 Compatible Object Storage and Google Cloud Storage over SFTP/SCP/FTP/WebDAV.
+- Per user protocols restrictions. You can configure the allowed protocols (SSH/FTP/WebDAV) for each user.
 - [Prometheus metrics](./docs/metrics.md) are exposed.
-- Support for HAProxy PROXY protocol: you can proxy and/or load balance the SFTP/SCP service without losing the information about the client's address.
+- Support for HAProxy PROXY protocol: you can proxy and/or load balance the SFTP/SCP/FTP/WebDAV service without losing the information about the client's address.
 - [REST API](./docs/rest-api.md) for users and folders management, backup, restore and real time reports of the active connections with possibility of forcibly closing a connection.
 - [Web based administration interface](./docs/web-admin.md) to easily manage users, folders and connections.
 - Easy [migration](./examples/rest-api-cli#convert-users-from-other-stores) from Linux system user accounts.
 - [Portable mode](./docs/portable-mode.md): a convenient way to share a single directory on demand.
+- [SFTP subsystem mode](./docs/sftp-subsystem.md): you can use SFTPGo as OpenSSH's SFTP subsystem.
 - Performance analysis using built-in [profiler](./docs/profiling.md).
 - Configuration format is at your choice: JSON, TOML, YAML, HCL, envfile are supported.
 - Log files are accurate and they are saved in the easily parsable JSON format ([more information](./docs/logs.md)).
 
 ## Platforms
 
-SFTPGo is developed and tested on Linux. After each commit, the code is automatically built and tested on Linux, macOS and Windows using a [GitHub Action](./.github/workflows/development.yml). Other UNIX variants such as \*BSD should work too.
+SFTPGo is developed and tested on Linux. After each commit, the code is automatically built and tested on Linux, macOS and Windows using a [GitHub Action](./.github/workflows/development.yml). The test cases are regularly manually executed and passed on FreeBSD. Other *BSD variants should work too.
 
 ## Requirements
 
-- Go 1.13 or higher as build only dependency.
-- A suitable SQL server or key/value store to use as data provider: PostgreSQL 9.4+ or MySQL 5.6+ or SQLite 3.x or bbolt 1.3.x
+- Go 1.14 or higher as build only dependency.
+- A suitable SQL server to use as data provider: PostgreSQL 9.4+ or MySQL 5.6+ or SQLite 3.x.
+- The SQL server is optional: you can choose to use an embedded bolt database as key/value store or an in memory data provider.
 
 ## Installation
 
 Binary releases for Linux, macOS, and Windows are available. Please visit the [releases](https://github.com/drakkan/sftpgo/releases "releases") page.
 
-Sample Dockerfiles for [Debian](https://www.debian.org) and [Alpine](https://alpinelinux.org) are available inside the source tree [docker](./docker) directory.
+An official Docker image is available. Documentation is [here](./docker/README.md).
 
 Some Linux distro packages are available:
 
@@ -65,6 +72,8 @@ Some Linux distro packages are available:
   - [sftpgo](https://aur.archlinux.org/packages/sftpgo/). This package follows stable releases. It requires `git`, `gcc` and `go` to build.
   - [sftpgo-bin](https://aur.archlinux.org/packages/sftpgo-bin/). This package follows stable releases downloading the prebuilt linux binary from GitHub. It does not require `git`, `gcc` and `go` to build.
   - [sftpgo-git](https://aur.archlinux.org/packages/sftpgo-git/). This package builds and installs the latest git master. It requires `git`, `gcc` and `go` to build.
+- Deb and RPM packages are built after each commit and for each release.
+- For Ubuntu a PPA is available [here](https://launchpad.net/~sftpgo/+archive/ubuntu/sftpgo).
 
 You can easily test new features selecting a commit from the [Actions](https://github.com/drakkan/sftpgo/actions) page and downloading the matching build artifacts for Linux, macOS or Windows. GitHub stores artifacts for 90 days.
 
@@ -76,7 +85,7 @@ A full explanation of all configuration methods can be found [here](./docs/full-
 
 Please make sure to [initialize the data provider](#data-provider-initialization) before running the daemon!
 
-To start the SFTP server with default settings, simply run:
+To start SFTPGo with the default settings, simply run:
 
 ```bash
 sftpgo serve
@@ -84,15 +93,17 @@ sftpgo serve
 
 Check out [this documentation](./docs/service.md) if you want to run SFTPGo as a service.
 
-### Data provider initialization
+### Data provider initialization and update
 
-Before starting the SFTPGo server, please ensure that the configured data provider is properly initialized.
+Before starting the SFTPGo server please ensure that the configured data provider is properly initialized/updated.
 
-SQL based data providers (SQLite, MySQL, PostgreSQL) require the creation of a database containing the required tables. Memory and bolt data providers do not require an initialization.
+SQL based data providers (SQLite, MySQL, PostgreSQL) require the creation of a database containing the required tables. Memory and bolt data providers do not require an initialization but they could require an update to the existing data after upgrading SFTPGo.
 
-After configuring the data provider using the configuration file, you can create the required database structure using the `initprovider` command.
-For SQLite provider, the `initprovider` command will auto create the database file, if missing, and the required tables.
-For PostgreSQL and MySQL providers, you need to create the configured database, and the `initprovider` command will create the required tables.
+For PostgreSQL and MySQL providers, you need to create the configured database.
+
+SFTPGo will attempt to automatically detect if the data provider is initialized/updated and if not, will attempt to initialize/ update it on startup as needed.
+
+Alternately, you can create/update the required data provider structures yourself using the `initprovider` command.
 
 For example, you can simply execute the following command from the configuration directory:
 
@@ -106,13 +117,21 @@ Take a look at the CLI usage to learn how to specify a different configuration f
 sftpgo initprovider --help
 ```
 
-The `initprovider` command is enough for new installations. From now on, the database structure will be automatically checked and updated, if required, at startup.
+You can disable automatic data provider checks/updates at startup by setting the `update_mode` configuration key to `1`.
 
-#### Upgrading
+## Users and folders management
 
-If you are upgrading from version 0.9.5 or before, you have to manually execute the SQL scripts to create the required database structure. These scripts can be found inside the source tree [sql](./sql "sql") directory. The SQL scripts filename is, by convention, the date as `YYYYMMDD` and the suffix `.sql`. You need to apply all the SQL scripts for your database ordered by name. For example, `20190828.sql` must be applied before `20191112.sql`, and so on.
-Example for SQLite: `find sql/sqlite/ -type f -iname '*.sql' -print | sort -n | xargs cat | sqlite3 sftpgo.db`.
-After applying these scripts, your database structure is the same as the one obtained using `initprovider` for new installations, so from now on, you don't have to manually upgrade your database anymore.
+After starting SFTPGo you can manage users and folders using:
+
+- the [web based administration interface](./docs/web-admin.md)
+- the [REST API](./docs/rest-api.md)
+- the sample [REST API CLI](./examples/rest-api-cli)
+
+To support embedded data providers like `bolt` and `SQLite` we can't have a CLI that directly write users and folders to the data provider, we always have to use the REST API.
+
+## Tutorials
+
+Some step-to-step tutorials can be found inside the source tree [howto](./docs/howto "How-to") directory.
 
 ## Authentication options
 
@@ -141,15 +160,24 @@ More information about custom actions can be found [here](./docs/custom-actions.
 
 Directories outside the user home directory can be exposed as virtual folders, more information [here](./docs/virtual-folders.md).
 
+## Other hooks
+
+You can get notified as soon as a new connection is established using the [Post-connect hook](./docs/post-connect-hook.md) and after each login using the [Post-login hook](./docs/post-login-hook.md).
+You can use your own hook to [check passwords](./docs/check-password-hook.md).
+
 ## Storage backends
 
-### S3 Compabible Object Storage backends
+### S3 Compatible Object Storage backends
 
-Each user can be mapped to whole bucket or to a bucket virtual folder. This way, the mapped bucket/virtual folder is exposed over SFTP/SCP. More information about S3 integration can be found [here](./docs/s3.md).
+Each user can be mapped to the whole bucket or to a bucket virtual folder. This way, the mapped bucket/virtual folder is exposed over SFTP/SCP/FTP/WebDAV. More information about S3 integration can be found [here](./docs/s3.md).
 
 ### Google Cloud Storage backend
 
-Each user can be mapped with a Google Cloud Storage bucket or a bucket virtual folder. This way, the mapped bucket/virtual folder is exposed over SFTP/SCP. More information about Google Cloud Storage integration can be found [here](./docs/google-cloud-storage.md).
+Each user can be mapped with a Google Cloud Storage bucket or a bucket virtual folder. This way, the mapped bucket/virtual folder is exposed over SFTP/SCP/FTP/WebDAV. More information about Google Cloud Storage integration can be found [here](./docs/google-cloud-storage.md).
+
+### Azure Blob Storage backend
+
+Each user can be mapped with an Azure Blob Storage container or a container virtual folder. This way, the mapped container/virtual folder is exposed over SFTP/SCP/FTP/WebDAV. More information about Azure Blob Storage integration can be found [here](./docs/azure-blob-storage.md).
 
 ### Other Storage backends
 
@@ -176,6 +204,10 @@ SFTPGo can easily saturate a Gigabit connection on low end hardware with no spec
 
 More in-depth analysis of performance can be found [here](./docs/performance.md).
 
+## Release Cadence
+
+SFTPGo releases are feature-driven, we don't have a fixed time based schedule. As a rough estimate, you can expect 1 or 2 new releases per year.
+
 ## Acknowledgements
 
 SFTPGo makes use of the third party libraries listed inside [go.mod](./go.mod).

SECURITY.md

@@ -0,0 +1,12 @@
+# Security Policy
+
+## Supported Versions
+
+Only the current release of the software is actively supported.  If you need
+help backporting fixes into an older release, feel free to ask.
+
+## Reporting a Vulnerability
+
+Email your vulnerability information to SFTPGo's maintainer:
+
+  Nicola Murino <nicola.murino@gmail.com>

cmd/gen.go

@@ -0,0 +1,12 @@
+package cmd
+
+import "github.com/spf13/cobra"
+
+var genCmd = &cobra.Command{
+	Use:   "gen",
+	Short: "A collection of useful generators",
+}
+
+func init() {
+	rootCmd.AddCommand(genCmd)
+}

cmd/gencompletion.go

@@ -0,0 +1,76 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/rs/zerolog"
+	"github.com/spf13/cobra"
+
+	"github.com/drakkan/sftpgo/logger"
+)
+
+var genCompletionCmd = &cobra.Command{
+	Use:   "completion [bash|zsh|fish|powershell]",
+	Short: "Generate shell completion script",
+	Long: `To load completions:
+
+Bash:
+
+$ source <(sftpgo gen completion bash)
+
+To load completions for each session, execute once:
+
+Linux:
+
+$ sudo sftpgo gen completion bash > /usr/share/bash-completion/completions/sftpgo
+
+MacOS:
+
+$ sudo sftpgo gen completion bash > /usr/local/etc/bash_completion.d/sftpgo
+
+Zsh:
+
+If shell completion is not already enabled in your environment you will need
+to enable it.  You can execute the following once:
+
+$ echo "autoload -U compinit; compinit" >> ~/.zshrc
+
+To load completions for each session, execute once:
+
+$ sftpgo gen completion zsh > "${fpath[1]}/_sftpgo"
+
+Fish:
+
+$ sftpgo gen completion fish | source
+
+To load completions for each session, execute once:
+
+$ sftpgo gen completion fish > ~/.config/fish/completions/sftpgo.fish
+`,
+	DisableFlagsInUseLine: true,
+	ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
+	Args:                  cobra.ExactValidArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		var err error
+		logger.DisableLogger()
+		logger.EnableConsoleLogger(zerolog.DebugLevel)
+		switch args[0] {
+		case "bash":
+			err = cmd.Root().GenBashCompletion(os.Stdout)
+		case "zsh":
+			err = cmd.Root().GenZshCompletion(os.Stdout)
+		case "fish":
+			err = cmd.Root().GenFishCompletion(os.Stdout, true)
+		case "powershell":
+			err = cmd.Root().GenPowerShellCompletion(os.Stdout)
+		}
+		if err != nil {
+			logger.WarnToConsole("Unable to generate shell completion script: %v", err)
+			os.Exit(1)
+		}
+	},
+}
+
+func init() {
+	genCmd.AddCommand(genCompletionCmd)
+}

cmd/genman.go

@@ -0,0 +1,52 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/rs/zerolog"
+	"github.com/spf13/cobra"
+	"github.com/spf13/cobra/doc"
+
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/version"
+)
+
+var (
+	manDir    string
+	genManCmd = &cobra.Command{
+		Use:   "man",
+		Short: "Generate man pages for SFTPGo CLI",
+		Long: `This command automatically generates up-to-date man pages of SFTPGo's
+command-line interface.  By default, it creates the man page files
+in the "man" directory under the current directory.
+`,
+		Run: func(cmd *cobra.Command, args []string) {
+			logger.DisableLogger()
+			logger.EnableConsoleLogger(zerolog.DebugLevel)
+			if _, err := os.Stat(manDir); os.IsNotExist(err) {
+				err = os.MkdirAll(manDir, os.ModePerm)
+				if err != nil {
+					logger.WarnToConsole("Unable to generate man page files: %v", err)
+					os.Exit(1)
+				}
+			}
+			header := &doc.GenManHeader{
+				Section: "1",
+				Manual:  "SFTPGo Manual",
+				Source:  fmt.Sprintf("SFTPGo %v", version.Get().Version),
+			}
+			cmd.Root().DisableAutoGenTag = true
+			err := doc.GenManTree(cmd.Root(), header, manDir)
+			if err != nil {
+				logger.WarnToConsole("Unable to generate man page files: %v", err)
+				os.Exit(1)
+			}
+		},
+	}
+)
+
+func init() {
+	genManCmd.Flags().StringVarP(&manDir, "dir", "d", "man", "The directory to write the man pages")
+	genCmd.AddCommand(genManCmd)
+}

cmd/initprovider.go

@@ -16,18 +16,22 @@ import (
 var (
 	initProviderCmd = &cobra.Command{
 		Use:   "initprovider",
-		Short: "Initializes the configured data provider",
-		Long: `This command reads the data provider connection details from the specified configuration file and creates the initial structure.
+		Short: "Initializes and/or updates the configured data provider",
+		Long: `This command reads the data provider connection details from the specified
+configuration file and creates the initial structure or update the existing one,
+as needed.
 
-Some data providers such as bolt and memory does not require an initialization.
+Some data providers such as bolt and memory does not require an initialization
+but they could require an update to the existing data after upgrading SFTPGo.
 
-For SQLite provider the database file will be auto created if missing.
+For SQLite/bolt providers the database file will be auto-created if missing.
 
-For PostgreSQL and MySQL providers you need to create the configured database, this command will create the required tables.
+For PostgreSQL and MySQL providers you need to create the configured database,
+this command will create/update the required tables as needed.
 
-To initialize the data provider from the configuration directory simply use:
+To initialize/update the data provider from the configuration directory simply use:
 
-sftpgo initprovider
+$ sftpgo initprovider
 
 Please take a look at the usage below to customize the options.`,
 		Run: func(cmd *cobra.Command, args []string) {
@@ -40,12 +44,14 @@ Please take a look at the usage below to customize the options.`,
 				return
 			}
 			providerConf := config.GetProviderConf()
-			logger.DebugToConsole("Initializing provider: %#v config file: %#v", providerConf.Driver, viper.ConfigFileUsed())
+			logger.InfoToConsole("Initializing provider: %#v config file: %#v", providerConf.Driver, viper.ConfigFileUsed())
 			err = dataprovider.InitializeDatabase(providerConf, configDir)
 			if err == nil {
-				logger.DebugToConsole("Data provider successfully initialized")
+				logger.InfoToConsole("Data provider successfully initialized/updated")
+			} else if err == dataprovider.ErrNoInitRequired {
+				logger.InfoToConsole("%v", err.Error())
 			} else {
-				logger.WarnToConsole("Unable to initialize data provider: %v", err)
+				logger.WarnToConsole("Unable to initialize/update the data provider: %v", err)
 				os.Exit(1)
 			}
 		},

cmd/install_windows.go

@@ -15,7 +15,8 @@ var (
 	installCmd = &cobra.Command{
 		Use:   "install",
 		Short: "Install SFTPGo as Windows Service",
-		Long: `To install the SFTPGo Windows Service with the default values for the command line flags simply use:
+		Long: `To install the SFTPGo Windows Service with the default values for the command
+line flags simply use:
 
 sftpgo service install
 

cmd/portable.go

@@ -3,7 +3,6 @@
 package cmd
 
 import (
-	"encoding/base64"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -13,6 +12,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/drakkan/sftpgo/common"
 	"github.com/drakkan/sftpgo/dataprovider"
 	"github.com/drakkan/sftpgo/service"
 	"github.com/drakkan/sftpgo/sftpd"
@@ -32,8 +32,8 @@ var (
 	portablePublicKeys           []string
 	portablePermissions          []string
 	portableSSHCommands          []string
-	portableAllowedExtensions    []string
-	portableDeniedExtensions     []string
+	portableAllowedPatterns      []string
+	portableDeniedPatterns       []string
 	portableFsProvider           int
 	portableS3Bucket             string
 	portableS3Region             string
@@ -49,18 +49,36 @@ var (
 	portableGCSAutoCredentials   int
 	portableGCSStorageClass      string
 	portableGCSKeyPrefix         string
+	portableFTPDPort             int
+	portableFTPSCert             string
+	portableFTPSKey              string
+	portableWebDAVPort           int
+	portableWebDAVCert           string
+	portableWebDAVKey            string
+	portableAzContainer          string
+	portableAzAccountName        string
+	portableAzAccountKey         string
+	portableAzEndpoint           string
+	portableAzAccessTier         string
+	portableAzSASURL             string
+	portableAzKeyPrefix          string
+	portableAzULPartSize         int
+	portableAzULConcurrency      int
+	portableAzUseEmulator        bool
 	portableCmd                  = &cobra.Command{
 		Use:   "portable",
 		Short: "Serve a single directory",
-		Long: `To serve the current working directory with auto generated credentials simply use:
+		Long: `To serve the current working directory with auto generated credentials simply
+use:
 
-sftpgo portable
+$ sftpgo portable
 
 Please take a look at the usage below to customize the serving parameters`,
 		Run: func(cmd *cobra.Command, args []string) {
 			portableDir := directoryToServe
+			fsProvider := dataprovider.FilesystemProvider(portableFsProvider)
 			if !filepath.IsAbs(portableDir) {
-				if portableFsProvider == 0 {
+				if fsProvider == dataprovider.LocalFilesystemProvider {
 					portableDir, _ = filepath.Abs(portableDir)
 				} else {
 					portableDir = os.TempDir()
@@ -68,8 +86,8 @@ Please take a look at the usage below to customize the serving parameters`,
 			}
 			permissions := make(map[string][]string)
 			permissions["/"] = portablePermissions
-			portableGCSCredentials := ""
-			if portableFsProvider == 2 && len(portableGCSCredentialsFile) > 0 {
+			var portableGCSCredentials []byte
+			if fsProvider == dataprovider.GCSFilesystemProvider && len(portableGCSCredentialsFile) > 0 {
 				fi, err := os.Stat(portableGCSCredentialsFile)
 				if err != nil {
 					fmt.Printf("Invalid GCS credentials file: %v\n", err)
@@ -84,9 +102,25 @@ Please take a look at the usage below to customize the serving parameters`,
 				if err != nil {
 					fmt.Printf("Unable to read credentials file: %v\n", err)
 				}
-				portableGCSCredentials = base64.StdEncoding.EncodeToString(creds)
+				portableGCSCredentials = creds
 				portableGCSAutoCredentials = 0
 			}
+			if portableFTPDPort >= 0 && len(portableFTPSCert) > 0 && len(portableFTPSKey) > 0 {
+				_, err := common.NewCertManager(portableFTPSCert, portableFTPSKey, "FTP portable")
+				if err != nil {
+					fmt.Printf("Unable to load FTPS key pair, cert file %#v key file %#v error: %v\n",
+						portableFTPSCert, portableFTPSKey, err)
+					os.Exit(1)
+				}
+			}
+			if portableWebDAVPort > 0 && len(portableWebDAVCert) > 0 && len(portableWebDAVKey) > 0 {
+				_, err := common.NewCertManager(portableWebDAVCert, portableWebDAVKey, "WebDAV portable")
+				if err != nil {
+					fmt.Printf("Unable to load WebDAV key pair, cert file %#v key file %#v error: %v\n",
+						portableWebDAVCert, portableWebDAVKey, err)
+					os.Exit(1)
+				}
+			}
 			service := service.Service{
 				ConfigDir:     filepath.Clean(defaultConfigDir),
 				ConfigFile:    defaultConfigName,
@@ -107,7 +141,7 @@ Please take a look at the usage below to customize the serving parameters`,
 					HomeDir:     portableDir,
 					Status:      1,
 					FsConfig: dataprovider.Filesystem{
-						Provider: portableFsProvider,
+						Provider: dataprovider.FilesystemProvider(portableFsProvider),
 						S3Config: vfs.S3FsConfig{
 							Bucket:            portableS3Bucket,
 							Region:            portableS3Region,
@@ -126,16 +160,30 @@ Please take a look at the usage below to customize the serving parameters`,
 							StorageClass:         portableGCSStorageClass,
 							KeyPrefix:            portableGCSKeyPrefix,
 						},
+						AzBlobConfig: vfs.AzBlobFsConfig{
+							Container:         portableAzContainer,
+							AccountName:       portableAzAccountName,
+							AccountKey:        portableAzAccountKey,
+							Endpoint:          portableAzEndpoint,
+							AccessTier:        portableAzAccessTier,
+							SASURL:            portableAzSASURL,
+							KeyPrefix:         portableAzKeyPrefix,
+							UseEmulator:       portableAzUseEmulator,
+							UploadPartSize:    int64(portableAzULPartSize),
+							UploadConcurrency: portableAzULConcurrency,
+						},
 					},
 					Filters: dataprovider.UserFilters{
-						FileExtensions: parseFileExtensionsFilters(),
+						FilePatterns: parsePatternsFilesFilters(),
 					},
 				},
 			}
-			if err := service.StartPortableMode(portableSFTPDPort, portableSSHCommands, portableAdvertiseService,
-				portableAdvertiseCredentials); err == nil {
+			if err := service.StartPortableMode(portableSFTPDPort, portableFTPDPort, portableWebDAVPort, portableSSHCommands, portableAdvertiseService,
+				portableAdvertiseCredentials, portableFTPSCert, portableFTPSKey, portableWebDAVCert, portableWebDAVKey); err == nil {
 				service.Wait()
-				os.Exit(0)
+				if service.Error == nil {
+					os.Exit(0)
+				}
 			}
 			os.Exit(1)
 		},
@@ -145,84 +193,136 @@ Please take a look at the usage below to customize the serving parameters`,
 func init() {
 	version.AddFeature("+portable")
 
-	portableCmd.Flags().StringVarP(&directoryToServe, "directory", "d", ".",
-		"Path to the directory to serve. This can be an absolute path or a path relative to the current directory")
-	portableCmd.Flags().IntVarP(&portableSFTPDPort, "sftpd-port", "s", 0, "0 means a random non privileged port")
+	portableCmd.Flags().StringVarP(&directoryToServe, "directory", "d", ".", `Path to the directory to serve.
+This can be an absolute path or a path
+relative to the current directory
+`)
+	portableCmd.Flags().IntVarP(&portableSFTPDPort, "sftpd-port", "s", 0, "0 means a random unprivileged port")
+	portableCmd.Flags().IntVar(&portableFTPDPort, "ftpd-port", -1, `0 means a random unprivileged port,
+< 0 disabled`)
+	portableCmd.Flags().IntVar(&portableWebDAVPort, "webdav-port", -1, `0 means a random unprivileged port,
+< 0 disabled`)
 	portableCmd.Flags().StringSliceVarP(&portableSSHCommands, "ssh-commands", "c", sftpd.GetDefaultSSHCommands(),
-		"SSH commands to enable. \"*\" means any supported SSH command including scp")
-	portableCmd.Flags().StringVarP(&portableUsername, "username", "u", "", "Leave empty to use an auto generated value")
-	portableCmd.Flags().StringVarP(&portablePassword, "password", "p", "", "Leave empty to use an auto generated value")
+		`SSH commands to enable.
+"*" means any supported SSH command
+including scp
+`)
+	portableCmd.Flags().StringVarP(&portableUsername, "username", "u", "", `Leave empty to use an auto generated
+value`)
+	portableCmd.Flags().StringVarP(&portablePassword, "password", "p", "", `Leave empty to use an auto generated
+value`)
 	portableCmd.Flags().StringVarP(&portableLogFile, logFilePathFlag, "l", "", "Leave empty to disable logging")
 	portableCmd.Flags().BoolVarP(&portableLogVerbose, logVerboseFlag, "v", false, "Enable verbose logs")
 	portableCmd.Flags().StringSliceVarP(&portablePublicKeys, "public-key", "k", []string{}, "")
 	portableCmd.Flags().StringSliceVarP(&portablePermissions, "permissions", "g", []string{"list", "download"},
-		"User's permissions. \"*\" means any permission")
-	portableCmd.Flags().StringArrayVar(&portableAllowedExtensions, "allowed-extensions", []string{},
-		"Allowed file extensions case insensitive. The format is /dir::ext1,ext2. For example: \"/somedir::.jpg,.png\"")
-	portableCmd.Flags().StringArrayVar(&portableDeniedExtensions, "denied-extensions", []string{},
-		"Denied file extensions case insensitive. The format is /dir::ext1,ext2. For example: \"/somedir::.jpg,.png\"")
+		`User's permissions. "*" means any
+permission`)
+	portableCmd.Flags().StringArrayVar(&portableAllowedPatterns, "allowed-patterns", []string{},
+		`Allowed file patterns case insensitive.
+The format is:
+/dir::pattern1,pattern2.
+For example: "/somedir::*.jpg,a*b?.png"`)
+	portableCmd.Flags().StringArrayVar(&portableDeniedPatterns, "denied-patterns", []string{},
+		`Denied file patterns case insensitive.
+The format is:
+/dir::pattern1,pattern2.
+For example: "/somedir::*.jpg,a*b?.png"`)
 	portableCmd.Flags().BoolVarP(&portableAdvertiseService, "advertise-service", "S", false,
-		"Advertise SFTP service using multicast DNS")
+		`Advertise SFTP/FTP service using
+multicast DNS`)
 	portableCmd.Flags().BoolVarP(&portableAdvertiseCredentials, "advertise-credentials", "C", false,
-		"If the SFTP service is advertised via multicast DNS, this flag allows to put username/password inside the advertised TXT record")
-	portableCmd.Flags().IntVarP(&portableFsProvider, "fs-provider", "f", 0, "0 means local filesystem, 1 Amazon S3 compatible, "+
-		"2 Google Cloud Storage")
+		`If the SFTP/FTP service is
+advertised via multicast DNS, this
+flag allows to put username/password
+inside the advertised TXT record`)
+	portableCmd.Flags().IntVarP(&portableFsProvider, "fs-provider", "f", int(dataprovider.LocalFilesystemProvider), `0 => local filesystem
+1 => AWS S3 compatible
+2 => Google Cloud Storage
+3 => Azure Blob Storage`)
 	portableCmd.Flags().StringVar(&portableS3Bucket, "s3-bucket", "", "")
 	portableCmd.Flags().StringVar(&portableS3Region, "s3-region", "", "")
 	portableCmd.Flags().StringVar(&portableS3AccessKey, "s3-access-key", "", "")
 	portableCmd.Flags().StringVar(&portableS3AccessSecret, "s3-access-secret", "", "")
 	portableCmd.Flags().StringVar(&portableS3Endpoint, "s3-endpoint", "", "")
 	portableCmd.Flags().StringVar(&portableS3StorageClass, "s3-storage-class", "", "")
-	portableCmd.Flags().StringVar(&portableS3KeyPrefix, "s3-key-prefix", "", "Allows to restrict access to the virtual folder "+
-		"identified by this prefix and its contents")
-	portableCmd.Flags().IntVar(&portableS3ULPartSize, "s3-upload-part-size", 5, "The buffer size for multipart uploads (MB)")
-	portableCmd.Flags().IntVar(&portableS3ULConcurrency, "s3-upload-concurrency", 2, "How many parts are uploaded in parallel")
+	portableCmd.Flags().StringVar(&portableS3KeyPrefix, "s3-key-prefix", "", `Allows to restrict access to the
+virtual folder identified by this
+prefix and its contents`)
+	portableCmd.Flags().IntVar(&portableS3ULPartSize, "s3-upload-part-size", 5, `The buffer size for multipart uploads
+(MB)`)
+	portableCmd.Flags().IntVar(&portableS3ULConcurrency, "s3-upload-concurrency", 2, `How many parts are uploaded in
+parallel`)
 	portableCmd.Flags().StringVar(&portableGCSBucket, "gcs-bucket", "", "")
 	portableCmd.Flags().StringVar(&portableGCSStorageClass, "gcs-storage-class", "", "")
-	portableCmd.Flags().StringVar(&portableGCSKeyPrefix, "gcs-key-prefix", "", "Allows to restrict access to the virtual folder "+
-		"identified by this prefix and its contents")
-	portableCmd.Flags().StringVar(&portableGCSCredentialsFile, "gcs-credentials-file", "", "Google Cloud Storage JSON credentials file")
-	portableCmd.Flags().IntVar(&portableGCSAutoCredentials, "gcs-automatic-credentials", 1, "0 means explicit credentials using a JSON "+
-		"credentials file, 1 automatic")
+	portableCmd.Flags().StringVar(&portableGCSKeyPrefix, "gcs-key-prefix", "", `Allows to restrict access to the
+virtual folder identified by this
+prefix and its contents`)
+	portableCmd.Flags().StringVar(&portableGCSCredentialsFile, "gcs-credentials-file", "", `Google Cloud Storage JSON credentials
+file`)
+	portableCmd.Flags().IntVar(&portableGCSAutoCredentials, "gcs-automatic-credentials", 1, `0 means explicit credentials using
+a JSON credentials file, 1 automatic
+`)
+	portableCmd.Flags().StringVar(&portableFTPSCert, "ftpd-cert", "", "Path to the certificate file for FTPS")
+	portableCmd.Flags().StringVar(&portableFTPSKey, "ftpd-key", "", "Path to the key file for FTPS")
+	portableCmd.Flags().StringVar(&portableWebDAVCert, "webdav-cert", "", `Path to the certificate file for WebDAV
+over HTTPS`)
+	portableCmd.Flags().StringVar(&portableWebDAVKey, "webdav-key", "", `Path to the key file for WebDAV over
+HTTPS`)
+	portableCmd.Flags().StringVar(&portableAzContainer, "az-container", "", "")
+	portableCmd.Flags().StringVar(&portableAzAccountName, "az-account-name", "", "")
+	portableCmd.Flags().StringVar(&portableAzAccountKey, "az-account-key", "", "")
+	portableCmd.Flags().StringVar(&portableAzSASURL, "az-sas-url", "", `Shared access signature URL`)
+	portableCmd.Flags().StringVar(&portableAzEndpoint, "az-endpoint", "", `Leave empty to use the default:
+"blob.core.windows.net"`)
+	portableCmd.Flags().StringVar(&portableAzAccessTier, "az-access-tier", "", `Leave empty to use the default
+container setting`)
+	portableCmd.Flags().StringVar(&portableAzKeyPrefix, "az-key-prefix", "", `Allows to restrict access to the
+virtual folder identified by this
+prefix and its contents`)
+	portableCmd.Flags().IntVar(&portableAzULPartSize, "az-upload-part-size", 4, `The buffer size for multipart uploads
+(MB)`)
+	portableCmd.Flags().IntVar(&portableAzULConcurrency, "az-upload-concurrency", 2, `How many parts are uploaded in
+parallel`)
+	portableCmd.Flags().BoolVar(&portableAzUseEmulator, "az-use-emulator", false, "")
 	rootCmd.AddCommand(portableCmd)
 }
 
-func parseFileExtensionsFilters() []dataprovider.ExtensionsFilter {
-	var extensions []dataprovider.ExtensionsFilter
-	for _, val := range portableAllowedExtensions {
-		p, exts := getExtensionsFilterValues(strings.TrimSpace(val))
+func parsePatternsFilesFilters() []dataprovider.PatternsFilter {
+	var patterns []dataprovider.PatternsFilter
+	for _, val := range portableAllowedPatterns {
+		p, exts := getPatternsFilterValues(strings.TrimSpace(val))
 		if len(p) > 0 {
-			extensions = append(extensions, dataprovider.ExtensionsFilter{
-				Path:              path.Clean(p),
-				AllowedExtensions: exts,
-				DeniedExtensions:  []string{},
+			patterns = append(patterns, dataprovider.PatternsFilter{
+				Path:            path.Clean(p),
+				AllowedPatterns: exts,
+				DeniedPatterns:  []string{},
 			})
 		}
 	}
-	for _, val := range portableDeniedExtensions {
-		p, exts := getExtensionsFilterValues(strings.TrimSpace(val))
+	for _, val := range portableDeniedPatterns {
+		p, exts := getPatternsFilterValues(strings.TrimSpace(val))
 		if len(p) > 0 {
 			found := false
-			for index, e := range extensions {
+			for index, e := range patterns {
 				if path.Clean(e.Path) == path.Clean(p) {
-					extensions[index].DeniedExtensions = append(extensions[index].DeniedExtensions, exts...)
+					patterns[index].DeniedPatterns = append(patterns[index].DeniedPatterns, exts...)
 					found = true
 					break
 				}
 			}
 			if !found {
-				extensions = append(extensions, dataprovider.ExtensionsFilter{
-					Path:              path.Clean(p),
-					AllowedExtensions: []string{},
-					DeniedExtensions:  exts,
+				patterns = append(patterns, dataprovider.PatternsFilter{
+					Path:            path.Clean(p),
+					AllowedPatterns: []string{},
+					DeniedPatterns:  exts,
 				})
 			}
 		}
 	}
-	return extensions
+	return patterns
 }
 
-func getExtensionsFilterValues(value string) (string, []string) {
+func getPatternsFilterValues(value string) (string, []string) {
 	if strings.Contains(value, "::") {
 		dirExts := strings.Split(value, "::")
 		if len(dirExts) > 1 {
@@ -234,7 +334,7 @@ func getExtensionsFilterValues(value string) (string, []string) {
 					exts = append(exts, cleanedExt)
 				}
 			}
-			if len(dir) > 0 && len(exts) > 0 {
+			if dir != "" && len(exts) > 0 {
 				return dir, exts
 			}
 		}

cmd/root.go

@@ -13,49 +13,65 @@ import (
 )
 
 const (
-	configDirFlag       = "config-dir"
-	configDirKey        = "config_dir"
-	configFileFlag      = "config-file"
-	configFileKey       = "config_file"
-	logFilePathFlag     = "log-file-path"
-	logFilePathKey      = "log_file_path"
-	logMaxSizeFlag      = "log-max-size"
-	logMaxSizeKey       = "log_max_size"
-	logMaxBackupFlag    = "log-max-backups"
-	logMaxBackupKey     = "log_max_backups"
-	logMaxAgeFlag       = "log-max-age"
-	logMaxAgeKey        = "log_max_age"
-	logCompressFlag     = "log-compress"
-	logCompressKey      = "log_compress"
-	logVerboseFlag      = "log-verbose"
-	logVerboseKey       = "log_verbose"
-	profilerFlag        = "profiler"
-	profilerKey         = "profiler"
-	defaultConfigDir    = "."
-	defaultConfigName   = config.DefaultConfigName
-	defaultLogFile      = "sftpgo.log"
-	defaultLogMaxSize   = 10
-	defaultLogMaxBackup = 5
-	defaultLogMaxAge    = 28
-	defaultLogCompress  = false
-	defaultLogVerbose   = true
-	defaultProfiler     = false
+	configDirFlag            = "config-dir"
+	configDirKey             = "config_dir"
+	configFileFlag           = "config-file"
+	configFileKey            = "config_file"
+	logFilePathFlag          = "log-file-path"
+	logFilePathKey           = "log_file_path"
+	logMaxSizeFlag           = "log-max-size"
+	logMaxSizeKey            = "log_max_size"
+	logMaxBackupFlag         = "log-max-backups"
+	logMaxBackupKey          = "log_max_backups"
+	logMaxAgeFlag            = "log-max-age"
+	logMaxAgeKey             = "log_max_age"
+	logCompressFlag          = "log-compress"
+	logCompressKey           = "log_compress"
+	logVerboseFlag           = "log-verbose"
+	logVerboseKey            = "log_verbose"
+	profilerFlag             = "profiler"
+	profilerKey              = "profiler"
+	loadDataFromFlag         = "loaddata-from"
+	loadDataFromKey          = "loaddata_from"
+	loadDataModeFlag         = "loaddata-mode"
+	loadDataModeKey          = "loaddata_mode"
+	loadDataQuotaScanFlag    = "loaddata-scan"
+	loadDataQuotaScanKey     = "loaddata_scan"
+	loadDataCleanFlag        = "loaddata-clean"
+	loadDataCleanKey         = "loaddata_clean"
+	defaultConfigDir         = "."
+	defaultConfigName        = config.DefaultConfigName
+	defaultLogFile           = "sftpgo.log"
+	defaultLogMaxSize        = 10
+	defaultLogMaxBackup      = 5
+	defaultLogMaxAge         = 28
+	defaultLogCompress       = false
+	defaultLogVerbose        = true
+	defaultProfiler          = false
+	defaultLoadDataFrom      = ""
+	defaultLoadDataMode      = 1
+	defaultLoadDataQuotaScan = 0
+	defaultLoadDataClean     = false
 )
 
 var (
-	configDir     string
-	configFile    string
-	logFilePath   string
-	logMaxSize    int
-	logMaxBackups int
-	logMaxAge     int
-	logCompress   bool
-	logVerbose    bool
-	profiler      bool
+	configDir         string
+	configFile        string
+	logFilePath       string
+	logMaxSize        int
+	logMaxBackups     int
+	logMaxAge         int
+	logCompress       bool
+	logVerbose        bool
+	profiler          bool
+	loadDataFrom      string
+	loadDataMode      int
+	loadDataQuotaScan int
+	loadDataClean     bool
 
 	rootCmd = &cobra.Command{
 		Use:   "sftpgo",
-		Short: "Full featured and highly configurable SFTP server",
+		Short: "Fully featured and highly configurable SFTP server",
 	}
 )
 
@@ -79,18 +95,29 @@ func addConfigFlags(cmd *cobra.Command) {
 	viper.SetDefault(configDirKey, defaultConfigDir)
 	viper.BindEnv(configDirKey, "SFTPGO_CONFIG_DIR") //nolint:errcheck // err is not nil only if the key to bind is missing
 	cmd.Flags().StringVarP(&configDir, configDirFlag, "c", viper.GetString(configDirKey),
-		"Location for SFTPGo config dir. This directory should contain the \"sftpgo\" configuration file or the configured "+
-			"config-file and it is used as the base for files with a relative path (eg. the private keys for the SFTP server, "+
-			"the SQLite database if you use SQLite as data provider). This flag can be set using SFTPGO_CONFIG_DIR env var too.")
+		`Location for SFTPGo config dir. This directory
+should contain the "sftpgo" configuration file
+or the configured config-file and it is used as
+the base for files with a relative path (eg. the
+private keys for the SFTP server, the SQLite
+database if you use SQLite as data provider).
+This flag can be set using SFTPGO_CONFIG_DIR
+env var too.`)
 	viper.BindPFlag(configDirKey, cmd.Flags().Lookup(configDirFlag)) //nolint:errcheck
 
 	viper.SetDefault(configFileKey, defaultConfigName)
 	viper.BindEnv(configFileKey, "SFTPGO_CONFIG_FILE") //nolint:errcheck
 	cmd.Flags().StringVarP(&configFile, configFileFlag, "f", viper.GetString(configFileKey),
-		"Name for SFTPGo configuration file. It must be the name of a file stored in config-dir not the absolute path to the "+
-			"configuration file. The specified file name must have no extension we automatically load JSON, YAML, TOML, HCL and "+
-			"Java properties. Therefore if you set \"sftpgo\" then \"sftpgo.json\", \"sftpgo.yaml\" and so on are searched. "+
-			"This flag can be set using SFTPGO_CONFIG_FILE env var too.")
+		`Name for SFTPGo configuration file. It must be
+the name of a file stored in config-dir not the
+absolute path to the configuration file. The
+specified file name must have no extension we
+automatically load JSON, YAML, TOML, HCL and
+Java properties. Therefore if you set "sftpgo"
+then "sftpgo.json", "sftpgo.yaml" and so on
+are searched.
+This flag can be set using SFTPGO_CONFIG_FILE
+env var too.`)
 	viper.BindPFlag(configFileKey, cmd.Flags().Lookup(configFileFlag)) //nolint:errcheck
 }
 
@@ -100,48 +127,112 @@ func addServeFlags(cmd *cobra.Command) {
 	viper.SetDefault(logFilePathKey, defaultLogFile)
 	viper.BindEnv(logFilePathKey, "SFTPGO_LOG_FILE_PATH") //nolint:errcheck
 	cmd.Flags().StringVarP(&logFilePath, logFilePathFlag, "l", viper.GetString(logFilePathKey),
-		"Location for the log file. Leave empty to write logs to the standard output. This flag can be set using SFTPGO_LOG_FILE_PATH "+
-			"env var too.")
+		`Location for the log file. Leave empty to write
+logs to the standard output. This flag can be
+set using SFTPGO_LOG_FILE_PATH env var too.
+`)
 	viper.BindPFlag(logFilePathKey, cmd.Flags().Lookup(logFilePathFlag)) //nolint:errcheck
 
 	viper.SetDefault(logMaxSizeKey, defaultLogMaxSize)
 	viper.BindEnv(logMaxSizeKey, "SFTPGO_LOG_MAX_SIZE") //nolint:errcheck
 	cmd.Flags().IntVarP(&logMaxSize, logMaxSizeFlag, "s", viper.GetInt(logMaxSizeKey),
-		"Maximum size in megabytes of the log file before it gets rotated. This flag can be set using SFTPGO_LOG_MAX_SIZE "+
-			"env var too. It is unused if log-file-path is empty.")
+		`Maximum size in megabytes of the log file
+before it gets rotated. This flag can be set
+using SFTPGO_LOG_MAX_SIZE env var too. It is
+unused if log-file-path is empty.
+`)
 	viper.BindPFlag(logMaxSizeKey, cmd.Flags().Lookup(logMaxSizeFlag)) //nolint:errcheck
 
 	viper.SetDefault(logMaxBackupKey, defaultLogMaxBackup)
 	viper.BindEnv(logMaxBackupKey, "SFTPGO_LOG_MAX_BACKUPS") //nolint:errcheck
 	cmd.Flags().IntVarP(&logMaxBackups, "log-max-backups", "b", viper.GetInt(logMaxBackupKey),
-		"Maximum number of old log files to retain. This flag can be set using SFTPGO_LOG_MAX_BACKUPS env var too. "+
-			"It is unused if log-file-path is empty.")
+		`Maximum number of old log files to retain.
+This flag can be set using SFTPGO_LOG_MAX_BACKUPS
+env var too. It is unused if log-file-path is
+empty.`)
 	viper.BindPFlag(logMaxBackupKey, cmd.Flags().Lookup(logMaxBackupFlag)) //nolint:errcheck
 
 	viper.SetDefault(logMaxAgeKey, defaultLogMaxAge)
 	viper.BindEnv(logMaxAgeKey, "SFTPGO_LOG_MAX_AGE") //nolint:errcheck
 	cmd.Flags().IntVarP(&logMaxAge, "log-max-age", "a", viper.GetInt(logMaxAgeKey),
-		"Maximum number of days to retain old log files. This flag can be set using SFTPGO_LOG_MAX_AGE env var too. "+
-			"It is unused if log-file-path is empty.")
+		`Maximum number of days to retain old log files.
+This flag can be set using SFTPGO_LOG_MAX_AGE env
+var too. It is unused if log-file-path is empty.
+`)
 	viper.BindPFlag(logMaxAgeKey, cmd.Flags().Lookup(logMaxAgeFlag)) //nolint:errcheck
 
 	viper.SetDefault(logCompressKey, defaultLogCompress)
 	viper.BindEnv(logCompressKey, "SFTPGO_LOG_COMPRESS") //nolint:errcheck
-	cmd.Flags().BoolVarP(&logCompress, logCompressFlag, "z", viper.GetBool(logCompressKey), "Determine if the rotated "+
-		"log files should be compressed using gzip. This flag can be set using SFTPGO_LOG_COMPRESS env var too. "+
-		"It is unused if log-file-path is empty.")
+	cmd.Flags().BoolVarP(&logCompress, logCompressFlag, "z", viper.GetBool(logCompressKey),
+		`Determine if the rotated log files
+should be compressed using gzip. This flag can
+be set using SFTPGO_LOG_COMPRESS env var too.
+It is unused if log-file-path is empty.
+`)
 	viper.BindPFlag(logCompressKey, cmd.Flags().Lookup(logCompressFlag)) //nolint:errcheck
 
 	viper.SetDefault(logVerboseKey, defaultLogVerbose)
 	viper.BindEnv(logVerboseKey, "SFTPGO_LOG_VERBOSE") //nolint:errcheck
-	cmd.Flags().BoolVarP(&logVerbose, logVerboseFlag, "v", viper.GetBool(logVerboseKey), "Enable verbose logs. "+
-		"This flag can be set using SFTPGO_LOG_VERBOSE env var too.")
+	cmd.Flags().BoolVarP(&logVerbose, logVerboseFlag, "v", viper.GetBool(logVerboseKey),
+		`Enable verbose logs. This flag can be set
+using SFTPGO_LOG_VERBOSE env var too.
+`)
 	viper.BindPFlag(logVerboseKey, cmd.Flags().Lookup(logVerboseFlag)) //nolint:errcheck
 
 	viper.SetDefault(profilerKey, defaultProfiler)
 	viper.BindEnv(profilerKey, "SFTPGO_PROFILER") //nolint:errcheck
-	cmd.Flags().BoolVarP(&profiler, profilerFlag, "p", viper.GetBool(profilerKey), "Enable the built-in profiler. "+
-		"The profiler will be accessible via HTTP/HTTPS using the base URL \"/debug/pprof/\". "+
-		"This flag can be set using SFTPGO_PROFILER env var too.")
+	cmd.Flags().BoolVarP(&profiler, profilerFlag, "p", viper.GetBool(profilerKey),
+		`Enable the built-in profiler. The profiler will
+be accessible via HTTP/HTTPS using the base URL
+"/debug/pprof/".
+This flag can be set using SFTPGO_PROFILER env
+var too.`)
 	viper.BindPFlag(profilerKey, cmd.Flags().Lookup(profilerFlag)) //nolint:errcheck
+
+	viper.SetDefault(loadDataFromKey, defaultLoadDataFrom)
+	viper.BindEnv(loadDataFromKey, "SFTPGO_LOADDATA_FROM") //nolint:errcheck
+	cmd.Flags().StringVar(&loadDataFrom, loadDataFromFlag, viper.GetString(loadDataFromKey),
+		`Load users and folders from this file.
+The file must be specified as absolute path
+and it must contain a backup obtained using
+the "dumpdata" REST API or compatible content.
+This flag can be set using SFTPGO_LOADDATA_FROM
+env var too.
+`)
+	viper.BindPFlag(loadDataFromKey, cmd.Flags().Lookup(loadDataFromFlag)) //nolint:errcheck
+
+	viper.SetDefault(loadDataModeKey, defaultLoadDataMode)
+	viper.BindEnv(loadDataModeKey, "SFTPGO_LOADDATA_MODE") //nolint:errcheck
+	cmd.Flags().IntVar(&loadDataMode, loadDataModeFlag, viper.GetInt(loadDataModeKey),
+		`Restore mode for data to load:
+  0 - new users are added, existing users are
+      updated
+  1 - New users are added, existing users are
+	  not modified
+This flag can be set using SFTPGO_LOADDATA_MODE
+env var too.
+`)
+	viper.BindPFlag(loadDataModeKey, cmd.Flags().Lookup(loadDataModeFlag)) //nolint:errcheck
+
+	viper.SetDefault(loadDataQuotaScanKey, defaultLoadDataQuotaScan)
+	viper.BindEnv(loadDataQuotaScanKey, "SFTPGO_LOADDATA_QUOTA_SCAN") //nolint:errcheck
+	cmd.Flags().IntVar(&loadDataQuotaScan, loadDataQuotaScanFlag, viper.GetInt(loadDataQuotaScanKey),
+		`Quota scan mode after data load:
+  0 - no quota scan
+  1 - scan quota
+  2 - scan quota if the user has quota restrictions
+This flag can be set using SFTPGO_LOADDATA_QUOTA_SCAN
+env var too.
+(default 0)`)
+	viper.BindPFlag(loadDataQuotaScanKey, cmd.Flags().Lookup(loadDataQuotaScanFlag)) //nolint:errcheck
+
+	viper.SetDefault(loadDataCleanKey, defaultLoadDataClean)
+	viper.BindEnv(loadDataCleanKey, "SFTPGO_LOADDATA_CLEAN") //nolint:errcheck
+	cmd.Flags().BoolVar(&loadDataClean, loadDataCleanFlag, viper.GetBool(loadDataCleanKey),
+		`Determine if the loaddata-from file should
+be removed after a successful load. This flag
+can be set using SFTPGO_LOADDATA_CLEAN env var
+too. (default "false")
+`)
+	viper.BindPFlag(logCompressKey, cmd.Flags().Lookup(logCompressFlag)) //nolint:errcheck
 }

cmd/rotatelogs_windows.go

@@ -12,7 +12,7 @@ import (
 var (
 	rotateLogCmd = &cobra.Command{
 		Use:   "rotatelogs",
-		Short: "Signal to the running service to close the existing log file and immediately create a new one",
+		Short: "Signal to the running service to rotate the logs",
 		Run: func(cmd *cobra.Command, args []string) {
 			s := service.WindowsService{
 				Service: service.Service{

cmd/serve.go

@@ -13,27 +13,34 @@ var (
 	serveCmd = &cobra.Command{
 		Use:   "serve",
 		Short: "Start the SFTP Server",
-		Long: `To start the SFTPGo with the default values for the command line flags simply use:
+		Long: `To start the SFTPGo with the default values for the command line flags simply
+use:
 
-sftpgo serve
+$ sftpgo serve
 
 Please take a look at the usage below to customize the startup options`,
 		Run: func(cmd *cobra.Command, args []string) {
 			service := service.Service{
-				ConfigDir:     utils.CleanDirInput(configDir),
-				ConfigFile:    configFile,
-				LogFilePath:   logFilePath,
-				LogMaxSize:    logMaxSize,
-				LogMaxBackups: logMaxBackups,
-				LogMaxAge:     logMaxAge,
-				LogCompress:   logCompress,
-				LogVerbose:    logVerbose,
-				Profiler:      profiler,
-				Shutdown:      make(chan bool),
+				ConfigDir:         utils.CleanDirInput(configDir),
+				ConfigFile:        configFile,
+				LogFilePath:       logFilePath,
+				LogMaxSize:        logMaxSize,
+				LogMaxBackups:     logMaxBackups,
+				LogMaxAge:         logMaxAge,
+				LogCompress:       logCompress,
+				LogVerbose:        logVerbose,
+				LoadDataFrom:      loadDataFrom,
+				LoadDataMode:      loadDataMode,
+				LoadDataQuotaScan: loadDataQuotaScan,
+				LoadDataClean:     loadDataClean,
+				Profiler:          profiler,
+				Shutdown:          make(chan bool),
 			}
 			if err := service.Start(); err == nil {
 				service.Wait()
-				os.Exit(0)
+				if service.Error == nil {
+					os.Exit(0)
+				}
 			}
 			os.Exit(1)
 		},

cmd/service_windows.go

@@ -7,7 +7,7 @@ import (
 var (
 	serviceCmd = &cobra.Command{
 		Use:   "service",
-		Short: "Install, Uninstall, Start, Stop, Reload and retrieve status for SFTPGo Windows Service",
+		Short: "Manage SFTPGo Windows Service",
 	}
 )
 

cmd/startsubsys.go

@@ -0,0 +1,180 @@
+package cmd
+
+import (
+	"io"
+	"os"
+	"os/user"
+	"path/filepath"
+
+	"github.com/rs/xid"
+	"github.com/rs/zerolog"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/drakkan/sftpgo/common"
+	"github.com/drakkan/sftpgo/config"
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/sftpd"
+	"github.com/drakkan/sftpgo/version"
+)
+
+var (
+	logJournalD     = false
+	preserveHomeDir = false
+	baseHomeDir     = ""
+	subsystemCmd    = &cobra.Command{
+		Use:   "startsubsys",
+		Short: "Use SFTPGo as SFTP file transfer subsystem",
+		Long: `In this mode SFTPGo speaks the server side of SFTP protocol to stdout and
+expects client requests from stdin.
+This mode is not intended to be called directly, but from sshd using the
+Subsystem option.
+For example adding a line like this one in "/etc/ssh/sshd_config":
+
+Subsystem	sftp	sftpgo startsubsys
+
+Command-line flags should be specified in the Subsystem declaration.
+`,
+		Run: func(cmd *cobra.Command, args []string) {
+			logSender := "startsubsys"
+			connectionID := xid.New().String()
+			logLevel := zerolog.DebugLevel
+			if !logVerbose {
+				logLevel = zerolog.InfoLevel
+			}
+			if logJournalD {
+				logger.InitJournalDLogger(logLevel)
+			} else {
+				logger.InitStdErrLogger(logLevel)
+			}
+			osUser, err := user.Current()
+			if err != nil {
+				logger.Error(logSender, connectionID, "unable to get the current user: %v", err)
+				os.Exit(1)
+			}
+			username := osUser.Username
+			homedir := osUser.HomeDir
+			logger.Info(logSender, connectionID, "starting SFTPGo %v as subsystem, user %#v home dir %#v config dir %#v base home dir %#v",
+				version.Get(), username, homedir, configDir, baseHomeDir)
+			err = config.LoadConfig(configDir, configFile)
+			if err != nil {
+				logger.Error(logSender, connectionID, "unable to load configuration: %v", err)
+				os.Exit(1)
+			}
+			commonConfig := config.GetCommonConfig()
+			// idle connection are managed externally
+			commonConfig.IdleTimeout = 0
+			config.SetCommonConfig(commonConfig)
+			common.Initialize(config.GetCommonConfig())
+			dataProviderConf := config.GetProviderConf()
+			if dataProviderConf.Driver == dataprovider.SQLiteDataProviderName || dataProviderConf.Driver == dataprovider.BoltDataProviderName {
+				logger.Debug(logSender, connectionID, "data provider %#v not supported in subsystem mode, using %#v provider",
+					dataProviderConf.Driver, dataprovider.MemoryDataProviderName)
+				dataProviderConf.Driver = dataprovider.MemoryDataProviderName
+				dataProviderConf.Name = ""
+				dataProviderConf.PreferDatabaseCredentials = true
+			}
+			config.SetProviderConf(dataProviderConf)
+			err = dataprovider.Initialize(dataProviderConf, configDir)
+			if err != nil {
+				logger.Error(logSender, connectionID, "unable to initialize the data provider: %v", err)
+				os.Exit(1)
+			}
+			httpConfig := config.GetHTTPConfig()
+			httpConfig.Initialize(configDir)
+			user, err := dataprovider.UserExists(username)
+			if err == nil {
+				if user.HomeDir != filepath.Clean(homedir) && !preserveHomeDir {
+					// update the user
+					user.HomeDir = filepath.Clean(homedir)
+					err = dataprovider.UpdateUser(user)
+					if err != nil {
+						logger.Error(logSender, connectionID, "unable to update user %#v: %v", username, err)
+						os.Exit(1)
+					}
+				}
+			} else {
+				user.Username = username
+				if baseHomeDir != "" && filepath.IsAbs(baseHomeDir) {
+					user.HomeDir = filepath.Join(baseHomeDir, username)
+				} else {
+					user.HomeDir = filepath.Clean(homedir)
+				}
+				logger.Debug(logSender, connectionID, "home dir for new user %#v", user.HomeDir)
+				user.Password = connectionID
+				user.Permissions = make(map[string][]string)
+				user.Permissions["/"] = []string{dataprovider.PermAny}
+				err = dataprovider.AddUser(user)
+				if err != nil {
+					logger.Error(logSender, connectionID, "unable to add user %#v: %v", username, err)
+					os.Exit(1)
+				}
+			}
+			err = sftpd.ServeSubSystemConnection(user, connectionID, os.Stdin, os.Stdout)
+			if err != nil && err != io.EOF {
+				logger.Warn(logSender, connectionID, "serving subsystem finished with error: %v", err)
+				os.Exit(1)
+			}
+			logger.Info(logSender, connectionID, "serving subsystem finished")
+			os.Exit(0)
+		},
+	}
+)
+
+func init() {
+	subsystemCmd.Flags().BoolVarP(&preserveHomeDir, "preserve-home", "p", false, `If the user already exists, the existing home
+directory will not be changed`)
+	subsystemCmd.Flags().StringVarP(&baseHomeDir, "base-home-dir", "d", "", `If the user does not exist specify an alternate
+starting directory. The home directory for a new
+user will be:
+
+[base-home-dir]/[username]
+
+base-home-dir must be an absolute path.`)
+	subsystemCmd.Flags().BoolVarP(&logJournalD, "log-to-journald", "j", false, `Send logs to journald. Only available on Linux.
+Use:
+
+$ journalctl -o verbose -f
+
+To see full logs.
+If not set, the logs will be sent to the standard
+error`)
+	viper.SetDefault(configDirKey, defaultConfigDir)
+	viper.BindEnv(configDirKey, "SFTPGO_CONFIG_DIR") //nolint:errcheck // err is not nil only if the key to bind is missing
+	subsystemCmd.Flags().StringVarP(&configDir, configDirFlag, "c", viper.GetString(configDirKey),
+		`Location for SFTPGo config dir. This directory
+should contain the "sftpgo" configuration file
+or the configured config-file and it is used as
+the base for files with a relative path (eg. the
+private keys for the SFTP server, the SQLite
+database if you use SQLite as data provider).
+This flag can be set using SFTPGO_CONFIG_DIR
+env var too.`)
+	viper.BindPFlag(configDirKey, subsystemCmd.Flags().Lookup(configDirFlag)) //nolint:errcheck
+
+	viper.SetDefault(configFileKey, defaultConfigName)
+	viper.BindEnv(configFileKey, "SFTPGO_CONFIG_FILE") //nolint:errcheck
+	subsystemCmd.Flags().StringVarP(&configFile, configFileFlag, "f", viper.GetString(configFileKey),
+		`Name for SFTPGo configuration file. It must be
+the name of a file stored in config-dir not the
+absolute path to the configuration file. The
+specified file name must have no extension we
+automatically load JSON, YAML, TOML, HCL and
+Java properties. Therefore if you set "sftpgo"
+then "sftpgo.json", "sftpgo.yaml" and so on
+are searched.
+This flag can be set using SFTPGO_CONFIG_FILE
+env var too.`)
+	viper.BindPFlag(configFileKey, subsystemCmd.Flags().Lookup(configFileFlag)) //nolint:errcheck
+
+	viper.SetDefault(logVerboseKey, defaultLogVerbose)
+	viper.BindEnv(logVerboseKey, "SFTPGO_LOG_VERBOSE") //nolint:errcheck
+	subsystemCmd.Flags().BoolVarP(&logVerbose, logVerboseFlag, "v", viper.GetBool(logVerboseKey),
+		`Enable verbose logs. This flag can be set
+using SFTPGO_LOG_VERBOSE env var too.
+`)
+	viper.BindPFlag(logVerboseKey, subsystemCmd.Flags().Lookup(logVerboseFlag)) //nolint:errcheck
+
+	rootCmd.AddCommand(subsystemCmd)
+}

common/actions.go

@@ -0,0 +1,205 @@
+package common
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/httpclient"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/utils"
+)
+
+var (
+	errUnconfiguredAction    = errors.New("no hook is configured for this action")
+	errNoHook                = errors.New("unable to execute action, no hook defined")
+	errUnexpectedHTTResponse = errors.New("unexpected HTTP response code")
+)
+
+// ProtocolActions defines the action to execute on file operations and SSH commands
+type ProtocolActions struct {
+	// Valid values are download, upload, pre-delete, delete, rename, ssh_cmd. Empty slice to disable
+	ExecuteOn []string `json:"execute_on" mapstructure:"execute_on"`
+	// Absolute path to an external program or an HTTP URL
+	Hook string `json:"hook" mapstructure:"hook"`
+}
+
+var actionHandler ActionHandler = defaultActionHandler{}
+
+// InitializeActionHandler lets the user choose an action handler implementation.
+//
+// Do NOT call this function after application initialization.
+func InitializeActionHandler(handler ActionHandler) {
+	actionHandler = handler
+}
+
+// SSHCommandActionNotification executes the defined action for the specified SSH command.
+func SSHCommandActionNotification(user *dataprovider.User, filePath, target, sshCmd string, err error) {
+	notification := newActionNotification(user, operationSSHCmd, filePath, target, sshCmd, ProtocolSSH, 0, err)
+
+	go actionHandler.Handle(notification) // nolint:errcheck
+}
+
+// ActionHandler handles a notification for a Protocol Action.
+type ActionHandler interface {
+	Handle(notification ActionNotification) error
+}
+
+// ActionNotification defines a notification for a Protocol Action.
+type ActionNotification struct {
+	Action     string `json:"action"`
+	Username   string `json:"username"`
+	Path       string `json:"path"`
+	TargetPath string `json:"target_path,omitempty"`
+	SSHCmd     string `json:"ssh_cmd,omitempty"`
+	FileSize   int64  `json:"file_size,omitempty"`
+	FsProvider int    `json:"fs_provider"`
+	Bucket     string `json:"bucket,omitempty"`
+	Endpoint   string `json:"endpoint,omitempty"`
+	Status     int    `json:"status"`
+	Protocol   string `json:"protocol"`
+}
+
+func newActionNotification(
+	user *dataprovider.User,
+	operation, filePath, target, sshCmd, protocol string,
+	fileSize int64,
+	err error,
+) ActionNotification {
+	var bucket, endpoint string
+	status := 1
+
+	if user.FsConfig.Provider == dataprovider.S3FilesystemProvider {
+		bucket = user.FsConfig.S3Config.Bucket
+		endpoint = user.FsConfig.S3Config.Endpoint
+	} else if user.FsConfig.Provider == dataprovider.GCSFilesystemProvider {
+		bucket = user.FsConfig.GCSConfig.Bucket
+	} else if user.FsConfig.Provider == dataprovider.AzureBlobFilesystemProvider {
+		bucket = user.FsConfig.AzBlobConfig.Container
+		if user.FsConfig.AzBlobConfig.SASURL != "" {
+			endpoint = user.FsConfig.AzBlobConfig.SASURL
+		} else {
+			endpoint = user.FsConfig.AzBlobConfig.Endpoint
+		}
+	}
+
+	if err == ErrQuotaExceeded {
+		status = 2
+	} else if err != nil {
+		status = 0
+	}
+
+	return ActionNotification{
+		Action:     operation,
+		Username:   user.Username,
+		Path:       filePath,
+		TargetPath: target,
+		SSHCmd:     sshCmd,
+		FileSize:   fileSize,
+		FsProvider: int(user.FsConfig.Provider),
+		Bucket:     bucket,
+		Endpoint:   endpoint,
+		Status:     status,
+		Protocol:   protocol,
+	}
+}
+
+type defaultActionHandler struct{}
+
+func (h defaultActionHandler) Handle(notification ActionNotification) error {
+	if !utils.IsStringInSlice(notification.Action, Config.Actions.ExecuteOn) {
+		return errUnconfiguredAction
+	}
+
+	if Config.Actions.Hook == "" {
+		logger.Warn(notification.Protocol, "", "Unable to send notification, no hook is defined")
+
+		return errNoHook
+	}
+
+	if strings.HasPrefix(Config.Actions.Hook, "http") {
+		return h.handleHTTP(notification)
+	}
+
+	return h.handleCommand(notification)
+}
+
+func (h defaultActionHandler) handleHTTP(notification ActionNotification) error {
+	u, err := url.Parse(Config.Actions.Hook)
+	if err != nil {
+		logger.Warn(notification.Protocol, "", "Invalid hook %#v for operation %#v: %v", Config.Actions.Hook, notification.Action, err)
+
+		return err
+	}
+
+	startTime := time.Now()
+	respCode := 0
+
+	httpClient := httpclient.GetHTTPClient()
+
+	var b bytes.Buffer
+	_ = json.NewEncoder(&b).Encode(notification)
+
+	resp, err := httpClient.Post(u.String(), "application/json", &b)
+	if err == nil {
+		respCode = resp.StatusCode
+		resp.Body.Close()
+
+		if respCode != http.StatusOK {
+			err = errUnexpectedHTTResponse
+		}
+	}
+
+	logger.Debug(notification.Protocol, "", "notified operation %#v to URL: %v status code: %v, elapsed: %v err: %v", notification.Action, u.String(), respCode, time.Since(startTime), err)
+
+	return err
+}
+
+func (h defaultActionHandler) handleCommand(notification ActionNotification) error {
+	if !filepath.IsAbs(Config.Actions.Hook) {
+		err := fmt.Errorf("invalid notification command %#v", Config.Actions.Hook)
+		logger.Warn(notification.Protocol, "", "unable to execute notification command: %v", err)
+
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, Config.Actions.Hook, notification.Action, notification.Username, notification.Path, notification.TargetPath, notification.SSHCmd)
+	cmd.Env = append(os.Environ(), notificationAsEnvVars(notification)...)
+
+	startTime := time.Now()
+	err := cmd.Run()
+
+	logger.Debug(notification.Protocol, "", "executed command %#v with arguments: %#v, %#v, %#v, %#v, %#v, elapsed: %v, error: %v",
+		Config.Actions.Hook, notification.Action, notification.Username, notification.Path, notification.TargetPath, notification.SSHCmd, time.Since(startTime), err)
+
+	return err
+}
+
+func notificationAsEnvVars(notification ActionNotification) []string {
+	return []string{
+		fmt.Sprintf("SFTPGO_ACTION=%v", notification.Action),
+		fmt.Sprintf("SFTPGO_ACTION_USERNAME=%v", notification.Username),
+		fmt.Sprintf("SFTPGO_ACTION_PATH=%v", notification.Path),
+		fmt.Sprintf("SFTPGO_ACTION_TARGET=%v", notification.TargetPath),
+		fmt.Sprintf("SFTPGO_ACTION_SSH_CMD=%v", notification.SSHCmd),
+		fmt.Sprintf("SFTPGO_ACTION_FILE_SIZE=%v", notification.FileSize),
+		fmt.Sprintf("SFTPGO_ACTION_FS_PROVIDER=%v", notification.FsProvider),
+		fmt.Sprintf("SFTPGO_ACTION_BUCKET=%v", notification.Bucket),
+		fmt.Sprintf("SFTPGO_ACTION_ENDPOINT=%v", notification.Endpoint),
+		fmt.Sprintf("SFTPGO_ACTION_STATUS=%v", notification.Status),
+		fmt.Sprintf("SFTPGO_ACTION_PROTOCOL=%v", notification.Protocol),
+	}
+}

common/actions_test.go

@@ -0,0 +1,222 @@
+package common
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+func TestNewActionNotification(t *testing.T) {
+	user := &dataprovider.User{
+		Username: "username",
+	}
+	user.FsConfig.Provider = dataprovider.LocalFilesystemProvider
+	user.FsConfig.S3Config = vfs.S3FsConfig{
+		Bucket:   "s3bucket",
+		Endpoint: "endpoint",
+	}
+	user.FsConfig.GCSConfig = vfs.GCSFsConfig{
+		Bucket: "gcsbucket",
+	}
+	user.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{
+		Container: "azcontainer",
+		SASURL:    "azsasurl",
+		Endpoint:  "azendpoint",
+	}
+	a := newActionNotification(user, operationDownload, "path", "target", "", ProtocolSFTP, 123, errors.New("fake error"))
+	assert.Equal(t, user.Username, a.Username)
+	assert.Equal(t, 0, len(a.Bucket))
+	assert.Equal(t, 0, len(a.Endpoint))
+	assert.Equal(t, 0, a.Status)
+
+	user.FsConfig.Provider = dataprovider.S3FilesystemProvider
+	a = newActionNotification(user, operationDownload, "path", "target", "", ProtocolSSH, 123, nil)
+	assert.Equal(t, "s3bucket", a.Bucket)
+	assert.Equal(t, "endpoint", a.Endpoint)
+	assert.Equal(t, 1, a.Status)
+
+	user.FsConfig.Provider = dataprovider.GCSFilesystemProvider
+	a = newActionNotification(user, operationDownload, "path", "target", "", ProtocolSCP, 123, ErrQuotaExceeded)
+	assert.Equal(t, "gcsbucket", a.Bucket)
+	assert.Equal(t, 0, len(a.Endpoint))
+	assert.Equal(t, 2, a.Status)
+
+	user.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider
+	a = newActionNotification(user, operationDownload, "path", "target", "", ProtocolSCP, 123, nil)
+	assert.Equal(t, "azcontainer", a.Bucket)
+	assert.Equal(t, "azsasurl", a.Endpoint)
+	assert.Equal(t, 1, a.Status)
+
+	user.FsConfig.AzBlobConfig.SASURL = ""
+	a = newActionNotification(user, operationDownload, "path", "target", "", ProtocolSCP, 123, nil)
+	assert.Equal(t, "azcontainer", a.Bucket)
+	assert.Equal(t, "azendpoint", a.Endpoint)
+	assert.Equal(t, 1, a.Status)
+}
+
+func TestActionHTTP(t *testing.T) {
+	actionsCopy := Config.Actions
+
+	Config.Actions = ProtocolActions{
+		ExecuteOn: []string{operationDownload},
+		Hook:      fmt.Sprintf("http://%v", httpAddr),
+	}
+	user := &dataprovider.User{
+		Username: "username",
+	}
+	a := newActionNotification(user, operationDownload, "path", "target", "", ProtocolSFTP, 123, nil)
+	err := actionHandler.Handle(a)
+	assert.NoError(t, err)
+
+	Config.Actions.Hook = "http://invalid:1234"
+	err = actionHandler.Handle(a)
+	assert.Error(t, err)
+
+	Config.Actions.Hook = fmt.Sprintf("http://%v/404", httpAddr)
+	err = actionHandler.Handle(a)
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, errUnexpectedHTTResponse.Error())
+	}
+
+	Config.Actions = actionsCopy
+}
+
+func TestActionCMD(t *testing.T) {
+	if runtime.GOOS == osWindows {
+		t.Skip("this test is not available on Windows")
+	}
+	actionsCopy := Config.Actions
+
+	hookCmd, err := exec.LookPath("true")
+	assert.NoError(t, err)
+
+	Config.Actions = ProtocolActions{
+		ExecuteOn: []string{operationDownload},
+		Hook:      hookCmd,
+	}
+	user := &dataprovider.User{
+		Username: "username",
+	}
+	a := newActionNotification(user, operationDownload, "path", "target", "", ProtocolSFTP, 123, nil)
+	err = actionHandler.Handle(a)
+	assert.NoError(t, err)
+
+	SSHCommandActionNotification(user, "path", "target", "sha1sum", nil)
+
+	Config.Actions = actionsCopy
+}
+
+func TestWrongActions(t *testing.T) {
+	actionsCopy := Config.Actions
+
+	badCommand := "/bad/command"
+	if runtime.GOOS == osWindows {
+		badCommand = "C:\\bad\\command"
+	}
+	Config.Actions = ProtocolActions{
+		ExecuteOn: []string{operationUpload},
+		Hook:      badCommand,
+	}
+	user := &dataprovider.User{
+		Username: "username",
+	}
+
+	a := newActionNotification(user, operationUpload, "", "", "", ProtocolSFTP, 123, nil)
+	err := actionHandler.Handle(a)
+	assert.Error(t, err, "action with bad command must fail")
+
+	a.Action = operationDelete
+	err = actionHandler.Handle(a)
+	assert.EqualError(t, err, errUnconfiguredAction.Error())
+
+	Config.Actions.Hook = "http://foo\x7f.com/"
+	a.Action = operationUpload
+	err = actionHandler.Handle(a)
+	assert.Error(t, err, "action with bad url must fail")
+
+	Config.Actions.Hook = ""
+	err = actionHandler.Handle(a)
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, errNoHook.Error())
+	}
+
+	Config.Actions.Hook = "relative path"
+	err = actionHandler.Handle(a)
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, fmt.Sprintf("invalid notification command %#v", Config.Actions.Hook))
+	}
+
+	Config.Actions = actionsCopy
+}
+
+func TestPreDeleteAction(t *testing.T) {
+	if runtime.GOOS == osWindows {
+		t.Skip("this test is not available on Windows")
+	}
+	actionsCopy := Config.Actions
+
+	hookCmd, err := exec.LookPath("true")
+	assert.NoError(t, err)
+	Config.Actions = ProtocolActions{
+		ExecuteOn: []string{operationPreDelete},
+		Hook:      hookCmd,
+	}
+	homeDir := filepath.Join(os.TempDir(), "test_user")
+	err = os.MkdirAll(homeDir, os.ModePerm)
+	assert.NoError(t, err)
+	user := dataprovider.User{
+		Username: "username",
+		HomeDir:  homeDir,
+	}
+	user.Permissions = make(map[string][]string)
+	user.Permissions["/"] = []string{dataprovider.PermAny}
+	fs := vfs.NewOsFs("id", homeDir, nil)
+	c := NewBaseConnection("id", ProtocolSFTP, user, fs)
+
+	testfile := filepath.Join(user.HomeDir, "testfile")
+	err = ioutil.WriteFile(testfile, []byte("test"), os.ModePerm)
+	assert.NoError(t, err)
+	info, err := os.Stat(testfile)
+	assert.NoError(t, err)
+	err = c.RemoveFile(testfile, "testfile", info)
+	assert.NoError(t, err)
+	assert.FileExists(t, testfile)
+
+	os.RemoveAll(homeDir)
+
+	Config.Actions = actionsCopy
+}
+
+type actionHandlerStub struct {
+	called bool
+}
+
+func (h *actionHandlerStub) Handle(notification ActionNotification) error {
+	h.called = true
+
+	return nil
+}
+
+func TestInitializeActionHandler(t *testing.T) {
+	handler := &actionHandlerStub{}
+
+	InitializeActionHandler(handler)
+	t.Cleanup(func() {
+		InitializeActionHandler(defaultActionHandler{})
+	})
+
+	err := actionHandler.Handle(ActionNotification{})
+
+	assert.NoError(t, err)
+	assert.True(t, handler.called)
+}

common/common.go

@@ -0,0 +1,739 @@
+// Package common defines code shared among file transfer packages and protocols
+package common
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/pires/go-proxyproto"
+
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/httpclient"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/metrics"
+	"github.com/drakkan/sftpgo/utils"
+)
+
+// constants
+const (
+	logSender                = "common"
+	uploadLogSender          = "Upload"
+	downloadLogSender        = "Download"
+	renameLogSender          = "Rename"
+	rmdirLogSender           = "Rmdir"
+	mkdirLogSender           = "Mkdir"
+	symlinkLogSender         = "Symlink"
+	removeLogSender          = "Remove"
+	chownLogSender           = "Chown"
+	chmodLogSender           = "Chmod"
+	chtimesLogSender         = "Chtimes"
+	truncateLogSender        = "Truncate"
+	operationDownload        = "download"
+	operationUpload          = "upload"
+	operationDelete          = "delete"
+	operationPreDelete       = "pre-delete"
+	operationRename          = "rename"
+	operationSSHCmd          = "ssh_cmd"
+	chtimesFormat            = "2006-01-02T15:04:05" // YYYY-MM-DDTHH:MM:SS
+	idleTimeoutCheckInterval = 3 * time.Minute
+)
+
+// Stat flags
+const (
+	StatAttrUIDGID = 1
+	StatAttrPerms  = 2
+	StatAttrTimes  = 4
+	StatAttrSize   = 8
+)
+
+// Transfer types
+const (
+	TransferUpload = iota
+	TransferDownload
+)
+
+// Supported protocols
+const (
+	ProtocolSFTP   = "SFTP"
+	ProtocolSCP    = "SCP"
+	ProtocolSSH    = "SSH"
+	ProtocolFTP    = "FTP"
+	ProtocolWebDAV = "DAV"
+)
+
+// Upload modes
+const (
+	UploadModeStandard = iota
+	UploadModeAtomic
+	UploadModeAtomicWithResume
+)
+
+// errors definitions
+var (
+	ErrPermissionDenied     = errors.New("permission denied")
+	ErrNotExist             = errors.New("no such file or directory")
+	ErrOpUnsupported        = errors.New("operation unsupported")
+	ErrGenericFailure       = errors.New("failure")
+	ErrQuotaExceeded        = errors.New("denying write due to space limit")
+	ErrSkipPermissionsCheck = errors.New("permission check skipped")
+	ErrConnectionDenied     = errors.New("You are not allowed to connect")
+	errNoTransfer           = errors.New("requested transfer not found")
+	errTransferMismatch     = errors.New("transfer mismatch")
+)
+
+var (
+	// Config is the configuration for the supported protocols
+	Config Configuration
+	// Connections is the list of active connections
+	Connections ActiveConnections
+	// QuotaScans is the list of active quota scans
+	QuotaScans            ActiveScans
+	idleTimeoutTicker     *time.Ticker
+	idleTimeoutTickerDone chan bool
+	supportedProtocols    = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP, ProtocolWebDAV}
+)
+
+// Initialize sets the common configuration
+func Initialize(c Configuration) {
+	Config = c
+	Config.idleLoginTimeout = 2 * time.Minute
+	Config.idleTimeoutAsDuration = time.Duration(Config.IdleTimeout) * time.Minute
+	if Config.IdleTimeout > 0 {
+		startIdleTimeoutTicker(idleTimeoutCheckInterval)
+	}
+}
+
+func startIdleTimeoutTicker(duration time.Duration) {
+	stopIdleTimeoutTicker()
+	idleTimeoutTicker = time.NewTicker(duration)
+	idleTimeoutTickerDone = make(chan bool)
+	go func() {
+		for {
+			select {
+			case <-idleTimeoutTickerDone:
+				return
+			case <-idleTimeoutTicker.C:
+				Connections.checkIdles()
+			}
+		}
+	}()
+}
+
+func stopIdleTimeoutTicker() {
+	if idleTimeoutTicker != nil {
+		idleTimeoutTicker.Stop()
+		idleTimeoutTickerDone <- true
+		idleTimeoutTicker = nil
+	}
+}
+
+// ActiveTransfer defines the interface for the current active transfers
+type ActiveTransfer interface {
+	GetID() uint64
+	GetType() int
+	GetSize() int64
+	GetVirtualPath() string
+	GetStartTime() time.Time
+	SignalClose()
+	Truncate(fsPath string, size int64) (int64, error)
+	GetRealFsPath(fsPath string) string
+}
+
+// ActiveConnection defines the interface for the current active connections
+type ActiveConnection interface {
+	GetID() string
+	GetUsername() string
+	GetRemoteAddress() string
+	GetClientVersion() string
+	GetProtocol() string
+	GetConnectionTime() time.Time
+	GetLastActivity() time.Time
+	GetCommand() string
+	Disconnect() error
+	AddTransfer(t ActiveTransfer)
+	RemoveTransfer(t ActiveTransfer)
+	GetTransfers() []ConnectionTransfer
+}
+
+// StatAttributes defines the attributes for set stat commands
+type StatAttributes struct {
+	Mode  os.FileMode
+	Atime time.Time
+	Mtime time.Time
+	UID   int
+	GID   int
+	Flags int
+	Size  int64
+}
+
+// ConnectionTransfer defines the trasfer details to expose
+type ConnectionTransfer struct {
+	ID            uint64 `json:"-"`
+	OperationType string `json:"operation_type"`
+	StartTime     int64  `json:"start_time"`
+	Size          int64  `json:"size"`
+	VirtualPath   string `json:"path"`
+}
+
+func (t *ConnectionTransfer) getConnectionTransferAsString() string {
+	result := ""
+	switch t.OperationType {
+	case operationUpload:
+		result += "UL "
+	case operationDownload:
+		result += "DL "
+	}
+	result += fmt.Sprintf("%#v ", t.VirtualPath)
+	if t.Size > 0 {
+		elapsed := time.Since(utils.GetTimeFromMsecSinceEpoch(t.StartTime))
+		speed := float64(t.Size) / float64(utils.GetTimeAsMsSinceEpoch(time.Now())-t.StartTime)
+		result += fmt.Sprintf("Size: %#v Elapsed: %#v Speed: \"%.1f KB/s\"", utils.ByteCountSI(t.Size),
+			utils.GetDurationAsString(elapsed), speed)
+	}
+	return result
+}
+
+// Configuration defines configuration parameters common to all supported protocols
+type Configuration struct {
+	// Maximum idle timeout as minutes. If a client is idle for a time that exceeds this setting it will be disconnected.
+	// 0 means disabled
+	IdleTimeout int `json:"idle_timeout" mapstructure:"idle_timeout"`
+	// UploadMode 0 means standard, the files are uploaded directly to the requested path.
+	// 1 means atomic: the files are uploaded to a temporary path and renamed to the requested path
+	// when the client ends the upload. Atomic mode avoid problems such as a web server that
+	// serves partial files when the files are being uploaded.
+	// In atomic mode if there is an upload error the temporary file is deleted and so the requested
+	// upload path will not contain a partial file.
+	// 2 means atomic with resume support: as atomic but if there is an upload error the temporary
+	// file is renamed to the requested path and not deleted, this way a client can reconnect and resume
+	// the upload.
+	UploadMode int `json:"upload_mode" mapstructure:"upload_mode"`
+	// Actions to execute for SFTP file operations and SSH commands
+	Actions ProtocolActions `json:"actions" mapstructure:"actions"`
+	// SetstatMode 0 means "normal mode": requests for changing permissions and owner/group are executed.
+	// 1 means "ignore mode": requests for changing permissions and owner/group are silently ignored.
+	// 2 means "ignore mode for cloud fs": requests for changing permissions and owner/group/time are
+	// silently ignored for cloud based filesystem such as S3, GCS, Azure Blob
+	SetstatMode int `json:"setstat_mode" mapstructure:"setstat_mode"`
+	// Support for HAProxy PROXY protocol.
+	// If you are running SFTPGo behind a proxy server such as HAProxy, AWS ELB or NGNIX, you can enable
+	// the proxy protocol. It provides a convenient way to safely transport connection information
+	// such as a client's address across multiple layers of NAT or TCP proxies to get the real
+	// client IP address instead of the proxy IP. Both protocol versions 1 and 2 are supported.
+	// - 0 means disabled
+	// - 1 means proxy protocol enabled. Proxy header will be used and requests without proxy header will be accepted.
+	// - 2 means proxy protocol required. Proxy header will be used and requests without proxy header will be rejected.
+	// If the proxy protocol is enabled in SFTPGo then you have to enable the protocol in your proxy configuration too,
+	// for example for HAProxy add "send-proxy" or "send-proxy-v2" to each server configuration line.
+	ProxyProtocol int `json:"proxy_protocol" mapstructure:"proxy_protocol"`
+	// List of IP addresses and IP ranges allowed to send the proxy header.
+	// If proxy protocol is set to 1 and we receive a proxy header from an IP that is not in the list then the
+	// connection will be accepted and the header will be ignored.
+	// If proxy protocol is set to 2 and we receive a proxy header from an IP that is not in the list then the
+	// connection will be rejected.
+	ProxyAllowed []string `json:"proxy_allowed" mapstructure:"proxy_allowed"`
+	// Absolute path to an external program or an HTTP URL to invoke after a user connects
+	// and before he tries to login. It allows you to reject the connection based on the source
+	// ip address. Leave empty do disable.
+	PostConnectHook       string `json:"post_connect_hook" mapstructure:"post_connect_hook"`
+	idleTimeoutAsDuration time.Duration
+	idleLoginTimeout      time.Duration
+}
+
+// IsAtomicUploadEnabled returns true if atomic upload is enabled
+func (c *Configuration) IsAtomicUploadEnabled() bool {
+	return c.UploadMode == UploadModeAtomic || c.UploadMode == UploadModeAtomicWithResume
+}
+
+// GetProxyListener returns a wrapper for the given listener that supports the
+// HAProxy Proxy Protocol or nil if the proxy protocol is not configured
+func (c *Configuration) GetProxyListener(listener net.Listener) (*proxyproto.Listener, error) {
+	var proxyListener *proxyproto.Listener
+	var err error
+	if c.ProxyProtocol > 0 {
+		var policyFunc func(upstream net.Addr) (proxyproto.Policy, error)
+		if c.ProxyProtocol == 1 && len(c.ProxyAllowed) > 0 {
+			policyFunc, err = proxyproto.LaxWhiteListPolicy(c.ProxyAllowed)
+			if err != nil {
+				return nil, err
+			}
+		}
+		if c.ProxyProtocol == 2 {
+			if len(c.ProxyAllowed) == 0 {
+				policyFunc = func(upstream net.Addr) (proxyproto.Policy, error) {
+					return proxyproto.REQUIRE, nil
+				}
+			} else {
+				policyFunc, err = proxyproto.StrictWhiteListPolicy(c.ProxyAllowed)
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+		proxyListener = &proxyproto.Listener{
+			Listener: listener,
+			Policy:   policyFunc,
+		}
+	}
+	return proxyListener, nil
+}
+
+// ExecutePostConnectHook executes the post connect hook if defined
+func (c *Configuration) ExecutePostConnectHook(remoteAddr, protocol string) error {
+	if len(c.PostConnectHook) == 0 {
+		return nil
+	}
+	ip := utils.GetIPFromRemoteAddress(remoteAddr)
+	if strings.HasPrefix(c.PostConnectHook, "http") {
+		var url *url.URL
+		url, err := url.Parse(c.PostConnectHook)
+		if err != nil {
+			logger.Warn(protocol, "", "Login from ip %#v denied, invalid post connect hook %#v: %v",
+				ip, c.PostConnectHook, err)
+			return err
+		}
+		httpClient := httpclient.GetHTTPClient()
+		q := url.Query()
+		q.Add("ip", ip)
+		q.Add("protocol", protocol)
+		url.RawQuery = q.Encode()
+
+		resp, err := httpClient.Get(url.String())
+		if err != nil {
+			logger.Warn(protocol, "", "Login from ip %#v denied, error executing post connect hook: %v", ip, err)
+			return err
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			logger.Warn(protocol, "", "Login from ip %#v denied, post connect hook response code: %v", ip, resp.StatusCode)
+			return errUnexpectedHTTResponse
+		}
+		return nil
+	}
+	if !filepath.IsAbs(c.PostConnectHook) {
+		err := fmt.Errorf("invalid post connect hook %#v", c.PostConnectHook)
+		logger.Warn(protocol, "", "Login from ip %#v denied: %v", ip, err)
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, c.PostConnectHook)
+	cmd.Env = append(os.Environ(),
+		fmt.Sprintf("SFTPGO_CONNECTION_IP=%v", ip),
+		fmt.Sprintf("SFTPGO_CONNECTION_PROTOCOL=%v", protocol))
+	err := cmd.Run()
+	if err != nil {
+		logger.Warn(protocol, "", "Login from ip %#v denied, connect hook error: %v", ip, err)
+	}
+	return err
+}
+
+// SSHConnection defines an ssh connection.
+// Each SSH connection can open several channels for SFTP or SSH commands
+type SSHConnection struct {
+	id           string
+	conn         net.Conn
+	lastActivity int64
+}
+
+// NewSSHConnection returns a new SSHConnection
+func NewSSHConnection(id string, conn net.Conn) *SSHConnection {
+	return &SSHConnection{
+		id:           id,
+		conn:         conn,
+		lastActivity: time.Now().UnixNano(),
+	}
+}
+
+// GetID returns the ID for this SSHConnection
+func (c *SSHConnection) GetID() string {
+	return c.id
+}
+
+// UpdateLastActivity updates last activity for this connection
+func (c *SSHConnection) UpdateLastActivity() {
+	atomic.StoreInt64(&c.lastActivity, time.Now().UnixNano())
+}
+
+// GetLastActivity returns the last connection activity
+func (c *SSHConnection) GetLastActivity() time.Time {
+	return time.Unix(0, atomic.LoadInt64(&c.lastActivity))
+}
+
+// Close closes the underlying network connection
+func (c *SSHConnection) Close() error {
+	return c.conn.Close()
+}
+
+// ActiveConnections holds the currect active connections with the associated transfers
+type ActiveConnections struct {
+	sync.RWMutex
+	connections    []ActiveConnection
+	sshConnections []*SSHConnection
+}
+
+// GetActiveSessions returns the number of active sessions for the given username.
+// We return the open sessions for any protocol
+func (conns *ActiveConnections) GetActiveSessions(username string) int {
+	conns.RLock()
+	defer conns.RUnlock()
+
+	numSessions := 0
+	for _, c := range conns.connections {
+		if c.GetUsername() == username {
+			numSessions++
+		}
+	}
+	return numSessions
+}
+
+// Add adds a new connection to the active ones
+func (conns *ActiveConnections) Add(c ActiveConnection) {
+	conns.Lock()
+	defer conns.Unlock()
+
+	conns.connections = append(conns.connections, c)
+	metrics.UpdateActiveConnectionsSize(len(conns.connections))
+	logger.Debug(c.GetProtocol(), c.GetID(), "connection added, num open connections: %v", len(conns.connections))
+}
+
+// Swap replaces an existing connection with the given one.
+// This method is useful if you have to change some connection details
+// for example for FTP is used to update the connection once the user
+// authenticates
+func (conns *ActiveConnections) Swap(c ActiveConnection) error {
+	conns.Lock()
+	defer conns.Unlock()
+
+	for idx, conn := range conns.connections {
+		if conn.GetID() == c.GetID() {
+			conn = nil
+			conns.connections[idx] = c
+			return nil
+		}
+	}
+	return errors.New("connection to swap not found")
+}
+
+// Remove removes a connection from the active ones
+func (conns *ActiveConnections) Remove(connectionID string) {
+	conns.Lock()
+	defer conns.Unlock()
+
+	for idx, conn := range conns.connections {
+		if conn.GetID() == connectionID {
+			lastIdx := len(conns.connections) - 1
+			conns.connections[idx] = conns.connections[lastIdx]
+			conns.connections[lastIdx] = nil
+			conns.connections = conns.connections[:lastIdx]
+			metrics.UpdateActiveConnectionsSize(lastIdx)
+			logger.Debug(conn.GetProtocol(), conn.GetID(), "connection removed, num open connections: %v", lastIdx)
+			return
+		}
+	}
+	logger.Warn(logSender, "", "connection id %#v to remove not found!", connectionID)
+}
+
+// Close closes an active connection.
+// It returns true on success
+func (conns *ActiveConnections) Close(connectionID string) bool {
+	conns.RLock()
+	result := false
+
+	for _, c := range conns.connections {
+		if c.GetID() == connectionID {
+			defer func(conn ActiveConnection) {
+				err := conn.Disconnect()
+				logger.Debug(conn.GetProtocol(), conn.GetID(), "close connection requested, close err: %v", err)
+			}(c)
+			result = true
+			break
+		}
+	}
+
+	conns.RUnlock()
+	return result
+}
+
+// AddSSHConnection adds a new ssh connection to the active ones
+func (conns *ActiveConnections) AddSSHConnection(c *SSHConnection) {
+	conns.Lock()
+	defer conns.Unlock()
+
+	conns.sshConnections = append(conns.sshConnections, c)
+	logger.Debug(logSender, c.GetID(), "ssh connection added, num open connections: %v", len(conns.sshConnections))
+}
+
+// RemoveSSHConnection removes a connection from the active ones
+func (conns *ActiveConnections) RemoveSSHConnection(connectionID string) {
+	conns.Lock()
+	defer conns.Unlock()
+
+	for idx, conn := range conns.sshConnections {
+		if conn.GetID() == connectionID {
+			lastIdx := len(conns.sshConnections) - 1
+			conns.sshConnections[idx] = conns.sshConnections[lastIdx]
+			conns.sshConnections[lastIdx] = nil
+			conns.sshConnections = conns.sshConnections[:lastIdx]
+			logger.Debug(logSender, conn.GetID(), "ssh connection removed, num open ssh connections: %v", lastIdx)
+			return
+		}
+	}
+	logger.Warn(logSender, "", "ssh connection to remove with id %#v not found!", connectionID)
+}
+
+func (conns *ActiveConnections) checkIdles() {
+	conns.RLock()
+
+	for _, sshConn := range conns.sshConnections {
+		idleTime := time.Since(sshConn.GetLastActivity())
+		if idleTime > Config.idleTimeoutAsDuration {
+			// we close the an ssh connection if it has no active connections associated
+			idToMatch := fmt.Sprintf("_%v_", sshConn.GetID())
+			toClose := true
+			for _, conn := range conns.connections {
+				if strings.Contains(conn.GetID(), idToMatch) {
+					toClose = false
+					break
+				}
+			}
+			if toClose {
+				defer func(c *SSHConnection) {
+					err := c.Close()
+					logger.Debug(logSender, c.GetID(), "close idle SSH connection, idle time: %v, close err: %v",
+						time.Since(c.GetLastActivity()), err)
+				}(sshConn)
+			}
+		}
+	}
+
+	for _, c := range conns.connections {
+		idleTime := time.Since(c.GetLastActivity())
+		isUnauthenticatedFTPUser := (c.GetProtocol() == ProtocolFTP && len(c.GetUsername()) == 0)
+
+		if idleTime > Config.idleTimeoutAsDuration || (isUnauthenticatedFTPUser && idleTime > Config.idleLoginTimeout) {
+			defer func(conn ActiveConnection, isFTPNoAuth bool) {
+				err := conn.Disconnect()
+				logger.Debug(conn.GetProtocol(), conn.GetID(), "close idle connection, idle time: %v, username: %#v close err: %v",
+					time.Since(conn.GetLastActivity()), conn.GetUsername(), err)
+				if isFTPNoAuth {
+					ip := utils.GetIPFromRemoteAddress(c.GetRemoteAddress())
+					logger.ConnectionFailedLog("", ip, dataprovider.LoginMethodNoAuthTryed, c.GetProtocol(), "client idle")
+					metrics.AddNoAuthTryed()
+					dataprovider.ExecutePostLoginHook("", dataprovider.LoginMethodNoAuthTryed, ip, c.GetProtocol(),
+						dataprovider.ErrNoAuthTryed)
+				}
+			}(c, isUnauthenticatedFTPUser)
+		}
+	}
+
+	conns.RUnlock()
+}
+
+// GetStats returns stats for active connections
+func (conns *ActiveConnections) GetStats() []ConnectionStatus {
+	conns.RLock()
+	defer conns.RUnlock()
+
+	stats := make([]ConnectionStatus, 0, len(conns.connections))
+	for _, c := range conns.connections {
+		stat := ConnectionStatus{
+			Username:       c.GetUsername(),
+			ConnectionID:   c.GetID(),
+			ClientVersion:  c.GetClientVersion(),
+			RemoteAddress:  c.GetRemoteAddress(),
+			ConnectionTime: utils.GetTimeAsMsSinceEpoch(c.GetConnectionTime()),
+			LastActivity:   utils.GetTimeAsMsSinceEpoch(c.GetLastActivity()),
+			Protocol:       c.GetProtocol(),
+			Command:        c.GetCommand(),
+			Transfers:      c.GetTransfers(),
+		}
+		stats = append(stats, stat)
+	}
+	return stats
+}
+
+// ConnectionStatus returns the status for an active connection
+type ConnectionStatus struct {
+	// Logged in username
+	Username string `json:"username"`
+	// Unique identifier for the connection
+	ConnectionID string `json:"connection_id"`
+	// client's version string
+	ClientVersion string `json:"client_version,omitempty"`
+	// Remote address for this connection
+	RemoteAddress string `json:"remote_address"`
+	// Connection time as unix timestamp in milliseconds
+	ConnectionTime int64 `json:"connection_time"`
+	// Last activity as unix timestamp in milliseconds
+	LastActivity int64 `json:"last_activity"`
+	// Protocol for this connection
+	Protocol string `json:"protocol"`
+	// active uploads/downloads
+	Transfers []ConnectionTransfer `json:"active_transfers,omitempty"`
+	// SSH command or WevDAV method
+	Command string `json:"command,omitempty"`
+}
+
+// GetConnectionDuration returns the connection duration as string
+func (c ConnectionStatus) GetConnectionDuration() string {
+	elapsed := time.Since(utils.GetTimeFromMsecSinceEpoch(c.ConnectionTime))
+	return utils.GetDurationAsString(elapsed)
+}
+
+// GetConnectionInfo returns connection info.
+// Protocol,Client Version and RemoteAddress are returned.
+// For SSH commands the issued command is returned too.
+func (c ConnectionStatus) GetConnectionInfo() string {
+	result := fmt.Sprintf("%v. Client: %#v From: %#v", c.Protocol, c.ClientVersion, c.RemoteAddress)
+	if c.Protocol == ProtocolSSH && len(c.Command) > 0 {
+		result += fmt.Sprintf(". Command: %#v", c.Command)
+	}
+	if c.Protocol == ProtocolWebDAV && len(c.Command) > 0 {
+		result += fmt.Sprintf(". Method: %#v", c.Command)
+	}
+	return result
+}
+
+// GetTransfersAsString returns the active transfers as string
+func (c ConnectionStatus) GetTransfersAsString() string {
+	result := ""
+	for _, t := range c.Transfers {
+		if len(result) > 0 {
+			result += ". "
+		}
+		result += t.getConnectionTransferAsString()
+	}
+	return result
+}
+
+// ActiveQuotaScan defines an active quota scan for a user home dir
+type ActiveQuotaScan struct {
+	// Username to which the quota scan refers
+	Username string `json:"username"`
+	// quota scan start time as unix timestamp in milliseconds
+	StartTime int64 `json:"start_time"`
+}
+
+// ActiveVirtualFolderQuotaScan defines an active quota scan for a virtual folder
+type ActiveVirtualFolderQuotaScan struct {
+	// folder path to which the quota scan refers
+	MappedPath string `json:"mapped_path"`
+	// quota scan start time as unix timestamp in milliseconds
+	StartTime int64 `json:"start_time"`
+}
+
+// ActiveScans holds the active quota scans
+type ActiveScans struct {
+	sync.RWMutex
+	UserHomeScans []ActiveQuotaScan
+	FolderScans   []ActiveVirtualFolderQuotaScan
+}
+
+// GetUsersQuotaScans returns the active quota scans for users home directories
+func (s *ActiveScans) GetUsersQuotaScans() []ActiveQuotaScan {
+	s.RLock()
+	defer s.RUnlock()
+
+	scans := make([]ActiveQuotaScan, len(s.UserHomeScans))
+	copy(scans, s.UserHomeScans)
+	return scans
+}
+
+// AddUserQuotaScan adds a user to the ones with active quota scans.
+// Returns false if the user has a quota scan already running
+func (s *ActiveScans) AddUserQuotaScan(username string) bool {
+	s.Lock()
+	defer s.Unlock()
+
+	for _, scan := range s.UserHomeScans {
+		if scan.Username == username {
+			return false
+		}
+	}
+	s.UserHomeScans = append(s.UserHomeScans, ActiveQuotaScan{
+		Username:  username,
+		StartTime: utils.GetTimeAsMsSinceEpoch(time.Now()),
+	})
+	return true
+}
+
+// RemoveUserQuotaScan removes a user from the ones with active quota scans.
+// Returns false if the user has no active quota scans
+func (s *ActiveScans) RemoveUserQuotaScan(username string) bool {
+	s.Lock()
+	defer s.Unlock()
+
+	indexToRemove := -1
+	for i, scan := range s.UserHomeScans {
+		if scan.Username == username {
+			indexToRemove = i
+			break
+		}
+	}
+	if indexToRemove >= 0 {
+		s.UserHomeScans[indexToRemove] = s.UserHomeScans[len(s.UserHomeScans)-1]
+		s.UserHomeScans = s.UserHomeScans[:len(s.UserHomeScans)-1]
+		return true
+	}
+	return false
+}
+
+// GetVFoldersQuotaScans returns the active quota scans for virtual folders
+func (s *ActiveScans) GetVFoldersQuotaScans() []ActiveVirtualFolderQuotaScan {
+	s.RLock()
+	defer s.RUnlock()
+	scans := make([]ActiveVirtualFolderQuotaScan, len(s.FolderScans))
+	copy(scans, s.FolderScans)
+	return scans
+}
+
+// AddVFolderQuotaScan adds a virtual folder to the ones with active quota scans.
+// Returns false if the folder has a quota scan already running
+func (s *ActiveScans) AddVFolderQuotaScan(folderPath string) bool {
+	s.Lock()
+	defer s.Unlock()
+
+	for _, scan := range s.FolderScans {
+		if scan.MappedPath == folderPath {
+			return false
+		}
+	}
+	s.FolderScans = append(s.FolderScans, ActiveVirtualFolderQuotaScan{
+		MappedPath: folderPath,
+		StartTime:  utils.GetTimeAsMsSinceEpoch(time.Now()),
+	})
+	return true
+}
+
+// RemoveVFolderQuotaScan removes a folder from the ones with active quota scans.
+// Returns false if the folder has no active quota scans
+func (s *ActiveScans) RemoveVFolderQuotaScan(folderPath string) bool {
+	s.Lock()
+	defer s.Unlock()
+
+	indexToRemove := -1
+	for i, scan := range s.FolderScans {
+		if scan.MappedPath == folderPath {
+			indexToRemove = i
+			break
+		}
+	}
+	if indexToRemove >= 0 {
+		s.FolderScans[indexToRemove] = s.FolderScans[len(s.FolderScans)-1]
+		s.FolderScans = s.FolderScans[:len(s.FolderScans)-1]
+		return true
+	}
+	return false
+}

common/common_test.go

@@ -0,0 +1,536 @@
+package common
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/rs/zerolog"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/httpclient"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+const (
+	logSenderTest    = "common_test"
+	httpAddr         = "127.0.0.1:9999"
+	httpProxyAddr    = "127.0.0.1:7777"
+	configDir        = ".."
+	osWindows        = "windows"
+	userTestUsername = "common_test_username"
+	userTestPwd      = "common_test_pwd"
+)
+
+type providerConf struct {
+	Config dataprovider.Config `json:"data_provider" mapstructure:"data_provider"`
+}
+
+type fakeConnection struct {
+	*BaseConnection
+	command string
+}
+
+func (c *fakeConnection) AddUser(user dataprovider.User) error {
+	fs, err := user.GetFilesystem(c.GetID())
+	if err != nil {
+		return err
+	}
+	c.BaseConnection.User = user
+	c.BaseConnection.Fs = fs
+	return nil
+}
+
+func (c *fakeConnection) Disconnect() error {
+	Connections.Remove(c.GetID())
+	return nil
+}
+
+func (c *fakeConnection) GetClientVersion() string {
+	return ""
+}
+
+func (c *fakeConnection) GetCommand() string {
+	return c.command
+}
+
+func (c *fakeConnection) GetRemoteAddress() string {
+	return ""
+}
+
+type customNetConn struct {
+	net.Conn
+	id       string
+	isClosed bool
+}
+
+func (c *customNetConn) Close() error {
+	Connections.RemoveSSHConnection(c.id)
+	c.isClosed = true
+	return c.Conn.Close()
+}
+
+func TestMain(m *testing.M) {
+	logfilePath := "common_test.log"
+	logger.InitLogger(logfilePath, 5, 1, 28, false, zerolog.DebugLevel)
+
+	viper.SetEnvPrefix("sftpgo")
+	replacer := strings.NewReplacer(".", "__")
+	viper.SetEnvKeyReplacer(replacer)
+	viper.SetConfigName("sftpgo")
+	viper.AutomaticEnv()
+	viper.AllowEmptyEnv(true)
+
+	driver, err := initializeDataprovider(-1)
+	if err != nil {
+		logger.WarnToConsole("error initializing data provider: %v", err)
+		os.Exit(1)
+	}
+	logger.InfoToConsole("Starting COMMON tests, provider: %v", driver)
+	Initialize(Configuration{})
+	httpConfig := httpclient.Config{
+		Timeout: 5,
+	}
+	httpConfig.Initialize(configDir)
+
+	go func() {
+		// start a test HTTP server to receive action notifications
+		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+			fmt.Fprintf(w, "OK\n")
+		})
+		http.HandleFunc("/404", func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusNotFound)
+			fmt.Fprintf(w, "Not found\n")
+		})
+		if err := http.ListenAndServe(httpAddr, nil); err != nil {
+			logger.ErrorToConsole("could not start HTTP notification server: %v", err)
+			os.Exit(1)
+		}
+	}()
+
+	go func() {
+		Config.ProxyProtocol = 2
+		listener, err := net.Listen("tcp", httpProxyAddr)
+		if err != nil {
+			logger.ErrorToConsole("error creating listener for proxy protocol server: %v", err)
+			os.Exit(1)
+		}
+		proxyListener, err := Config.GetProxyListener(listener)
+		if err != nil {
+			logger.ErrorToConsole("error creating proxy protocol listener: %v", err)
+			os.Exit(1)
+		}
+		Config.ProxyProtocol = 0
+
+		s := &http.Server{}
+		if err := s.Serve(proxyListener); err != nil {
+			logger.ErrorToConsole("could not start HTTP proxy protocol server: %v", err)
+			os.Exit(1)
+		}
+	}()
+
+	waitTCPListening(httpAddr)
+	waitTCPListening(httpProxyAddr)
+	exitCode := m.Run()
+	os.Remove(logfilePath) //nolint:errcheck
+	os.Exit(exitCode)
+}
+
+func waitTCPListening(address string) {
+	for {
+		conn, err := net.Dial("tcp", address)
+		if err != nil {
+			logger.WarnToConsole("tcp server %v not listening: %v\n", address, err)
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+		logger.InfoToConsole("tcp server %v now listening\n", address)
+		conn.Close()
+		break
+	}
+}
+
+func initializeDataprovider(trackQuota int) (string, error) {
+	configDir := ".."
+	viper.AddConfigPath(configDir)
+	if err := viper.ReadInConfig(); err != nil {
+		return "", err
+	}
+	var cfg providerConf
+	if err := viper.Unmarshal(&cfg); err != nil {
+		return "", err
+	}
+	if trackQuota >= 0 && trackQuota <= 2 {
+		cfg.Config.TrackQuota = trackQuota
+	}
+	return cfg.Config.Driver, dataprovider.Initialize(cfg.Config, configDir)
+}
+
+func closeDataprovider() error {
+	return dataprovider.Close()
+}
+
+func TestSSHConnections(t *testing.T) {
+	conn1, conn2 := net.Pipe()
+	now := time.Now()
+	sshConn1 := NewSSHConnection("id1", conn1)
+	sshConn2 := NewSSHConnection("id2", conn2)
+	sshConn3 := NewSSHConnection("id3", conn2)
+	assert.Equal(t, "id1", sshConn1.GetID())
+	assert.Equal(t, "id2", sshConn2.GetID())
+	assert.Equal(t, "id3", sshConn3.GetID())
+	sshConn1.UpdateLastActivity()
+	assert.GreaterOrEqual(t, sshConn1.GetLastActivity().UnixNano(), now.UnixNano())
+	Connections.AddSSHConnection(sshConn1)
+	Connections.AddSSHConnection(sshConn2)
+	Connections.AddSSHConnection(sshConn3)
+	Connections.RLock()
+	assert.Len(t, Connections.sshConnections, 3)
+	Connections.RUnlock()
+	Connections.RemoveSSHConnection(sshConn1.id)
+	Connections.RLock()
+	assert.Len(t, Connections.sshConnections, 2)
+	assert.Equal(t, sshConn3.id, Connections.sshConnections[0].id)
+	assert.Equal(t, sshConn2.id, Connections.sshConnections[1].id)
+	Connections.RUnlock()
+	Connections.RemoveSSHConnection(sshConn1.id)
+	Connections.RLock()
+	assert.Len(t, Connections.sshConnections, 2)
+	assert.Equal(t, sshConn3.id, Connections.sshConnections[0].id)
+	assert.Equal(t, sshConn2.id, Connections.sshConnections[1].id)
+	Connections.RUnlock()
+	Connections.RemoveSSHConnection(sshConn2.id)
+	Connections.RLock()
+	assert.Len(t, Connections.sshConnections, 1)
+	assert.Equal(t, sshConn3.id, Connections.sshConnections[0].id)
+	Connections.RUnlock()
+	Connections.RemoveSSHConnection(sshConn3.id)
+	Connections.RLock()
+	assert.Len(t, Connections.sshConnections, 0)
+	Connections.RUnlock()
+	assert.NoError(t, sshConn1.Close())
+	assert.NoError(t, sshConn2.Close())
+	assert.NoError(t, sshConn3.Close())
+}
+
+func TestIdleConnections(t *testing.T) {
+	configCopy := Config
+
+	Config.IdleTimeout = 1
+	Initialize(Config)
+
+	conn1, conn2 := net.Pipe()
+	customConn1 := &customNetConn{
+		Conn: conn1,
+		id:   "id1",
+	}
+	customConn2 := &customNetConn{
+		Conn: conn2,
+		id:   "id2",
+	}
+	sshConn1 := NewSSHConnection(customConn1.id, customConn1)
+	sshConn2 := NewSSHConnection(customConn2.id, customConn2)
+
+	username := "test_user"
+	user := dataprovider.User{
+		Username: username,
+	}
+	c := NewBaseConnection(sshConn1.id+"_1", ProtocolSFTP, user, nil)
+	c.lastActivity = time.Now().Add(-24 * time.Hour).UnixNano()
+	fakeConn := &fakeConnection{
+		BaseConnection: c,
+	}
+	// both ssh connections are expired but they should get removed only
+	// if there is no associated connection
+	sshConn1.lastActivity = c.lastActivity
+	sshConn2.lastActivity = c.lastActivity
+	Connections.AddSSHConnection(sshConn1)
+	Connections.Add(fakeConn)
+	assert.Equal(t, Connections.GetActiveSessions(username), 1)
+	c = NewBaseConnection(sshConn2.id+"_1", ProtocolSSH, user, nil)
+	fakeConn = &fakeConnection{
+		BaseConnection: c,
+	}
+	Connections.AddSSHConnection(sshConn2)
+	Connections.Add(fakeConn)
+	assert.Equal(t, Connections.GetActiveSessions(username), 2)
+
+	cFTP := NewBaseConnection("id2", ProtocolFTP, dataprovider.User{}, nil)
+	cFTP.lastActivity = time.Now().UnixNano()
+	fakeConn = &fakeConnection{
+		BaseConnection: cFTP,
+	}
+	Connections.Add(fakeConn)
+	assert.Equal(t, Connections.GetActiveSessions(username), 2)
+	assert.Len(t, Connections.GetStats(), 3)
+	Connections.RLock()
+	assert.Len(t, Connections.sshConnections, 2)
+	Connections.RUnlock()
+
+	startIdleTimeoutTicker(100 * time.Millisecond)
+	assert.Eventually(t, func() bool { return Connections.GetActiveSessions(username) == 1 }, 1*time.Second, 200*time.Millisecond)
+	assert.Eventually(t, func() bool {
+		Connections.RLock()
+		defer Connections.RUnlock()
+		return len(Connections.sshConnections) == 1
+	}, 1*time.Second, 200*time.Millisecond)
+	stopIdleTimeoutTicker()
+	assert.Len(t, Connections.GetStats(), 2)
+	c.lastActivity = time.Now().Add(-24 * time.Hour).UnixNano()
+	cFTP.lastActivity = time.Now().Add(-24 * time.Hour).UnixNano()
+	sshConn2.lastActivity = c.lastActivity
+	startIdleTimeoutTicker(100 * time.Millisecond)
+	assert.Eventually(t, func() bool { return len(Connections.GetStats()) == 0 }, 1*time.Second, 200*time.Millisecond)
+	assert.Eventually(t, func() bool {
+		Connections.RLock()
+		defer Connections.RUnlock()
+		return len(Connections.sshConnections) == 0
+	}, 1*time.Second, 200*time.Millisecond)
+	stopIdleTimeoutTicker()
+	assert.True(t, customConn1.isClosed)
+	assert.True(t, customConn2.isClosed)
+
+	Config = configCopy
+}
+
+func TestCloseConnection(t *testing.T) {
+	c := NewBaseConnection("id", ProtocolSFTP, dataprovider.User{}, nil)
+	fakeConn := &fakeConnection{
+		BaseConnection: c,
+	}
+	Connections.Add(fakeConn)
+	assert.Len(t, Connections.GetStats(), 1)
+	res := Connections.Close(fakeConn.GetID())
+	assert.True(t, res)
+	assert.Eventually(t, func() bool { return len(Connections.GetStats()) == 0 }, 300*time.Millisecond, 50*time.Millisecond)
+	res = Connections.Close(fakeConn.GetID())
+	assert.False(t, res)
+	Connections.Remove(fakeConn.GetID())
+}
+
+func TestSwapConnection(t *testing.T) {
+	c := NewBaseConnection("id", ProtocolFTP, dataprovider.User{}, nil)
+	fakeConn := &fakeConnection{
+		BaseConnection: c,
+	}
+	Connections.Add(fakeConn)
+	if assert.Len(t, Connections.GetStats(), 1) {
+		assert.Equal(t, "", Connections.GetStats()[0].Username)
+	}
+	c = NewBaseConnection("id", ProtocolFTP, dataprovider.User{
+		Username: userTestUsername,
+	}, nil)
+	fakeConn = &fakeConnection{
+		BaseConnection: c,
+	}
+	err := Connections.Swap(fakeConn)
+	assert.NoError(t, err)
+	if assert.Len(t, Connections.GetStats(), 1) {
+		assert.Equal(t, userTestUsername, Connections.GetStats()[0].Username)
+	}
+	res := Connections.Close(fakeConn.GetID())
+	assert.True(t, res)
+	assert.Eventually(t, func() bool { return len(Connections.GetStats()) == 0 }, 300*time.Millisecond, 50*time.Millisecond)
+	err = Connections.Swap(fakeConn)
+	assert.Error(t, err)
+}
+
+func TestAtomicUpload(t *testing.T) {
+	configCopy := Config
+
+	Config.UploadMode = UploadModeStandard
+	assert.False(t, Config.IsAtomicUploadEnabled())
+	Config.UploadMode = UploadModeAtomic
+	assert.True(t, Config.IsAtomicUploadEnabled())
+	Config.UploadMode = UploadModeAtomicWithResume
+	assert.True(t, Config.IsAtomicUploadEnabled())
+
+	Config = configCopy
+}
+
+func TestConnectionStatus(t *testing.T) {
+	username := "test_user"
+	user := dataprovider.User{
+		Username: username,
+	}
+	fs := vfs.NewOsFs("", os.TempDir(), nil)
+	c1 := NewBaseConnection("id1", ProtocolSFTP, user, fs)
+	fakeConn1 := &fakeConnection{
+		BaseConnection: c1,
+	}
+	t1 := NewBaseTransfer(nil, c1, nil, "/p1", "/r1", TransferUpload, 0, 0, 0, true, fs)
+	t1.BytesReceived = 123
+	t2 := NewBaseTransfer(nil, c1, nil, "/p2", "/r2", TransferDownload, 0, 0, 0, true, fs)
+	t2.BytesSent = 456
+	c2 := NewBaseConnection("id2", ProtocolSSH, user, nil)
+	fakeConn2 := &fakeConnection{
+		BaseConnection: c2,
+		command:        "md5sum",
+	}
+	c3 := NewBaseConnection("id3", ProtocolWebDAV, user, nil)
+	fakeConn3 := &fakeConnection{
+		BaseConnection: c3,
+		command:        "PROPFIND",
+	}
+	t3 := NewBaseTransfer(nil, c3, nil, "/p2", "/r2", TransferDownload, 0, 0, 0, true, fs)
+	Connections.Add(fakeConn1)
+	Connections.Add(fakeConn2)
+	Connections.Add(fakeConn3)
+
+	stats := Connections.GetStats()
+	assert.Len(t, stats, 3)
+	for _, stat := range stats {
+		assert.Equal(t, stat.Username, username)
+		assert.True(t, strings.HasPrefix(stat.GetConnectionInfo(), stat.Protocol))
+		assert.True(t, strings.HasPrefix(stat.GetConnectionDuration(), "00:"))
+		if stat.ConnectionID == "SFTP_id1" {
+			assert.Len(t, stat.Transfers, 2)
+			assert.Greater(t, len(stat.GetTransfersAsString()), 0)
+			for _, tr := range stat.Transfers {
+				if tr.OperationType == operationDownload {
+					assert.True(t, strings.HasPrefix(tr.getConnectionTransferAsString(), "DL"))
+				} else if tr.OperationType == operationUpload {
+					assert.True(t, strings.HasPrefix(tr.getConnectionTransferAsString(), "UL"))
+				}
+			}
+		} else if stat.ConnectionID == "DAV_id3" {
+			assert.Len(t, stat.Transfers, 1)
+			assert.Greater(t, len(stat.GetTransfersAsString()), 0)
+		} else {
+			assert.Equal(t, 0, len(stat.GetTransfersAsString()))
+		}
+	}
+
+	err := t1.Close()
+	assert.NoError(t, err)
+	err = t2.Close()
+	assert.NoError(t, err)
+
+	err = fakeConn3.SignalTransfersAbort()
+	assert.NoError(t, err)
+	assert.Equal(t, int32(1), atomic.LoadInt32(&t3.AbortTransfer))
+	err = t3.Close()
+	assert.NoError(t, err)
+	err = fakeConn3.SignalTransfersAbort()
+	assert.Error(t, err)
+
+	Connections.Remove(fakeConn1.GetID())
+	stats = Connections.GetStats()
+	assert.Len(t, stats, 2)
+	assert.Equal(t, fakeConn3.GetID(), stats[0].ConnectionID)
+	assert.Equal(t, fakeConn2.GetID(), stats[1].ConnectionID)
+	Connections.Remove(fakeConn2.GetID())
+	stats = Connections.GetStats()
+	assert.Len(t, stats, 1)
+	assert.Equal(t, fakeConn3.GetID(), stats[0].ConnectionID)
+	Connections.Remove(fakeConn3.GetID())
+	stats = Connections.GetStats()
+	assert.Len(t, stats, 0)
+}
+
+func TestQuotaScans(t *testing.T) {
+	username := "username"
+	assert.True(t, QuotaScans.AddUserQuotaScan(username))
+	assert.False(t, QuotaScans.AddUserQuotaScan(username))
+	if assert.Len(t, QuotaScans.GetUsersQuotaScans(), 1) {
+		assert.Equal(t, QuotaScans.GetUsersQuotaScans()[0].Username, username)
+	}
+
+	assert.True(t, QuotaScans.RemoveUserQuotaScan(username))
+	assert.False(t, QuotaScans.RemoveUserQuotaScan(username))
+	assert.Len(t, QuotaScans.GetUsersQuotaScans(), 0)
+
+	folderName := "/folder"
+	assert.True(t, QuotaScans.AddVFolderQuotaScan(folderName))
+	assert.False(t, QuotaScans.AddVFolderQuotaScan(folderName))
+	if assert.Len(t, QuotaScans.GetVFoldersQuotaScans(), 1) {
+		assert.Equal(t, QuotaScans.GetVFoldersQuotaScans()[0].MappedPath, folderName)
+	}
+
+	assert.True(t, QuotaScans.RemoveVFolderQuotaScan(folderName))
+	assert.False(t, QuotaScans.RemoveVFolderQuotaScan(folderName))
+	assert.Len(t, QuotaScans.GetVFoldersQuotaScans(), 0)
+}
+
+func TestProxyProtocolVersion(t *testing.T) {
+	c := Configuration{
+		ProxyProtocol: 1,
+	}
+	proxyListener, err := c.GetProxyListener(nil)
+	assert.NoError(t, err)
+	assert.Nil(t, proxyListener.Policy)
+
+	c.ProxyProtocol = 2
+	proxyListener, err = c.GetProxyListener(nil)
+	assert.NoError(t, err)
+	assert.NotNil(t, proxyListener.Policy)
+
+	c.ProxyProtocol = 1
+	c.ProxyAllowed = []string{"invalid"}
+	_, err = c.GetProxyListener(nil)
+	assert.Error(t, err)
+
+	c.ProxyProtocol = 2
+	_, err = c.GetProxyListener(nil)
+	assert.Error(t, err)
+}
+
+func TestProxyProtocol(t *testing.T) {
+	httpClient := httpclient.GetHTTPClient()
+	resp, err := httpClient.Get(fmt.Sprintf("http://%v", httpProxyAddr))
+	if assert.NoError(t, err) {
+		defer resp.Body.Close()
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+	}
+}
+
+func TestPostConnectHook(t *testing.T) {
+	Config.PostConnectHook = ""
+
+	remoteAddr := &net.IPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Zone: "",
+	}
+
+	assert.NoError(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolFTP))
+
+	Config.PostConnectHook = "http://foo\x7f.com/"
+	assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP))
+
+	Config.PostConnectHook = "http://invalid:1234/"
+	assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP))
+
+	Config.PostConnectHook = fmt.Sprintf("http://%v/404", httpAddr)
+	assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolFTP))
+
+	Config.PostConnectHook = fmt.Sprintf("http://%v", httpAddr)
+	assert.NoError(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolFTP))
+
+	Config.PostConnectHook = "invalid"
+	assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolFTP))
+
+	if runtime.GOOS == osWindows {
+		Config.PostConnectHook = "C:\\bad\\command"
+		assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP))
+	} else {
+		Config.PostConnectHook = "/invalid/path"
+		assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP))
+
+		hookCmd, err := exec.LookPath("true")
+		assert.NoError(t, err)
+		Config.PostConnectHook = hookCmd
+		assert.NoError(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP))
+	}
+
+	Config.PostConnectHook = ""
+}

common/connection.go

@@ -0,0 +1,989 @@
+package common
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/pkg/sftp"
+
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/utils"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+// BaseConnection defines common fields for a connection using any supported protocol
+type BaseConnection struct {
+	// Unique identifier for the connection
+	ID string
+	// user associated with this connection if any
+	User dataprovider.User
+	// start time for this connection
+	startTime time.Time
+	protocol  string
+	Fs        vfs.Fs
+	sync.RWMutex
+	// last activity for this connection
+	lastActivity    int64
+	transferID      uint64
+	activeTransfers []ActiveTransfer
+}
+
+// NewBaseConnection returns a new BaseConnection
+func NewBaseConnection(ID, protocol string, user dataprovider.User, fs vfs.Fs) *BaseConnection {
+	connID := ID
+	if utils.IsStringInSlice(protocol, supportedProtocols) {
+		connID = fmt.Sprintf("%v_%v", protocol, ID)
+	}
+	return &BaseConnection{
+		ID:           connID,
+		User:         user,
+		startTime:    time.Now(),
+		protocol:     protocol,
+		Fs:           fs,
+		lastActivity: time.Now().UnixNano(),
+		transferID:   0,
+	}
+}
+
+// Log outputs a log entry to the configured logger
+func (c *BaseConnection) Log(level logger.LogLevel, format string, v ...interface{}) {
+	logger.Log(level, c.protocol, c.ID, format, v...)
+}
+
+// GetTransferID returns an unique transfer ID for this connection
+func (c *BaseConnection) GetTransferID() uint64 {
+	return atomic.AddUint64(&c.transferID, 1)
+}
+
+// GetID returns the connection ID
+func (c *BaseConnection) GetID() string {
+	return c.ID
+}
+
+// GetUsername returns the authenticated username associated with this connection if any
+func (c *BaseConnection) GetUsername() string {
+	return c.User.Username
+}
+
+// GetProtocol returns the protocol for the connection
+func (c *BaseConnection) GetProtocol() string {
+	return c.protocol
+}
+
+// SetProtocol sets the protocol for this connection
+func (c *BaseConnection) SetProtocol(protocol string) {
+	c.protocol = protocol
+	if utils.IsStringInSlice(c.protocol, supportedProtocols) {
+		c.ID = fmt.Sprintf("%v_%v", c.protocol, c.ID)
+	}
+}
+
+// GetConnectionTime returns the initial connection time
+func (c *BaseConnection) GetConnectionTime() time.Time {
+	return c.startTime
+}
+
+// UpdateLastActivity updates last activity for this connection
+func (c *BaseConnection) UpdateLastActivity() {
+	atomic.StoreInt64(&c.lastActivity, time.Now().UnixNano())
+}
+
+// GetLastActivity returns the last connection activity
+func (c *BaseConnection) GetLastActivity() time.Time {
+	return time.Unix(0, atomic.LoadInt64(&c.lastActivity))
+}
+
+// AddTransfer associates a new transfer to this connection
+func (c *BaseConnection) AddTransfer(t ActiveTransfer) {
+	c.Lock()
+	defer c.Unlock()
+
+	c.activeTransfers = append(c.activeTransfers, t)
+	c.Log(logger.LevelDebug, "transfer added, id: %v, active transfers: %v", t.GetID(), len(c.activeTransfers))
+}
+
+// RemoveTransfer removes the specified transfer from the active ones
+func (c *BaseConnection) RemoveTransfer(t ActiveTransfer) {
+	c.Lock()
+	defer c.Unlock()
+
+	indexToRemove := -1
+	for i, v := range c.activeTransfers {
+		if v.GetID() == t.GetID() {
+			indexToRemove = i
+			break
+		}
+	}
+	if indexToRemove >= 0 {
+		c.activeTransfers[indexToRemove] = c.activeTransfers[len(c.activeTransfers)-1]
+		c.activeTransfers[len(c.activeTransfers)-1] = nil
+		c.activeTransfers = c.activeTransfers[:len(c.activeTransfers)-1]
+		c.Log(logger.LevelDebug, "transfer removed, id: %v active transfers: %v", t.GetID(), len(c.activeTransfers))
+	} else {
+		c.Log(logger.LevelWarn, "transfer to remove not found!")
+	}
+}
+
+// GetTransfers returns the active transfers
+func (c *BaseConnection) GetTransfers() []ConnectionTransfer {
+	c.RLock()
+	defer c.RUnlock()
+
+	transfers := make([]ConnectionTransfer, 0, len(c.activeTransfers))
+	for _, t := range c.activeTransfers {
+		var operationType string
+		switch t.GetType() {
+		case TransferDownload:
+			operationType = operationDownload
+		case TransferUpload:
+			operationType = operationUpload
+		}
+		transfers = append(transfers, ConnectionTransfer{
+			ID:            t.GetID(),
+			OperationType: operationType,
+			StartTime:     utils.GetTimeAsMsSinceEpoch(t.GetStartTime()),
+			Size:          t.GetSize(),
+			VirtualPath:   t.GetVirtualPath(),
+		})
+	}
+
+	return transfers
+}
+
+// SignalTransfersAbort signals to the active transfers to exit as soon as possible
+func (c *BaseConnection) SignalTransfersAbort() error {
+	c.RLock()
+	defer c.RUnlock()
+
+	if len(c.activeTransfers) == 0 {
+		return errors.New("no active transfer found")
+	}
+
+	for _, t := range c.activeTransfers {
+		t.SignalClose()
+	}
+	return nil
+}
+
+func (c *BaseConnection) getRealFsPath(fsPath string) string {
+	c.RLock()
+	defer c.RUnlock()
+
+	for _, t := range c.activeTransfers {
+		if p := t.GetRealFsPath(fsPath); len(p) > 0 {
+			return p
+		}
+	}
+	return fsPath
+}
+
+func (c *BaseConnection) truncateOpenHandle(fsPath string, size int64) (int64, error) {
+	c.RLock()
+	defer c.RUnlock()
+
+	for _, t := range c.activeTransfers {
+		initialSize, err := t.Truncate(fsPath, size)
+		if err != errTransferMismatch {
+			return initialSize, err
+		}
+	}
+
+	return 0, errNoTransfer
+}
+
+// ListDir reads the directory named by fsPath and returns a list of directory entries
+func (c *BaseConnection) ListDir(fsPath, virtualPath string) ([]os.FileInfo, error) {
+	if !c.User.HasPerm(dataprovider.PermListItems, virtualPath) {
+		return nil, c.GetPermissionDeniedError()
+	}
+	files, err := c.Fs.ReadDir(fsPath)
+	if err != nil {
+		c.Log(logger.LevelWarn, "error listing directory: %+v", err)
+		return nil, c.GetFsError(err)
+	}
+	return c.User.AddVirtualDirs(files, virtualPath), nil
+}
+
+// CreateDir creates a new directory at the specified fsPath
+func (c *BaseConnection) CreateDir(fsPath, virtualPath string) error {
+	if !c.User.HasPerm(dataprovider.PermCreateDirs, path.Dir(virtualPath)) {
+		return c.GetPermissionDeniedError()
+	}
+	if c.User.IsVirtualFolder(virtualPath) {
+		c.Log(logger.LevelWarn, "mkdir not allowed %#v is a virtual folder", virtualPath)
+		return c.GetPermissionDeniedError()
+	}
+	if err := c.Fs.Mkdir(fsPath); err != nil {
+		c.Log(logger.LevelWarn, "error creating dir: %#v error: %+v", fsPath, err)
+		return c.GetFsError(err)
+	}
+	vfs.SetPathPermissions(c.Fs, fsPath, c.User.GetUID(), c.User.GetGID())
+
+	logger.CommandLog(mkdirLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1)
+	return nil
+}
+
+// IsRemoveFileAllowed returns an error if removing this file is not allowed
+func (c *BaseConnection) IsRemoveFileAllowed(fsPath, virtualPath string) error {
+	if !c.User.HasPerm(dataprovider.PermDelete, path.Dir(virtualPath)) {
+		return c.GetPermissionDeniedError()
+	}
+	if !c.User.IsFileAllowed(virtualPath) {
+		c.Log(logger.LevelDebug, "removing file %#v is not allowed", fsPath)
+		return c.GetPermissionDeniedError()
+	}
+	return nil
+}
+
+// RemoveFile removes a file at the specified fsPath
+func (c *BaseConnection) RemoveFile(fsPath, virtualPath string, info os.FileInfo) error {
+	if err := c.IsRemoveFileAllowed(fsPath, virtualPath); err != nil {
+		return err
+	}
+	size := info.Size()
+	action := newActionNotification(&c.User, operationPreDelete, fsPath, "", "", c.protocol, size, nil)
+	actionErr := actionHandler.Handle(action)
+	if actionErr == nil {
+		c.Log(logger.LevelDebug, "remove for path %#v handled by pre-delete action", fsPath)
+	} else {
+		if err := c.Fs.Remove(fsPath, false); err != nil {
+			c.Log(logger.LevelWarn, "failed to remove a file/symlink %#v: %+v", fsPath, err)
+			return c.GetFsError(err)
+		}
+	}
+
+	logger.CommandLog(removeLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1)
+	if info.Mode()&os.ModeSymlink == 0 {
+		vfolder, err := c.User.GetVirtualFolderForPath(path.Dir(virtualPath))
+		if err == nil {
+			dataprovider.UpdateVirtualFolderQuota(vfolder.BaseVirtualFolder, -1, -size, false) //nolint:errcheck
+			if vfolder.IsIncludedInUserQuota() {
+				dataprovider.UpdateUserQuota(c.User, -1, -size, false) //nolint:errcheck
+			}
+		} else {
+			dataprovider.UpdateUserQuota(c.User, -1, -size, false) //nolint:errcheck
+		}
+	}
+	if actionErr != nil {
+		action := newActionNotification(&c.User, operationDelete, fsPath, "", "", c.protocol, size, nil)
+		go actionHandler.Handle(action) // nolint:errcheck
+	}
+	return nil
+}
+
+// IsRemoveDirAllowed returns an error if removing this directory is not allowed
+func (c *BaseConnection) IsRemoveDirAllowed(fsPath, virtualPath string) error {
+	if c.Fs.GetRelativePath(fsPath) == "/" {
+		c.Log(logger.LevelWarn, "removing root dir is not allowed")
+		return c.GetPermissionDeniedError()
+	}
+	if c.User.IsVirtualFolder(virtualPath) {
+		c.Log(logger.LevelWarn, "removing a virtual folder is not allowed: %#v", virtualPath)
+		return c.GetPermissionDeniedError()
+	}
+	if c.User.HasVirtualFoldersInside(virtualPath) {
+		c.Log(logger.LevelWarn, "removing a directory with a virtual folder inside is not allowed: %#v", virtualPath)
+		return c.GetOpUnsupportedError()
+	}
+	if c.User.IsMappedPath(fsPath) {
+		c.Log(logger.LevelWarn, "removing a directory mapped as virtual folder is not allowed: %#v", fsPath)
+		return c.GetPermissionDeniedError()
+	}
+	if !c.User.HasPerm(dataprovider.PermDelete, path.Dir(virtualPath)) {
+		return c.GetPermissionDeniedError()
+	}
+	return nil
+}
+
+// RemoveDir removes a directory at the specified fsPath
+func (c *BaseConnection) RemoveDir(fsPath, virtualPath string) error {
+	if err := c.IsRemoveDirAllowed(fsPath, virtualPath); err != nil {
+		return err
+	}
+
+	var fi os.FileInfo
+	var err error
+	if fi, err = c.Fs.Lstat(fsPath); err != nil {
+		// see #149
+		if c.Fs.IsNotExist(err) && c.Fs.HasVirtualFolders() {
+			return nil
+		}
+		c.Log(logger.LevelWarn, "failed to remove a dir %#v: stat error: %+v", fsPath, err)
+		return c.GetFsError(err)
+	}
+	if !fi.IsDir() || fi.Mode()&os.ModeSymlink != 0 {
+		c.Log(logger.LevelDebug, "cannot remove %#v is not a directory", fsPath)
+		return c.GetGenericError(nil)
+	}
+
+	if err := c.Fs.Remove(fsPath, true); err != nil {
+		c.Log(logger.LevelWarn, "failed to remove directory %#v: %+v", fsPath, err)
+		return c.GetFsError(err)
+	}
+
+	logger.CommandLog(rmdirLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1)
+	return nil
+}
+
+// Rename renames (moves) fsSourcePath to fsTargetPath
+func (c *BaseConnection) Rename(fsSourcePath, fsTargetPath, virtualSourcePath, virtualTargetPath string) error {
+	if c.User.IsMappedPath(fsSourcePath) {
+		c.Log(logger.LevelWarn, "renaming a directory mapped as virtual folder is not allowed: %#v", fsSourcePath)
+		return c.GetPermissionDeniedError()
+	}
+	if c.User.IsMappedPath(fsTargetPath) {
+		c.Log(logger.LevelWarn, "renaming to a directory mapped as virtual folder is not allowed: %#v", fsTargetPath)
+		return c.GetPermissionDeniedError()
+	}
+	srcInfo, err := c.Fs.Lstat(fsSourcePath)
+	if err != nil {
+		return c.GetFsError(err)
+	}
+	if !c.isRenamePermitted(fsSourcePath, virtualSourcePath, virtualTargetPath, srcInfo) {
+		return c.GetPermissionDeniedError()
+	}
+	initialSize := int64(-1)
+	if dstInfo, err := c.Fs.Lstat(fsTargetPath); err == nil {
+		if dstInfo.IsDir() {
+			c.Log(logger.LevelWarn, "attempted to rename %#v overwriting an existing directory %#v",
+				fsSourcePath, fsTargetPath)
+			return c.GetOpUnsupportedError()
+		}
+		// we are overwriting an existing file/symlink
+		if dstInfo.Mode().IsRegular() {
+			initialSize = dstInfo.Size()
+		}
+		if !c.User.HasPerm(dataprovider.PermOverwrite, path.Dir(virtualTargetPath)) {
+			c.Log(logger.LevelDebug, "renaming is not allowed, %#v -> %#v. Target exists but the user "+
+				"has no overwrite permission", virtualSourcePath, virtualTargetPath)
+			return c.GetPermissionDeniedError()
+		}
+	}
+	if srcInfo.IsDir() {
+		if c.User.HasVirtualFoldersInside(virtualSourcePath) {
+			c.Log(logger.LevelDebug, "renaming the folder %#v is not supported: it has virtual folders inside it",
+				virtualSourcePath)
+			return c.GetOpUnsupportedError()
+		}
+		if err = c.checkRecursiveRenameDirPermissions(fsSourcePath, fsTargetPath); err != nil {
+			c.Log(logger.LevelDebug, "error checking recursive permissions before renaming %#v: %+v", fsSourcePath, err)
+			return c.GetFsError(err)
+		}
+	}
+	if !c.hasSpaceForRename(virtualSourcePath, virtualTargetPath, initialSize, fsSourcePath) {
+		c.Log(logger.LevelInfo, "denying cross rename due to space limit")
+		return c.GetGenericError(ErrQuotaExceeded)
+	}
+	if err := c.Fs.Rename(fsSourcePath, fsTargetPath); err != nil {
+		c.Log(logger.LevelWarn, "failed to rename %#v -> %#v: %+v", fsSourcePath, fsTargetPath, err)
+		return c.GetFsError(err)
+	}
+	if dataprovider.GetQuotaTracking() > 0 {
+		c.updateQuotaAfterRename(virtualSourcePath, virtualTargetPath, fsTargetPath, initialSize) //nolint:errcheck
+	}
+	logger.CommandLog(renameLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1,
+		"", "", "", -1)
+	action := newActionNotification(&c.User, operationRename, fsSourcePath, fsTargetPath, "", c.protocol, 0, nil)
+	// the returned error is used in test cases only, we already log the error inside action.execute
+	go actionHandler.Handle(action) // nolint:errcheck
+
+	return nil
+}
+
+// CreateSymlink creates fsTargetPath as a symbolic link to fsSourcePath
+func (c *BaseConnection) CreateSymlink(fsSourcePath, fsTargetPath, virtualSourcePath, virtualTargetPath string) error {
+	if c.Fs.GetRelativePath(fsSourcePath) == "/" {
+		c.Log(logger.LevelWarn, "symlinking root dir is not allowed")
+		return c.GetPermissionDeniedError()
+	}
+	if c.User.IsVirtualFolder(virtualTargetPath) {
+		c.Log(logger.LevelWarn, "symlinking a virtual folder is not allowed")
+		return c.GetPermissionDeniedError()
+	}
+	if !c.User.HasPerm(dataprovider.PermCreateSymlinks, path.Dir(virtualTargetPath)) {
+		return c.GetPermissionDeniedError()
+	}
+	if c.isCrossFoldersRequest(virtualSourcePath, virtualTargetPath) {
+		c.Log(logger.LevelWarn, "cross folder symlink is not supported, src: %v dst: %v", virtualSourcePath, virtualTargetPath)
+		return c.GetOpUnsupportedError()
+	}
+	if c.User.IsMappedPath(fsSourcePath) {
+		c.Log(logger.LevelWarn, "symlinking a directory mapped as virtual folder is not allowed: %#v", fsSourcePath)
+		return c.GetPermissionDeniedError()
+	}
+	if c.User.IsMappedPath(fsTargetPath) {
+		c.Log(logger.LevelWarn, "symlinking to a directory mapped as virtual folder is not allowed: %#v", fsTargetPath)
+		return c.GetPermissionDeniedError()
+	}
+	if err := c.Fs.Symlink(fsSourcePath, fsTargetPath); err != nil {
+		c.Log(logger.LevelWarn, "failed to create symlink %#v -> %#v: %+v", fsSourcePath, fsTargetPath, err)
+		return c.GetFsError(err)
+	}
+	logger.CommandLog(symlinkLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1)
+	return nil
+}
+
+func (c *BaseConnection) getPathForSetStatPerms(fsPath, virtualPath string) string {
+	pathForPerms := virtualPath
+	if fi, err := c.Fs.Lstat(fsPath); err == nil {
+		if fi.IsDir() {
+			pathForPerms = path.Dir(virtualPath)
+		}
+	}
+	return pathForPerms
+}
+
+// DoStat execute a Stat if mode = 0, Lstat if mode = 1
+func (c *BaseConnection) DoStat(fsPath string, mode int) (os.FileInfo, error) {
+	if mode == 1 {
+		return c.Fs.Lstat(c.getRealFsPath(fsPath))
+	}
+	return c.Fs.Stat(c.getRealFsPath(fsPath))
+}
+
+func (c *BaseConnection) ignoreSetStat() bool {
+	if Config.SetstatMode == 1 {
+		return true
+	}
+	if Config.SetstatMode == 2 && !vfs.IsLocalOsFs(c.Fs) {
+		return true
+	}
+	return false
+}
+
+func (c *BaseConnection) handleChmod(fsPath, pathForPerms string, attributes *StatAttributes) error {
+	if !c.User.HasPerm(dataprovider.PermChmod, pathForPerms) {
+		return c.GetPermissionDeniedError()
+	}
+	if c.ignoreSetStat() {
+		return nil
+	}
+	if err := c.Fs.Chmod(c.getRealFsPath(fsPath), attributes.Mode); err != nil {
+		c.Log(logger.LevelWarn, "failed to chmod path %#v, mode: %v, err: %+v", fsPath, attributes.Mode.String(), err)
+		return c.GetFsError(err)
+	}
+	logger.CommandLog(chmodLogSender, fsPath, "", c.User.Username, attributes.Mode.String(), c.ID, c.protocol,
+		-1, -1, "", "", "", -1)
+	return nil
+}
+
+func (c *BaseConnection) handleChown(fsPath, pathForPerms string, attributes *StatAttributes) error {
+	if !c.User.HasPerm(dataprovider.PermChown, pathForPerms) {
+		return c.GetPermissionDeniedError()
+	}
+	if c.ignoreSetStat() {
+		return nil
+	}
+	if err := c.Fs.Chown(c.getRealFsPath(fsPath), attributes.UID, attributes.GID); err != nil {
+		c.Log(logger.LevelWarn, "failed to chown path %#v, uid: %v, gid: %v, err: %+v", fsPath, attributes.UID,
+			attributes.GID, err)
+		return c.GetFsError(err)
+	}
+	logger.CommandLog(chownLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, attributes.UID, attributes.GID,
+		"", "", "", -1)
+	return nil
+}
+
+func (c *BaseConnection) handleChtimes(fsPath, pathForPerms string, attributes *StatAttributes) error {
+	if !c.User.HasPerm(dataprovider.PermChtimes, pathForPerms) {
+		return c.GetPermissionDeniedError()
+	}
+	if c.ignoreSetStat() {
+		return nil
+	}
+	if err := c.Fs.Chtimes(c.getRealFsPath(fsPath), attributes.Atime, attributes.Mtime); err != nil {
+		c.Log(logger.LevelWarn, "failed to chtimes for path %#v, access time: %v, modification time: %v, err: %+v",
+			fsPath, attributes.Atime, attributes.Mtime, err)
+		return c.GetFsError(err)
+	}
+	accessTimeString := attributes.Atime.Format(chtimesFormat)
+	modificationTimeString := attributes.Mtime.Format(chtimesFormat)
+	logger.CommandLog(chtimesLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1,
+		accessTimeString, modificationTimeString, "", -1)
+	return nil
+}
+
+// SetStat set StatAttributes for the specified fsPath
+func (c *BaseConnection) SetStat(fsPath, virtualPath string, attributes *StatAttributes) error {
+	pathForPerms := c.getPathForSetStatPerms(fsPath, virtualPath)
+
+	if attributes.Flags&StatAttrPerms != 0 {
+		return c.handleChmod(fsPath, pathForPerms, attributes)
+	}
+
+	if attributes.Flags&StatAttrUIDGID != 0 {
+		return c.handleChown(fsPath, pathForPerms, attributes)
+	}
+
+	if attributes.Flags&StatAttrTimes != 0 {
+		return c.handleChtimes(fsPath, pathForPerms, attributes)
+	}
+
+	if attributes.Flags&StatAttrSize != 0 {
+		if !c.User.HasPerm(dataprovider.PermOverwrite, pathForPerms) {
+			return c.GetPermissionDeniedError()
+		}
+
+		if err := c.truncateFile(fsPath, virtualPath, attributes.Size); err != nil {
+			c.Log(logger.LevelWarn, "failed to truncate path %#v, size: %v, err: %+v", fsPath, attributes.Size, err)
+			return c.GetFsError(err)
+		}
+		logger.CommandLog(truncateLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", attributes.Size)
+	}
+
+	return nil
+}
+
+func (c *BaseConnection) truncateFile(fsPath, virtualPath string, size int64) error {
+	// check first if we have an open transfer for the given path and try to truncate the file already opened
+	// if we found no transfer we truncate by path.
+	var initialSize int64
+	var err error
+	initialSize, err = c.truncateOpenHandle(fsPath, size)
+	if err == errNoTransfer {
+		c.Log(logger.LevelDebug, "file path %#v not found in active transfers, execute trucate by path", fsPath)
+		var info os.FileInfo
+		info, err = c.Fs.Stat(fsPath)
+		if err != nil {
+			return err
+		}
+		initialSize = info.Size()
+		err = c.Fs.Truncate(fsPath, size)
+	}
+	if err == nil && vfs.IsLocalOsFs(c.Fs) {
+		sizeDiff := initialSize - size
+		vfolder, err := c.User.GetVirtualFolderForPath(path.Dir(virtualPath))
+		if err == nil {
+			dataprovider.UpdateVirtualFolderQuota(vfolder.BaseVirtualFolder, 0, -sizeDiff, false) //nolint:errcheck
+			if vfolder.IsIncludedInUserQuota() {
+				dataprovider.UpdateUserQuota(c.User, 0, -sizeDiff, false) //nolint:errcheck
+			}
+		} else {
+			dataprovider.UpdateUserQuota(c.User, 0, -sizeDiff, false) //nolint:errcheck
+		}
+	}
+	return err
+}
+
+func (c *BaseConnection) checkRecursiveRenameDirPermissions(sourcePath, targetPath string) error {
+	dstPerms := []string{
+		dataprovider.PermCreateDirs,
+		dataprovider.PermUpload,
+		dataprovider.PermCreateSymlinks,
+	}
+
+	err := c.Fs.Walk(sourcePath, func(walkedPath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		dstPath := strings.Replace(walkedPath, sourcePath, targetPath, 1)
+		virtualSrcPath := c.Fs.GetRelativePath(walkedPath)
+		virtualDstPath := c.Fs.GetRelativePath(dstPath)
+		// walk scans the directory tree in order, checking the parent directory permissions we are sure that all contents
+		// inside the parent path was checked. If the current dir has no subdirs with defined permissions inside it
+		// and it has all the possible permissions we can stop scanning
+		if !c.User.HasPermissionsInside(path.Dir(virtualSrcPath)) && !c.User.HasPermissionsInside(path.Dir(virtualDstPath)) {
+			if c.User.HasPerm(dataprovider.PermRename, path.Dir(virtualSrcPath)) &&
+				c.User.HasPerm(dataprovider.PermRename, path.Dir(virtualDstPath)) {
+				return ErrSkipPermissionsCheck
+			}
+			if c.User.HasPerm(dataprovider.PermDelete, path.Dir(virtualSrcPath)) &&
+				c.User.HasPerms(dstPerms, path.Dir(virtualDstPath)) {
+				return ErrSkipPermissionsCheck
+			}
+		}
+		if !c.isRenamePermitted(walkedPath, virtualSrcPath, virtualDstPath, info) {
+			c.Log(logger.LevelInfo, "rename %#v -> %#v is not allowed, virtual destination path: %#v",
+				walkedPath, dstPath, virtualDstPath)
+			return os.ErrPermission
+		}
+		return nil
+	})
+	if err == ErrSkipPermissionsCheck {
+		err = nil
+	}
+	return err
+}
+
+func (c *BaseConnection) isRenamePermitted(fsSourcePath, virtualSourcePath, virtualTargetPath string, fi os.FileInfo) bool {
+	if c.Fs.GetRelativePath(fsSourcePath) == "/" {
+		c.Log(logger.LevelWarn, "renaming root dir is not allowed")
+		return false
+	}
+	if c.User.IsVirtualFolder(virtualSourcePath) || c.User.IsVirtualFolder(virtualTargetPath) {
+		c.Log(logger.LevelWarn, "renaming a virtual folder is not allowed")
+		return false
+	}
+	if !c.User.IsFileAllowed(virtualSourcePath) || !c.User.IsFileAllowed(virtualTargetPath) {
+		if fi != nil && fi.Mode().IsRegular() {
+			c.Log(logger.LevelDebug, "renaming file is not allowed, source: %#v target: %#v",
+				virtualSourcePath, virtualTargetPath)
+			return false
+		}
+	}
+	if c.User.HasPerm(dataprovider.PermRename, path.Dir(virtualSourcePath)) &&
+		c.User.HasPerm(dataprovider.PermRename, path.Dir(virtualTargetPath)) {
+		return true
+	}
+	if !c.User.HasPerm(dataprovider.PermDelete, path.Dir(virtualSourcePath)) {
+		return false
+	}
+	if fi != nil {
+		if fi.IsDir() {
+			return c.User.HasPerm(dataprovider.PermCreateDirs, path.Dir(virtualTargetPath))
+		} else if fi.Mode()&os.ModeSymlink != 0 {
+			return c.User.HasPerm(dataprovider.PermCreateSymlinks, path.Dir(virtualTargetPath))
+		}
+	}
+	return c.User.HasPerm(dataprovider.PermUpload, path.Dir(virtualTargetPath))
+}
+
+func (c *BaseConnection) hasSpaceForRename(virtualSourcePath, virtualTargetPath string, initialSize int64,
+	fsSourcePath string) bool {
+	if dataprovider.GetQuotaTracking() == 0 {
+		return true
+	}
+	sourceFolder, errSrc := c.User.GetVirtualFolderForPath(path.Dir(virtualSourcePath))
+	dstFolder, errDst := c.User.GetVirtualFolderForPath(path.Dir(virtualTargetPath))
+	if errSrc != nil && errDst != nil {
+		// rename inside the user home dir
+		return true
+	}
+	if errSrc == nil && errDst == nil {
+		// rename between virtual folders
+		if sourceFolder.MappedPath == dstFolder.MappedPath {
+			// rename inside the same virtual folder
+			return true
+		}
+	}
+	if errSrc != nil && dstFolder.IsIncludedInUserQuota() {
+		// rename between user root dir and a virtual folder included in user quota
+		return true
+	}
+	quotaResult := c.HasSpace(true, virtualTargetPath)
+	return c.hasSpaceForCrossRename(quotaResult, initialSize, fsSourcePath)
+}
+
+// hasSpaceForCrossRename checks the quota after a rename between different folders
+func (c *BaseConnection) hasSpaceForCrossRename(quotaResult vfs.QuotaCheckResult, initialSize int64, sourcePath string) bool {
+	if !quotaResult.HasSpace && initialSize == -1 {
+		// we are over quota and this is not a file replace
+		return false
+	}
+	fi, err := c.Fs.Lstat(sourcePath)
+	if err != nil {
+		c.Log(logger.LevelWarn, "cross rename denied, stat error for path %#v: %v", sourcePath, err)
+		return false
+	}
+	var sizeDiff int64
+	var filesDiff int
+	if fi.Mode().IsRegular() {
+		sizeDiff = fi.Size()
+		filesDiff = 1
+		if initialSize != -1 {
+			sizeDiff -= initialSize
+			filesDiff = 0
+		}
+	} else if fi.IsDir() {
+		filesDiff, sizeDiff, err = c.Fs.GetDirSize(sourcePath)
+		if err != nil {
+			c.Log(logger.LevelWarn, "cross rename denied, error getting size for directory %#v: %v", sourcePath, err)
+			return false
+		}
+	}
+	if !quotaResult.HasSpace && initialSize != -1 {
+		// we are over quota but we are overwriting an existing file so we check if the quota size after the rename is ok
+		if quotaResult.QuotaSize == 0 {
+			return true
+		}
+		c.Log(logger.LevelDebug, "cross rename overwrite, source %#v, used size %v, size to add %v",
+			sourcePath, quotaResult.UsedSize, sizeDiff)
+		quotaResult.UsedSize += sizeDiff
+		return quotaResult.GetRemainingSize() >= 0
+	}
+	if quotaResult.QuotaFiles > 0 {
+		remainingFiles := quotaResult.GetRemainingFiles()
+		c.Log(logger.LevelDebug, "cross rename, source %#v remaining file %v to add %v", sourcePath,
+			remainingFiles, filesDiff)
+		if remainingFiles < filesDiff {
+			return false
+		}
+	}
+	if quotaResult.QuotaSize > 0 {
+		remainingSize := quotaResult.GetRemainingSize()
+		c.Log(logger.LevelDebug, "cross rename, source %#v remaining size %v to add %v", sourcePath,
+			remainingSize, sizeDiff)
+		if remainingSize < sizeDiff {
+			return false
+		}
+	}
+	return true
+}
+
+// GetMaxWriteSize returns the allowed size for an upload or an error
+// if no enough size is available for a resume/append
+func (c *BaseConnection) GetMaxWriteSize(quotaResult vfs.QuotaCheckResult, isResume bool, fileSize int64) (int64, error) {
+	maxWriteSize := quotaResult.GetRemainingSize()
+
+	if isResume {
+		if !c.Fs.IsUploadResumeSupported() {
+			return 0, c.GetOpUnsupportedError()
+		}
+		if c.User.Filters.MaxUploadFileSize > 0 && c.User.Filters.MaxUploadFileSize <= fileSize {
+			return 0, ErrQuotaExceeded
+		}
+		if c.User.Filters.MaxUploadFileSize > 0 {
+			maxUploadSize := c.User.Filters.MaxUploadFileSize - fileSize
+			if maxUploadSize < maxWriteSize || maxWriteSize == 0 {
+				maxWriteSize = maxUploadSize
+			}
+		}
+	} else {
+		if maxWriteSize > 0 {
+			maxWriteSize += fileSize
+		}
+		if c.User.Filters.MaxUploadFileSize > 0 && (c.User.Filters.MaxUploadFileSize < maxWriteSize || maxWriteSize == 0) {
+			maxWriteSize = c.User.Filters.MaxUploadFileSize
+		}
+	}
+
+	return maxWriteSize, nil
+}
+
+// HasSpace checks user's quota usage
+func (c *BaseConnection) HasSpace(checkFiles bool, requestPath string) vfs.QuotaCheckResult {
+	result := vfs.QuotaCheckResult{
+		HasSpace:     true,
+		AllowedSize:  0,
+		AllowedFiles: 0,
+		UsedSize:     0,
+		UsedFiles:    0,
+		QuotaSize:    0,
+		QuotaFiles:   0,
+	}
+
+	if dataprovider.GetQuotaTracking() == 0 {
+		return result
+	}
+	var err error
+	var vfolder vfs.VirtualFolder
+	vfolder, err = c.User.GetVirtualFolderForPath(path.Dir(requestPath))
+	if err == nil && !vfolder.IsIncludedInUserQuota() {
+		if vfolder.HasNoQuotaRestrictions(checkFiles) {
+			return result
+		}
+		result.QuotaSize = vfolder.QuotaSize
+		result.QuotaFiles = vfolder.QuotaFiles
+		result.UsedFiles, result.UsedSize, err = dataprovider.GetUsedVirtualFolderQuota(vfolder.MappedPath)
+	} else {
+		if c.User.HasNoQuotaRestrictions(checkFiles) {
+			return result
+		}
+		result.QuotaSize = c.User.QuotaSize
+		result.QuotaFiles = c.User.QuotaFiles
+		result.UsedFiles, result.UsedSize, err = dataprovider.GetUsedQuota(c.User.Username)
+	}
+	if err != nil {
+		c.Log(logger.LevelWarn, "error getting used quota for %#v request path %#v: %v", c.User.Username, requestPath, err)
+		result.HasSpace = false
+		return result
+	}
+	result.AllowedFiles = result.QuotaFiles - result.UsedFiles
+	result.AllowedSize = result.QuotaSize - result.UsedSize
+	if (checkFiles && result.QuotaFiles > 0 && result.UsedFiles >= result.QuotaFiles) ||
+		(result.QuotaSize > 0 && result.UsedSize >= result.QuotaSize) {
+		c.Log(logger.LevelDebug, "quota exceed for user %#v, request path %#v, num files: %v/%v, size: %v/%v check files: %v",
+			c.User.Username, requestPath, result.UsedFiles, result.QuotaFiles, result.UsedSize, result.QuotaSize, checkFiles)
+		result.HasSpace = false
+		return result
+	}
+	return result
+}
+
+func (c *BaseConnection) isCrossFoldersRequest(virtualSourcePath, virtualTargetPath string) bool {
+	sourceFolder, errSrc := c.User.GetVirtualFolderForPath(virtualSourcePath)
+	dstFolder, errDst := c.User.GetVirtualFolderForPath(virtualTargetPath)
+	if errSrc != nil && errDst != nil {
+		return false
+	}
+	if errSrc == nil && errDst == nil {
+		return sourceFolder.MappedPath != dstFolder.MappedPath
+	}
+	return true
+}
+
+func (c *BaseConnection) updateQuotaMoveBetweenVFolders(sourceFolder, dstFolder vfs.VirtualFolder, initialSize,
+	filesSize int64, numFiles int) {
+	if sourceFolder.MappedPath == dstFolder.MappedPath {
+		// both files are inside the same virtual folder
+		if initialSize != -1 {
+			dataprovider.UpdateVirtualFolderQuota(dstFolder.BaseVirtualFolder, -numFiles, -initialSize, false) //nolint:errcheck
+			if dstFolder.IsIncludedInUserQuota() {
+				dataprovider.UpdateUserQuota(c.User, -numFiles, -initialSize, false) //nolint:errcheck
+			}
+		}
+		return
+	}
+	// files are inside different virtual folders
+	dataprovider.UpdateVirtualFolderQuota(sourceFolder.BaseVirtualFolder, -numFiles, -filesSize, false) //nolint:errcheck
+	if sourceFolder.IsIncludedInUserQuota() {
+		dataprovider.UpdateUserQuota(c.User, -numFiles, -filesSize, false) //nolint:errcheck
+	}
+	if initialSize == -1 {
+		dataprovider.UpdateVirtualFolderQuota(dstFolder.BaseVirtualFolder, numFiles, filesSize, false) //nolint:errcheck
+		if dstFolder.IsIncludedInUserQuota() {
+			dataprovider.UpdateUserQuota(c.User, numFiles, filesSize, false) //nolint:errcheck
+		}
+	} else {
+		// we cannot have a directory here, initialSize != -1 only for files
+		dataprovider.UpdateVirtualFolderQuota(dstFolder.BaseVirtualFolder, 0, filesSize-initialSize, false) //nolint:errcheck
+		if dstFolder.IsIncludedInUserQuota() {
+			dataprovider.UpdateUserQuota(c.User, 0, filesSize-initialSize, false) //nolint:errcheck
+		}
+	}
+}
+
+func (c *BaseConnection) updateQuotaMoveFromVFolder(sourceFolder vfs.VirtualFolder, initialSize, filesSize int64, numFiles int) {
+	// move between a virtual folder and the user home dir
+	dataprovider.UpdateVirtualFolderQuota(sourceFolder.BaseVirtualFolder, -numFiles, -filesSize, false) //nolint:errcheck
+	if sourceFolder.IsIncludedInUserQuota() {
+		dataprovider.UpdateUserQuota(c.User, -numFiles, -filesSize, false) //nolint:errcheck
+	}
+	if initialSize == -1 {
+		dataprovider.UpdateUserQuota(c.User, numFiles, filesSize, false) //nolint:errcheck
+	} else {
+		// we cannot have a directory here, initialSize != -1 only for files
+		dataprovider.UpdateUserQuota(c.User, 0, filesSize-initialSize, false) //nolint:errcheck
+	}
+}
+
+func (c *BaseConnection) updateQuotaMoveToVFolder(dstFolder vfs.VirtualFolder, initialSize, filesSize int64, numFiles int) {
+	// move between the user home dir and a virtual folder
+	dataprovider.UpdateUserQuota(c.User, -numFiles, -filesSize, false) //nolint:errcheck
+	if initialSize == -1 {
+		dataprovider.UpdateVirtualFolderQuota(dstFolder.BaseVirtualFolder, numFiles, filesSize, false) //nolint:errcheck
+		if dstFolder.IsIncludedInUserQuota() {
+			dataprovider.UpdateUserQuota(c.User, numFiles, filesSize, false) //nolint:errcheck
+		}
+	} else {
+		// we cannot have a directory here, initialSize != -1 only for files
+		dataprovider.UpdateVirtualFolderQuota(dstFolder.BaseVirtualFolder, 0, filesSize-initialSize, false) //nolint:errcheck
+		if dstFolder.IsIncludedInUserQuota() {
+			dataprovider.UpdateUserQuota(c.User, 0, filesSize-initialSize, false) //nolint:errcheck
+		}
+	}
+}
+
+func (c *BaseConnection) updateQuotaAfterRename(virtualSourcePath, virtualTargetPath, targetPath string, initialSize int64) error {
+	// we don't allow to overwrite an existing directory so targetPath can be:
+	// - a new file, a symlink is as a new file here
+	// - a file overwriting an existing one
+	// - a new directory
+	// initialSize != -1 only when overwriting files
+	sourceFolder, errSrc := c.User.GetVirtualFolderForPath(path.Dir(virtualSourcePath))
+	dstFolder, errDst := c.User.GetVirtualFolderForPath(path.Dir(virtualTargetPath))
+	if errSrc != nil && errDst != nil {
+		// both files are contained inside the user home dir
+		if initialSize != -1 {
+			// we cannot have a directory here
+			dataprovider.UpdateUserQuota(c.User, -1, -initialSize, false) //nolint:errcheck
+		}
+		return nil
+	}
+
+	filesSize := int64(0)
+	numFiles := 1
+	if fi, err := c.Fs.Stat(targetPath); err == nil {
+		if fi.Mode().IsDir() {
+			numFiles, filesSize, err = c.Fs.GetDirSize(targetPath)
+			if err != nil {
+				c.Log(logger.LevelWarn, "failed to update quota after rename, error scanning moved folder %#v: %v",
+					targetPath, err)
+				return err
+			}
+		} else {
+			filesSize = fi.Size()
+		}
+	} else {
+		c.Log(logger.LevelWarn, "failed to update quota after rename, file %#v stat error: %+v", targetPath, err)
+		return err
+	}
+	if errSrc == nil && errDst == nil {
+		c.updateQuotaMoveBetweenVFolders(sourceFolder, dstFolder, initialSize, filesSize, numFiles)
+	}
+	if errSrc == nil && errDst != nil {
+		c.updateQuotaMoveFromVFolder(sourceFolder, initialSize, filesSize, numFiles)
+	}
+	if errSrc != nil && errDst == nil {
+		c.updateQuotaMoveToVFolder(dstFolder, initialSize, filesSize, numFiles)
+	}
+	return nil
+}
+
+// GetPermissionDeniedError returns an appropriate permission denied error for the connection protocol
+func (c *BaseConnection) GetPermissionDeniedError() error {
+	switch c.protocol {
+	case ProtocolSFTP:
+		return sftp.ErrSSHFxPermissionDenied
+	case ProtocolWebDAV:
+		return os.ErrPermission
+	default:
+		return ErrPermissionDenied
+	}
+}
+
+// GetNotExistError returns an appropriate not exist error for the connection protocol
+func (c *BaseConnection) GetNotExistError() error {
+	switch c.protocol {
+	case ProtocolSFTP:
+		return sftp.ErrSSHFxNoSuchFile
+	case ProtocolWebDAV:
+		return os.ErrNotExist
+	default:
+		return ErrNotExist
+	}
+}
+
+// GetOpUnsupportedError returns an appropriate operation not supported error for the connection protocol
+func (c *BaseConnection) GetOpUnsupportedError() error {
+	switch c.protocol {
+	case ProtocolSFTP:
+		return sftp.ErrSSHFxOpUnsupported
+	default:
+		return ErrOpUnsupported
+	}
+}
+
+// GetGenericError returns an appropriate generic error for the connection protocol
+func (c *BaseConnection) GetGenericError(err error) error {
+	switch c.protocol {
+	case ProtocolSFTP:
+		return sftp.ErrSSHFxFailure
+	default:
+		if err == ErrPermissionDenied || err == ErrNotExist || err == ErrOpUnsupported || err == ErrQuotaExceeded {
+			return err
+		}
+		return ErrGenericFailure
+	}
+}
+
+// GetFsError converts a filesystem error to a protocol error
+func (c *BaseConnection) GetFsError(err error) error {
+	if c.Fs.IsNotExist(err) {
+		return c.GetNotExistError()
+	} else if c.Fs.IsPermission(err) {
+		return c.GetPermissionDeniedError()
+	} else if c.Fs.IsNotSupported(err) {
+		return c.GetOpUnsupportedError()
+	} else if err != nil {
+		return c.GetGenericError(err)
+	}
+	return nil
+}

common/tlsutils.go

@@ -0,0 +1,54 @@
+package common
+
+import (
+	"crypto/tls"
+	"sync"
+
+	"github.com/drakkan/sftpgo/logger"
+)
+
+// CertManager defines a TLS certificate manager
+type CertManager struct {
+	certPath string
+	keyPath  string
+	sync.RWMutex
+	cert *tls.Certificate
+}
+
+// LoadCertificate loads the configured x509 key pair
+func (m *CertManager) LoadCertificate(logSender string) error {
+	newCert, err := tls.LoadX509KeyPair(m.certPath, m.keyPath)
+	if err != nil {
+		logger.Warn(logSender, "", "unable to load X509 key pair, cert file %#v key file %#v error: %v",
+			m.certPath, m.keyPath, err)
+		return err
+	}
+	logger.Debug(logSender, "", "TLS certificate %#v successfully loaded", m.certPath)
+	m.Lock()
+	defer m.Unlock()
+	m.cert = &newCert
+	return nil
+}
+
+// GetCertificateFunc returns the loaded certificate
+func (m *CertManager) GetCertificateFunc() func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+	return func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+		m.RLock()
+		defer m.RUnlock()
+		return m.cert, nil
+	}
+}
+
+// NewCertManager creates a new certificate manager
+func NewCertManager(certificateFile, certificateKeyFile, logSender string) (*CertManager, error) {
+	manager := &CertManager{
+		cert:     nil,
+		certPath: certificateFile,
+		keyPath:  certificateKeyFile,
+	}
+	err := manager.LoadCertificate(logSender)
+	if err != nil {
+		return nil, err
+	}
+	return manager, nil
+}

common/tlsutils_test.go

@@ -0,0 +1,69 @@
+package common
+
+import (
+	"crypto/tls"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	httpsCert = `-----BEGIN CERTIFICATE-----
+MIICHTCCAaKgAwIBAgIUHnqw7QnB1Bj9oUsNpdb+ZkFPOxMwCgYIKoZIzj0EAwIw
+RTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGElu
+dGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMDAyMDQwOTUzMDRaFw0zMDAyMDEw
+OTUzMDRaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYD
+VQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwdjAQBgcqhkjOPQIBBgUrgQQA
+IgNiAARCjRMqJ85rzMC998X5z761nJ+xL3bkmGVqWvrJ51t5OxV0v25NsOgR82CA
+NXUgvhVYs7vNFN+jxtb2aj6Xg+/2G/BNxkaFspIVCzgWkxiz7XE4lgUwX44FCXZM
+3+JeUbKjUzBRMB0GA1UdDgQWBBRhLw+/o3+Z02MI/d4tmaMui9W16jAfBgNVHSME
+GDAWgBRhLw+/o3+Z02MI/d4tmaMui9W16jAPBgNVHRMBAf8EBTADAQH/MAoGCCqG
+SM49BAMCA2kAMGYCMQDqLt2lm8mE+tGgtjDmtFgdOcI72HSbRQ74D5rYTzgST1rY
+/8wTi5xl8TiFUyLMUsICMQC5ViVxdXbhuG7gX6yEqSkMKZICHpO8hqFwOD/uaFVI
+dV4vKmHUzwK/eIx+8Ay3neE=
+-----END CERTIFICATE-----`
+	httpsKey = `-----BEGIN EC PARAMETERS-----
+BgUrgQQAIg==
+-----END EC PARAMETERS-----
+-----BEGIN EC PRIVATE KEY-----
+MIGkAgEBBDCfMNsN6miEE3rVyUPwElfiJSWaR5huPCzUenZOfJT04GAcQdWvEju3
+UM2lmBLIXpGgBwYFK4EEACKhZANiAARCjRMqJ85rzMC998X5z761nJ+xL3bkmGVq
+WvrJ51t5OxV0v25NsOgR82CANXUgvhVYs7vNFN+jxtb2aj6Xg+/2G/BNxkaFspIV
+CzgWkxiz7XE4lgUwX44FCXZM3+JeUbI=
+-----END EC PRIVATE KEY-----`
+)
+
+func TestLoadCertificate(t *testing.T) {
+	certPath := filepath.Join(os.TempDir(), "test.crt")
+	keyPath := filepath.Join(os.TempDir(), "test.key")
+	err := ioutil.WriteFile(certPath, []byte(httpsCert), os.ModePerm)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(keyPath, []byte(httpsKey), os.ModePerm)
+	assert.NoError(t, err)
+	certManager, err := NewCertManager(certPath, keyPath, logSenderTest)
+	assert.NoError(t, err)
+	certFunc := certManager.GetCertificateFunc()
+	if assert.NotNil(t, certFunc) {
+		hello := &tls.ClientHelloInfo{
+			ServerName:   "localhost",
+			CipherSuites: []uint16{tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305},
+		}
+		cert, err := certFunc(hello)
+		assert.NoError(t, err)
+		assert.Equal(t, certManager.cert, cert)
+	}
+
+	err = os.Remove(certPath)
+	assert.NoError(t, err)
+	err = os.Remove(keyPath)
+	assert.NoError(t, err)
+}
+
+func TestLoadInvalidCert(t *testing.T) {
+	certManager, err := NewCertManager("test.crt", "test.key", logSenderTest)
+	assert.Error(t, err)
+	assert.Nil(t, certManager)
+}

common/transfer.go

@@ -0,0 +1,290 @@
+package common
+
+import (
+	"errors"
+	"os"
+	"path"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/metrics"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+var (
+	// ErrTransferClosed defines the error returned for a closed transfer
+	ErrTransferClosed = errors.New("transfer already closed")
+)
+
+// BaseTransfer contains protocols common transfer details for an upload or a download.
+type BaseTransfer struct { //nolint:maligned
+	ID             uint64
+	Fs             vfs.Fs
+	File           vfs.File
+	Connection     *BaseConnection
+	cancelFn       func()
+	fsPath         string
+	start          time.Time
+	transferType   int
+	MinWriteOffset int64
+	InitialSize    int64
+	isNewFile      bool
+	requestPath    string
+	BytesSent      int64
+	BytesReceived  int64
+	MaxWriteSize   int64
+	AbortTransfer  int32
+	sync.Mutex
+	ErrTransfer error
+}
+
+// NewBaseTransfer returns a new BaseTransfer and adds it to the given connection
+func NewBaseTransfer(file vfs.File, conn *BaseConnection, cancelFn func(), fsPath, requestPath string, transferType int,
+	minWriteOffset, initialSize, maxWriteSize int64, isNewFile bool, fs vfs.Fs) *BaseTransfer {
+	t := &BaseTransfer{
+		ID:             conn.GetTransferID(),
+		File:           file,
+		Connection:     conn,
+		cancelFn:       cancelFn,
+		fsPath:         fsPath,
+		start:          time.Now(),
+		transferType:   transferType,
+		MinWriteOffset: minWriteOffset,
+		InitialSize:    initialSize,
+		isNewFile:      isNewFile,
+		requestPath:    requestPath,
+		BytesSent:      0,
+		BytesReceived:  0,
+		MaxWriteSize:   maxWriteSize,
+		AbortTransfer:  0,
+		Fs:             fs,
+	}
+
+	conn.AddTransfer(t)
+	return t
+}
+
+// GetID returns the transfer ID
+func (t *BaseTransfer) GetID() uint64 {
+	return t.ID
+}
+
+// GetType returns the transfer type
+func (t *BaseTransfer) GetType() int {
+	return t.transferType
+}
+
+// GetSize returns the transferred size
+func (t *BaseTransfer) GetSize() int64 {
+	if t.transferType == TransferDownload {
+		return atomic.LoadInt64(&t.BytesSent)
+	}
+	return atomic.LoadInt64(&t.BytesReceived)
+}
+
+// GetStartTime returns the start time
+func (t *BaseTransfer) GetStartTime() time.Time {
+	return t.start
+}
+
+// SignalClose signals that the transfer should be closed.
+// For same protocols, for example WebDAV, we have no
+// access to the network connection, so we use this method
+// to make the next read or write to fail
+func (t *BaseTransfer) SignalClose() {
+	atomic.StoreInt32(&(t.AbortTransfer), 1)
+}
+
+// GetVirtualPath returns the transfer virtual path
+func (t *BaseTransfer) GetVirtualPath() string {
+	return t.requestPath
+}
+
+// GetFsPath returns the transfer filesystem path
+func (t *BaseTransfer) GetFsPath() string {
+	return t.fsPath
+}
+
+// GetRealFsPath returns the real transfer filesystem path.
+// If atomic uploads are enabled this differ from fsPath
+func (t *BaseTransfer) GetRealFsPath(fsPath string) string {
+	if fsPath == t.GetFsPath() {
+		if t.File != nil {
+			return t.File.Name()
+		}
+		return t.fsPath
+	}
+	return ""
+}
+
+// SetCancelFn sets the cancel function for the transfer
+func (t *BaseTransfer) SetCancelFn(cancelFn func()) {
+	t.cancelFn = cancelFn
+}
+
+// Truncate changes the size of the opened file.
+// Supported for local fs only
+func (t *BaseTransfer) Truncate(fsPath string, size int64) (int64, error) {
+	if fsPath == t.GetFsPath() {
+		if t.File != nil {
+			initialSize := t.InitialSize
+			err := t.File.Truncate(size)
+			if err == nil {
+				t.Lock()
+				t.InitialSize = size
+				if t.MaxWriteSize > 0 {
+					sizeDiff := initialSize - size
+					t.MaxWriteSize += sizeDiff
+					metrics.TransferCompleted(atomic.LoadInt64(&t.BytesSent), atomic.LoadInt64(&t.BytesReceived), t.transferType, t.ErrTransfer)
+					atomic.StoreInt64(&t.BytesReceived, 0)
+				}
+				t.Unlock()
+			}
+			t.Connection.Log(logger.LevelDebug, "file %#v truncated to size %v max write size %v new initial size %v err: %v",
+				fsPath, size, t.MaxWriteSize, t.InitialSize, err)
+			return initialSize, err
+		}
+		if size == 0 && atomic.LoadInt64(&t.BytesSent) == 0 {
+			// for cloud providers the file is always truncated to zero, we don't support append/resume for uploads
+			return 0, nil
+		}
+		return 0, ErrOpUnsupported
+	}
+	return 0, errTransferMismatch
+}
+
+// TransferError is called if there is an unexpected error.
+// For example network or client issues
+func (t *BaseTransfer) TransferError(err error) {
+	t.Lock()
+	defer t.Unlock()
+	if t.ErrTransfer != nil {
+		return
+	}
+	t.ErrTransfer = err
+	if t.cancelFn != nil {
+		t.cancelFn()
+	}
+	elapsed := time.Since(t.start).Nanoseconds() / 1000000
+	t.Connection.Log(logger.LevelWarn, "Unexpected error for transfer, path: %#v, error: \"%v\" bytes sent: %v, "+
+		"bytes received: %v transfer running since %v ms", t.fsPath, t.ErrTransfer, atomic.LoadInt64(&t.BytesSent),
+		atomic.LoadInt64(&t.BytesReceived), elapsed)
+}
+
+// Close it is called when the transfer is completed.
+// It logs the transfer info, updates the user quota (for uploads)
+// and executes any defined action.
+// If there is an error no action will be executed and, in atomic mode,
+// we try to delete the temporary file
+func (t *BaseTransfer) Close() error {
+	defer t.Connection.RemoveTransfer(t)
+
+	var err error
+	numFiles := 0
+	if t.isNewFile {
+		numFiles = 1
+	}
+	metrics.TransferCompleted(atomic.LoadInt64(&t.BytesSent), atomic.LoadInt64(&t.BytesReceived), t.transferType, t.ErrTransfer)
+	if t.ErrTransfer == ErrQuotaExceeded && t.File != nil {
+		// if quota is exceeded we try to remove the partial file for uploads to local filesystem
+		err = os.Remove(t.File.Name())
+		if err == nil {
+			numFiles--
+			atomic.StoreInt64(&t.BytesReceived, 0)
+			t.MinWriteOffset = 0
+		}
+		t.Connection.Log(logger.LevelWarn, "upload denied due to space limit, delete temporary file: %#v, deletion error: %v",
+			t.File.Name(), err)
+	} else if t.transferType == TransferUpload && t.File != nil && t.File.Name() != t.fsPath {
+		if t.ErrTransfer == nil || Config.UploadMode == UploadModeAtomicWithResume {
+			err = os.Rename(t.File.Name(), t.fsPath)
+			t.Connection.Log(logger.LevelDebug, "atomic upload completed, rename: %#v -> %#v, error: %v",
+				t.File.Name(), t.fsPath, err)
+		} else {
+			err = os.Remove(t.File.Name())
+			t.Connection.Log(logger.LevelWarn, "atomic upload completed with error: \"%v\", delete temporary file: %#v, "+
+				"deletion error: %v", t.ErrTransfer, t.File.Name(), err)
+			if err == nil {
+				numFiles--
+				atomic.StoreInt64(&t.BytesReceived, 0)
+				t.MinWriteOffset = 0
+			}
+		}
+	}
+	elapsed := time.Since(t.start).Nanoseconds() / 1000000
+	if t.transferType == TransferDownload {
+		logger.TransferLog(downloadLogSender, t.fsPath, elapsed, atomic.LoadInt64(&t.BytesSent), t.Connection.User.Username,
+			t.Connection.ID, t.Connection.protocol)
+		action := newActionNotification(&t.Connection.User, operationDownload, t.fsPath, "", "", t.Connection.protocol,
+			atomic.LoadInt64(&t.BytesSent), t.ErrTransfer)
+		go actionHandler.Handle(action) //nolint:errcheck
+	} else {
+		fileSize := atomic.LoadInt64(&t.BytesReceived) + t.MinWriteOffset
+		info, err := t.Fs.Stat(t.fsPath)
+		if err == nil {
+			fileSize = info.Size()
+		}
+		t.Connection.Log(logger.LevelDebug, "uploaded file size %v stat error: %v", fileSize, err)
+		t.updateQuota(numFiles, fileSize)
+		logger.TransferLog(uploadLogSender, t.fsPath, elapsed, atomic.LoadInt64(&t.BytesReceived), t.Connection.User.Username,
+			t.Connection.ID, t.Connection.protocol)
+		action := newActionNotification(&t.Connection.User, operationUpload, t.fsPath, "", "", t.Connection.protocol,
+			fileSize, t.ErrTransfer)
+		go actionHandler.Handle(action) //nolint:errcheck
+	}
+	if t.ErrTransfer != nil {
+		t.Connection.Log(logger.LevelWarn, "transfer error: %v, path: %#v", t.ErrTransfer, t.fsPath)
+		if err == nil {
+			err = t.ErrTransfer
+		}
+	}
+	return err
+}
+
+func (t *BaseTransfer) updateQuota(numFiles int, fileSize int64) bool {
+	// S3 uploads are atomic, if there is an error nothing is uploaded
+	if t.File == nil && t.ErrTransfer != nil {
+		return false
+	}
+	sizeDiff := fileSize - t.InitialSize
+	if t.transferType == TransferUpload && (numFiles != 0 || sizeDiff > 0) {
+		vfolder, err := t.Connection.User.GetVirtualFolderForPath(path.Dir(t.requestPath))
+		if err == nil {
+			dataprovider.UpdateVirtualFolderQuota(vfolder.BaseVirtualFolder, numFiles, //nolint:errcheck
+				sizeDiff, false)
+			if vfolder.IsIncludedInUserQuota() {
+				dataprovider.UpdateUserQuota(t.Connection.User, numFiles, sizeDiff, false) //nolint:errcheck
+			}
+		} else {
+			dataprovider.UpdateUserQuota(t.Connection.User, numFiles, sizeDiff, false) //nolint:errcheck
+		}
+		return true
+	}
+	return false
+}
+
+// HandleThrottle manage bandwidth throttling
+func (t *BaseTransfer) HandleThrottle() {
+	var wantedBandwidth int64
+	var trasferredBytes int64
+	if t.transferType == TransferDownload {
+		wantedBandwidth = t.Connection.User.DownloadBandwidth
+		trasferredBytes = atomic.LoadInt64(&t.BytesSent)
+	} else {
+		wantedBandwidth = t.Connection.User.UploadBandwidth
+		trasferredBytes = atomic.LoadInt64(&t.BytesReceived)
+	}
+	if wantedBandwidth > 0 {
+		// real and wanted elapsed as milliseconds, bytes as kilobytes
+		realElapsed := time.Since(t.start).Nanoseconds() / 1000000
+		// trasferredBytes / 1000 = KB/s, we multiply for 1000 to get milliseconds
+		wantedElapsed := 1000 * (trasferredBytes / 1000) / wantedBandwidth
+		if wantedElapsed > realElapsed {
+			toSleep := time.Duration(wantedElapsed - realElapsed)
+			time.Sleep(toSleep * time.Millisecond)
+		}
+	}
+}

common/transfer_test.go

@@ -0,0 +1,254 @@
+package common
+
+import (
+	"errors"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+func TestTransferUpdateQuota(t *testing.T) {
+	conn := NewBaseConnection("", ProtocolSFTP, dataprovider.User{}, nil)
+	transfer := BaseTransfer{
+		Connection:    conn,
+		transferType:  TransferUpload,
+		BytesReceived: 123,
+		Fs:            vfs.NewOsFs("", os.TempDir(), nil),
+	}
+	errFake := errors.New("fake error")
+	transfer.TransferError(errFake)
+	assert.False(t, transfer.updateQuota(1, 0))
+	err := transfer.Close()
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, errFake.Error())
+	}
+	mappedPath := filepath.Join(os.TempDir(), "vdir")
+	vdirPath := "/vdir"
+	conn.User.VirtualFolders = append(conn.User.VirtualFolders, vfs.VirtualFolder{
+		BaseVirtualFolder: vfs.BaseVirtualFolder{
+			MappedPath: mappedPath,
+		},
+		VirtualPath: vdirPath,
+		QuotaFiles:  -1,
+		QuotaSize:   -1,
+	})
+	transfer.ErrTransfer = nil
+	transfer.BytesReceived = 1
+	transfer.requestPath = "/vdir/file"
+	assert.True(t, transfer.updateQuota(1, 0))
+	err = transfer.Close()
+	assert.NoError(t, err)
+}
+
+func TestTransferThrottling(t *testing.T) {
+	u := dataprovider.User{
+		Username:          "test",
+		UploadBandwidth:   50,
+		DownloadBandwidth: 40,
+	}
+	fs := vfs.NewOsFs("", os.TempDir(), nil)
+	testFileSize := int64(131072)
+	wantedUploadElapsed := 1000 * (testFileSize / 1000) / u.UploadBandwidth
+	wantedDownloadElapsed := 1000 * (testFileSize / 1000) / u.DownloadBandwidth
+	// some tolerance
+	wantedUploadElapsed -= wantedDownloadElapsed / 10
+	wantedDownloadElapsed -= wantedDownloadElapsed / 10
+	conn := NewBaseConnection("id", ProtocolSCP, u, nil)
+	transfer := NewBaseTransfer(nil, conn, nil, "", "", TransferUpload, 0, 0, 0, true, fs)
+	transfer.BytesReceived = testFileSize
+	transfer.Connection.UpdateLastActivity()
+	startTime := transfer.Connection.GetLastActivity()
+	transfer.HandleThrottle()
+	elapsed := time.Since(startTime).Nanoseconds() / 1000000
+	assert.GreaterOrEqual(t, elapsed, wantedUploadElapsed, "upload bandwidth throttling not respected")
+	err := transfer.Close()
+	assert.NoError(t, err)
+
+	transfer = NewBaseTransfer(nil, conn, nil, "", "", TransferDownload, 0, 0, 0, true, fs)
+	transfer.BytesSent = testFileSize
+	transfer.Connection.UpdateLastActivity()
+	startTime = transfer.Connection.GetLastActivity()
+
+	transfer.HandleThrottle()
+	elapsed = time.Since(startTime).Nanoseconds() / 1000000
+	assert.GreaterOrEqual(t, elapsed, wantedDownloadElapsed, "download bandwidth throttling not respected")
+	err = transfer.Close()
+	assert.NoError(t, err)
+}
+
+func TestRealPath(t *testing.T) {
+	testFile := filepath.Join(os.TempDir(), "afile.txt")
+	fs := vfs.NewOsFs("123", os.TempDir(), nil)
+	u := dataprovider.User{
+		Username: "user",
+		HomeDir:  os.TempDir(),
+	}
+	u.Permissions = make(map[string][]string)
+	u.Permissions["/"] = []string{dataprovider.PermAny}
+	file, err := os.Create(testFile)
+	require.NoError(t, err)
+	conn := NewBaseConnection(fs.ConnectionID(), ProtocolSFTP, u, fs)
+	transfer := NewBaseTransfer(file, conn, nil, testFile, "/transfer_test_file", TransferUpload, 0, 0, 0, true, fs)
+	rPath := transfer.GetRealFsPath(testFile)
+	assert.Equal(t, testFile, rPath)
+	rPath = conn.getRealFsPath(testFile)
+	assert.Equal(t, testFile, rPath)
+	err = transfer.Close()
+	assert.NoError(t, err)
+	err = file.Close()
+	assert.NoError(t, err)
+	transfer.File = nil
+	rPath = transfer.GetRealFsPath(testFile)
+	assert.Equal(t, testFile, rPath)
+	rPath = transfer.GetRealFsPath("")
+	assert.Empty(t, rPath)
+	err = os.Remove(testFile)
+	assert.NoError(t, err)
+	assert.Len(t, conn.GetTransfers(), 0)
+}
+
+func TestTruncate(t *testing.T) {
+	testFile := filepath.Join(os.TempDir(), "transfer_test_file")
+	fs := vfs.NewOsFs("123", os.TempDir(), nil)
+	u := dataprovider.User{
+		Username: "user",
+		HomeDir:  os.TempDir(),
+	}
+	u.Permissions = make(map[string][]string)
+	u.Permissions["/"] = []string{dataprovider.PermAny}
+	file, err := os.Create(testFile)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, "unable to open test file")
+	}
+	_, err = file.Write([]byte("hello"))
+	assert.NoError(t, err)
+	conn := NewBaseConnection(fs.ConnectionID(), ProtocolSFTP, u, fs)
+	transfer := NewBaseTransfer(file, conn, nil, testFile, "/transfer_test_file", TransferUpload, 0, 5, 100, false, fs)
+
+	err = conn.SetStat(testFile, "/transfer_test_file", &StatAttributes{
+		Size:  2,
+		Flags: StatAttrSize,
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, int64(103), transfer.MaxWriteSize)
+	err = transfer.Close()
+	assert.NoError(t, err)
+	err = file.Close()
+	assert.NoError(t, err)
+	fi, err := os.Stat(testFile)
+	if assert.NoError(t, err) {
+		assert.Equal(t, int64(2), fi.Size())
+	}
+
+	transfer = NewBaseTransfer(file, conn, nil, testFile, "/transfer_test_file", TransferUpload, 0, 0, 100, true, fs)
+	// file.Stat will fail on a closed file
+	err = conn.SetStat(testFile, "/transfer_test_file", &StatAttributes{
+		Size:  2,
+		Flags: StatAttrSize,
+	})
+	assert.Error(t, err)
+	err = transfer.Close()
+	assert.NoError(t, err)
+
+	transfer = NewBaseTransfer(nil, conn, nil, testFile, "", TransferUpload, 0, 0, 0, true, fs)
+	_, err = transfer.Truncate("mismatch", 0)
+	assert.EqualError(t, err, errTransferMismatch.Error())
+	_, err = transfer.Truncate(testFile, 0)
+	assert.NoError(t, err)
+	_, err = transfer.Truncate(testFile, 1)
+	assert.EqualError(t, err, ErrOpUnsupported.Error())
+
+	err = transfer.Close()
+	assert.NoError(t, err)
+
+	err = os.Remove(testFile)
+	assert.NoError(t, err)
+
+	assert.Len(t, conn.GetTransfers(), 0)
+}
+
+func TestTransferErrors(t *testing.T) {
+	isCancelled := false
+	cancelFn := func() {
+		isCancelled = true
+	}
+	testFile := filepath.Join(os.TempDir(), "transfer_test_file")
+	fs := vfs.NewOsFs("id", os.TempDir(), nil)
+	u := dataprovider.User{
+		Username: "test",
+		HomeDir:  os.TempDir(),
+	}
+	err := ioutil.WriteFile(testFile, []byte("test data"), os.ModePerm)
+	assert.NoError(t, err)
+	file, err := os.Open(testFile)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, "unable to open test file")
+	}
+	conn := NewBaseConnection("id", ProtocolSFTP, u, fs)
+	transfer := NewBaseTransfer(file, conn, nil, testFile, "/transfer_test_file", TransferUpload, 0, 0, 0, true, fs)
+	assert.Nil(t, transfer.cancelFn)
+	assert.Equal(t, testFile, transfer.GetFsPath())
+	transfer.SetCancelFn(cancelFn)
+	errFake := errors.New("err fake")
+	transfer.BytesReceived = 9
+	transfer.TransferError(ErrQuotaExceeded)
+	assert.True(t, isCancelled)
+	transfer.TransferError(errFake)
+	assert.Error(t, transfer.ErrTransfer, ErrQuotaExceeded.Error())
+	// the file is closed from the embedding struct before to call close
+	err = file.Close()
+	assert.NoError(t, err)
+	err = transfer.Close()
+	if assert.Error(t, err) {
+		assert.Error(t, err, ErrQuotaExceeded.Error())
+	}
+	assert.NoFileExists(t, testFile)
+
+	err = ioutil.WriteFile(testFile, []byte("test data"), os.ModePerm)
+	assert.NoError(t, err)
+	file, err = os.Open(testFile)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, "unable to open test file")
+	}
+	fsPath := filepath.Join(os.TempDir(), "test_file")
+	transfer = NewBaseTransfer(file, conn, nil, fsPath, "/test_file", TransferUpload, 0, 0, 0, true, fs)
+	transfer.BytesReceived = 9
+	transfer.TransferError(errFake)
+	assert.Error(t, transfer.ErrTransfer, errFake.Error())
+	// the file is closed from the embedding struct before to call close
+	err = file.Close()
+	assert.NoError(t, err)
+	err = transfer.Close()
+	if assert.Error(t, err) {
+		assert.Error(t, err, errFake.Error())
+	}
+	assert.NoFileExists(t, testFile)
+
+	err = ioutil.WriteFile(testFile, []byte("test data"), os.ModePerm)
+	assert.NoError(t, err)
+	file, err = os.Open(testFile)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, "unable to open test file")
+	}
+	transfer = NewBaseTransfer(file, conn, nil, fsPath, "/test_file", TransferUpload, 0, 0, 0, true, fs)
+	transfer.BytesReceived = 9
+	// the file is closed from the embedding struct before to call close
+	err = file.Close()
+	assert.NoError(t, err)
+	err = transfer.Close()
+	assert.NoError(t, err)
+	assert.NoFileExists(t, testFile)
+	assert.FileExists(t, fsPath)
+	err = os.Remove(fsPath)
+	assert.NoError(t, err)
+
+	assert.Len(t, conn.GetTransfers(), 0)
+}

config/config.go

@@ -1,8 +1,4 @@
-// Package config manages the configuration.
-// Configuration is loaded from sftpgo.conf file.
-// If sftpgo.conf is not found or cannot be readed or decoded as json the default configuration is used.
-// The default configuration an be found inside the source tree:
-// https://github.com/drakkan/sftpgo/blob/master/sftpgo.conf
+// Package config manages the configuration
 package config
 
 import (
@@ -11,13 +7,16 @@ import (
 
 	"github.com/spf13/viper"
 
+	"github.com/drakkan/sftpgo/common"
 	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/ftpd"
 	"github.com/drakkan/sftpgo/httpclient"
 	"github.com/drakkan/sftpgo/httpd"
 	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/sftpd"
 	"github.com/drakkan/sftpgo/utils"
 	"github.com/drakkan/sftpgo/version"
+	"github.com/drakkan/sftpgo/webdavd"
 )
 
 const (
@@ -31,32 +30,40 @@ const (
 )
 
 var (
-	globalConf    globalConfig
-	defaultBanner = fmt.Sprintf("SFTPGo_%v", version.Get().Version)
+	globalConf         globalConfig
+	defaultSFTPDBanner = fmt.Sprintf("SFTPGo_%v", version.Get().Version)
+	defaultFTPDBanner  = fmt.Sprintf("SFTPGo %v ready", version.Get().Version)
 )
 
 type globalConfig struct {
-	SFTPD        sftpd.Configuration `json:"sftpd" mapstructure:"sftpd"`
-	ProviderConf dataprovider.Config `json:"data_provider" mapstructure:"data_provider"`
-	HTTPDConfig  httpd.Conf          `json:"httpd" mapstructure:"httpd"`
-	HTTPConfig   httpclient.Config   `json:"http" mapstructure:"http"`
+	Common       common.Configuration  `json:"common" mapstructure:"common"`
+	SFTPD        sftpd.Configuration   `json:"sftpd" mapstructure:"sftpd"`
+	FTPD         ftpd.Configuration    `json:"ftpd" mapstructure:"ftpd"`
+	WebDAVD      webdavd.Configuration `json:"webdavd" mapstructure:"webdavd"`
+	ProviderConf dataprovider.Config   `json:"data_provider" mapstructure:"data_provider"`
+	HTTPDConfig  httpd.Conf            `json:"httpd" mapstructure:"httpd"`
+	HTTPConfig   httpclient.Config     `json:"http" mapstructure:"http"`
 }
 
 func init() {
 	// create a default configuration to use if no config file is provided
 	globalConf = globalConfig{
-		SFTPD: sftpd.Configuration{
-			Banner:       defaultBanner,
-			BindPort:     2022,
-			BindAddress:  "",
-			IdleTimeout:  15,
-			MaxAuthTries: 0,
-			Umask:        "0022",
-			UploadMode:   0,
-			Actions: sftpd.Actions{
+		Common: common.Configuration{
+			IdleTimeout: 15,
+			UploadMode:  0,
+			Actions: common.ProtocolActions{
 				ExecuteOn: []string{},
 				Hook:      "",
 			},
+			SetstatMode:   0,
+			ProxyProtocol: 0,
+			ProxyAllowed:  []string{},
+		},
+		SFTPD: sftpd.Configuration{
+			Banner:                  defaultSFTPDBanner,
+			BindPort:                2022,
+			BindAddress:             "",
+			MaxAuthTries:            0,
 			HostKeys:                []string{},
 			KexAlgorithms:           []string{},
 			Ciphers:                 []string{},
@@ -65,8 +72,46 @@ func init() {
 			LoginBannerFile:         "",
 			EnabledSSHCommands:      sftpd.GetDefaultSSHCommands(),
 			KeyboardInteractiveHook: "",
-			ProxyProtocol:           0,
-			ProxyAllowed:            []string{},
+			PasswordAuthentication:  true,
+		},
+		FTPD: ftpd.Configuration{
+			BindPort:                 0,
+			BindAddress:              "",
+			Banner:                   defaultFTPDBanner,
+			BannerFile:               "",
+			ActiveTransfersPortNon20: false,
+			ForcePassiveIP:           "",
+			PassivePortRange: ftpd.PortRange{
+				Start: 50000,
+				End:   50100,
+			},
+			CertificateFile:    "",
+			CertificateKeyFile: "",
+		},
+		WebDAVD: webdavd.Configuration{
+			BindPort:           0,
+			BindAddress:        "",
+			CertificateFile:    "",
+			CertificateKeyFile: "",
+			Cors: webdavd.Cors{
+				Enabled:          false,
+				AllowedOrigins:   []string{},
+				AllowedMethods:   []string{},
+				AllowedHeaders:   []string{},
+				ExposedHeaders:   []string{},
+				AllowCredentials: false,
+				MaxAge:           0,
+			},
+			Cache: webdavd.Cache{
+				Users: webdavd.UsersCacheConfig{
+					ExpirationTime: 0,
+					MaxSize:        50,
+				},
+				MimeTypes: webdavd.MimeCacheConfig{
+					Enabled: true,
+					MaxSize: 1000,
+				},
+			},
 		},
 		ProviderConf: dataprovider.Config{
 			Driver:           "sqlite",
@@ -82,14 +127,27 @@ func init() {
 			TrackQuota:       1,
 			PoolSize:         0,
 			UsersBaseDir:     "",
-			Actions: dataprovider.Actions{
+			Actions: dataprovider.UserActions{
 				ExecuteOn: []string{},
 				Hook:      "",
 			},
-			ExternalAuthHook:  "",
-			ExternalAuthScope: 0,
-			CredentialsPath:   "credentials",
-			PreLoginHook:      "",
+			ExternalAuthHook:   "",
+			ExternalAuthScope:  0,
+			CredentialsPath:    "credentials",
+			PreLoginHook:       "",
+			PostLoginHook:      "",
+			PostLoginScope:     0,
+			CheckPasswordHook:  "",
+			CheckPasswordScope: 0,
+			PasswordHashing: dataprovider.PasswordHashing{
+				Argon2Options: dataprovider.Argon2Options{
+					Memory:      65536,
+					Iterations:  1,
+					Parallelism: 2,
+				},
+			},
+			UpdateMode:                0,
+			PreferDatabaseCredentials: false,
 		},
 		HTTPDConfig: httpd.Conf{
 			BindPort:           8080,
@@ -112,10 +170,21 @@ func init() {
 	replacer := strings.NewReplacer(".", "__")
 	viper.SetEnvKeyReplacer(replacer)
 	viper.SetConfigName(DefaultConfigName)
+	setViperDefaults()
 	viper.AutomaticEnv()
 	viper.AllowEmptyEnv(true)
 }
 
+// GetCommonConfig returns the common protocols configuration
+func GetCommonConfig() common.Configuration {
+	return globalConf.Common
+}
+
+// SetCommonConfig sets the common protocols configuration
+func SetCommonConfig(config common.Configuration) {
+	globalConf.Common = config
+}
+
 // GetSFTPDConfig returns the configuration for the SFTP server
 func GetSFTPDConfig() sftpd.Configuration {
 	return globalConf.SFTPD
@@ -126,6 +195,26 @@ func SetSFTPDConfig(config sftpd.Configuration) {
 	globalConf.SFTPD = config
 }
 
+// GetFTPDConfig returns the configuration for the FTP server
+func GetFTPDConfig() ftpd.Configuration {
+	return globalConf.FTPD
+}
+
+// SetFTPDConfig sets the configuration for the FTP server
+func SetFTPDConfig(config ftpd.Configuration) {
+	globalConf.FTPD = config
+}
+
+// GetWebDAVDConfig returns the configuration for the WebDAV server
+func GetWebDAVDConfig() webdavd.Configuration {
+	return globalConf.WebDAVD
+}
+
+// SetWebDAVDConfig sets the configuration for the WebDAV server
+func SetWebDAVDConfig(config webdavd.Configuration) {
+	globalConf.WebDAVD = config
+}
+
 // GetHTTPDConfig returns the configuration for the HTTP server
 func GetHTTPDConfig() httpd.Conf {
 	return globalConf.HTTPDConfig
@@ -169,10 +258,8 @@ func LoadConfig(configDir, configName string) error {
 	viper.AddConfigPath(".")
 	viper.SetConfigName(configName)
 	if err = viper.ReadInConfig(); err != nil {
-		logger.Warn(logSender, "", "error loading configuration file: %v. Default configuration will be used: %+v",
-			err, getRedactedGlobalConf())
-		logger.WarnToConsole("error loading configuration file: %v. Default configuration will be used.", err)
-		return err
+		logger.Warn(logSender, "", "error loading configuration file: %v", err)
+		logger.WarnToConsole("error loading configuration file: %v", err)
 	}
 	err = viper.Unmarshal(&globalConf)
 	if err != nil {
@@ -181,8 +268,12 @@ func LoadConfig(configDir, configName string) error {
 		logger.WarnToConsole("error parsing configuration file: %v. Default configuration will be used.", err)
 		return err
 	}
+	checkCommonParamsCompatibility()
 	if strings.TrimSpace(globalConf.SFTPD.Banner) == "" {
-		globalConf.SFTPD.Banner = defaultBanner
+		globalConf.SFTPD.Banner = defaultSFTPDBanner
+	}
+	if strings.TrimSpace(globalConf.FTPD.Banner) == "" {
+		globalConf.FTPD.Banner = defaultFTPDBanner
 	}
 	if len(globalConf.ProviderConf.UsersBaseDir) > 0 && !utils.IsFileInputValid(globalConf.ProviderConf.UsersBaseDir) {
 		err = fmt.Errorf("invalid users base dir %#v will be ignored", globalConf.ProviderConf.UsersBaseDir)
@@ -190,17 +281,17 @@ func LoadConfig(configDir, configName string) error {
 		logger.Warn(logSender, "", "Configuration error: %v", err)
 		logger.WarnToConsole("Configuration error: %v", err)
 	}
-	if globalConf.SFTPD.UploadMode < 0 || globalConf.SFTPD.UploadMode > 2 {
+	if globalConf.Common.UploadMode < 0 || globalConf.Common.UploadMode > 2 {
 		err = fmt.Errorf("invalid upload_mode 0, 1 and 2 are supported, configured: %v reset upload_mode to 0",
-			globalConf.SFTPD.UploadMode)
-		globalConf.SFTPD.UploadMode = 0
+			globalConf.Common.UploadMode)
+		globalConf.Common.UploadMode = 0
 		logger.Warn(logSender, "", "Configuration error: %v", err)
 		logger.WarnToConsole("Configuration error: %v", err)
 	}
-	if globalConf.SFTPD.ProxyProtocol < 0 || globalConf.SFTPD.ProxyProtocol > 2 {
+	if globalConf.Common.ProxyProtocol < 0 || globalConf.Common.ProxyProtocol > 2 {
 		err = fmt.Errorf("invalid proxy_protocol 0, 1 and 2 are supported, configured: %v reset proxy_protocol to 0",
-			globalConf.SFTPD.ProxyProtocol)
-		globalConf.SFTPD.ProxyProtocol = 0
+			globalConf.Common.ProxyProtocol)
+		globalConf.Common.ProxyProtocol = 0
 		logger.Warn(logSender, "", "Configuration error: %v", err)
 		logger.WarnToConsole("Configuration error: %v", err)
 	}
@@ -216,53 +307,11 @@ func LoadConfig(configDir, configName string) error {
 		logger.Warn(logSender, "", "Configuration error: %v", err)
 		logger.WarnToConsole("Configuration error: %v", err)
 	}
-	checkHooksCompatibility()
 	checkHostKeyCompatibility()
 	logger.Debug(logSender, "", "config file used: '%#v', config loaded: %+v", viper.ConfigFileUsed(), getRedactedGlobalConf())
 	return err
 }
 
-func checkHooksCompatibility() {
-	// we copy deprecated fields to new ones to keep backward compatibility so lint is disabled
-	if len(globalConf.ProviderConf.ExternalAuthProgram) > 0 && len(globalConf.ProviderConf.ExternalAuthHook) == 0 { //nolint:staticcheck
-		logger.Warn(logSender, "", "external_auth_program is deprecated, please use external_auth_hook")
-		logger.WarnToConsole("external_auth_program is deprecated, please use external_auth_hook")
-		globalConf.ProviderConf.ExternalAuthHook = globalConf.ProviderConf.ExternalAuthProgram //nolint:staticcheck
-	}
-	if len(globalConf.ProviderConf.PreLoginProgram) > 0 && len(globalConf.ProviderConf.PreLoginHook) == 0 { //nolint:staticcheck
-		logger.Warn(logSender, "", "pre_login_program is deprecated, please use pre_login_hook")
-		logger.WarnToConsole("pre_login_program is deprecated, please use pre_login_hook")
-		globalConf.ProviderConf.PreLoginHook = globalConf.ProviderConf.PreLoginProgram //nolint:staticcheck
-	}
-	if len(globalConf.SFTPD.KeyboardInteractiveProgram) > 0 && len(globalConf.SFTPD.KeyboardInteractiveHook) == 0 { //nolint:staticcheck
-		logger.Warn(logSender, "", "keyboard_interactive_auth_program is deprecated, please use keyboard_interactive_auth_hook")
-		logger.WarnToConsole("keyboard_interactive_auth_program is deprecated, please use keyboard_interactive_auth_hook")
-		globalConf.SFTPD.KeyboardInteractiveHook = globalConf.SFTPD.KeyboardInteractiveProgram //nolint:staticcheck
-	}
-	if len(globalConf.SFTPD.Actions.Hook) == 0 {
-		if len(globalConf.SFTPD.Actions.HTTPNotificationURL) > 0 { //nolint:staticcheck
-			logger.Warn(logSender, "", "http_notification_url is deprecated, please use hook")
-			logger.WarnToConsole("http_notification_url is deprecated, please use hook")
-			globalConf.SFTPD.Actions.Hook = globalConf.SFTPD.Actions.HTTPNotificationURL //nolint:staticcheck
-		} else if len(globalConf.SFTPD.Actions.Command) > 0 { //nolint:staticcheck
-			logger.Warn(logSender, "", "command is deprecated, please use hook")
-			logger.WarnToConsole("command is deprecated, please use hook")
-			globalConf.SFTPD.Actions.Hook = globalConf.SFTPD.Actions.Command //nolint:staticcheck
-		}
-	}
-	if len(globalConf.ProviderConf.Actions.Hook) == 0 {
-		if len(globalConf.ProviderConf.Actions.HTTPNotificationURL) > 0 { //nolint:staticcheck
-			logger.Warn(logSender, "", "http_notification_url is deprecated, please use hook")
-			logger.WarnToConsole("http_notification_url is deprecated, please use hook")
-			globalConf.ProviderConf.Actions.Hook = globalConf.ProviderConf.Actions.HTTPNotificationURL //nolint:staticcheck
-		} else if len(globalConf.ProviderConf.Actions.Command) > 0 { //nolint:staticcheck
-			logger.Warn(logSender, "", "command is deprecated, please use hook")
-			logger.WarnToConsole("command is deprecated, please use hook")
-			globalConf.ProviderConf.Actions.Hook = globalConf.ProviderConf.Actions.Command //nolint:staticcheck
-		}
-	}
-}
-
 func checkHostKeyCompatibility() {
 	// we copy deprecated fields to new ones to keep backward compatibility so lint is disabled
 	if len(globalConf.SFTPD.Keys) > 0 && len(globalConf.SFTPD.HostKeys) == 0 { //nolint:staticcheck
@@ -273,3 +322,123 @@ func checkHostKeyCompatibility() {
 		}
 	}
 }
+
+func checkCommonParamsCompatibility() {
+	// we copy deprecated fields to new ones to keep backward compatibility so lint is disabled
+	if globalConf.SFTPD.IdleTimeout > 0 { //nolint:staticcheck
+		logger.Warn(logSender, "", "sftpd.idle_timeout is deprecated, please use common.idle_timeout")
+		logger.WarnToConsole("sftpd.idle_timeout is deprecated, please use common.idle_timeout")
+		globalConf.Common.IdleTimeout = globalConf.SFTPD.IdleTimeout //nolint:staticcheck
+	}
+	if len(globalConf.SFTPD.Actions.Hook) > 0 && len(globalConf.Common.Actions.Hook) == 0 { //nolint:staticcheck
+		logger.Warn(logSender, "", "sftpd.actions is deprecated, please use common.actions")
+		logger.WarnToConsole("sftpd.actions is deprecated, please use common.actions")
+		globalConf.Common.Actions.ExecuteOn = globalConf.SFTPD.Actions.ExecuteOn //nolint:staticcheck
+		globalConf.Common.Actions.Hook = globalConf.SFTPD.Actions.Hook           //nolint:staticcheck
+	}
+	if globalConf.SFTPD.SetstatMode > 0 && globalConf.Common.SetstatMode == 0 { //nolint:staticcheck
+		logger.Warn(logSender, "", "sftpd.setstat_mode is deprecated, please use common.setstat_mode")
+		logger.WarnToConsole("sftpd.setstat_mode is deprecated, please use common.setstat_mode")
+		globalConf.Common.SetstatMode = globalConf.SFTPD.SetstatMode //nolint:staticcheck
+	}
+	if globalConf.SFTPD.UploadMode > 0 && globalConf.Common.UploadMode == 0 { //nolint:staticcheck
+		logger.Warn(logSender, "", "sftpd.upload_mode is deprecated, please use common.upload_mode")
+		logger.WarnToConsole("sftpd.upload_mode is deprecated, please use common.upload_mode")
+		globalConf.Common.UploadMode = globalConf.SFTPD.UploadMode //nolint:staticcheck
+	}
+	if globalConf.SFTPD.ProxyProtocol > 0 && globalConf.Common.ProxyProtocol == 0 { //nolint:staticcheck
+		logger.Warn(logSender, "", "sftpd.proxy_protocol is deprecated, please use common.proxy_protocol")
+		logger.WarnToConsole("sftpd.proxy_protocol is deprecated, please use common.proxy_protocol")
+		globalConf.Common.ProxyProtocol = globalConf.SFTPD.ProxyProtocol //nolint:staticcheck
+		globalConf.Common.ProxyAllowed = globalConf.SFTPD.ProxyAllowed   //nolint:staticcheck
+	}
+}
+
+func setViperDefaults() {
+	viper.SetDefault("common.idle_timeout", globalConf.Common.IdleTimeout)
+	viper.SetDefault("common.upload_mode", globalConf.Common.UploadMode)
+	viper.SetDefault("common.actions.execute_on", globalConf.Common.Actions.ExecuteOn)
+	viper.SetDefault("common.actions.hook", globalConf.Common.Actions.Hook)
+	viper.SetDefault("common.setstat_mode", globalConf.Common.SetstatMode)
+	viper.SetDefault("common.proxy_protocol", globalConf.Common.ProxyProtocol)
+	viper.SetDefault("common.proxy_allowed", globalConf.Common.ProxyAllowed)
+	viper.SetDefault("common.post_connect_hook", globalConf.Common.PostConnectHook)
+	viper.SetDefault("sftpd.bind_port", globalConf.SFTPD.BindPort)
+	viper.SetDefault("sftpd.bind_address", globalConf.SFTPD.BindAddress)
+	viper.SetDefault("sftpd.max_auth_tries", globalConf.SFTPD.MaxAuthTries)
+	viper.SetDefault("sftpd.banner", globalConf.SFTPD.Banner)
+	viper.SetDefault("sftpd.host_keys", globalConf.SFTPD.HostKeys)
+	viper.SetDefault("sftpd.kex_algorithms", globalConf.SFTPD.KexAlgorithms)
+	viper.SetDefault("sftpd.ciphers", globalConf.SFTPD.Ciphers)
+	viper.SetDefault("sftpd.macs", globalConf.SFTPD.MACs)
+	viper.SetDefault("sftpd.trusted_user_ca_keys", globalConf.SFTPD.TrustedUserCAKeys)
+	viper.SetDefault("sftpd.login_banner_file", globalConf.SFTPD.LoginBannerFile)
+	viper.SetDefault("sftpd.enabled_ssh_commands", globalConf.SFTPD.EnabledSSHCommands)
+	viper.SetDefault("sftpd.keyboard_interactive_auth_hook", globalConf.SFTPD.KeyboardInteractiveHook)
+	viper.SetDefault("sftpd.password_authentication", globalConf.SFTPD.PasswordAuthentication)
+	viper.SetDefault("ftpd.bind_port", globalConf.FTPD.BindPort)
+	viper.SetDefault("ftpd.bind_address", globalConf.FTPD.BindAddress)
+	viper.SetDefault("ftpd.banner", globalConf.FTPD.Banner)
+	viper.SetDefault("ftpd.banner_file", globalConf.FTPD.BannerFile)
+	viper.SetDefault("ftpd.active_transfers_port_non_20", globalConf.FTPD.ActiveTransfersPortNon20)
+	viper.SetDefault("ftpd.force_passive_ip", globalConf.FTPD.ForcePassiveIP)
+	viper.SetDefault("ftpd.passive_port_range.start", globalConf.FTPD.PassivePortRange.Start)
+	viper.SetDefault("ftpd.passive_port_range.end", globalConf.FTPD.PassivePortRange.End)
+	viper.SetDefault("ftpd.certificate_file", globalConf.FTPD.CertificateFile)
+	viper.SetDefault("ftpd.certificate_key_file", globalConf.FTPD.CertificateKeyFile)
+	viper.SetDefault("ftpd.tls_mode", globalConf.FTPD.TLSMode)
+	viper.SetDefault("webdavd.bind_port", globalConf.WebDAVD.BindPort)
+	viper.SetDefault("webdavd.bind_address", globalConf.WebDAVD.BindAddress)
+	viper.SetDefault("webdavd.certificate_file", globalConf.WebDAVD.CertificateFile)
+	viper.SetDefault("webdavd.certificate_key_file", globalConf.WebDAVD.CertificateKeyFile)
+	viper.SetDefault("webdavd.cors.enabled", globalConf.WebDAVD.Cors.Enabled)
+	viper.SetDefault("webdavd.cors.allowed_origins", globalConf.WebDAVD.Cors.AllowedOrigins)
+	viper.SetDefault("webdavd.cors.allowed_methods", globalConf.WebDAVD.Cors.AllowedMethods)
+	viper.SetDefault("webdavd.cors.allowed_headers", globalConf.WebDAVD.Cors.AllowedHeaders)
+	viper.SetDefault("webdavd.cors.exposed_headers", globalConf.WebDAVD.Cors.ExposedHeaders)
+	viper.SetDefault("webdavd.cors.allow_credentials", globalConf.WebDAVD.Cors.AllowCredentials)
+	viper.SetDefault("webdavd.cors.max_age", globalConf.WebDAVD.Cors.MaxAge)
+	viper.SetDefault("webdavd.cache.users.expiration_time", globalConf.WebDAVD.Cache.Users.ExpirationTime)
+	viper.SetDefault("webdavd.cache.users.max_size", globalConf.WebDAVD.Cache.Users.MaxSize)
+	viper.SetDefault("webdavd.cache.mime_types.enabled", globalConf.WebDAVD.Cache.MimeTypes.Enabled)
+	viper.SetDefault("webdavd.cache.mime_types.max_size", globalConf.WebDAVD.Cache.MimeTypes.MaxSize)
+	viper.SetDefault("data_provider.driver", globalConf.ProviderConf.Driver)
+	viper.SetDefault("data_provider.name", globalConf.ProviderConf.Name)
+	viper.SetDefault("data_provider.host", globalConf.ProviderConf.Host)
+	viper.SetDefault("data_provider.port", globalConf.ProviderConf.Port)
+	viper.SetDefault("data_provider.username", globalConf.ProviderConf.Username)
+	viper.SetDefault("data_provider.password", globalConf.ProviderConf.Password)
+	viper.SetDefault("data_provider.sslmode", globalConf.ProviderConf.SSLMode)
+	viper.SetDefault("data_provider.connection_string", globalConf.ProviderConf.ConnectionString)
+	viper.SetDefault("data_provider.sql_tables_prefix", globalConf.ProviderConf.SQLTablesPrefix)
+	viper.SetDefault("data_provider.manage_users", globalConf.ProviderConf.ManageUsers)
+	viper.SetDefault("data_provider.track_quota", globalConf.ProviderConf.TrackQuota)
+	viper.SetDefault("data_provider.pool_size", globalConf.ProviderConf.PoolSize)
+	viper.SetDefault("data_provider.users_base_dir", globalConf.ProviderConf.UsersBaseDir)
+	viper.SetDefault("data_provider.actions.execute_on", globalConf.ProviderConf.Actions.ExecuteOn)
+	viper.SetDefault("data_provider.actions.hook", globalConf.ProviderConf.Actions.Hook)
+	viper.SetDefault("data_provider.external_auth_hook", globalConf.ProviderConf.ExternalAuthHook)
+	viper.SetDefault("data_provider.external_auth_scope", globalConf.ProviderConf.ExternalAuthScope)
+	viper.SetDefault("data_provider.credentials_path", globalConf.ProviderConf.CredentialsPath)
+	viper.SetDefault("data_provider.prefer_database_credentials", globalConf.ProviderConf.PreferDatabaseCredentials)
+	viper.SetDefault("data_provider.pre_login_hook", globalConf.ProviderConf.PreLoginHook)
+	viper.SetDefault("data_provider.post_login_hook", globalConf.ProviderConf.PostLoginHook)
+	viper.SetDefault("data_provider.post_login_scope", globalConf.ProviderConf.PostLoginScope)
+	viper.SetDefault("data_provider.check_password_hook", globalConf.ProviderConf.CheckPasswordHook)
+	viper.SetDefault("data_provider.check_password_scope", globalConf.ProviderConf.CheckPasswordScope)
+	viper.SetDefault("data_provider.password_hashing.argon2_options.memory", globalConf.ProviderConf.PasswordHashing.Argon2Options.Memory)
+	viper.SetDefault("data_provider.password_hashing.argon2_options.iterations", globalConf.ProviderConf.PasswordHashing.Argon2Options.Iterations)
+	viper.SetDefault("data_provider.password_hashing.argon2_options.parallelism", globalConf.ProviderConf.PasswordHashing.Argon2Options.Parallelism)
+	viper.SetDefault("data_provider.update_mode", globalConf.ProviderConf.UpdateMode)
+	viper.SetDefault("httpd.bind_port", globalConf.HTTPDConfig.BindPort)
+	viper.SetDefault("httpd.bind_address", globalConf.HTTPDConfig.BindAddress)
+	viper.SetDefault("httpd.templates_path", globalConf.HTTPDConfig.TemplatesPath)
+	viper.SetDefault("httpd.static_files_path", globalConf.HTTPDConfig.StaticFilesPath)
+	viper.SetDefault("httpd.backups_path", globalConf.HTTPDConfig.BackupsPath)
+	viper.SetDefault("httpd.auth_user_file", globalConf.HTTPDConfig.AuthUserFile)
+	viper.SetDefault("httpd.certificate_file", globalConf.HTTPDConfig.CertificateFile)
+	viper.SetDefault("httpd.certificate_key_file", globalConf.HTTPDConfig.CertificateKeyFile)
+	viper.SetDefault("http.timeout", globalConf.HTTPConfig.Timeout)
+	viper.SetDefault("http.ca_certificates", globalConf.HTTPConfig.CACertificates)
+	viper.SetDefault("http.skip_tls_verify", globalConf.HTTPConfig.SkipTLSVerify)
+}

config/config_test.go

@@ -10,8 +10,10 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/drakkan/sftpgo/common"
 	"github.com/drakkan/sftpgo/config"
 	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/ftpd"
 	"github.com/drakkan/sftpgo/httpclient"
 	"github.com/drakkan/sftpgo/httpd"
 	"github.com/drakkan/sftpgo/sftpd"
@@ -34,12 +36,12 @@ func TestLoadConfigTest(t *testing.T) {
 	confName := tempConfigName + ".json"
 	configFilePath := filepath.Join(configDir, confName)
 	err = config.LoadConfig(configDir, tempConfigName)
-	assert.NotNil(t, err)
-	err = ioutil.WriteFile(configFilePath, []byte("{invalid json}"), 0666)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(configFilePath, []byte("{invalid json}"), os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
-	assert.NotNil(t, err)
-	err = ioutil.WriteFile(configFilePath, []byte("{\"sftpd\": {\"bind_port\": \"a\"}}"), 0666)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(configFilePath, []byte("{\"sftpd\": {\"bind_port\": \"a\"}}"), os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
@@ -58,7 +60,7 @@ func TestEmptyBanner(t *testing.T) {
 	c := make(map[string]sftpd.Configuration)
 	c["sftpd"] = sftpdConf
 	jsonConf, _ := json.Marshal(c)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NoError(t, err)
@@ -66,6 +68,20 @@ func TestEmptyBanner(t *testing.T) {
 	assert.NotEmpty(t, strings.TrimSpace(sftpdConf.Banner))
 	err = os.Remove(configFilePath)
 	assert.NoError(t, err)
+
+	ftpdConf := config.GetFTPDConfig()
+	ftpdConf.Banner = " "
+	c1 := make(map[string]ftpd.Configuration)
+	c1["ftpd"] = ftpdConf
+	jsonConf, _ = json.Marshal(c1)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, tempConfigName)
+	assert.NoError(t, err)
+	ftpdConf = config.GetFTPDConfig()
+	assert.NotEmpty(t, strings.TrimSpace(ftpdConf.Banner))
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
 }
 
 func TestInvalidUploadMode(t *testing.T) {
@@ -74,13 +90,13 @@ func TestInvalidUploadMode(t *testing.T) {
 	configFilePath := filepath.Join(configDir, confName)
 	err := config.LoadConfig(configDir, configName)
 	assert.NoError(t, err)
-	sftpdConf := config.GetSFTPDConfig()
-	sftpdConf.UploadMode = 10
-	c := make(map[string]sftpd.Configuration)
-	c["sftpd"] = sftpdConf
+	commonConf := config.GetCommonConfig()
+	commonConf.UploadMode = 10
+	c := make(map[string]common.Configuration)
+	c["common"] = commonConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
@@ -100,7 +116,7 @@ func TestInvalidExternalAuthScope(t *testing.T) {
 	c["data_provider"] = providerConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
@@ -120,7 +136,7 @@ func TestInvalidCredentialsPath(t *testing.T) {
 	c["data_provider"] = providerConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
@@ -134,13 +150,13 @@ func TestInvalidProxyProtocol(t *testing.T) {
 	configFilePath := filepath.Join(configDir, confName)
 	err := config.LoadConfig(configDir, configName)
 	assert.NoError(t, err)
-	sftpdConf := config.GetSFTPDConfig()
-	sftpdConf.ProxyProtocol = 10
-	c := make(map[string]sftpd.Configuration)
-	c["sftpd"] = sftpdConf
+	commonConf := config.GetCommonConfig()
+	commonConf.ProxyProtocol = 10
+	c := make(map[string]common.Configuration)
+	c["common"] = commonConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
@@ -160,7 +176,7 @@ func TestInvalidUsersBaseDir(t *testing.T) {
 	c["data_provider"] = providerConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
@@ -168,72 +184,37 @@ func TestInvalidUsersBaseDir(t *testing.T) {
 	assert.NoError(t, err)
 }
 
-func TestHookCompatibity(t *testing.T) {
+func TestCommonParamsCompatibility(t *testing.T) {
 	configDir := ".."
 	confName := tempConfigName + ".json"
 	configFilePath := filepath.Join(configDir, confName)
 	err := config.LoadConfig(configDir, configName)
 	assert.NoError(t, err)
-	providerConf := config.GetProviderConf()
-	providerConf.ExternalAuthProgram = "ext_auth_program" //nolint:staticcheck
-	providerConf.PreLoginProgram = "pre_login_program"    //nolint:staticcheck
-	providerConf.Actions.Command = "/tmp/test_cmd"        //nolint:staticcheck
-	c := make(map[string]dataprovider.Config)
-	c["data_provider"] = providerConf
+	sftpdConf := config.GetSFTPDConfig()
+	sftpdConf.IdleTimeout = 21 //nolint:staticcheck
+	sftpdConf.Actions.Hook = "http://hook"
+	sftpdConf.Actions.ExecuteOn = []string{"upload"}
+	sftpdConf.SetstatMode = 1                                //nolint:staticcheck
+	sftpdConf.UploadMode = common.UploadModeAtomicWithResume //nolint:staticcheck
+	sftpdConf.ProxyProtocol = 1                              //nolint:staticcheck
+	sftpdConf.ProxyAllowed = []string{"192.168.1.1"}         //nolint:staticcheck
+	c := make(map[string]sftpd.Configuration)
+	c["sftpd"] = sftpdConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NoError(t, err)
-	providerConf = config.GetProviderConf()
-	assert.Equal(t, "ext_auth_program", providerConf.ExternalAuthHook)
-	assert.Equal(t, "pre_login_program", providerConf.PreLoginHook)
-	assert.Equal(t, "/tmp/test_cmd", providerConf.Actions.Hook)
-	err = os.Remove(configFilePath)
-	assert.NoError(t, err)
-	providerConf.Actions.Hook = ""
-	providerConf.Actions.HTTPNotificationURL = "http://example.com/notify" //nolint:staticcheck
-	c = make(map[string]dataprovider.Config)
-	c["data_provider"] = providerConf
-	jsonConf, err = json.Marshal(c)
-	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
-	assert.NoError(t, err)
-	err = config.LoadConfig(configDir, tempConfigName)
-	assert.NoError(t, err)
-	providerConf = config.GetProviderConf()
-	assert.Equal(t, "http://example.com/notify", providerConf.Actions.Hook)
-	err = os.Remove(configFilePath)
-	assert.NoError(t, err)
-	sftpdConf := config.GetSFTPDConfig()
-	sftpdConf.KeyboardInteractiveProgram = "key_int_program" //nolint:staticcheck
-	sftpdConf.Actions.Command = "/tmp/sftp_cmd"              //nolint:staticcheck
-	cnf := make(map[string]sftpd.Configuration)
-	cnf["sftpd"] = sftpdConf
-	jsonConf, err = json.Marshal(cnf)
-	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
-	assert.NoError(t, err)
-	err = config.LoadConfig(configDir, tempConfigName)
-	assert.NoError(t, err)
-	sftpdConf = config.GetSFTPDConfig()
-	assert.Equal(t, "key_int_program", sftpdConf.KeyboardInteractiveHook)
-	assert.Equal(t, "/tmp/sftp_cmd", sftpdConf.Actions.Hook)
-	err = os.Remove(configFilePath)
-	assert.NoError(t, err)
-	sftpdConf.Actions.Hook = ""
-	sftpdConf.Actions.HTTPNotificationURL = "http://example.com/sftp" //nolint:staticcheck
-	cnf = make(map[string]sftpd.Configuration)
-	cnf["sftpd"] = sftpdConf
-	jsonConf, err = json.Marshal(cnf)
-	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
-	assert.NoError(t, err)
-	err = config.LoadConfig(configDir, tempConfigName)
-	assert.NoError(t, err)
-	sftpdConf = config.GetSFTPDConfig()
-	assert.Equal(t, "http://example.com/sftp", sftpdConf.Actions.Hook)
+	commonConf := config.GetCommonConfig()
+	assert.Equal(t, 21, commonConf.IdleTimeout)
+	assert.Equal(t, "http://hook", commonConf.Actions.Hook)
+	assert.Len(t, commonConf.Actions.ExecuteOn, 1)
+	assert.True(t, utils.IsStringInSlice("upload", commonConf.Actions.ExecuteOn))
+	assert.Equal(t, 1, commonConf.SetstatMode)
+	assert.Equal(t, 1, commonConf.ProxyProtocol)
+	assert.Len(t, commonConf.ProxyAllowed, 1)
+	assert.True(t, utils.IsStringInSlice("192.168.1.1", commonConf.ProxyAllowed))
 	err = os.Remove(configFilePath)
 	assert.NoError(t, err)
 }
@@ -257,7 +238,7 @@ func TestHostKeyCompatibility(t *testing.T) {
 	c["sftpd"] = sftpdConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NoError(t, err)
@@ -271,9 +252,9 @@ func TestHostKeyCompatibility(t *testing.T) {
 
 func TestSetGetConfig(t *testing.T) {
 	sftpdConf := config.GetSFTPDConfig()
-	sftpdConf.IdleTimeout = 3
+	sftpdConf.MaxAuthTries = 10
 	config.SetSFTPDConfig(sftpdConf)
-	assert.Equal(t, sftpdConf.IdleTimeout, config.GetSFTPDConfig().IdleTimeout)
+	assert.Equal(t, sftpdConf.MaxAuthTries, config.GetSFTPDConfig().MaxAuthTries)
 	dataProviderConf := config.GetProviderConf()
 	dataProviderConf.Host = "test host"
 	config.SetProviderConf(dataProviderConf)
@@ -282,4 +263,42 @@ func TestSetGetConfig(t *testing.T) {
 	httpdConf.BindAddress = "0.0.0.0"
 	config.SetHTTPDConfig(httpdConf)
 	assert.Equal(t, httpdConf.BindAddress, config.GetHTTPDConfig().BindAddress)
+	commonConf := config.GetCommonConfig()
+	commonConf.IdleTimeout = 10
+	config.SetCommonConfig(commonConf)
+	assert.Equal(t, commonConf.IdleTimeout, config.GetCommonConfig().IdleTimeout)
+	ftpdConf := config.GetFTPDConfig()
+	ftpdConf.CertificateFile = "cert"
+	ftpdConf.CertificateKeyFile = "key"
+	config.SetFTPDConfig(ftpdConf)
+	assert.Equal(t, ftpdConf.CertificateFile, config.GetFTPDConfig().CertificateFile)
+	assert.Equal(t, ftpdConf.CertificateKeyFile, config.GetFTPDConfig().CertificateKeyFile)
+	webDavConf := config.GetWebDAVDConfig()
+	webDavConf.CertificateFile = "dav_cert"
+	webDavConf.CertificateKeyFile = "dav_key"
+	config.SetWebDAVDConfig(webDavConf)
+	assert.Equal(t, webDavConf.CertificateFile, config.GetWebDAVDConfig().CertificateFile)
+	assert.Equal(t, webDavConf.CertificateKeyFile, config.GetWebDAVDConfig().CertificateKeyFile)
+}
+
+func TestConfigFromEnv(t *testing.T) {
+	os.Setenv("SFTPGO_SFTPD__BIND_ADDRESS", "127.0.0.1")
+	os.Setenv("SFTPGO_DATA_PROVIDER__PASSWORD_HASHING__ARGON2_OPTIONS__ITERATIONS", "41")
+	os.Setenv("SFTPGO_DATA_PROVIDER__POOL_SIZE", "10")
+	os.Setenv("SFTPGO_DATA_PROVIDER__ACTIONS__EXECUTE_ON", "add")
+	t.Cleanup(func() {
+		os.Unsetenv("SFTPGO_SFTPD__BIND_ADDRESS")
+		os.Unsetenv("SFTPGO_DATA_PROVIDER__PASSWORD_HASHING__ARGON2_OPTIONS__ITERATIONS")
+		os.Unsetenv("SFTPGO_DATA_PROVIDER__POOL_SIZE")
+		os.Unsetenv("SFTPGO_DATA_PROVIDER__ACTIONS__EXECUTE_ON")
+	})
+	err := config.LoadConfig(".", "invalid config")
+	assert.NoError(t, err)
+	sftpdConfig := config.GetSFTPDConfig()
+	assert.Equal(t, "127.0.0.1", sftpdConfig.BindAddress)
+	dataProviderConf := config.GetProviderConf()
+	assert.Equal(t, uint32(41), dataProviderConf.PasswordHashing.Argon2Options.Iterations)
+	assert.Equal(t, 10, dataProviderConf.PoolSize)
+	assert.Len(t, dataProviderConf.Actions.ExecuteOn, 1)
+	assert.Contains(t, dataProviderConf.Actions.ExecuteOn, "add")
 }

dataprovider/bolt.go

@@ -121,7 +121,7 @@ func (p BoltProvider) checkAvailability() error {
 	return err
 }
 
-func (p BoltProvider) validateUserAndPass(username string, password string) (User, error) {
+func (p BoltProvider) validateUserAndPass(username, password, ip, protocol string) (User, error) {
 	var user User
 	if len(password) == 0 {
 		return user, errors.New("Credentials cannot be null or empty")
@@ -131,7 +131,7 @@ func (p BoltProvider) validateUserAndPass(username string, password string) (Use
 		providerLog(logger.LevelWarn, "error authenticating user: %v, error: %v", username, err)
 		return user, err
 	}
-	return checkUserAndPass(user, password)
+	return checkUserAndPass(user, password, ip, protocol)
 }
 
 func (p BoltProvider) validateUserAndPubKey(username string, pubKey []byte) (User, string, error) {
@@ -348,6 +348,7 @@ func (p BoltProvider) updateUser(user User) error {
 				return err
 			}
 		}
+		user.ID = oldUser.ID
 		user.LastQuotaUpdate = oldUser.LastQuotaUpdate
 		user.UsedQuotaSize = oldUser.UsedQuotaSize
 		user.UsedQuotaFiles = oldUser.UsedQuotaFiles
@@ -703,7 +704,7 @@ func (p BoltProvider) reloadConfig() error {
 
 // initializeDatabase does nothing, no initilization is needed for bolt provider
 func (p BoltProvider) initializeDatabase() error {
-	return errNoInitRequired
+	return ErrNoInitRequired
 }
 
 func (p BoltProvider) migrateDatabase() error {
@@ -712,8 +713,8 @@ func (p BoltProvider) migrateDatabase() error {
 		return err
 	}
 	if dbVersion.Version == boltDatabaseVersion {
-		providerLog(logger.LevelDebug, "bolt database is updated, current version: %v", dbVersion.Version)
-		return nil
+		providerLog(logger.LevelDebug, "bolt database is up to date, current version: %v", dbVersion.Version)
+		return ErrNoInitRequired
 	}
 	switch dbVersion.Version {
 	case 1:
@@ -866,6 +867,7 @@ func getFolderBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
 }
 
 func updateDatabaseFrom1To2(dbHandle *bolt.DB) error {
+	logger.InfoToConsole("updating bolt database version: 1 -> 2")
 	providerLog(logger.LevelInfo, "updating bolt database version: 1 -> 2")
 	usernames, err := getBoltAvailableUsernames(dbHandle)
 	if err != nil {
@@ -887,6 +889,7 @@ func updateDatabaseFrom1To2(dbHandle *bolt.DB) error {
 }
 
 func updateDatabaseFrom2To3(dbHandle *bolt.DB) error {
+	logger.InfoToConsole("updating bolt database version: 2 -> 3")
 	providerLog(logger.LevelInfo, "updating bolt database version: 2 -> 3")
 	users := []User{}
 	err := dbHandle.View(func(tx *bolt.Tx) error {
@@ -941,6 +944,7 @@ func updateDatabaseFrom2To3(dbHandle *bolt.DB) error {
 }
 
 func updateDatabaseFrom3To4(dbHandle *bolt.DB) error {
+	logger.InfoToConsole("updating bolt database version: 3 -> 4")
 	providerLog(logger.LevelInfo, "updating bolt database version: 3 -> 4")
 	foldersToScan := []string{}
 	users := []userCompactVFolders{}

dataprovider/memory.go

@@ -21,6 +21,9 @@ var (
 )
 
 type memoryProviderHandle struct {
+	// configuration file to use for loading users
+	configFile string
+	sync.Mutex
 	isClosed bool
 	// slice with ordered usernames
 	usernames []string
@@ -32,9 +35,6 @@ type memoryProviderHandle struct {
 	vfolders map[string]vfs.BaseVirtualFolder
 	// slice with ordered folders mapped path
 	vfoldersPaths []string
-	// configuration file to use for loading users
-	configFile string
-	lock       *sync.Mutex
 }
 
 // MemoryProvider auth provider for a memory store
@@ -60,15 +60,14 @@ func initializeMemoryProvider(basePath string) error {
 			vfolders:      make(map[string]vfs.BaseVirtualFolder),
 			vfoldersPaths: []string{},
 			configFile:    configFile,
-			lock:          new(sync.Mutex),
 		},
 	}
 	return provider.reloadConfig()
 }
 
 func (p MemoryProvider) checkAvailability() error {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return errMemoryProviderClosed
 	}
@@ -76,8 +75,8 @@ func (p MemoryProvider) checkAvailability() error {
 }
 
 func (p MemoryProvider) close() error {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return errMemoryProviderClosed
 	}
@@ -85,7 +84,7 @@ func (p MemoryProvider) close() error {
 	return nil
 }
 
-func (p MemoryProvider) validateUserAndPass(username string, password string) (User, error) {
+func (p MemoryProvider) validateUserAndPass(username, password, ip, protocol string) (User, error) {
 	var user User
 	if len(password) == 0 {
 		return user, errors.New("Credentials cannot be null or empty")
@@ -95,7 +94,7 @@ func (p MemoryProvider) validateUserAndPass(username string, password string) (U
 		providerLog(logger.LevelWarn, "error authenticating user %#v, error: %v", username, err)
 		return user, err
 	}
-	return checkUserAndPass(user, password)
+	return checkUserAndPass(user, password, ip, protocol)
 }
 
 func (p MemoryProvider) validateUserAndPubKey(username string, pubKey []byte) (User, string, error) {
@@ -112,8 +111,8 @@ func (p MemoryProvider) validateUserAndPubKey(username string, pubKey []byte) (U
 }
 
 func (p MemoryProvider) getUserByID(ID int64) (User, error) {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return User{}, errMemoryProviderClosed
 	}
@@ -124,8 +123,8 @@ func (p MemoryProvider) getUserByID(ID int64) (User, error) {
 }
 
 func (p MemoryProvider) updateLastLogin(username string) error {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return errMemoryProviderClosed
 	}
@@ -139,8 +138,8 @@ func (p MemoryProvider) updateLastLogin(username string) error {
 }
 
 func (p MemoryProvider) updateQuota(username string, filesAdd int, sizeAdd int64, reset bool) error {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return errMemoryProviderClosed
 	}
@@ -164,8 +163,8 @@ func (p MemoryProvider) updateQuota(username string, filesAdd int, sizeAdd int64
 }
 
 func (p MemoryProvider) getUsedQuota(username string) (int, int64, error) {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return 0, 0, errMemoryProviderClosed
 	}
@@ -178,8 +177,8 @@ func (p MemoryProvider) getUsedQuota(username string) (int, int64, error) {
 }
 
 func (p MemoryProvider) addUser(user User) error {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return errMemoryProviderClosed
 	}
@@ -205,8 +204,8 @@ func (p MemoryProvider) addUser(user User) error {
 }
 
 func (p MemoryProvider) updateUser(user User) error {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return errMemoryProviderClosed
 	}
@@ -231,8 +230,8 @@ func (p MemoryProvider) updateUser(user User) error {
 }
 
 func (p MemoryProvider) deleteUser(user User) error {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return errMemoryProviderClosed
 	}
@@ -255,8 +254,8 @@ func (p MemoryProvider) deleteUser(user User) error {
 }
 
 func (p MemoryProvider) dumpUsers() ([]User, error) {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	users := make([]User, 0, len(p.dbHandle.usernames))
 	var err error
 	if p.dbHandle.isClosed {
@@ -274,8 +273,8 @@ func (p MemoryProvider) dumpUsers() ([]User, error) {
 }
 
 func (p MemoryProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	folders := make([]vfs.BaseVirtualFolder, 0, len(p.dbHandle.vfoldersPaths))
 	if p.dbHandle.isClosed {
 		return folders, errMemoryProviderClosed
@@ -289,8 +288,8 @@ func (p MemoryProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
 func (p MemoryProvider) getUsers(limit int, offset int, order string, username string) ([]User, error) {
 	users := make([]User, 0, limit)
 	var err error
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return users, errMemoryProviderClosed
 	}
@@ -337,8 +336,8 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s
 }
 
 func (p MemoryProvider) userExists(username string) (User, error) {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return User{}, errMemoryProviderClosed
 	}
@@ -353,8 +352,8 @@ func (p MemoryProvider) userExistsInternal(username string) (User, error) {
 }
 
 func (p MemoryProvider) updateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool) error {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return errMemoryProviderClosed
 	}
@@ -376,8 +375,8 @@ func (p MemoryProvider) updateFolderQuota(mappedPath string, filesAdd int, sizeA
 }
 
 func (p MemoryProvider) getUsedFolderQuota(mappedPath string) (int, int64, error) {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return 0, 0, errMemoryProviderClosed
 	}
@@ -458,8 +457,8 @@ func (p MemoryProvider) folderExistsInternal(mappedPath string) (vfs.BaseVirtual
 func (p MemoryProvider) getFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualFolder, error) {
 	folders := make([]vfs.BaseVirtualFolder, 0, limit)
 	var err error
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return folders, errMemoryProviderClosed
 	}
@@ -507,8 +506,8 @@ func (p MemoryProvider) getFolders(limit, offset int, order, folderPath string)
 }
 
 func (p MemoryProvider) getFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error) {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return vfs.BaseVirtualFolder{}, errMemoryProviderClosed
 	}
@@ -516,8 +515,8 @@ func (p MemoryProvider) getFolderByPath(mappedPath string) (vfs.BaseVirtualFolde
 }
 
 func (p MemoryProvider) addFolder(folder vfs.BaseVirtualFolder) error {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return errMemoryProviderClosed
 	}
@@ -537,8 +536,8 @@ func (p MemoryProvider) addFolder(folder vfs.BaseVirtualFolder) error {
 }
 
 func (p MemoryProvider) deleteFolder(folder vfs.BaseVirtualFolder) error {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	if p.dbHandle.isClosed {
 		return errMemoryProviderClosed
 	}
@@ -590,8 +589,8 @@ func (p MemoryProvider) getNextFolderID() int64 {
 }
 
 func (p MemoryProvider) clear() {
-	p.dbHandle.lock.Lock()
-	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
 	p.dbHandle.usernames = []string{}
 	p.dbHandle.usersIdx = make(map[int64]string)
 	p.dbHandle.users = make(map[string]User)
@@ -668,9 +667,9 @@ func (p MemoryProvider) reloadConfig() error {
 
 // initializeDatabase does nothing, no initilization is needed for memory provider
 func (p MemoryProvider) initializeDatabase() error {
-	return errNoInitRequired
+	return ErrNoInitRequired
 }
 
 func (p MemoryProvider) migrateDatabase() error {
-	return nil
+	return ErrNoInitRequired
 }

dataprovider/mysql.go

@@ -3,6 +3,7 @@
 package dataprovider
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"strings"
@@ -83,8 +84,8 @@ func (p MySQLProvider) checkAvailability() error {
 	return sqlCommonCheckAvailability(p.dbHandle)
 }
 
-func (p MySQLProvider) validateUserAndPass(username string, password string) (User, error) {
-	return sqlCommonValidateUserAndPass(username, password, p.dbHandle)
+func (p MySQLProvider) validateUserAndPass(username, password, ip, protocol string) (User, error) {
+	return sqlCommonValidateUserAndPass(username, password, ip, protocol, p.dbHandle)
 }
 
 func (p MySQLProvider) validateUserAndPubKey(username string, publicKey []byte) (User, string, error) {
@@ -140,7 +141,9 @@ func (p MySQLProvider) getFolders(limit, offset int, order, folderPath string) (
 }
 
 func (p MySQLProvider) getFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error) {
-	return sqlCommonCheckFolderExists(mappedPath, p.dbHandle)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
+	return sqlCommonCheckFolderExists(ctx, mappedPath, p.dbHandle)
 }
 
 func (p MySQLProvider) addFolder(folder vfs.BaseVirtualFolder) error {
@@ -169,6 +172,10 @@ func (p MySQLProvider) reloadConfig() error {
 
 // initializeDatabase creates the initial database structure
 func (p MySQLProvider) initializeDatabase() error {
+	dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle, false)
+	if err == nil && dbVersion.Version > 0 {
+		return ErrNoInitRequired
+	}
 	sqlUsers := strings.Replace(mysqlUsersTableSQL, "{{users}}", sqlTableUsers, 1)
 	tx, err := p.dbHandle.Begin()
 	if err != nil {
@@ -193,13 +200,13 @@ func (p MySQLProvider) initializeDatabase() error {
 }
 
 func (p MySQLProvider) migrateDatabase() error {
-	dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle)
+	dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle, true)
 	if err != nil {
 		return err
 	}
 	if dbVersion.Version == sqlDatabaseVersion {
-		providerLog(logger.LevelDebug, "sql database is updated, current version: %v", dbVersion.Version)
-		return nil
+		providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", dbVersion.Version)
+		return ErrNoInitRequired
 	}
 	switch dbVersion.Version {
 	case 1:
@@ -226,12 +233,14 @@ func (p MySQLProvider) migrateDatabase() error {
 }
 
 func updateMySQLDatabaseFrom1To2(dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 1 -> 2")
 	providerLog(logger.LevelInfo, "updating database version: 1 -> 2")
 	sql := strings.Replace(mysqlV2SQL, "{{users}}", sqlTableUsers, 1)
 	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 2)
 }
 
 func updateMySQLDatabaseFrom2To3(dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 2 -> 3")
 	providerLog(logger.LevelInfo, "updating database version: 2 -> 3")
 	sql := strings.Replace(mysqlV3SQL, "{{users}}", sqlTableUsers, 1)
 	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 3)

dataprovider/pgsql.go

@@ -3,6 +3,7 @@
 package dataprovider
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"strings"
@@ -82,8 +83,8 @@ func (p PGSQLProvider) checkAvailability() error {
 	return sqlCommonCheckAvailability(p.dbHandle)
 }
 
-func (p PGSQLProvider) validateUserAndPass(username string, password string) (User, error) {
-	return sqlCommonValidateUserAndPass(username, password, p.dbHandle)
+func (p PGSQLProvider) validateUserAndPass(username, password, ip, protocol string) (User, error) {
+	return sqlCommonValidateUserAndPass(username, password, ip, protocol, p.dbHandle)
 }
 
 func (p PGSQLProvider) validateUserAndPubKey(username string, publicKey []byte) (User, string, error) {
@@ -139,7 +140,9 @@ func (p PGSQLProvider) getFolders(limit, offset int, order, folderPath string) (
 }
 
 func (p PGSQLProvider) getFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error) {
-	return sqlCommonCheckFolderExists(mappedPath, p.dbHandle)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
+	return sqlCommonCheckFolderExists(ctx, mappedPath, p.dbHandle)
 }
 
 func (p PGSQLProvider) addFolder(folder vfs.BaseVirtualFolder) error {
@@ -168,6 +171,10 @@ func (p PGSQLProvider) reloadConfig() error {
 
 // initializeDatabase creates the initial database structure
 func (p PGSQLProvider) initializeDatabase() error {
+	dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle, false)
+	if err == nil && dbVersion.Version > 0 {
+		return ErrNoInitRequired
+	}
 	sqlUsers := strings.Replace(pgsqlUsersTableSQL, "{{users}}", sqlTableUsers, 1)
 	tx, err := p.dbHandle.Begin()
 	if err != nil {
@@ -192,13 +199,13 @@ func (p PGSQLProvider) initializeDatabase() error {
 }
 
 func (p PGSQLProvider) migrateDatabase() error {
-	dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle)
+	dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle, true)
 	if err != nil {
 		return err
 	}
 	if dbVersion.Version == sqlDatabaseVersion {
-		providerLog(logger.LevelDebug, "sql database is updated, current version: %v", dbVersion.Version)
-		return nil
+		providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", dbVersion.Version)
+		return ErrNoInitRequired
 	}
 	switch dbVersion.Version {
 	case 1:
@@ -225,12 +232,14 @@ func (p PGSQLProvider) migrateDatabase() error {
 }
 
 func updatePGSQLDatabaseFrom1To2(dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 1 -> 2")
 	providerLog(logger.LevelInfo, "updating database version: 1 -> 2")
 	sql := strings.Replace(pgsqlV2SQL, "{{users}}", sqlTableUsers, 1)
 	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 2)
 }
 
 func updatePGSQLDatabaseFrom2To3(dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 2 -> 3")
 	providerLog(logger.LevelInfo, "updating database version: 2 -> 3")
 	sql := strings.Replace(pgsqlV3SQL, "{{users}}", sqlTableUsers, 1)
 	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 3)

dataprovider/sqlcommon.go

@@ -14,27 +14,31 @@ import (
 )
 
 const (
-	sqlDatabaseVersion  = 4
-	initialDBVersionSQL = "INSERT INTO {{schema_version}} (version) VALUES (1);"
+	sqlDatabaseVersion     = 4
+	initialDBVersionSQL    = "INSERT INTO {{schema_version}} (version) VALUES (1);"
+	defaultSQLQueryTimeout = 10 * time.Second
+	longSQLQueryTimeout    = 60 * time.Second
 )
 
 var errSQLFoldersAssosaction = errors.New("unable to associate virtual folders to user")
 
 type sqlQuerier interface {
-	Prepare(query string) (*sql.Stmt, error)
+	PrepareContext(ctx context.Context, query string) (*sql.Stmt, error)
 }
 
 func getUserByUsername(username string, dbHandle sqlQuerier) (User, error) {
 	var user User
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getUserByUsernameQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return user, err
 	}
 	defer stmt.Close()
 
-	row := stmt.QueryRow(username)
+	row := stmt.QueryRowContext(ctx, username)
 	user, err = getUserFromDbRow(row, nil)
 	if err != nil {
 		return user, err
@@ -42,7 +46,7 @@ func getUserByUsername(username string, dbHandle sqlQuerier) (User, error) {
 	return getUserWithVirtualFolders(user, dbHandle)
 }
 
-func sqlCommonValidateUserAndPass(username string, password string, dbHandle *sql.DB) (User, error) {
+func sqlCommonValidateUserAndPass(username, password, ip, protocol string, dbHandle *sql.DB) (User, error) {
 	var user User
 	if len(password) == 0 {
 		return user, errors.New("Credentials cannot be null or empty")
@@ -52,7 +56,7 @@ func sqlCommonValidateUserAndPass(username string, password string, dbHandle *sq
 		providerLog(logger.LevelWarn, "error authenticating user: %v, error: %v", username, err)
 		return user, err
 	}
-	return checkUserAndPass(user, password)
+	return checkUserAndPass(user, password, ip, protocol)
 }
 
 func sqlCommonValidateUserAndPubKey(username string, pubKey []byte, dbHandle *sql.DB) (User, string, error) {
@@ -69,22 +73,24 @@ func sqlCommonValidateUserAndPubKey(username string, pubKey []byte, dbHandle *sq
 }
 
 func sqlCommonCheckAvailability(dbHandle *sql.DB) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
 	defer cancel()
 	return dbHandle.PingContext(ctx)
 }
 
 func sqlCommonGetUserByID(ID int64, dbHandle *sql.DB) (User, error) {
 	var user User
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getUserByIDQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return user, err
 	}
 	defer stmt.Close()
 
-	row := stmt.QueryRow(ID)
+	row := stmt.QueryRowContext(ctx, ID)
 	user, err = getUserFromDbRow(row, nil)
 	if err != nil {
 		return user, err
@@ -93,14 +99,16 @@ func sqlCommonGetUserByID(ID int64, dbHandle *sql.DB) (User, error) {
 }
 
 func sqlCommonUpdateQuota(username string, filesAdd int, sizeAdd int64, reset bool, dbHandle *sql.DB) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getUpdateQuotaQuery(reset)
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return err
 	}
 	defer stmt.Close()
-	_, err = stmt.Exec(sizeAdd, filesAdd, utils.GetTimeAsMsSinceEpoch(time.Now()), username)
+	_, err = stmt.ExecContext(ctx, sizeAdd, filesAdd, utils.GetTimeAsMsSinceEpoch(time.Now()), username)
 	if err == nil {
 		providerLog(logger.LevelDebug, "quota updated for user %#v, files increment: %v size increment: %v is reset? %v",
 			username, filesAdd, sizeAdd, reset)
@@ -111,8 +119,10 @@ func sqlCommonUpdateQuota(username string, filesAdd int, sizeAdd int64, reset bo
 }
 
 func sqlCommonGetUsedQuota(username string, dbHandle *sql.DB) (int, int64, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getQuotaQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return 0, 0, err
@@ -121,7 +131,7 @@ func sqlCommonGetUsedQuota(username string, dbHandle *sql.DB) (int, int64, error
 
 	var usedFiles int
 	var usedSize int64
-	err = stmt.QueryRow(username).Scan(&usedSize, &usedFiles)
+	err = stmt.QueryRowContext(ctx, username).Scan(&usedSize, &usedFiles)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error getting quota for user: %v, error: %v", username, err)
 		return 0, 0, err
@@ -130,14 +140,16 @@ func sqlCommonGetUsedQuota(username string, dbHandle *sql.DB) (int, int64, error
 }
 
 func sqlCommonUpdateLastLogin(username string, dbHandle *sql.DB) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getUpdateLastLoginQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return err
 	}
 	defer stmt.Close()
-	_, err = stmt.Exec(utils.GetTimeAsMsSinceEpoch(time.Now()), username)
+	_, err = stmt.ExecContext(ctx, utils.GetTimeAsMsSinceEpoch(time.Now()), username)
 	if err == nil {
 		providerLog(logger.LevelDebug, "last login updated for user %#v", username)
 	} else {
@@ -148,14 +160,16 @@ func sqlCommonUpdateLastLogin(username string, dbHandle *sql.DB) error {
 
 func sqlCommonCheckUserExists(username string, dbHandle *sql.DB) (User, error) {
 	var user User
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getUserByUsernameQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return user, err
 	}
 	defer stmt.Close()
-	row := stmt.QueryRow(username)
+	row := stmt.QueryRowContext(ctx, username)
 	user, err = getUserFromDbRow(row, nil)
 	if err != nil {
 		return user, err
@@ -168,12 +182,14 @@ func sqlCommonAddUser(user User, dbHandle *sql.DB) error {
 	if err != nil {
 		return err
 	}
-	tx, err := dbHandle.Begin()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
+	tx, err := dbHandle.BeginTx(ctx, nil)
 	if err != nil {
 		return err
 	}
 	q := getAddUserQuery()
-	stmt, err := tx.Prepare(q)
+	stmt, err := tx.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		sqlCommonRollbackTransaction(tx)
@@ -200,14 +216,14 @@ func sqlCommonAddUser(user User, dbHandle *sql.DB) error {
 		sqlCommonRollbackTransaction(tx)
 		return err
 	}
-	_, err = stmt.Exec(user.Username, user.Password, string(publicKeys), user.HomeDir, user.UID, user.GID, user.MaxSessions, user.QuotaSize,
+	_, err = stmt.ExecContext(ctx, user.Username, user.Password, string(publicKeys), user.HomeDir, user.UID, user.GID, user.MaxSessions, user.QuotaSize,
 		user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.Status, user.ExpirationDate, string(filters),
 		string(fsConfig))
 	if err != nil {
 		sqlCommonRollbackTransaction(tx)
 		return err
 	}
-	err = generateVirtualFoldersMapping(user, tx)
+	err = generateVirtualFoldersMapping(ctx, user, tx)
 	if err != nil {
 		sqlCommonRollbackTransaction(tx)
 		return err
@@ -220,12 +236,14 @@ func sqlCommonUpdateUser(user User, dbHandle *sql.DB) error {
 	if err != nil {
 		return err
 	}
-	tx, err := dbHandle.Begin()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
+	tx, err := dbHandle.BeginTx(ctx, nil)
 	if err != nil {
 		return err
 	}
 	q := getUpdateUserQuery()
-	stmt, err := tx.Prepare(q)
+	stmt, err := tx.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		sqlCommonRollbackTransaction(tx)
@@ -252,14 +270,14 @@ func sqlCommonUpdateUser(user User, dbHandle *sql.DB) error {
 		sqlCommonRollbackTransaction(tx)
 		return err
 	}
-	_, err = stmt.Exec(user.Password, string(publicKeys), user.HomeDir, user.UID, user.GID, user.MaxSessions, user.QuotaSize,
+	_, err = stmt.ExecContext(ctx, user.Password, string(publicKeys), user.HomeDir, user.UID, user.GID, user.MaxSessions, user.QuotaSize,
 		user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.Status, user.ExpirationDate,
 		string(filters), string(fsConfig), user.ID)
 	if err != nil {
 		sqlCommonRollbackTransaction(tx)
 		return err
 	}
-	err = generateVirtualFoldersMapping(user, tx)
+	err = generateVirtualFoldersMapping(ctx, user, tx)
 	if err != nil {
 		sqlCommonRollbackTransaction(tx)
 		return err
@@ -268,27 +286,31 @@ func sqlCommonUpdateUser(user User, dbHandle *sql.DB) error {
 }
 
 func sqlCommonDeleteUser(user User, dbHandle *sql.DB) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getDeleteUserQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return err
 	}
 	defer stmt.Close()
-	_, err = stmt.Exec(user.ID)
+	_, err = stmt.ExecContext(ctx, user.ID)
 	return err
 }
 
 func sqlCommonDumpUsers(dbHandle sqlQuerier) ([]User, error) {
 	users := make([]User, 0, 100)
+	ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout)
+	defer cancel()
 	q := getDumpUsersQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return nil, err
 	}
 	defer stmt.Close()
-	rows, err := stmt.Query()
+	rows, err := stmt.QueryContext(ctx)
 	if err != nil {
 		return users, err
 	}
@@ -310,8 +332,10 @@ func sqlCommonDumpUsers(dbHandle sqlQuerier) ([]User, error) {
 
 func sqlCommonGetUsers(limit int, offset int, order string, username string, dbHandle sqlQuerier) ([]User, error) {
 	users := make([]User, 0, limit)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getUsersQuery(order, username)
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return nil, err
@@ -319,9 +343,9 @@ func sqlCommonGetUsers(limit int, offset int, order string, username string, dbH
 	defer stmt.Close()
 	var rows *sql.Rows
 	if len(username) > 0 {
-		rows, err = stmt.Query(username, limit, offset) //nolint:rowserrcheck // rows.Err() is checked
+		rows, err = stmt.QueryContext(ctx, username, limit, offset) //nolint:rowserrcheck // rows.Err() is checked
 	} else {
-		rows, err = stmt.Query(limit, offset) //nolint:rowserrcheck // rows.Err() is checked
+		rows, err = stmt.QueryContext(ctx, limit, offset) //nolint:rowserrcheck // rows.Err() is checked
 	}
 	if err == nil {
 		defer rows.Close()
@@ -418,16 +442,16 @@ func getUserFromDbRow(row *sql.Row, rows *sql.Rows) (User, error) {
 	return user, err
 }
 
-func sqlCommonCheckFolderExists(name string, dbHandle sqlQuerier) (vfs.BaseVirtualFolder, error) {
+func sqlCommonCheckFolderExists(ctx context.Context, name string, dbHandle sqlQuerier) (vfs.BaseVirtualFolder, error) {
 	var folder vfs.BaseVirtualFolder
 	q := getFolderByPathQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return folder, err
 	}
 	defer stmt.Close()
-	row := stmt.QueryRow(name)
+	row := stmt.QueryRowContext(ctx, name)
 	err = row.Scan(&folder.ID, &folder.MappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles, &folder.LastQuotaUpdate)
 	if err == sql.ErrNoRows {
 		return folder, &RecordNotFoundError{err: err.Error()}
@@ -435,8 +459,8 @@ func sqlCommonCheckFolderExists(name string, dbHandle sqlQuerier) (vfs.BaseVirtu
 	return folder, err
 }
 
-func sqlCommonAddOrGetFolder(name string, usedQuotaSize int64, usedQuotaFiles int, lastQuotaUpdate int64, dbHandle sqlQuerier) (vfs.BaseVirtualFolder, error) {
-	folder, err := sqlCommonCheckFolderExists(name, dbHandle)
+func sqlCommonAddOrGetFolder(ctx context.Context, name string, usedQuotaSize int64, usedQuotaFiles int, lastQuotaUpdate int64, dbHandle sqlQuerier) (vfs.BaseVirtualFolder, error) {
+	folder, err := sqlCommonCheckFolderExists(ctx, name, dbHandle)
 	if _, ok := err.(*RecordNotFoundError); ok {
 		f := vfs.BaseVirtualFolder{
 			MappedPath:      name,
@@ -448,7 +472,7 @@ func sqlCommonAddOrGetFolder(name string, usedQuotaSize int64, usedQuotaFiles in
 		if err != nil {
 			return folder, err
 		}
-		return sqlCommonCheckFolderExists(name, dbHandle)
+		return sqlCommonCheckFolderExists(ctx, name, dbHandle)
 	}
 	return folder, err
 }
@@ -458,39 +482,45 @@ func sqlCommonAddFolder(folder vfs.BaseVirtualFolder, dbHandle sqlQuerier) error
 	if err != nil {
 		return err
 	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getAddFolderQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return err
 	}
 	defer stmt.Close()
-	_, err = stmt.Exec(folder.MappedPath, folder.UsedQuotaSize, folder.UsedQuotaFiles, folder.LastQuotaUpdate)
+	_, err = stmt.ExecContext(ctx, folder.MappedPath, folder.UsedQuotaSize, folder.UsedQuotaFiles, folder.LastQuotaUpdate)
 	return err
 }
 
 func sqlCommonDeleteFolder(folder vfs.BaseVirtualFolder, dbHandle sqlQuerier) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getDeleteFolderQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return err
 	}
 	defer stmt.Close()
-	_, err = stmt.Exec(folder.ID)
+	_, err = stmt.ExecContext(ctx, folder.ID)
 	return err
 }
 
 func sqlCommonDumpFolders(dbHandle sqlQuerier) ([]vfs.BaseVirtualFolder, error) {
 	folders := make([]vfs.BaseVirtualFolder, 0, 50)
+	ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout)
+	defer cancel()
 	q := getDumpFoldersQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return nil, err
 	}
 	defer stmt.Close()
-	rows, err := stmt.Query()
+	rows, err := stmt.QueryContext(ctx)
 	if err != nil {
 		return folders, err
 	}
@@ -512,8 +542,10 @@ func sqlCommonDumpFolders(dbHandle sqlQuerier) ([]vfs.BaseVirtualFolder, error)
 
 func sqlCommonGetFolders(limit, offset int, order, folderPath string, dbHandle sqlQuerier) ([]vfs.BaseVirtualFolder, error) {
 	folders := make([]vfs.BaseVirtualFolder, 0, limit)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getFoldersQuery(order, folderPath)
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return nil, err
@@ -521,9 +553,9 @@ func sqlCommonGetFolders(limit, offset int, order, folderPath string, dbHandle s
 	defer stmt.Close()
 	var rows *sql.Rows
 	if len(folderPath) > 0 {
-		rows, err = stmt.Query(folderPath, limit, offset) //nolint:rowserrcheck // rows.Err() is checked
+		rows, err = stmt.QueryContext(ctx, folderPath, limit, offset) //nolint:rowserrcheck // rows.Err() is checked
 	} else {
-		rows, err = stmt.Query(limit, offset) //nolint:rowserrcheck // rows.Err() is checked
+		rows, err = stmt.QueryContext(ctx, limit, offset) //nolint:rowserrcheck // rows.Err() is checked
 	}
 	if err != nil {
 		return folders, err
@@ -545,42 +577,42 @@ func sqlCommonGetFolders(limit, offset int, order, folderPath string, dbHandle s
 	return getVirtualFoldersWithUsers(folders, dbHandle)
 }
 
-func sqlCommonClearFolderMapping(user User, dbHandle sqlQuerier) error {
+func sqlCommonClearFolderMapping(ctx context.Context, user User, dbHandle sqlQuerier) error {
 	q := getClearFolderMappingQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return err
 	}
 	defer stmt.Close()
-	_, err = stmt.Exec(user.Username)
+	_, err = stmt.ExecContext(ctx, user.Username)
 	return err
 }
 
-func sqlCommonAddFolderMapping(user User, folder vfs.VirtualFolder, dbHandle sqlQuerier) error {
+func sqlCommonAddFolderMapping(ctx context.Context, user User, folder vfs.VirtualFolder, dbHandle sqlQuerier) error {
 	q := getAddFolderMappingQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return err
 	}
 	defer stmt.Close()
-	_, err = stmt.Exec(folder.VirtualPath, folder.QuotaSize, folder.QuotaFiles, folder.ID, user.Username)
+	_, err = stmt.ExecContext(ctx, folder.VirtualPath, folder.QuotaSize, folder.QuotaFiles, folder.ID, user.Username)
 	return err
 }
 
-func generateVirtualFoldersMapping(user User, dbHandle sqlQuerier) error {
-	err := sqlCommonClearFolderMapping(user, dbHandle)
+func generateVirtualFoldersMapping(ctx context.Context, user User, dbHandle sqlQuerier) error {
+	err := sqlCommonClearFolderMapping(ctx, user, dbHandle)
 	if err != nil {
 		return err
 	}
 	for _, vfolder := range user.VirtualFolders {
-		f, err := sqlCommonAddOrGetFolder(vfolder.MappedPath, 0, 0, 0, dbHandle)
+		f, err := sqlCommonAddOrGetFolder(ctx, vfolder.MappedPath, 0, 0, 0, dbHandle)
 		if err != nil {
 			return err
 		}
 		vfolder.BaseVirtualFolder = f
-		err = sqlCommonAddFolderMapping(user, vfolder, dbHandle)
+		err = sqlCommonAddFolderMapping(ctx, user, vfolder, dbHandle)
 		if err != nil {
 			return err
 		}
@@ -605,14 +637,16 @@ func getUsersWithVirtualFolders(users []User, dbHandle sqlQuerier) ([]User, erro
 	if len(users) == 0 {
 		return users, err
 	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getRelatedFoldersForUsersQuery(users)
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return nil, err
 	}
 	defer stmt.Close()
-	rows, err := stmt.Query()
+	rows, err := stmt.QueryContext(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -647,14 +681,16 @@ func getVirtualFoldersWithUsers(folders []vfs.BaseVirtualFolder, dbHandle sqlQue
 	if len(folders) == 0 {
 		return folders, err
 	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getRelatedUsersForFoldersQuery(folders)
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return nil, err
 	}
 	defer stmt.Close()
-	rows, err := stmt.Query()
+	rows, err := stmt.QueryContext(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -683,14 +719,16 @@ func getVirtualFoldersWithUsers(folders []vfs.BaseVirtualFolder, dbHandle sqlQue
 }
 
 func sqlCommonUpdateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool, dbHandle *sql.DB) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getUpdateFolderQuotaQuery(reset)
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return err
 	}
 	defer stmt.Close()
-	_, err = stmt.Exec(sizeAdd, filesAdd, utils.GetTimeAsMsSinceEpoch(time.Now()), mappedPath)
+	_, err = stmt.ExecContext(ctx, sizeAdd, filesAdd, utils.GetTimeAsMsSinceEpoch(time.Now()), mappedPath)
 	if err == nil {
 		providerLog(logger.LevelDebug, "quota updated for folder %#v, files increment: %v size increment: %v is reset? %v",
 			mappedPath, filesAdd, sizeAdd, reset)
@@ -701,8 +739,10 @@ func sqlCommonUpdateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64,
 }
 
 func sqlCommonGetFolderUsedQuota(mappedPath string, dbHandle *sql.DB) (int, int64, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getQuotaFolderQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return 0, 0, err
@@ -711,7 +751,7 @@ func sqlCommonGetFolderUsedQuota(mappedPath string, dbHandle *sql.DB) (int, int6
 
 	var usedFiles int
 	var usedSize int64
-	err = stmt.QueryRow(mappedPath).Scan(&usedSize, &usedFiles)
+	err = stmt.QueryRowContext(ctx, mappedPath).Scan(&usedSize, &usedFiles)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error getting quota for folder: %v, error: %v", mappedPath, err)
 		return 0, 0, err
@@ -726,34 +766,41 @@ func sqlCommonRollbackTransaction(tx *sql.Tx) {
 	}
 }
 
-func sqlCommonGetDatabaseVersion(dbHandle *sql.DB) (schemaVersion, error) {
+func sqlCommonGetDatabaseVersion(dbHandle *sql.DB, showInitWarn bool) (schemaVersion, error) {
 	var result schemaVersion
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getDatabaseVersionQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		if showInitWarn && strings.Contains(err.Error(), sqlTableSchemaVersion) {
+			logger.WarnToConsole("database query error, did you forgot to run the \"initprovider\" command?")
+		}
 		return result, err
 	}
 	defer stmt.Close()
-	row := stmt.QueryRow()
+	row := stmt.QueryRowContext(ctx)
 	err = row.Scan(&result.Version)
 	return result, err
 }
 
-func sqlCommonUpdateDatabaseVersion(dbHandle sqlQuerier, version int) error {
+func sqlCommonUpdateDatabaseVersion(ctx context.Context, dbHandle sqlQuerier, version int) error {
 	q := getUpdateDBVersionQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return err
 	}
 	defer stmt.Close()
-	_, err = stmt.Exec(version)
+	_, err = stmt.ExecContext(ctx, version)
 	return err
 }
 
 func sqlCommonExecSQLAndUpdateDBVersion(dbHandle *sql.DB, sql []string, newVersion int) error {
-	tx, err := dbHandle.Begin()
+	ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout)
+	defer cancel()
+	tx, err := dbHandle.BeginTx(ctx, nil)
 	if err != nil {
 		return err
 	}
@@ -761,13 +808,13 @@ func sqlCommonExecSQLAndUpdateDBVersion(dbHandle *sql.DB, sql []string, newVersi
 		if len(strings.TrimSpace(q)) == 0 {
 			continue
 		}
-		_, err = tx.Exec(q)
+		_, err = tx.ExecContext(ctx, q)
 		if err != nil {
 			sqlCommonRollbackTransaction(tx)
 			return err
 		}
 	}
-	err = sqlCommonUpdateDatabaseVersion(tx, newVersion)
+	err = sqlCommonUpdateDatabaseVersion(ctx, tx, newVersion)
 	if err != nil {
 		sqlCommonRollbackTransaction(tx)
 		return err
@@ -777,14 +824,16 @@ func sqlCommonExecSQLAndUpdateDBVersion(dbHandle *sql.DB, sql []string, newVersi
 
 func sqlCommonGetCompatVirtualFolders(dbHandle *sql.DB) ([]userCompactVFolders, error) {
 	users := []userCompactVFolders{}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
 	q := getCompatVirtualFoldersQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
 		return nil, err
 	}
 	defer stmt.Close()
-	rows, err := stmt.Query()
+	rows, err := stmt.QueryContext(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -808,7 +857,7 @@ func sqlCommonGetCompatVirtualFolders(dbHandle *sql.DB) ([]userCompactVFolders,
 	return users, rows.Err()
 }
 
-func sqlCommonRestoreCompatVirtualFolders(users []userCompactVFolders, dbHandle sqlQuerier) ([]string, error) {
+func sqlCommonRestoreCompatVirtualFolders(ctx context.Context, users []userCompactVFolders, dbHandle sqlQuerier) ([]string, error) {
 	foldersToScan := []string{}
 	for _, user := range users {
 		for _, vfolder := range user.VirtualFolders {
@@ -820,7 +869,7 @@ func sqlCommonRestoreCompatVirtualFolders(users []userCompactVFolders, dbHandle
 				quotaFiles = 0
 				quotaSize = 0
 			}
-			b, err := sqlCommonAddOrGetFolder(vfolder.MappedPath, 0, 0, 0, dbHandle)
+			b, err := sqlCommonAddOrGetFolder(ctx, vfolder.MappedPath, 0, 0, 0, dbHandle)
 			if err != nil {
 				providerLog(logger.LevelWarn, "error restoring virtual folder for user %#v: %v", user.Username, err)
 				return foldersToScan, err
@@ -835,7 +884,7 @@ func sqlCommonRestoreCompatVirtualFolders(users []userCompactVFolders, dbHandle
 				QuotaSize:         quotaSize,
 				QuotaFiles:        quotaFiles,
 			}
-			err = sqlCommonAddFolderMapping(u, f, dbHandle)
+			err = sqlCommonAddFolderMapping(ctx, u, f, dbHandle)
 			if err != nil {
 				providerLog(logger.LevelWarn, "error adding virtual folder mapping for user %#v: %v", user.Username, err)
 				return foldersToScan, err
@@ -850,6 +899,7 @@ func sqlCommonRestoreCompatVirtualFolders(users []userCompactVFolders, dbHandle
 }
 
 func sqlCommonUpdateDatabaseFrom3To4(sqlV4 string, dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 3 -> 4")
 	providerLog(logger.LevelInfo, "updating database version: 3 -> 4")
 	users, err := sqlCommonGetCompatVirtualFolders(dbHandle)
 	if err != nil {
@@ -858,7 +908,9 @@ func sqlCommonUpdateDatabaseFrom3To4(sqlV4 string, dbHandle *sql.DB) error {
 	sql := strings.ReplaceAll(sqlV4, "{{users}}", sqlTableUsers)
 	sql = strings.ReplaceAll(sql, "{{folders}}", sqlTableFolders)
 	sql = strings.ReplaceAll(sql, "{{folders_mapping}}", sqlTableFoldersMapping)
-	tx, err := dbHandle.Begin()
+	ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout)
+	defer cancel()
+	tx, err := dbHandle.BeginTx(ctx, nil)
 	if err != nil {
 		return err
 	}
@@ -866,18 +918,18 @@ func sqlCommonUpdateDatabaseFrom3To4(sqlV4 string, dbHandle *sql.DB) error {
 		if len(strings.TrimSpace(q)) == 0 {
 			continue
 		}
-		_, err = tx.Exec(q)
+		_, err = tx.ExecContext(ctx, q)
 		if err != nil {
 			sqlCommonRollbackTransaction(tx)
 			return err
 		}
 	}
-	foldersToScan, err := sqlCommonRestoreCompatVirtualFolders(users, tx)
+	foldersToScan, err := sqlCommonRestoreCompatVirtualFolders(ctx, users, tx)
 	if err != nil {
 		sqlCommonRollbackTransaction(tx)
 		return err
 	}
-	err = sqlCommonUpdateDatabaseVersion(tx, 4)
+	err = sqlCommonUpdateDatabaseVersion(ctx, tx, 4)
 	if err != nil {
 		sqlCommonRollbackTransaction(tx)
 		return err

dataprovider/sqlite.go

@@ -3,6 +3,7 @@
 package dataprovider
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"path/filepath"
@@ -105,8 +106,8 @@ func (p SQLiteProvider) checkAvailability() error {
 	return sqlCommonCheckAvailability(p.dbHandle)
 }
 
-func (p SQLiteProvider) validateUserAndPass(username string, password string) (User, error) {
-	return sqlCommonValidateUserAndPass(username, password, p.dbHandle)
+func (p SQLiteProvider) validateUserAndPass(username, password, ip, protocol string) (User, error) {
+	return sqlCommonValidateUserAndPass(username, password, ip, protocol, p.dbHandle)
 }
 
 func (p SQLiteProvider) validateUserAndPubKey(username string, publicKey []byte) (User, string, error) {
@@ -162,7 +163,9 @@ func (p SQLiteProvider) getFolders(limit, offset int, order, folderPath string)
 }
 
 func (p SQLiteProvider) getFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error) {
-	return sqlCommonCheckFolderExists(mappedPath, p.dbHandle)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
+	defer cancel()
+	return sqlCommonCheckFolderExists(ctx, mappedPath, p.dbHandle)
 }
 
 func (p SQLiteProvider) addFolder(folder vfs.BaseVirtualFolder) error {
@@ -191,6 +194,10 @@ func (p SQLiteProvider) reloadConfig() error {
 
 // initializeDatabase creates the initial database structure
 func (p SQLiteProvider) initializeDatabase() error {
+	dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle, false)
+	if err == nil && dbVersion.Version > 0 {
+		return ErrNoInitRequired
+	}
 	sqlUsers := strings.Replace(sqliteUsersTableSQL, "{{users}}", sqlTableUsers, 1)
 	tx, err := p.dbHandle.Begin()
 	if err != nil {
@@ -215,13 +222,13 @@ func (p SQLiteProvider) initializeDatabase() error {
 }
 
 func (p SQLiteProvider) migrateDatabase() error {
-	dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle)
+	dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle, true)
 	if err != nil {
 		return err
 	}
 	if dbVersion.Version == sqlDatabaseVersion {
-		providerLog(logger.LevelDebug, "sql database is updated, current version: %v", dbVersion.Version)
-		return nil
+		providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", dbVersion.Version)
+		return ErrNoInitRequired
 	}
 	switch dbVersion.Version {
 	case 1:
@@ -248,12 +255,14 @@ func (p SQLiteProvider) migrateDatabase() error {
 }
 
 func updateSQLiteDatabaseFrom1To2(dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 1 -> 2")
 	providerLog(logger.LevelInfo, "updating database version: 1 -> 2")
 	sql := strings.Replace(sqliteV2SQL, "{{users}}", sqlTableUsers, 1)
 	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 2)
 }
 
 func updateSQLiteDatabaseFrom2To3(dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 2 -> 3")
 	providerLog(logger.LevelInfo, "updating database version: 2 -> 3")
 	sql := strings.ReplaceAll(sqliteV3SQL, "{{users}}", sqlTableUsers)
 	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 3)

dataprovider/user.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 	"time"
 
+	"golang.org/x/net/webdav"
+
 	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/utils"
 	"github.com/drakkan/sftpgo/vfs"
@@ -46,10 +48,11 @@ const (
 	PermChtimes = "chtimes"
 )
 
-// Available SSH login methods
+// Available login methods
 const (
+	LoginMethodNoAuthTryed            = "no_auth_tryed"
+	LoginMethodPassword               = "password"
 	SSHLoginMethodPublicKey           = "publickey"
-	SSHLoginMethodPassword            = "password"
 	SSHLoginMethodKeyboardInteractive = "keyboard-interactive"
 	SSHLoginMethodKeyAndPassword      = "publickey+password"
 	SSHLoginMethodKeyAndKeyboardInt   = "publickey+keyboard-interactive"
@@ -59,28 +62,65 @@ var (
 	errNoMatchingVirtualFolder = errors.New("no matching virtual folder found")
 )
 
+// CachedUser adds fields useful for caching to a SFTPGo user
+type CachedUser struct {
+	User       User
+	Expiration time.Time
+	Password   string
+	LockSystem webdav.LockSystem
+}
+
+// IsExpired returns true if the cached user is expired
+func (c *CachedUser) IsExpired() bool {
+	if c.Expiration.IsZero() {
+		return false
+	}
+	return c.Expiration.Before(time.Now())
+}
+
 // ExtensionsFilter defines filters based on file extensions.
 // These restrictions do not apply to files listing for performance reasons, so
 // a denied file cannot be downloaded/overwritten/renamed but will still be
-// it will still be listed in the list of files.
+// in the list of files.
 // System commands such as Git and rsync interacts with the filesystem directly
 // and they are not aware about these restrictions so they are not allowed
 // inside paths with extensions filters
 type ExtensionsFilter struct {
-	// SFTP/SCP path, if no other specific filter is defined, the filter apply for
+	// Virtual path, if no other specific filter is defined, the filter apply for
 	// sub directories too.
 	// For example if filters are defined for the paths "/" and "/sub" then the
 	// filters for "/" are applied for any file outside the "/sub" directory
 	Path string `json:"path"`
 	// only files with these, case insensitive, extensions are allowed.
 	// Shell like expansion is not supported so you have to specify ".jpg" and
-	// not "*.jpg"
+	// not "*.jpg". If you want shell like patterns use pattern filters
 	AllowedExtensions []string `json:"allowed_extensions,omitempty"`
 	// files with these, case insensitive, extensions are not allowed.
 	// Denied file extensions are evaluated before the allowed ones
 	DeniedExtensions []string `json:"denied_extensions,omitempty"`
 }
 
+// PatternsFilter defines filters based on shell like patterns.
+// These restrictions do not apply to files listing for performance reasons, so
+// a denied file cannot be downloaded/overwritten/renamed but will still be
+// in the list of files.
+// System commands such as Git and rsync interacts with the filesystem directly
+// and they are not aware about these restrictions so they are not allowed
+// inside paths with extensions filters
+type PatternsFilter struct {
+	// Virtual path, if no other specific filter is defined, the filter apply for
+	// sub directories too.
+	// For example if filters are defined for the paths "/" and "/sub" then the
+	// filters for "/" are applied for any file outside the "/sub" directory
+	Path string `json:"path"`
+	// files with these, case insensitive, patterns are allowed.
+	// Denied file patterns are evaluated before the allowed ones
+	AllowedPatterns []string `json:"allowed_patterns,omitempty"`
+	// files with these, case insensitive, patterns are not allowed.
+	// Denied file patterns are evaluated before the allowed ones
+	DeniedPatterns []string `json:"denied_patterns,omitempty"`
+}
+
 // UserFilters defines additional restrictions for a user
 type UserFilters struct {
 	// only clients connecting from these IP/Mask are allowed.
@@ -93,20 +133,38 @@ type UserFilters struct {
 	// these login methods are not allowed.
 	// If null or empty any available login method is allowed
 	DeniedLoginMethods []string `json:"denied_login_methods,omitempty"`
+	// these protocols are not allowed.
+	// If null or empty any available protocol is allowed
+	DeniedProtocols []string `json:"denied_protocols,omitempty"`
 	// filters based on file extensions.
 	// Please note that these restrictions can be easily bypassed.
 	FileExtensions []ExtensionsFilter `json:"file_extensions,omitempty"`
+	// filter based on shell patterns
+	FilePatterns []PatternsFilter `json:"file_patterns,omitempty"`
+	// max size allowed for a single upload, 0 means unlimited
+	MaxUploadFileSize int64 `json:"max_upload_file_size,omitempty"`
 }
 
+// FilesystemProvider defines the supported storages
+type FilesystemProvider int
+
+// supported values for FilesystemProvider
+const (
+	LocalFilesystemProvider     FilesystemProvider = iota // Local
+	S3FilesystemProvider                                  // AWS S3 compatible
+	GCSFilesystemProvider                                 // Google Cloud Storage
+	AzureBlobFilesystemProvider                           // Azure Blob Storage
+)
+
 // Filesystem defines cloud storage filesystem details
 type Filesystem struct {
-	// 0 local filesystem, 1 Amazon S3 compatible, 2 Google Cloud Storage
-	Provider  int             `json:"provider"`
-	S3Config  vfs.S3FsConfig  `json:"s3config,omitempty"`
-	GCSConfig vfs.GCSFsConfig `json:"gcsconfig,omitempty"`
+	Provider     FilesystemProvider `json:"provider"`
+	S3Config     vfs.S3FsConfig     `json:"s3config,omitempty"`
+	GCSConfig    vfs.GCSFsConfig    `json:"gcsconfig,omitempty"`
+	AzBlobConfig vfs.AzBlobFsConfig `json:"azblobconfig,omitempty"`
 }
 
-// User defines an SFTP user
+// User defines a SFTPGo user
 type User struct {
 	// Database unique identifier
 	ID int64 `json:"id"`
@@ -160,12 +218,14 @@ type User struct {
 
 // GetFilesystem returns the filesystem for this user
 func (u *User) GetFilesystem(connectionID string) (vfs.Fs, error) {
-	if u.FsConfig.Provider == 1 {
+	if u.FsConfig.Provider == S3FilesystemProvider {
 		return vfs.NewS3Fs(connectionID, u.GetHomeDir(), u.FsConfig.S3Config)
-	} else if u.FsConfig.Provider == 2 {
+	} else if u.FsConfig.Provider == GCSFilesystemProvider {
 		config := u.FsConfig.GCSConfig
 		config.CredentialFile = u.getGCSCredentialsFilePath()
 		return vfs.NewGCSFs(connectionID, u.GetHomeDir(), config)
+	} else if u.FsConfig.Provider == AzureBlobFilesystemProvider {
+		return vfs.NewAzBlobFs(connectionID, u.GetHomeDir(), u.FsConfig.AzBlobConfig)
 	}
 	return vfs.NewOsFs(connectionID, u.GetHomeDir(), u.VirtualFolders), nil
 }
@@ -200,7 +260,7 @@ func (u *User) GetPermissionsForPath(p string) []string {
 // If the path is not inside a virtual folder an error is returned
 func (u *User) GetVirtualFolderForPath(sftpPath string) (vfs.VirtualFolder, error) {
 	var folder vfs.VirtualFolder
-	if len(u.VirtualFolders) == 0 || u.FsConfig.Provider != 0 {
+	if len(u.VirtualFolders) == 0 || u.FsConfig.Provider != LocalFilesystemProvider {
 		return folder, errNoMatchingVirtualFolder
 	}
 	dirsForPath := utils.GetDirsForSFTPPath(sftpPath)
@@ -221,7 +281,7 @@ func (u *User) AddVirtualDirs(list []os.FileInfo, sftpPath string) []os.FileInfo
 	}
 	for _, v := range u.VirtualFolders {
 		if path.Dir(v.VirtualPath) == sftpPath {
-			fi := vfs.NewFileInfo(path.Base(v.VirtualPath), true, 0, time.Time{})
+			fi := vfs.NewFileInfo(v.VirtualPath, true, 0, time.Now(), false)
 			found := false
 			for index, f := range list {
 				if f.Name() == fi.Name() {
@@ -345,7 +405,7 @@ func (u *User) IsLoginMethodAllowed(loginMethod string, partialSuccessMethods []
 		return true
 	}
 	if len(partialSuccessMethods) == 1 {
-		for _, method := range u.GetNextAuthMethods(partialSuccessMethods) {
+		for _, method := range u.GetNextAuthMethods(partialSuccessMethods, true) {
 			if method == loginMethod {
 				return true
 			}
@@ -359,7 +419,7 @@ func (u *User) IsLoginMethodAllowed(loginMethod string, partialSuccessMethods []
 
 // GetNextAuthMethods returns the list of authentications methods that
 // can continue for multi-step authentication
-func (u *User) GetNextAuthMethods(partialSuccessMethods []string) []string {
+func (u *User) GetNextAuthMethods(partialSuccessMethods []string, isPasswordAuthEnabled bool) []string {
 	var methods []string
 	if len(partialSuccessMethods) != 1 {
 		return methods
@@ -368,8 +428,8 @@ func (u *User) GetNextAuthMethods(partialSuccessMethods []string) []string {
 		return methods
 	}
 	for _, method := range u.GetAllowedLoginMethods() {
-		if method == SSHLoginMethodKeyAndPassword {
-			methods = append(methods, SSHLoginMethodPassword)
+		if method == SSHLoginMethodKeyAndPassword && isPasswordAuthEnabled {
+			methods = append(methods, LoginMethodPassword)
 		}
 		if method == SSHLoginMethodKeyAndKeyboardInt {
 			methods = append(methods, SSHLoginMethodKeyboardInteractive)
@@ -407,11 +467,15 @@ func (u *User) GetAllowedLoginMethods() []string {
 }
 
 // IsFileAllowed returns true if the specified file is allowed by the file restrictions filters
-func (u *User) IsFileAllowed(sftpPath string) bool {
+func (u *User) IsFileAllowed(virtualPath string) bool {
+	return u.isFilePatternAllowed(virtualPath) && u.isFileExtensionAllowed(virtualPath)
+}
+
+func (u *User) isFileExtensionAllowed(virtualPath string) bool {
 	if len(u.Filters.FileExtensions) == 0 {
 		return true
 	}
-	dirsForPath := utils.GetDirsForSFTPPath(path.Dir(sftpPath))
+	dirsForPath := utils.GetDirsForSFTPPath(path.Dir(virtualPath))
 	var filter ExtensionsFilter
 	for _, dir := range dirsForPath {
 		for _, f := range u.Filters.FileExtensions {
@@ -420,12 +484,12 @@ func (u *User) IsFileAllowed(sftpPath string) bool {
 				break
 			}
 		}
-		if len(filter.Path) > 0 {
+		if filter.Path != "" {
 			break
 		}
 	}
-	if len(filter.Path) > 0 {
-		toMatch := strings.ToLower(sftpPath)
+	if filter.Path != "" {
+		toMatch := strings.ToLower(virtualPath)
 		for _, denied := range filter.DeniedExtensions {
 			if strings.HasSuffix(toMatch, denied) {
 				return false
@@ -441,6 +505,42 @@ func (u *User) IsFileAllowed(sftpPath string) bool {
 	return true
 }
 
+func (u *User) isFilePatternAllowed(virtualPath string) bool {
+	if len(u.Filters.FilePatterns) == 0 {
+		return true
+	}
+	dirsForPath := utils.GetDirsForSFTPPath(path.Dir(virtualPath))
+	var filter PatternsFilter
+	for _, dir := range dirsForPath {
+		for _, f := range u.Filters.FilePatterns {
+			if f.Path == dir {
+				filter = f
+				break
+			}
+		}
+		if filter.Path != "" {
+			break
+		}
+	}
+	if filter.Path != "" {
+		toMatch := strings.ToLower(path.Base(virtualPath))
+		for _, denied := range filter.DeniedPatterns {
+			matched, err := path.Match(denied, toMatch)
+			if err != nil || matched {
+				return false
+			}
+		}
+		for _, allowed := range filter.AllowedPatterns {
+			matched, err := path.Match(allowed, toMatch)
+			if err == nil && matched {
+				return true
+			}
+		}
+		return len(filter.AllowedPatterns) == 0
+	}
+	return true
+}
+
 // IsLoginFromAddrAllowed returns true if the login is allowed from the specified remoteAddr.
 // If AllowedIP is defined only the specified IP/Mask can login.
 // If DeniedIP is defined the specified IP/Mask cannot login.
@@ -592,10 +692,12 @@ func (u *User) GetInfoString() string {
 		t := utils.GetTimeFromMsecSinceEpoch(u.LastLogin)
 		result += fmt.Sprintf("Last login: %v ", t.Format("2006-01-02 15:04:05")) // YYYY-MM-DD HH:MM:SS
 	}
-	if u.FsConfig.Provider == 1 {
+	if u.FsConfig.Provider == S3FilesystemProvider {
 		result += "Storage: S3 "
-	} else if u.FsConfig.Provider == 2 {
+	} else if u.FsConfig.Provider == GCSFilesystemProvider {
 		result += "Storage: GCS "
+	} else if u.FsConfig.Provider == AzureBlobFilesystemProvider {
+		result += "Storage: Azure "
 	}
 	if len(u.PublicKeys) > 0 {
 		result += fmt.Sprintf("Public keys: %v ", len(u.PublicKeys))
@@ -663,6 +765,7 @@ func (u *User) getACopy() User {
 		permissions[k] = perms
 	}
 	filters := UserFilters{}
+	filters.MaxUploadFileSize = u.Filters.MaxUploadFileSize
 	filters.AllowedIP = make([]string, len(u.Filters.AllowedIP))
 	copy(filters.AllowedIP, u.Filters.AllowedIP)
 	filters.DeniedIP = make([]string, len(u.Filters.DeniedIP))
@@ -671,6 +774,10 @@ func (u *User) getACopy() User {
 	copy(filters.DeniedLoginMethods, u.Filters.DeniedLoginMethods)
 	filters.FileExtensions = make([]ExtensionsFilter, len(u.Filters.FileExtensions))
 	copy(filters.FileExtensions, u.Filters.FileExtensions)
+	filters.FilePatterns = make([]PatternsFilter, len(u.Filters.FilePatterns))
+	copy(filters.FilePatterns, u.Filters.FilePatterns)
+	filters.DeniedProtocols = make([]string, len(u.Filters.DeniedProtocols))
+	copy(filters.DeniedProtocols, u.Filters.DeniedProtocols)
 	fsConfig := Filesystem{
 		Provider: u.FsConfig.Provider,
 		S3Config: vfs.S3FsConfig{
@@ -687,10 +794,22 @@ func (u *User) getACopy() User {
 		GCSConfig: vfs.GCSFsConfig{
 			Bucket:               u.FsConfig.GCSConfig.Bucket,
 			CredentialFile:       u.FsConfig.GCSConfig.CredentialFile,
+			Credentials:          u.FsConfig.GCSConfig.Credentials,
 			AutomaticCredentials: u.FsConfig.GCSConfig.AutomaticCredentials,
 			StorageClass:         u.FsConfig.GCSConfig.StorageClass,
 			KeyPrefix:            u.FsConfig.GCSConfig.KeyPrefix,
 		},
+		AzBlobConfig: vfs.AzBlobFsConfig{
+			Container:         u.FsConfig.AzBlobConfig.Container,
+			AccountName:       u.FsConfig.AzBlobConfig.AccountName,
+			AccountKey:        u.FsConfig.AzBlobConfig.AccountKey,
+			Endpoint:          u.FsConfig.AzBlobConfig.Endpoint,
+			SASURL:            u.FsConfig.AzBlobConfig.SASURL,
+			KeyPrefix:         u.FsConfig.AzBlobConfig.KeyPrefix,
+			UploadPartSize:    u.FsConfig.AzBlobConfig.UploadPartSize,
+			UploadConcurrency: u.FsConfig.AzBlobConfig.UploadConcurrency,
+			UseEmulator:       u.FsConfig.AzBlobConfig.UseEmulator,
+		},
 	}
 
 	return User{

docker/README.md

@@ -1,5 +1,134 @@
-# Dockerfile examples
+# Official Docker image
 
-Sample Dockerfiles for `sftpgo` daemon and the REST API CLI.
+SFTPGo provides an official Docker image, it is available on both [Docker Hub](https://hub.docker.com/r/drakkan/sftpgo) and on [GitHub Container Registry](https://github.com/users/drakkan/packages/container/package/sftpgo).
 
-We don't want to add a `Dockerfile` for each single `sftpgo` configuration options or data provider. You can use the docker configurations here as starting point that you can customize to run `sftpgo` with [Docker](http://www.docker.io "Docker").
+## Supported tags and respective Dockerfile links
+
+- [v1.2.2, v1.2, v1, latest](https://github.com/drakkan/sftpgo/blob/v1.2.2/Dockerfile.full)
+- [v1.2.2-alpine, v1.2-alpine, v1-alpine, alpine](https://github.com/drakkan/sftpgo/blob/v1.2.2/Dockerfile.full.alpine)
+- [v1.2.2-slim, v1.2-slim, v1-slim, slim](https://github.com/drakkan/sftpgo/blob/v1.2.2/Dockerfile)
+- [v1.2.2-alpine-slim, v1.2-alpine-slim, v1-alpine-slim, alpine-slim](https://github.com/drakkan/sftpgo/blob/v1.2.2/Dockerfile.alpine)
+- [edge](../Dockerfile.full)
+- [edge-alpine](../Dockerfile.full.alpine)
+- [edge-slim](../Dockerfile)
+- [edge-alpine-slim](../Dockerfile.alpine)
+
+## How to use the SFTPGo image
+
+### Start a `sftpgo` server instance
+
+Starting a SFTPGo instance is simple:
+
+```shell
+docker run --name some-sftpgo -p 127.0.0.1:8080:8080 -p 2022:2022 -d "drakkan/sftpgo:tag"
+```
+
+... where `some-sftpgo` is the name you want to assign to your container, and `tag` is the tag specifying the SFTPGo version you want. See the list above for relevant tags.
+
+Now visit [http://localhost:8080/](http://localhost:8080/) and create a new SFTPGo user. The SFTP service is available on port 2022.
+
+If you prefer GitHub Container Registry to Docker Hub replace `drakkan/sftpgo:tag` with `ghcr.io/drakkan/sftpgo:tag`.
+
+### Container shell access and viewing SFTPGo logs
+
+The docker exec command allows you to run commands inside a Docker container. The following command line will give you a shell inside your `sftpgo` container:
+
+```shell
+docker exec -it some-sftpgo sh
+```
+
+The logs are available through Docker's container log:
+
+```shell
+docker logs some-sftpgo
+```
+
+### Where to Store Data
+
+Important note: There are several ways to store data used by applications that run in Docker containers. We encourage users of the SFTPGo images to familiarize themselves with the options available, including:
+
+- Let Docker manage the storage for SFTPGo data by [writing them to disk on the host system using its own internal volume management](https://docs.docker.com/engine/tutorials/dockervolumes/#adding-a-data-volume). This is the default and is easy and fairly transparent to the user. The downside is that the files may be hard to locate for tools and applications that run directly on the host system, i.e. outside containers.
+- Create a data directory on the host system (outside the container) and [mount this to a directory visible from inside the container]((https://docs.docker.com/engine/tutorials/dockervolumes/#mount-a-host-directory-as-a-data-volume)). This places the SFTPGo files in a known location on the host system, and makes it easy for tools and applications on the host system to access the files. The downside is that the user needs to make sure that the directory exists, and that e.g. directory permissions and other security mechanisms on the host system are set up correctly. The SFTPGo image runs using `1000` as UID/GID by default.
+
+The Docker documentation is a good starting point for understanding the different storage options and variations, and there are multiple blogs and forum postings that discuss and give advice in this area. We will simply show the basic procedure here for the latter option above:
+
+1. Create a data directory on a suitable volume on your host system, e.g. `/my/own/sftpgodata`.
+2. Create a home directory for the sftpgo container user on your host system e.g. `/my/own/sftpgohome`.
+3. Start your SFTPGo container like this:
+
+```shell
+docker run --name some-sftpgo \
+    -p 127.0.0.1:8080:8090 \
+    -p 2022:2022 \
+    --mount type=bind,source=/my/own/sftpgodata,target=/srv/sftpgo \
+    --mount type=bind,source=/my/own/sftpgohome,target=/var/lib/sftpgo \
+    -e SFTPGO_HTTPD__BIND_PORT=8090 \
+    -d "drakkan/sftpgo:tag"
+```
+
+As you can see SFTPGo uses two volumes:
+
+- `/srv/sftpgo` to handle persistent data. The default home directory for SFTP/FTP/WebDAV users is `/srv/sftpgo/data/<username>`. Backups are stored in `/srv/sftpgo/backups`
+- `/var/lib/sftpgo` is the home directory for the sftpgo system user defined inside the container. This is the container working directory too, host keys will be created here when using the default configuration.
+
+### Configuration
+
+The runtime configuration can be customized via environment variables that you can set passing the `-e` option to the `docker run` command or inside the `environment` section if you are using [docker stack deploy](https://docs.docker.com/engine/reference/commandline/stack_deploy/) or [docker-compose](https://github.com/docker/compose).
+
+Please take a look [here](../docs/full-configuration.md#environment-variables) to learn how to configure SFTPGo via environment variables.
+
+Alternately you can mount your custom configuration file to `/var/lib/sftpgo` or `/var/lib/sftpgo/.config/sftpgo`.
+
+### Running as an arbitrary user
+
+The SFTPGo image runs using `1000` as UID/GID by default. If you know the permissions of your data and/or configuration directory are already set appropriately or you have need of running SFTPGo with a specific UID/GID, it is possible to invoke this image with `--user` set to any value (other than `root/0`) in order to achieve the desired access/configuration:
+
+```shell
+$ ls -lnd data
+drwxr-xr-x 2 1100 11000 6  6 nov 09.09 data
+$ ls -lnd config
+drwxr-xr-x 2 1100 11000 6  6 nov 09.19 config
+```
+
+With the above directory permissions, you can start a SFTPGo instance like this:
+
+```shell
+docker run --name some-sftpgo \
+    --user 1100:1100 \
+    -p 127.0.0.1:8080:8080 \
+    -p 2022:2022 \
+    --mount type=bind,source="${PWD}/data",target=/srv/sftpgo \
+    --mount type=bind,source="${PWD}/config",target=/var/lib/sftpgo \
+    -d "drakkan/sftpgo:tag"
+```
+
+Alternately build your own image using the official one as a base, here is a sample Dockerfile:
+
+```shell
+FROM drakkan/sftpgo:tag
+USER root
+RUN chown -R 1100:1100 /etc/sftpgo && chown 1100:1100 /var/lib/sftpgo /srv/sftpgo
+USER 1100:1100
+```
+
+## Image Variants
+
+The `sftpgo` images comes in many flavors, each designed for a specific use case. The `edge` and `edge-alpine`tags are updated after each new commit.
+
+### `sftpgo:<version>`
+
+This is the defacto image, it is based on [Debian](https://www.debian.org/), available in [the `debian` official image](https://hub.docker.com/_/debian). If you are unsure about what your needs are, you probably want to use this one.
+
+### `sftpgo:<version>-alpine`
+
+This image is based on the popular [Alpine Linux project](https://alpinelinux.org/), available in [the `alpine` official image](https://hub.docker.com/_/alpine). Alpine Linux is much smaller than most distribution base images (~5MB), and thus leads to much slimmer images in general.
+
+This variant is highly recommended when final image size being as small as possible is desired. The main caveat to note is that it does use [musl libc](https://musl.libc.org/) instead of [glibc and friends](https://www.etalabs.net/compare_libcs.html), so certain software might run into issues depending on the depth of their libc requirements. However, most software doesn't have an issue with this, so this variant is usually a very safe choice. See [this Hacker News comment thread](https://news.ycombinator.com/item?id=10782897) for more discussion of the issues that might arise and some pro/con comparisons of using Alpine-based images.
+
+### `sftpgo:<suite>-slim`
+
+These tags provide a slimmer image that does not include the optional `git` and `rsync` dependencies.
+
+## Helm Chart
+
+An helm chart is [available](https://artifacthub.io/packages/helm/sagikazarmark/sftpgo). You can find the source code [here](https://github.com/sagikazarmark/helm-charts/tree/master/charts/sftpgo).

docker/rest-api-cli/Dockerfile

@@ -2,7 +2,7 @@ FROM debian:latest
 LABEL maintainer="nicola.murino@gmail.com"
 RUN apt-get update && apt-get install -y curl python3-requests python3-pygments
 
-RUN curl https://raw.githubusercontent.com/drakkan/sftpgo/master/examples/rest-api-cli/sftpgo_api_cli.py --output /usr/bin/sftpgo_api_cli.py
+RUN curl https://raw.githubusercontent.com/drakkan/sftpgo/master/examples/rest-api-cli/sftpgo_api_cli --output /usr/bin/sftpgo_api_cli
 
-ENTRYPOINT ["python3", "/usr/bin/sftpgo_api_cli.py" ]
+ENTRYPOINT ["python3", "/usr/bin/sftpgo_api_cli" ]
 CMD []

docker/scripts/entrypoint-alpine.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+SFTPGO_PUID=${SFTPGO_PUID:-1000}
+SFTPGO_PGID=${SFTPGO_PGID:-1000}
+
+if [ "$1" = 'sftpgo' ]; then
+    if [ "$(id -u)" = '0' ]; then
+        for DIR in "/etc/sftpgo" "/var/lib/sftpgo" "/srv/sftpgo"
+        do
+            DIR_UID=$(stat -c %u ${DIR})
+            DIR_GID=$(stat -c %g ${DIR})
+            if [ ${DIR_UID} != ${SFTPGO_PUID} ] || [ ${DIR_GID} != ${SFTPGO_PGID} ]; then
+                echo '{"level":"info","time":"'`date +%Y-%m-%dT%H:%M:%S.000`'","sender":"entrypoint","message":"change owner for \"'${DIR}'\" UID: '${SFTPGO_PUID}' GID: '${SFTPGO_PGID}'"}'
+                if [ ${DIR} = "/etc/sftpgo" ]; then
+                    chown -R ${SFTPGO_PUID}:${SFTPGO_PGID} ${DIR}
+                else
+                    chown ${SFTPGO_PUID}:${SFTPGO_PGID} ${DIR}
+                fi
+            fi
+        done
+        echo '{"level":"info","time":"'`date +%Y-%m-%dT%H:%M:%S.000`'","sender":"entrypoint","message":"run as UID: '${SFTPGO_PUID}' GID: '${SFTPGO_PGID}'"}'
+        exec su-exec ${SFTPGO_PUID}:${SFTPGO_PGID} "$@"
+    fi
+
+    exec "$@"
+fi
+
+exec "$@"

docker/scripts/entrypoint.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+SFTPGO_PUID=${SFTPGO_PUID:-1000}
+SFTPGO_PGID=${SFTPGO_PGID:-1000}
+
+if [ "$1" = 'sftpgo' ]; then
+    if [ "$(id -u)" = '0' ]; then
+        getent passwd ${SFTPGO_PUID} > /dev/null
+	    HAS_PUID=$?
+	    getent group ${SFTPGO_PGID} > /dev/null
+	    HAS_PGID=$?
+        if [ ${HAS_PUID} -ne 0 ] || [ ${HAS_PGID} -ne 0 ]; then
+            echo '{"level":"info","time":"'`date +%Y-%m-%dT%H:%M:%S.%3N`'","sender":"entrypoint","message":"prepare to run as UID: '${SFTPGO_PUID}' GID: '${SFTPGO_PGID}'"}'
+            if [ ${HAS_PGID} -ne 0 ];  then
+                echo '{"level":"info","time":"'`date +%Y-%m-%dT%H:%M:%S.%3N`'","sender":"entrypoint","message":"set GID to: '${SFTPGO_PGID}'"}'
+                groupmod -g ${SFTPGO_PGID} sftpgo
+            fi
+            if [ ${HAS_PUID} -ne 0 ]; then
+                echo '{"level":"info","time":"'`date +%Y-%m-%dT%H:%M:%S.%3N`'","sender":"entrypoint","message":"set UID to: '${SFTPGO_PUID}'"}'
+                usermod -u ${SFTPGO_PUID} sftpgo
+            fi
+            chown -R ${SFTPGO_PUID}:${SFTPGO_PGID} /etc/sftpgo
+            chown ${SFTPGO_PUID}:${SFTPGO_PGID} /var/lib/sftpgo /srv/sftpgo
+        fi
+        echo '{"level":"info","time":"'`date +%Y-%m-%dT%H:%M:%S.%3N`'","sender":"entrypoint","message":"run as UID: '${SFTPGO_PUID}' GID: '${SFTPGO_PGID}'"}'
+        exec gosu ${SFTPGO_PUID}:${SFTPGO_PGID} "$@"
+    fi
+
+    exec "$@"
+fi
+
+exec "$@"

docker/sftpgo/alpine/Dockerfile

@@ -1,13 +1,13 @@
 FROM golang:alpine as builder
 
 RUN apk add --no-cache git gcc g++ ca-certificates \
-  && go get -d github.com/drakkan/sftpgo
+  && go get -v -d github.com/drakkan/sftpgo
 WORKDIR /go/src/github.com/drakkan/sftpgo
 ARG TAG
 ARG FEATURES
-# Use --build-arg TAG=LATEST for latest tag. Use e.g. --build-arg TAG=0.9.6 for a specific tag/commit. Otherwise HEAD (master) is built.
+# Use --build-arg TAG=LATEST for latest tag. Use e.g. --build-arg TAG=v1.0.0 for a specific tag/commit. Otherwise HEAD (master) is built.
 RUN git checkout $(if [ "${TAG}" = LATEST ]; then echo `git rev-list --tags --max-count=1`; elif [ -n "${TAG}" ]; then echo "${TAG}"; else echo HEAD; fi)
-RUN go build -i $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o /go/bin/sftpgo
+RUN go build $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -v -o /go/bin/sftpgo
 
 FROM alpine:latest
 
@@ -27,5 +27,24 @@ RUN chmod +x /bin/entrypoint.sh
 VOLUME [ "/data", "/srv/sftpgo/config", "/srv/sftpgo/backups" ]
 EXPOSE 2022 8080
 
+# uncomment the following settings to enable FTP support
+#ENV SFTPGO_FTPD__BIND_PORT=2121
+#ENV SFTPGO_FTPD__FORCE_PASSIVE_IP=<your FTP visibile IP here>
+#EXPOSE 2121
+
+# we need to expose the passive ports range too
+#EXPOSE 50000-50100
+
+# it is a good idea to provide certificates to enable FTPS too
+#ENV SFTPGO_FTPD__CERTIFICATE_FILE=/srv/sftpgo/config/mycert.crt
+#ENV SFTPGO_FTPD__CERTIFICATE_KEY_FILE=/srv/sftpgo/config/mycert.key
+
+# uncomment the following setting to enable WebDAV support
+#ENV SFTPGO_WEBDAVD__BIND_PORT=8090
+
+# it is a good idea to provide certificates to enable WebDAV over HTTPS
+#ENV SFTPGO_WEBDAVD__CERTIFICATE_FILE=${CONFIG_DIR}/mycert.crt
+#ENV SFTPGO_WEBDAVD__CERTIFICATE_KEY_FILE=${CONFIG_DIR}/mycert.key
+
 ENTRYPOINT ["/bin/entrypoint.sh"]
 CMD ["serve"]

docker/sftpgo/alpine/README.md

@@ -1,5 +1,7 @@
 # SFTPGo with Docker and Alpine
 
+:warning: The recommended way to run SFTPGo on Docker is to use the official [images](https://hub.docker.com/r/drakkan/sftpgo). The documentation here is now obsolete.
+
 This DockerFile is made to build image to host multiple instances of SFTPGo started with different users.
 
 ## Example
@@ -16,7 +18,7 @@ sudo groupadd -g 1003 sftpgrp && \
 # Edit sftpgo.json as you need
 
 # Get and build SFTPGo image.
-# Add --build-arg TAG=LATEST to build the latest tag or e.g. TAG=0.9.6 for a specific tag/commit.
+# Add --build-arg TAG=LATEST to build the latest tag or e.g. TAG=v1.0.0 for a specific tag/commit.
 # Add --build-arg FEATURES=<build features comma separated> to specify the features to build.
 git clone https://github.com/drakkan/sftpgo.git && \
   cd sftpgo && \
@@ -46,6 +48,8 @@ sudo docker rm sftpgo && sudo docker run --name sftpgo \
   sftpgo
 ```
 
+If you want to enable FTP/S you also need the publish the FTP port and the FTP passive port range, defined in your `Dockerfile`, by adding, for example, the following options to the `docker run` command `-p 2121:2121 -p 50000-50100:50000-50100`. The same goes for WebDAV, you need to publish the configured port.
+
 The script `entrypoint.sh` makes sure to correct the permissions of directories and start the process with the right user.
 
 Several images can be run with different parameters.

docker/sftpgo/alpine/sftpgo.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=SFTPGo sftp server
+Description=SFTPGo server
 After=docker.service
 
 [Service]
@@ -15,12 +15,16 @@ ExecStart=docker run --name sftpgo \
   --env-file sftpgo-${PUID}.env \
   -e PUID=${PUID} \
   -e GUID=${GUID} \
+  -e SFTPGO_LOG_FILE_PATH= \
+  -e SFTPGO_CONFIG_DIR=/srv/sftpgo/config \
   -e SFTPGO_HTTPD__TEMPLATES_PATH=/srv/sftpgo/web/templates \
   -e SFTPGO_HTTPD__STATIC_FILES_PATH=/srv/sftpgo/web/static \
+  -e SFTPGO_HTTPD__BACKUPS_PATH=/srv/sftpgo/backups \
   -p 8080:8080 \
   -p 2022:2022 \
   -v /home/sftpuser/conf/:/srv/sftpgo/config \
   -v /home/sftpuser/data:/data \
+  -v /home/sftpuser/backups:/srv/sftpgo/backups \
   sftpgo
 ExecStop=docker stop sftpgo
 SyslogIdentifier=sftpgo

docker/sftpgo/debian/Dockerfile

@@ -1,22 +1,22 @@
 # we use a multi stage build to have a separate build and run env
 FROM golang:latest as buildenv
 LABEL maintainer="nicola.murino@gmail.com"
-RUN go get -d github.com/drakkan/sftpgo
+RUN go get -v -d github.com/drakkan/sftpgo
 WORKDIR /go/src/github.com/drakkan/sftpgo
 ARG TAG
 ARG FEATURES
-# Use --build-arg TAG=LATEST for latest tag. Use e.g. --build-arg TAG=0.9.6 for a specific tag/commit. Otherwise HEAD (master) is built.
+# Use --build-arg TAG=LATEST for latest tag. Use e.g. --build-arg TAG=v1.0.0 for a specific tag/commit. Otherwise HEAD (master) is built.
 RUN git checkout $(if [ "${TAG}" = LATEST ]; then echo `git rev-list --tags --max-count=1`; elif [ -n "${TAG}" ]; then echo "${TAG}"; else echo HEAD; fi)
-RUN go build -i $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
+RUN go build $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -v -o sftpgo
 
 # now define the run environment
 FROM debian:latest
 
-# ca-certificates is needed for Cloud Storage Support and to expose the REST API over HTTPS.
-RUN apt-get update && apt-get install -y ca-certificates
+# ca-certificates is needed for Cloud Storage Support and for HTTPS/FTPS.
+RUN apt-get update && apt-get install -y ca-certificates && apt-get clean
 
 # git and rsync are optional, uncomment the next line to add support for them if needed.
-#RUN apt-get update && apt-get install -y git rsync
+#RUN apt-get update && apt-get install -y git rsync && apt-get clean
 
 ARG BASE_DIR=/app
 ARG DATA_REL_DIR=data
@@ -40,7 +40,7 @@ ENV WEB_DIR=${BASE_DIR}/${WEB_REL_PATH}
 
 RUN mkdir -p ${DATA_DIR} ${CONFIG_DIR} ${WEB_DIR} ${BACKUPS_DIR}
 RUN groupadd --system -g ${GID} ${GROUPNAME}
-RUN useradd --system --create-home --no-log-init --home-dir ${HOME_DIR} --comment "SFTPGo user" --shell /bin/false --gid ${GID} --uid ${UID} ${USERNAME}
+RUN useradd --system --create-home --no-log-init --home-dir ${HOME_DIR} --comment "SFTPGo user" --shell /usr/sbin/nologin --gid ${GID} --uid ${UID} ${USERNAME}
 
 WORKDIR ${HOME_DIR}
 RUN mkdir -p bin .config/sftpgo
@@ -71,5 +71,23 @@ ENV SFTPGO_HTTPD__STATIC_FILES_PATH=${WEB_DIR}/static
 ENV SFTPGO_DATA_PROVIDER__USERS_BASE_DIR=${DATA_DIR}
 ENV SFTPGO_HTTPD__BACKUPS_PATH=${BACKUPS_DIR}
 
+# uncomment the following settings to enable FTP support
+#ENV SFTPGO_FTPD__BIND_PORT=2121
+#ENV SFTPGO_FTPD__FORCE_PASSIVE_IP=<your FTP visibile IP here>
+#EXPOSE 2121
+# we need to expose the passive ports range too
+#EXPOSE 50000-50100
+
+# it is a good idea to provide certificates to enable FTPS too
+#ENV SFTPGO_FTPD__CERTIFICATE_FILE=${CONFIG_DIR}/mycert.crt
+#ENV SFTPGO_FTPD__CERTIFICATE_KEY_FILE=${CONFIG_DIR}/mycert.key
+
+# uncomment the following setting to enable WebDAV support
+#ENV SFTPGO_WEBDAVD__BIND_PORT=8090
+
+# it is a good idea to provide certificates to enable WebDAV over HTTPS
+#ENV SFTPGO_WEBDAVD__CERTIFICATE_FILE=${CONFIG_DIR}/mycert.crt
+#ENV SFTPGO_WEBDAVD__CERTIFICATE_KEY_FILE=${CONFIG_DIR}/mycert.key
+
 ENTRYPOINT ["sftpgo"]
 CMD ["serve"]

docker/sftpgo/debian/README.md

@@ -1,5 +1,7 @@
 # Dockerfile based on Debian stable
 
+:warning: The recommended way to run SFTPGo on Docker is to use the official [images](https://hub.docker.com/r/drakkan/sftpgo). The documentation here is now obsolete.
+
 Please read the comments inside the `Dockerfile` to learn how to customize things for your setup.
 
 You can build the container image using `docker build`, for example:
@@ -10,10 +12,10 @@ docker build -t="drakkan/sftpgo" .
 
 This will build master of github.com/drakkan/sftpgo.
 
-To build the latest tag you can add `--build-arg TAG=LATEST` and to build a specific tag/commit you can use for example `TAG=0.9.6`, like this:
+To build the latest tag you can add `--build-arg TAG=LATEST` and to build a specific tag/commit you can use for example `TAG=v1.0.0`, like this:
 
 ```bash
-docker build -t="drakkan/sftpgo" --build-arg TAG=0.9.6 .
+docker build -t="drakkan/sftpgo" --build-arg TAG=v1.0.0 .
 ```
 
 To specify the features to build you can add `--build-arg FEATURES=<build features comma separated>`. For example you can disable SQLite and S3 support like this:
@@ -53,3 +55,5 @@ and finally you can run the image using something like this:
 ```bash
 docker rm sftpgo && docker run --name sftpgo -p 8080:8080 -p 2022:2022 --mount type=bind,source=/srv/sftpgo/data,target=/app/data --mount type=bind,source=/srv/sftpgo/config,target=/app/config --mount type=bind,source=/srv/sftpgo/backups,target=/app/backups drakkan/sftpgo
 ```
+
+If you want to enable FTP/S you also need the publish the FTP port and the FTP passive port range, defined in your `Dockerfile`, by adding, for example, the following options to the `docker run` command `-p 2121:2121 -p 50000-50100:50000-50100`. The same goes for WebDAV, you need to publish the configured port.

docs/account.md

@@ -30,17 +30,26 @@ For each account, the following properties can be configured:
 - `download_bandwidth` maximum download bandwidth as KB/s, 0 means unlimited.
 - `allowed_ip`, List of IP/Mask allowed to login. Any IP address not contained in this list cannot login. IP/Mask must be in CIDR notation as defined in RFC 4632 and RFC 4291, for example "192.0.2.0/24" or "2001:db8::/32"
 - `denied_ip`, List of IP/Mask not allowed to login. If an IP address is both allowed and denied then login will be denied
-- `denied_login_methods`, List of login methods not allowed. To enable multi-step authentication you have to allow only multi-step login methods. The following login methods are supported:
+- `max_upload_file_size`, max allowed size, as bytes, for a single file upload. The upload will be aborted if/when the size of the file being sent exceeds this limit. 0 means unlimited. This restriction does not apply for SSH system commands such as `git` and `rsync`
+- `denied_login_methods`, List of login methods not allowed. To enable multi-step authentication you have to allow only multi-step login methods. If password login method is denied or no password is set then FTP and WebDAV users cannot login. The following login methods are supported:
   - `publickey`
   - `password`
   - `keyboard-interactive`
   - `publickey+password`
   - `publickey+keyboard-interactive`
-- `file_extensions`, list of struct. These restrictions do not apply to files listing for performance reasons, so a denied file cannot be downloaded/overwritten/renamed but it will still be listed in the list of files. Please note that these restrictions can be easily bypassed. Each struct contains the following fields:
-  - `allowed_extensions`, list of, case insensitive, allowed files extension. Shell like expansion is not supported so you have to specify `.jpg` and not `*.jpg`. Any file that does not end with this suffix will be denied
-  - `denied_extensions`, list of, case insensitive, denied files extension. Denied file extensions are evaluated before the allowed ones
-  - `path`, SFTP/SCP path, if no other specific filter is defined, the filter apply for sub directories too. For example if filters are defined for the paths `/` and `/sub` then the filters for `/` are applied for any file outside the `/sub` directory
-- `fs_provider`, filesystem to serve via SFTP. Local filesystem and S3 Compatible Object Storage are supported
+- `denied_protocols`, list of protocols not allowed. The following protocols are supported:
+  - `SSH`
+  - `FTP`
+  - `DAV`
+- `file_extensions`, list of struct. Deprecated, please use `file_patterns`. These restrictions do not apply to files listing for performance reasons, so a denied file cannot be downloaded/overwritten/renamed but it will still be in the list of files. Please note that these restrictions can be easily bypassed. Each struct contains the following fields:
+  - `allowed_extensions`, list of, case insensitive, allowed file extensions. Shell like expansion is not supported so you have to specify `.jpg` and not `*.jpg`. Any file that does not end with this suffix will be denied
+  - `denied_extensions`, list of, case insensitive, denied file extensions. Denied file extensions are evaluated before the allowed ones
+  - `path`, exposed virtual path, if no other specific filter is defined, the filter apply for sub directories too. For example if filters are defined for the paths `/` and `/sub` then the filters for `/` are applied for any file outside the `/sub` directory
+- `file_patterns`, list of struct. These restrictions do not apply to files listing for performance reasons, so a denied file cannot be downloaded/overwritten/renamed but it will still be in the list of files. Please note that these restrictions can be easily bypassed. For syntax details take a look [here](https://golang.org/pkg/path/#Match). Each struct contains the following fields:
+  - `allowed_patterns`, list of, case insensitive, allowed file patterns. Examples: `*.jpg`, `a*b?.png`. Any non matching file will be denied
+  - `denied_patterns`, list of, case insensitive, denied file patterns. Denied file patterns are evaluated before the allowed ones
+  - `path`, exposed virtual path, if no other specific filter is defined, the filter apply for sub directories too. For example if filters are defined for the paths `/` and `/sub` then the filters for `/` are applied for any file outside the `/sub` directory
+- `fs_provider`, filesystem to serve via SFTP. Local filesystem (0), S3 Compatible Object Storage (1), Google Cloud Storage (2) and Azure Blob Storage (3) are supported
 - `s3_bucket`, required for S3 filesystem
 - `s3_region`, required for S3 filesystem. Must match the region for your bucket. You can find here the list of available [AWS regions](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). For example if your bucket is at `Frankfurt` you have to set the region to `eu-central-1`
 - `s3_access_key`
@@ -55,11 +64,28 @@ For each account, the following properties can be configured:
 - `gcs_automatic_credentials`, integer. Set to 1 to use Application Default Credentials strategy or set to 0 to use explicit credentials via `gcs_credentials`
 - `gcs_storage_class`
 - `gcs_key_prefix`, allows to restrict access to the folder identified by this prefix and its contents
+- `az_container`, Azure Blob Storage container
+- `az_account_name`, Azure account name. leave blank to use SAS URL
+- `az_account_key`, Azure account key. leave blank to use SAS URL. If provided it is stored encrypted (AES-256-GCM)
+- `az_sas_url`, Azure shared access signature URL
+- `az_endpoint`, Default is "blob.core.windows.net". If you use the emulator the endpoint must include the protocol, for example "http://127.0.0.1:10000"
+- `az_upload_part_size`, the buffer size for multipart uploads (MB). Zero means the default (4 MB)
+- `az_upload_concurrency`,  how many parts are uploaded in parallel. Zero means the default (2)
+- `az_key_prefix`,  allows to restrict access to the folder identified by this prefix and its contents
+- `az_use_emulator`, boolean
 
 These properties are stored inside the data provider.
 
 If you want to use your existing accounts, you have these options:
 
-- If your accounts are aleady stored inside a supported database, you can create a database view. Since a view is read only, you have to disable user management and quota tracking so SFTPGo will never try to write to the view
-- you can import your users inside SFTPGo. Take a look at [sftpgo_api_cli.py](../examples/rest-api-cli#convert-users-from-other-stores "SFTPGo API CLI example"), it can convert and import users from Linux system users and Pure-FTPd/ProFTPD virtual users
+- you can import your users inside SFTPGo. Take a look at [sftpgo_api_cli](../examples/rest-api-cli#convert-users-from-other-stores "SFTPGo API CLI example"), it can convert and import users from Linux system users and Pure-FTPd/ProFTPD virtual users
 - you can use an external authentication program
+
+Please take a look at the [OpenAPI schema](../httpd/schema/openapi.yaml) for the exact definitions of user and folder fields.
+If you need an example you can export a dump using the REST API CLI client or by invoking the `dumpdata` endpoint directly, for example:
+
+```shell
+curl "http://127.0.0.1:8080/api/v1/dumpdata?output_file=dump.json&indent=1"
+```
+
+the dump is a JSON with users and folder.

docs/azure-blob-storage.md

@@ -0,0 +1,20 @@
+# Azure Blob Storage backend
+
+To connect SFTPGo to Azure Blob Storage, you need to specify the access credentials. Azure Blob Storage has different options for credentials, we support:
+
+1. Providing an account name and account key.
+2. Providing a shared access signature (SAS).
+
+If you authenticate using account and key you also need to specify a container. The endpoint can generally be left blank, the default is `blob.core.windows.net`.
+
+If you provide a SAS URL the container is optional and if given it must match the one inside the shared access signature.
+
+If you want to connect to an emulator such as [Azurite](https://github.com/Azure/Azurite) you need to provide the account name/key pair and an endpoint prefixed with the protocol, for example `http://127.0.0.1:10000`.
+
+Specifying a different `key_prefix`, you can assign different "folders" of the same container to different users. This is similar to a chroot directory for local filesystem. Each SFTPGo user can only access the assigned folder and its contents. The folder identified by `key_prefix` does not need to be pre-created.
+
+For multipart uploads you can customize the parts size and the upload concurrency. Please note that if the upload bandwidth between the client and SFTPGo is greater than the upload bandwidth between SFTPGo and the Azure Blob service then the client should wait for the last parts to be uploaded to Azure after finishing uploading the file to SFTPGo, and it may time out. Keep this in mind if you customize these parameters.
+
+The configured container must exist.
+
+This backend is very similar to the [S3](./s3.md) backend, and it has the same limitations.

docs/build-from-source.md

@@ -14,6 +14,7 @@ The following build tags are available:
 
 - `nogcs`, disable Google Cloud Storage backend, default enabled
 - `nos3`, disable S3 Compabible Object Storage backends, default enabled
+- `noazblob`, disable Azure Blob Storage backend, default enabled
 - `nobolt`, disable Bolt data provider, default enabled
 - `nomysql`, disable MySQL data provider, default enabled
 - `nopgsql`, disable PostgreSQL data provider, default enabled
@@ -36,7 +37,7 @@ Version info, such as git commit and build date, can be embedded setting the fol
 For example, you can build using the following command:
 
 ```bash
-go build -i -tags nogcs,nos3,nosqlite -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
+go build -tags nogcs,nos3,nosqlite -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
 ```
 
 You should get a version that includes git commit, build date and available features like this one:

docs/check-password-hook.md

@@ -0,0 +1,45 @@
+# Check password hook
+
+This hook allows you to externally check the provided password, its main use case is to allow to easily support things like password+OTP for protocols without keyboard interactive support such as FTP and WebDAV. You can ask your users to login using a string consisting of a fixed password and a One Time Token, you can verify the token inside the hook and ask to SFTPGo to verify the fixed part.
+
+The same thing can be achieved using [External authentication](./external-auth.md) but using this hook is simpler in some use cases.
+
+The `check password hook` can be defined as the absolute path of your program or an HTTP URL.
+
+The expected response is a JSON serialized struct containing the following keys:
+
+- `status` integer. 0 means KO, 1 means OK, 2 means partial success
+- `to_verify` string. For `status` = 2 SFTPGo will check this password against the one stored inside SFTPGo data provider
+
+If the hook defines an external program it can read the following environment variables:
+
+- `SFTPGO_AUTHD_USERNAME`
+- `SFTPGO_AUTHD_PASSWORD`
+- `SFTPGO_AUTHD_IP`
+- `SFTPGO_AUTHD_PROTOCOL`, possible values are `SSH`, `FTP`, `DAV`
+
+Previous global environment variables aren't cleared when the script is called. The content of these variables is _not_ quoted. They may contain special characters. They are under the control of a possibly malicious remote user.
+
+The program must write, on its standard output, the expected JSON serialized response described above.
+
+If the hook is an HTTP URL then it will be invoked as HTTP POST. The request body will contain a JSON serialized struct with the following fields:
+
+- `username`
+- `password`
+- `ip`
+- `protocol`, possible values are `SSH`, `FTP`, `DAV`
+
+If authentication succeeds the HTTP response code must be 200 and the response body must contain the expected JSON serialized response described above.
+
+The program hook must finish within 30 seconds, the HTTP hook timeout will use the global configuration for HTTP clients.
+
+You can also restrict the hook scope using the `check_password_scope` configuration key:
+
+- `0` means all supported protocols.
+- `1` means SSH only
+- `2` means FTP only
+- `4` means WebDAV only
+
+You can combine the scopes. For example, 6 means FTP and WebDAV.
+
+An example check password program allowing 2FA using password + one time token can be found inside the source tree [checkpwd](../examples/OTP/authy/checkpwd) directory.

docs/custom-actions.md

@@ -1,6 +1,6 @@
 # Custom Actions
 
-The `actions` struct inside the "sftpd" configuration section allows to configure the actions for file operations and SSH commands.
+The `actions` struct inside the "common" configuration section allows to configure the actions for file operations and SSH commands.
 The `hook` can be defined as the absolute path of your program or an HTTP URL.
 
 The `upload` condition includes both uploads to new files and overwrite of existing files. If an upload is aborted for quota limits SFTPGo tries to remove the partial file, so if the notification reports a zero size file and a quota exceeded error the file has been deleted. The `ssh_cmd` condition will be triggered after a command is successfully executed via SSH. `scp` will trigger the `download` and `upload` conditions and not `ssh_cmd`.
@@ -23,10 +23,11 @@ The external program can also read the following environment variables:
 - `SFTPGO_ACTION_TARGET`, non-empty for `rename` `SFTPGO_ACTION`
 - `SFTPGO_ACTION_SSH_CMD`, non-empty for `ssh_cmd` `SFTPGO_ACTION`
 - `SFTPGO_ACTION_FILE_SIZE`, non-empty for `upload`, `download` and `delete` `SFTPGO_ACTION`
-- `SFTPGO_ACTION_FS_PROVIDER`, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend
-- `SFTPGO_ACTION_BUCKET`, non-empty for S3 and GCS backends
-- `SFTPGO_ACTION_ENDPOINT`, non-empty for S3 backend if configured
+- `SFTPGO_ACTION_FS_PROVIDER`, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend, `3` for Azure Blob Storage backend
+- `SFTPGO_ACTION_BUCKET`, non-empty for S3, GCS and Azure backends
+- `SFTPGO_ACTION_ENDPOINT`, non-empty for S3 and Azure backend if configured. For Azure this is the SAS URL, if configured otherwise the endpoint
 - `SFTPGO_ACTION_STATUS`, integer. 0 means a generic error occurred. 1 means no error, 2 means quota exceeded error
+- `SFTPGO_ACTION_PROTOCOL`, string. Possible values are `SSH`, `SFTP`, `SCP`, `FTP`, `DAV`
 
 Previous global environment variables aren't cleared when the script is called.
 The program must finish within 30 seconds.
@@ -39,10 +40,11 @@ If the `hook` defines an HTTP URL then this URL will be invoked as HTTP POST. Th
 - `target_path`, not null for `rename` action
 - `ssh_cmd`, not null for `ssh_cmd` action
 - `file_size`, not null for `upload`, `download`, `delete` actions
-- `fs_provider`, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend
-- `bucket`, not null for S3 and GCS backends
-- `endpoint`, not null for S3 backend if configured
+- `fs_provider`, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend, `3` for Azure Blob Storage backend
+- `bucket`, not null for S3, GCS and Azure backends
+- `endpoint`, not null for S3 and Azure backend if configured. For Azure this is the SAS URL, if configured otherwise the endpoint
 - `status`, integer. 0 means a generic error occurred. 1 means no error, 2 means quota exceeded error
+- `protocol`, string. Possible values are `SSH`, `FTP`, `DAV`
 
 The HTTP request will use the global configuration for HTTP clients.
 

docs/dynamic-user-mod.md

@@ -5,18 +5,20 @@ To enable dynamic user modification, you must set the absolute path of your prog
 
 The external program can read the following environment variables to get info about the user trying to login:
 
-- `SFTPGO_LOGIND_USER`, it contains the user trying to login serialized as JSON. A JSON serialized user id equal to zero means the user does not exists inside SFTPGo
+- `SFTPGO_LOGIND_USER`, it contains the user trying to login serialized as JSON. A JSON serialized user id equal to zero means the user does not exist inside SFTPGo
 - `SFTPGO_LOGIND_METHOD`, possible values are: `password`, `publickey` and `keyboard-interactive`
+- `SFTPGO_LOGIND_IP`, ip address of the user trying to login
+- `SFTPGO_LOGIND_PROTOCOL`, possible values are `SSH`, `FTP`, `DAV`
 
-The program must write, on its the standard output:
+The program must write, on its standard output:
 
 - an empty string (or no response at all) if the user should not be created/updated
-- or the SFTPGo user, JSON serialized, if you want create or update the given user
+- or the SFTPGo user, JSON serialized, if you want to create or update the given user
 
-If the hook is an HTTP URL then it will be invoked as HTTP POST. The login method is added to the query string, for example `<http_url>?login_method=password`.
+If the hook is an HTTP URL then it will be invoked as HTTP POST. The login method, the used protocol and the ip address of the user trying to login are added to the query string, for example `<http_url>?login_method=password&ip=1.2.3.4&protocol=SSH`.
 The request body will contain the user trying to login serialized as JSON. If no modification is needed the HTTP response code must be 204, otherwise the response code must be 200 and the response body a valid SFTPGo user serialized as JSON.
 
-Actions defined for user's updates will not be executed in this case.
+Actions defined for user's updates will not be executed in this case and an already logged in user with the same username will not be disconnected, you have to handle these things yourself.
 
 The JSON response can include only the fields to update instead of the full user. For example, if you want to disable the user, you can return a response like this:
 
@@ -30,8 +32,8 @@ The program hook must finish within 30 seconds, the HTTP hook will use the globa
 
 If an error happens while executing the hook then login will be denied.
 
-"Dynamic user creation or modification" and "External Authentication" are mutally exclusive, they are quite similar, the difference is that "External Authentication" returns an already authenticated user while using "Dynamic users modification" you simply create or update a user. The authentication will be checked inside SFTPGo.
-In other words while using "External Authentication" the external program receives the credentials of the user trying to login (for example the clear text password) and it need to validate them. While using "Dynamic users modification" the pre-login program receives the user stored inside the dataprovider (it includes the hashed password if any) and it can modify it, after the modification SFTPGo will check the credentials of the user trying to login.
+"Dynamic user creation or modification" and "External Authentication" are mutually exclusive, they are quite similar, the difference is that "External Authentication" returns an already authenticated user while using "Dynamic users modification" you simply create or update a user. The authentication will be checked inside SFTPGo.
+In other words while using "External Authentication" the external program receives the credentials of the user trying to login (for example the cleartext password) and it needs to validate them. While using "Dynamic users modification" the pre-login program receives the user stored inside the dataprovider (it includes the hashed password if any) and it can modify it, after the modification SFTPGo will check the credentials of the user trying to login.
 
 Let's see a very basic example. Our sample program will grant access to the existing user `test_user` only in the time range 10:00-18:00. Other users will not be modified since the program will terminate with no output.
 

docs/external-auth.md

@@ -5,33 +5,37 @@ To enable external authentication, you must set the absolute path of your authen
 The external program can read the following environment variables to get info about the user trying to authenticate:
 
 - `SFTPGO_AUTHD_USERNAME`
+- `SFTPGO_AUTHD_IP`
+- `SFTPGO_AUTHD_PROTOCOL`, possible values are `SSH`, `FTP`, `DAV`
 - `SFTPGO_AUTHD_PASSWORD`, not empty for password authentication
 - `SFTPGO_AUTHD_PUBLIC_KEY`, not empty for public key authentication
 - `SFTPGO_AUTHD_KEYBOARD_INTERACTIVE`, not empty for keyboard interactive authentication
 
 Previous global environment variables aren't cleared when the script is called. The content of these variables is _not_ quoted. They may contain special characters. They are under the control of a possibly malicious remote user.
-The program must write, on its standard output, a valid SFTPGo user serialized as JSON if the authentication succeed or a user with an empty username if the authentication fails.
+The program must write, on its standard output, a valid SFTPGo user serialized as JSON if the authentication succeeds or a user with an empty username if the authentication fails.
 
 If the hook is an HTTP URL then it will be invoked as HTTP POST. The request body will contain a JSON serialized struct with the following fields:
 
 - `username`
+- `ip`
+- `protocol`, possible values are `SSH`, `FTP`, `DAV`
 - `password`, not empty for password authentication
 - `public_key`, not empty for public key authentication
 - `keyboard_interactive`, not empty for keyboard interactive authentication
 
-If authentication succeed the HTTP response code must be 200 and the response body a valid SFTPGo user serialized as JSON. If the authentication fails the HTTP response code must be != 200 or the response body must be empty.
+If authentication succeeds the HTTP response code must be 200 and the response body a valid SFTPGo user serialized as JSON. If the authentication fails the HTTP response code must be != 200 or the response body must be empty.
+
+If the authentication succeeds, the user will be automatically added/updated inside the defined data provider. Actions defined for users added/updated will not be executed in this case and an already logged in user with the same username will not be disconnected, you have to handle these things yourself.
 
-If the authentication succeeds, the user will be automatically added/updated inside the defined data provider. Actions defined for users added/updated will not be executed in this case.
-The external hook should check authentication only. If there are login restrictions such as user disabled, expired, or login allowed only from specific IP addresses, it is enough to populate the matching user fields, and these conditions will be checked in the same way as for built-in users.
 The program hook must finish within 30 seconds, the HTTP hook timeout will use the global configuration for HTTP clients.
 
 This method is slower than built-in authentication, but it's very flexible as anyone can easily write his own authentication hooks.
 You can also restrict the authentication scope for the hook using the `external_auth_scope` configuration key:
 
-- 0 means all supported authetication scopes. The external hook will be used for password, public key and keyboard interactive authentication
-- 1 means passwords only
-- 2 means public keys only
-- 4 means keyboard interactive only
+- `0` means all supported authentication scopes. The external hook will be used for password, public key and keyboard interactive authentication
+- `1` means passwords only
+- `2` means public keys only
+- `4` means keyboard interactive only
 
 You can combine the scopes. For example, 3 means password and public key, 5 means password and keyboard interactive, and so on.
 

docs/full-configuration.md

@@ -9,8 +9,9 @@ Usage:
   sftpgo [command]
 
 Available Commands:
+  gen          A collection of useful generators
   help         Help about any command
-  initprovider Initializes the configured data provider
+  initprovider Initializes and/or updates the configured data provider
   portable     Serve a single directory
   serve        Start the SFTP Server
 
@@ -25,46 +26,35 @@ The `serve` command supports the following flags:
 
 - `--config-dir` string. Location of the config dir. This directory should contain the configuration file and is used as the base directory for any files that use a relative path (eg. the private keys for the SFTP server, the SQLite or bblot database if you use SQLite or bbolt as data provider). The default value is "." or the value of `SFTPGO_CONFIG_DIR` environment variable.
 - `--config-file` string. Name of the configuration file. It must be the name of a file stored in `config-dir`, not the absolute path to the configuration file. The specified file name must have no extension because we automatically append JSON, YAML, TOML, HCL and Java extensions when we search for the file. The default value is "sftpgo" (and therefore `sftpgo.json`, `sftpgo.yaml` and so on are searched) or the value of `SFTPGO_CONFIG_FILE` environment variable.
+- `--loaddata-from` string. Load users and folders from this file. The file must be specified as absolute path and it must contain a backup obtained using the `dumpdata` REST API or compatible content. The default value is empty or the value of `SFTPGO_LOADDATA_FROM` environment variable.
+- `--loaddata-clean` boolean. Determine if the loaddata-from file should be removed after a successful load. Default `false` or the value of `SFTPGO_LOADDATA_CLEAN` environment variable (1 or `true`, 0 or `false`).
+- `--loaddata-mode`, integer. Restore mode for data to load. 0 means new users are added, existing users are updated. 1 means new users are added, existing users are not modified. Default 1 or the value of `SFTPGO_LOADDATA_MODE` environment variable.
+- `--loaddata-scan`, integer. Quota scan mode after data load. 0 means no quota scan. 1 means quota scan. 2 means scan quota if the user has quota restrictions. Default 0 or the value of `SFTPGO_LOADDATA_QUOTA_SCAN` environment variable.
 - `--log-compress` boolean. Determine if the rotated log files should be compressed using gzip. Default `false` or the value of `SFTPGO_LOG_COMPRESS` environment variable (1 or `true`, 0 or `false`). It is unused if `log-file-path` is empty.
 - `--log-file-path` string. Location for the log file, default "sftpgo.log" or the value of `SFTPGO_LOG_FILE_PATH` environment variable. Leave empty to write logs to the standard error.
 - `--log-max-age` int. Maximum number of days to retain old log files. Default 28 or the value of `SFTPGO_LOG_MAX_AGE` environment variable. It is unused if `log-file-path` is empty.
 - `--log-max-backups` int. Maximum number of old log files to retain. Default 5 or the value of `SFTPGO_LOG_MAX_BACKUPS` environment variable. It is unused if `log-file-path` is empty.
 - `--log-max-size` int. Maximum size in megabytes of the log file before it gets rotated. Default 10 or the value of `SFTPGO_LOG_MAX_SIZE` environment variable. It is unused if `log-file-path` is empty.
 - `--log-verbose` boolean. Enable verbose logs. Default `true` or the value of `SFTPGO_LOG_VERBOSE` environment variable (1 or `true`, 0 or `false`).
+- `--profiler` boolean. Enable the built-in profiler. The profiler will be accessible via HTTP/HTTPS using the base URL "/debug/pprof/". Default `false` or the value of `SFTPGO_PROFILER` environment variable (1 or `true`, 0 or `false`).
 
 Log file can be rotated on demand sending a `SIGUSR1` signal on Unix based systems and using the command `sftpgo service rotatelogs` on Windows.
 
-If you don't configure any private host key, the daemon will use `id_rsa` and `id_ecdsa` in the configuration directory. If these files don't exist, the daemon will attempt to autogenerate them (if the user that executes SFTPGo has write access to the `config-dir`). The server supports any private key format supported by [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/keys.go#L33).
+If you don't configure any private host key, the daemon will use `id_rsa`, `id_ecdsa` and `id_ed25519` in the configuration directory. If these files don't exist, the daemon will attempt to autogenerate them. The server supports any private key format supported by [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/keys.go#L33).
+
+The `gen` command allows to generate completion scripts for your shell and man pages.
 
 ## Configuration file
 
 The configuration file contains the following sections:
 
-- **"sftpd"**, the configuration for the SFTP server
-  - `bind_port`, integer. The port used for serving SFTP requests. Default: 2022
-  - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: ""
+- **"common"**, configuration parameters shared among all the supported protocols
   - `idle_timeout`, integer. Time in minutes after which an idle client will be disconnected. 0 means disabled. Default: 15
-  - `max_auth_tries` integer. Maximum number of authentication attempts permitted per connection. If set to a negative number, the number of attempts is unlimited. If set to zero, the number of attempts are limited to 6.
-  - `umask`, string. Umask for the new files and directories. This setting has no effect on Windows. Default: "0022"
-  - `banner`, string. Identification string used by the server. Leave empty to use the default banner. Default `SFTPGo_<version>`, for example `SSH-2.0-SFTPGo_0.9.5`
   - `upload_mode` integer. 0 means standard: the files are uploaded directly to the requested path. 1 means atomic: files are uploaded to a temporary path and renamed to the requested path when the client ends the upload. Atomic mode avoids problems such as a web server that serves partial files when the files are being uploaded. In atomic mode, if there is an upload error, the temporary file is deleted and so the requested upload path will not contain a partial file. 2 means atomic with resume support: same as atomic but if there is an upload error, the temporary file is renamed to the requested path and not deleted. This way, a client can reconnect and resume the upload.
-  - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions. See the "Custom Actions" paragraph for more details
+  - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions. See [Custom Actions](./custom-actions.md) for more details
     - `execute_on`, list of strings. Valid values are `download`, `upload`, `pre-delete`, `delete`, `rename`, `ssh_cmd`. Leave empty to disable actions.
-    - `command`, string. Deprecated please use `hook`.
-    - `http_notification_url`, a valid URL. Deprecated please use `hook`.
     - `hook`, string. Absolute path to the command to execute or HTTP URL to notify.
-  - `keys`, struct array. Deprecated, please use `host_keys`.
-    - `private_key`, path to the private key file. It can be a path relative to the config dir or an absolute one.
-  - `host_keys`, list of strings. It contains the daemon's private host keys. Each host key can be defined as a path relative to the configuration directory or an absolute one. If empty, the daemon will search or try to generate `id_rsa` and `id_ecdsa` keys inside the configuration directory. If you configure absolute paths to files named `id_rsa` and/or `id_ecdsa` then SFTPGo will try to generate these keys using the default settings.
-  - `kex_algorithms`, list of strings. Available KEX (Key Exchange) algorithms in preference order. Leave empty to use default values. The supported values can be found here: [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/common.go#L46 "Supported kex algos")
-  - `ciphers`, list of strings. Allowed ciphers. Leave empty to use default values. The supported values can be found here: [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/common.go#L28 "Supported ciphers")
-  - `macs`, list of strings. Available MAC (message authentication code) algorithms in preference order. Leave empty to use default values. The supported values can be found here: [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/common.go#L84 "Supported MACs")
-  - `trusted_user_ca_keys`, list of public keys paths of certificate authorities that are trusted to sign user certificates for authentication. The paths can be absolute or relative to the configuration directory.
-  - `login_banner_file`, path to the login banner file. The contents of the specified file, if any, are sent to the remote user before authentication is allowed. It can be a path relative to the config dir or an absolute one. Leave empty to disable login banner.
-  - `setstat_mode`, integer. 0 means "normal mode": requests for changing permissions, owner/group and access/modification times are executed. 1 means "ignore mode": requests for changing permissions, owner/group and access/modification times are silently ignored.
-  - `enabled_ssh_commands`, list of enabled SSH commands. `*` enables all supported commands. More information can be found [here](./ssh-commands.md).
-  - `keyboard_interactive_auth_program`, string. Deprecated, please use `keyboard_interactive_auth_hook`.
-  - `keyboard_interactive_auth_hook`, string. Absolute path to an external program or an HTTP URL to invoke for keyboard interactive authentication. See the "Keyboard Interactive Authentication" paragraph for more details.
+  - `setstat_mode`, integer. 0 means "normal mode": requests for changing permissions, owner/group and access/modification times are executed. 1 means "ignore mode": requests for changing permissions, owner/group and access/modification times are silently ignored. 2 means "ignore mode for cloud based filesystems": requests for changing permissions, owner/group and access/modification times are silently ignored for cloud filesystems and executed for local filesystem.
   - `proxy_protocol`, integer. Support for [HAProxy PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt). If you are running SFTPGo behind a proxy server such as HAProxy, AWS ELB or NGNIX, you can enable the proxy protocol. It provides a convenient way to safely transport connection information such as a client's address across multiple layers of NAT or TCP proxies to get the real client IP address instead of the proxy IP. Both protocol versions 1 and 2 are supported. If the proxy protocol is enabled in SFTPGo then you have to enable the protocol in your proxy configuration too. For example, for HAProxy, add `send-proxy` or `send-proxy-v2` to each server configuration line. The following modes are supported:
     - 0, disabled
     - 1, enabled. Proxy header will be used and requests without proxy header will be accepted
@@ -72,6 +62,57 @@ The configuration file contains the following sections:
   - `proxy_allowed`, List of IP addresses and IP ranges allowed to send the proxy header:
     - If `proxy_protocol` is set to 1 and we receive a proxy header from an IP that is not in the list then the connection will be accepted and the header will be ignored
     - If `proxy_protocol` is set to 2 and we receive a proxy header from an IP that is not in the list then the connection will be rejected
+  - `post_connect_hook`, string. Absolute path to the command to execute or HTTP URL to notify. See [Post connect hook](./post-connect-hook.md) for more details. Leave empty to disable
+- **"sftpd"**, the configuration for the SFTP server
+  - `bind_port`, integer. The port used for serving SFTP requests. Default: 2022
+  - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: ""
+  - `idle_timeout`, integer. Deprecated, please use the same key in `common` section.
+  - `max_auth_tries` integer. Maximum number of authentication attempts permitted per connection. If set to a negative number, the number of attempts is unlimited. If set to zero, the number of attempts is limited to 6.
+  - `banner`, string. Identification string used by the server. Leave empty to use the default banner. Default `SFTPGo_<version>`, for example `SSH-2.0-SFTPGo_0.9.5`
+  - `upload_mode` integer. Deprecated, please use the same key in `common` section.
+  - `actions`, struct. Deprecated, please use the same key in `common` section.
+  - `keys`, struct array. Deprecated, please use `host_keys`.
+    - `private_key`, path to the private key file. It can be a path relative to the config dir or an absolute one.
+  - `host_keys`, list of strings. It contains the daemon's private host keys. Each host key can be defined as a path relative to the configuration directory or an absolute one. If empty, the daemon will search or try to generate `id_rsa`, `id_ecdsa` and `id_ed25519` keys inside the configuration directory. If you configure absolute paths to files named `id_rsa`, `id_ecdsa` and/or `id_ed25519` then SFTPGo will try to generate these keys using the default settings.
+  - `kex_algorithms`, list of strings. Available KEX (Key Exchange) algorithms in preference order. Leave empty to use default values. The supported values can be found here: [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/common.go#L46 "Supported kex algos")
+  - `ciphers`, list of strings. Allowed ciphers. Leave empty to use default values. The supported values can be found here: [crypto/ssh](https://github.com/golang/crypto/blob/master/ssh/common.go#L28 "Supported ciphers")
+  - `macs`, list of strings. Available MAC (message authentication code) algorithms in preference order. Leave empty to use default values. The supported values can be found here: [crypto/ssh](https://github.com/golang/crypto/blob/master/ssh/common.go#L84 "Supported MACs")
+  - `trusted_user_ca_keys`, list of public keys paths of certificate authorities that are trusted to sign user certificates for authentication. The paths can be absolute or relative to the configuration directory.
+  - `login_banner_file`, path to the login banner file. The contents of the specified file, if any, are sent to the remote user before authentication is allowed. It can be a path relative to the config dir or an absolute one. Leave empty to disable login banner.
+  - `setstat_mode`, integer. Deprecated, please use the same key in `common` section.
+  - `enabled_ssh_commands`, list of enabled SSH commands. `*` enables all supported commands. More information can be found [here](./ssh-commands.md).
+  - `keyboard_interactive_auth_hook`, string. Absolute path to an external program or an HTTP URL to invoke for keyboard interactive authentication. See [Keyboard Interactive Authentication](./keyboard-interactive.md) for more details.
+  - `password_authentication`, boolean. Set to false to disable password authentication. This setting will disable multi-step authentication method using public key + password too. It is useful for public key only configurations if you need to manage old clients that will not attempt to authenticate with public keys if the password login method is advertised. Default: true.
+  - `proxy_protocol`, integer.  Deprecated, please use the same key in `common` section.
+  - `proxy_allowed`, list of strings. Deprecated, please use the same key in `common` section.
+- **"ftpd"**, the configuration for the FTP server
+  - `bind_port`, integer. The port used for serving FTP requests. 0 means disabled. Default: 0.
+  - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "".
+  - `banner`, string. Greeting banner displayed when a connection first comes in. Leave empty to use the default banner. Default `SFTPGo <version> ready`, for example `SFTPGo 1.0.0-dev ready`.
+  - `banner_file`, path to the banner file. The contents of the specified file, if any, are displayed when someone connects to the server. It can be a path relative to the config dir or an absolute one. If set, it overrides the banner string provided by the `banner` option. Leave empty to disable.
+  - `active_transfers_port_non_20`, boolean. Do not impose the port 20 for active data transfers. Enabling this option allows to run SFTPGo with less privilege. Default: false.
+  - `force_passive_ip`, ip address. External IP address to expose for passive connections. Leavy empty to autodetect. Defaut: "".
+  - `passive_port_range`, struct containing the key `start` and `end`. Port Range for data connections. Random if not specified. Default range is 50000-50100.
+  - `certificate_file`, string. Certificate for FTPS. This can be an absolute path or a path relative to the config dir.
+  - `certificate_key_file`, string. Private key matching the above certificate. This can be an absolute path or a path relative to the config dir. If both the certificate and the private key are provided the server will accept both plain FTP an explicit FTP over TLS. Certificate and key files can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows.
+  - `tls_mode`, integer. 0 means accept both cleartext and encrypted sessions. 1 means TLS is required for both control and data connection. Do not enable this blindly, please check that a proper TLS config is in place or no login will be allowed if `tls_mode` is 1.
+- **webdavd**, the configuration for the WebDAV server, more info [here](./webdav.md)
+  - `bind_port`, integer. The port used for serving WebDAV requests. 0 means disabled. Default: 0.
+  - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "".
+  - `certificate_file`, string. Certificate for WebDAV over HTTPS. This can be an absolute path or a path relative to the config dir.
+  - `certificate_key_file`, string. Private key matching the above certificate. This can be an absolute path or a path relative to the config dir. If both the certificate and the private key are provided the server will expect HTTPS connections. Certificate and key files can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows.
+  - `cors` struct containing CORS configuration. SFTPGo uses [Go CORS handler](https://github.com/rs/cors), please refer to upstream documentation for fields meaning and their default values.
+    - `enabled`, boolean, set to true to enable CORS.
+    - `allowed_origins`, list of strings.
+    - `allowed_methods`, list of strings.
+    - `allowed_headers`, list of strings.
+    - `exposed_headers`, list of strings.
+    - `allow_credentials` boolean.
+    - `max_age`, integer.
+  - `cache` struct containing cache configuration for the authenticated users.
+    - `enabled`, boolean, set to true to enable user caching. Default: true.
+    - `expiration_time`, integer. Expiration time, in minutes, for the cached users. 0 means unlimited. Default: 0.
+    - `max_size`, integer. Maximum number of users to cache. 0 means unlimited. Default: 50.
 - **"data_provider"**, the configuration for the data provider
   - `driver`, string. Supported drivers are `sqlite`, `mysql`, `postgresql`, `bolt`, `memory`
   - `name`, string. Database name. For driver `sqlite` this can be the database name relative to the config dir or the absolute path to the SQLite database. For driver `memory` this is the (optional) path relative to the config dir or the absolute path to the users dump, obtained using the `dumpdata` REST API, to load. This dump will be loaded at startup and can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. The `memory` provider will not modify the provided file so quota usage and last login will not be persisted
@@ -89,17 +130,26 @@ The configuration file contains the following sections:
     - 2, quota is updated each time a user uploads or deletes a file, but only for users with quota restrictions and for virtual folders. With this configuration, the `quota scan` and `folder_quota_scan` REST API can still be used to periodically update space usage for users without quota restrictions and for folders
   - `pool_size`, integer. Sets the maximum number of open connections for `mysql` and `postgresql` driver. Default 0 (unlimited)
   - `users_base_dir`, string. Users default base directory. If no home dir is defined while adding a new user, and this value is a valid absolute path, then the user home dir will be automatically defined as the path obtained joining the base dir and the username
-  - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions. See the "Custom Actions" paragraph for more details
+  - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions. See [Custom Actions](./custom-actions.md) for more details
     - `execute_on`, list of strings. Valid values are `add`, `update`, `delete`. `update` action will not be fired for internal updates such as the last login or the user quota fields.
-    - `command`, string. Deprecated please use `hook`.
-    - `http_notification_url`, a valid URL. Deprecated please use `hook`.
     - `hook`, string. Absolute path to the command to execute or HTTP URL to notify.
   - `external_auth_program`, string. Deprecated, please use `external_auth_hook`.
-  - `external_auth_hook`, string. Absolute path to an external program or an HTTP URL to invoke for users authentication. See the "External Authentication" paragraph for more details. Leave empty to disable.
-  - `external_auth_scope`, integer. 0 means all supported authetication scopes (passwords, public keys and keyboard interactive). 1 means passwords only. 2 means public keys only. 4 means key keyboard interactive only. The flags can be combined, for example 6 means public keys and keyboard interactive
+  - `external_auth_hook`, string. Absolute path to an external program or an HTTP URL to invoke for users authentication. See [External Authentication](./external-auth.md) for more details. Leave empty to disable.
+  - `external_auth_scope`, integer. 0 means all supported authentication scopes (passwords, public keys and keyboard interactive). 1 means passwords only. 2 means public keys only. 4 means key keyboard interactive only. The flags can be combined, for example 6 means public keys and keyboard interactive
   - `credentials_path`, string. It defines the directory for storing user provided credential files such as Google Cloud Storage credentials. This can be an absolute path or a path relative to the config dir
+  - `prefer_database_credentials`, boolean. When true, users' Google Cloud Storage credentials will be written to the data provider instead of disk, though pre-existing credentials on disk will be used as a fallback. When false, they will be written to the directory specified by `credentials_path`.
   - `pre_login_program`, string. Deprecated, please use `pre_login_hook`.
-  - `pre_login_hook`, string. Absolute path to an external program or an HTTP URL to invoke to modify user details just before the login. See the "Dynamic user modification" paragraph for more details. Leave empty to disable.
+  - `pre_login_hook`, string. Absolute path to an external program or an HTTP URL to invoke to modify user details just before the login. See [Dynamic user modification](./dynamic-user-mod.md) for more details. Leave empty to disable.
+  - `post_login_hook`, string. Absolute path to an external program or an HTTP URL to invoke to notify a successful or failed login. See [Post-login hook](./post-login-hook.md) for more details. Leave empty to disable.
+  - `post_login_scope`, defines the scope for the post-login hook. 0 means notify both failed and successful logins. 1 means notify failed logins. 2 means notify successful logins.
+  - `check_password_hook`, string.  Absolute path to an external program or an HTTP URL to invoke to check the user provided password. See [Check password hook](./check-password-hook.md) for more details. Leave empty to disable.
+  - `check_password_scope`, defines the scope for the check password hook. 0 means all protocols, 1 means SSH, 2 means FTP, 4 means WebDAV. You can combine the scopes, for example 6 means FTP and WebDAV.
+  - `password_hashing`, struct. It contains the configuration parameters to be used to generate the password hash. SFTPGo can verify passwords in several formats and uses the `argon2id` algorithm to hash passwords in plain-text before storing them inside the data provider. These options allow you to customize how the hash is generated.
+    - `argon2_options` struct containing the options for argon2id hashing algorithm. The `memory` and `iterations` parameters control the computational cost of hashing the password. The higher these figures are, the greater the cost of generating the hash and the longer the runtime. It also follows that the greater the cost will be for any attacker trying to guess the password. If the code is running on a machine with multiple cores, then you can decrease the runtime without reducing the cost by increasing the `parallelism` parameter. This controls the number of threads that the work is spread across.
+      - `memory`, unsigned integer. The amount of memory used by the algorithm (in kibibytes). Default: 65536.
+      - `iterations`, unsigned integer. The number of iterations over the memory. Default: 1.
+      - `parallelism`. unsigned 8 bit integer. The number of threads (or lanes) used by the algorithm. Default: 2.
+  - `update_mode`, integer. Defines how the database will be initialized/updated. 0 means automatically. 1 means manually using the initprovider sub-command.
 - **"httpd"**, the configuration for the HTTP server used to serve REST API and to expose the built-in web interface
   - `bind_port`, integer. The port used for serving HTTP requests. Set to 0 to disable HTTP server. Default: 8080
   - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "127.0.0.1"
@@ -116,7 +166,7 @@ The configuration file contains the following sections:
 
 A full example showing the default config (in JSON format) can be found [here](../sftpgo.json).
 
-If you want to use a private host key that use an algorithm/setting different from the auto generated RSA/ECDSA keys, or more than two private keys, you can generate your own keys and replace the empty `keys` array with something like this:
+If you want to use a private host key that uses an algorithm/setting different from the auto generated RSA/ECDSA keys, or more than two private keys, you can generate your own keys and replace the empty `keys` array with something like this:
 
 ```json
 "host_keys": [
@@ -126,18 +176,19 @@ If you want to use a private host key that use an algorithm/setting different fr
 ]
 ```
 
-where `id_rsa`, `id_ecdsa` and `id_ed25519`, in this example, are files containing your generated keys. You can use absolute paths or paths relative to the configuration directory.
+where `id_rsa`, `id_ecdsa` and `id_ed25519`, in this example, are files containing your generated keys. You can use absolute paths or paths relative to the configuration directory specified via the `--config-dir` serve flag. By default the configuration directory is the working directory.
 
-If you want the default host keys generation in a directory different from the config dir, please specify absolute paths to files named `id_rsa` or `id_ecdsa` like this:
+If you want the default host keys generation in a directory different from the config dir, please specify absolute paths to files named `id_rsa`, `id_ecdsa` or `id_ed25519` like this:
 
 ```json
 "host_keys": [
   "/etc/sftpgo/keys/id_rsa",
-  "/etc/sftpgo/keys/id_ecdsa"
+  "/etc/sftpgo/keys/id_ecdsa",
+  "/etc/sftpgo/keys/id_ed25519"
 ]
 ```
 
-then SFTPGo will try to create `id_rsa` and `id_ecdsa`, if they are missing, inside the existing directory `/etc/sftpgo/keys`.
+then SFTPGo will try to create `id_rsa`, `id_ecdsa` and `id_ed25519`, if they are missing, inside the directory `/etc/sftpgo/keys`.
 
 The configuration can be read from JSON, TOML, YAML, HCL, envfile and Java properties config files. If your `config-file` flag is set to `sftpgo` (default value), you need to create a configuration file called `sftpgo.json` or `sftpgo.yaml` and so on inside `config-dir`.
 
@@ -148,6 +199,4 @@ You can also override all the available configuration options using environment
 Let's see some examples:
 
 - To set sftpd `bind_port`, you need to define the env var `SFTPGO_SFTPD__BIND_PORT`
-- To set the `execute_on` actions, you need to define the env var `SFTPGO_SFTPD__ACTIONS__EXECUTE_ON`. For example `SFTPGO_SFTPD__ACTIONS__EXECUTE_ON=upload,download`
-
-Please note that, to override configuration options with environment variables, a configuration file containing the options to override is required. You can, for example, deploy the default configuration file and then override the options you need to customize using environment variables.
+- To set the `execute_on` actions, you need to define the env var `SFTPGO_COMMON__ACTIONS__EXECUTE_ON`. For example `SFTPGO_COMMON__ACTIONS__EXECUTE_ON=upload,download`

docs/google-cloud-storage.md

@@ -8,6 +8,4 @@ You can optionally specify a [storage class](https://cloud.google.com/storage/do
 
 The configured bucket must exist.
 
-Google Cloud Storage is exposed over HTTPS so if you are running SFTPGo as docker image please be sure to uncomment the line that install `ca-certificates`, inside your `Dockerfile`, to be able to properly verify certificate authorities.
-
 This backend is very similar to the [S3](./s3.md) backend, and it has the same limitations.

docs/howto/README.md

@@ -0,0 +1,6 @@
+# Tutorials
+
+Here we collect step-to-step tutorials. SFTPGo users are encouraged to contribute!
+
+- [SFTPGo with PostgreSQL data provider and S3 backend](./postgresql-s3.md)
+- [Expose Web Admin and REST API over HTTPS and password protected](./rest-api-https-auth.md)

docs/howto/postgresql-s3.md

@@ -0,0 +1,215 @@
+# SFTPGo with PostgreSQL data provider and S3 backend
+
+This tutorial shows the installation of SFTPGo on Ubuntu 20.04 (Focal Fossa) with PostgreSQL data provider and S3 backend. SFTPGo will run as an unprivileged (non-root) user. We assume that you want to serve a single S3 bucket and you want to assign different "virtual folders" of this bucket to different SFTPGo virtual users.
+
+## Preliminary Note
+
+Before proceeding further you need to have a basic minimal installation of Ubuntu 20.04.
+
+## Install PostgreSQL
+
+Before installing any packages on the Ubuntu system, update and upgrade all packages using the `apt` commands below.
+
+```shell
+sudo apt update
+sudo apt upgrade
+```
+
+Install PostgreSQL with this `apt` command.
+
+```shell
+sudo apt -y install postgresql
+```
+
+Once installation is completed, start the PostgreSQL service and add it to the system boot.
+
+```shell
+sudo systemctl start postgresql
+sudo systemctl enable postgresql
+```
+
+Next, check the PostgreSQL service using the following command.
+
+```shell
+systemctl status postgresql
+```
+
+## Configure PostgreSQL
+
+PostgreSQL uses roles for user authentication and authorization, it just like Unix-Style permissions. By default, PostgreSQL creates a new user called `postgres` for basic authentication.
+
+In this step, we will create a new PostgreSQL user for SFTPGo.
+
+Login to the PostgreSQL shell using the command below.
+
+```shell
+sudo -i -u postgres psql
+```
+
+Next, create a new role `sftpgo` with the password `sftpgo_pg_pwd` using the following query.
+
+```sql
+create user "sftpgo" with encrypted password 'sftpgo_pg_pwd';
+```
+
+Next, create a new database `sftpgo.db` for the SFTPGo service using the following queries.
+
+```sql
+create database "sftpgo.db";
+grant all privileges on database "sftpgo.db" to "sftpgo";
+```
+
+Exit from the PostgreSQL shell typing `\q`.
+
+## Install SFTPGo
+
+To install SFTPGo you can use the PPA [here](https://launchpad.net/~sftpgo/+archive/ubuntu/sftpgo).
+
+Start by adding the PPA.
+
+```shell
+sudo add-apt-repository ppa:sftpgo/sftpgo
+sudo apt-get update
+```
+
+Next install SFTPGo.
+
+```shell
+sudo apt install sftpgo
+```
+
+After installation SFTPGo should already be running with default configuration and configured to start automatically at boot, check its status using the following command.
+
+```shell
+systemctl status sftpgo
+```
+
+## Configure AWS credentials
+
+We assume that you want to serve a single S3 bucket and you want to assign different "virtual folders" of this bucket to different SFTPGo virtual users. In this case is very convenient to configure a credential file so SFTPGo will automatically use it and you don't need to specify the same AWS credentials for each user.
+
+You can manually create the `/var/lib/sftpgo/.aws/credentials` file and write your AWS credentials like this.
+
+```shell
+[default]
+aws_access_key_id=AKIAIOSFODNN7EXAMPLE
+aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+```
+
+Alternately you can install `AWS CLI` and manage the credential using this tool.
+
+```shell
+sudo apt install awscli
+```
+
+and now set your credentials, region, and output format with the following command.
+
+```shell
+aws configure
+```
+
+Confirm that you can list your bucket contents with the following command.
+
+```shell
+aws s3 ls s3://mybucket
+```
+
+The AWS CLI will create the credential file in `~/.aws/credentials`. The SFTPGo service runs using the `sftpgo` system user whose home directory is `/var/lib/sftpgo` so you need to copy the credentials file to the sftpgo home directory and assign it the proper permissions.
+
+```shell
+sudo mkdir /var/lib/sftpgo/.aws
+sudo cp ~/.aws/credentials /var/lib/sftpgo/.aws/
+sudo chown -R sftpgo:sftpgo /var/lib/sftpgo/.aws
+```
+
+## Configure SFTPGo
+
+Now open the SFTPGo configuration.
+
+```shell
+sudo vi /etc/sftpgo/sftpgo.json
+```
+
+Search for the `data_provider` section and change it as follow.
+
+```json
+  "data_provider": {
+    "driver": "postgresql",
+    "name": "sftpgo.db",
+    "host": "127.0.0.1",
+    "port": 5432,
+    "username": "sftpgo",
+    "password": "sftpgo_pg_pwd",
+    ...
+}
+```
+
+This way we set the PostgreSQL connection parameters.
+
+If you want to connect to PostgreSQL over a Unix Domain socket you have to set the value `/var/run/postgresql` for the `host` configuration key instead of `127.0.0.1`.
+
+You can further customize your configuration adding custom actions and other hooks. A full explanation of all configuration parameters can be found [here](../full-configuration.md).
+
+Next, initialize the data provider with the following command.
+
+```shell
+$ sudo su - sftpgo -s /bin/bash -c 'sftpgo initprovider -c /etc/sftpgo'
+2020-10-09T21:07:50.000 INF Initializing provider: "postgresql" config file: "/etc/sftpgo/sftpgo.json"
+2020-10-09T21:07:50.000 INF updating database version: 1 -> 2
+2020-10-09T21:07:50.000 INF updating database version: 2 -> 3
+2020-10-09T21:07:50.000 INF updating database version: 3 -> 4
+2020-10-09T21:07:50.000 INF Data provider successfully initialized/updated
+```
+
+The default sftpgo systemd service will start after the network target, in this setup it is more appropriate to start it after the PostgreSQL service, so edit the service using the following command.
+
+```shell
+sudo systemctl edit sftpgo.service
+```
+
+And override the unit definition with the following snippet.
+
+```shell
+[Unit]
+After=postgresql.service
+```
+
+Confirm that `sftpgo.service` will start after `postgresql.service` with the next command.
+
+```shell
+$ systemctl show sftpgo.service | grep After=
+After=postgresql.service systemd-journald.socket system.slice -.mount systemd-tmpfiles-setup.service network.target sysinit.target basic.target
+```
+
+Next restart the sftpgo service to use the new configuration and check that it is running.
+
+```shell
+sudo systemctl restart sftpgo
+systemctl status sftpgo
+```
+
+## Add virtual users
+
+The easiest way to add virtual users is to use the built-in Web interface.
+
+You can expose the Web Admin interface over the network replacing `"bind_address": "127.0.0.1"` in the `httpd` configuration section with `"bind_address": ""` and apply the change restarting the SFTPGo service with the following command.
+
+```shell
+sudo systemctl restart sftpgo
+```
+
+So now open the Web Admin URL.
+
+[http://127.0.0.1:8080/web](http://127.0.0.1:8080/web)
+
+Click `Add` and fill the user details, the minimum required parameters are:
+
+- `Username`
+- `Password` or `Public keys`
+- `Permissions`
+- `Home Dir` can be empty since we defined a default base dir
+- Select `AWS S3 (Compatible)` as storage and then set `Bucket`, `Region` and optionally a `Key Prefix` if you want to restrict the user to a specific virtual folder in the bucket. The specified virtual folder does not need to be pre-created. You can leave `Access Key` and `Access Secret` empty since we defined global credentials for the `sftpgo` user and we use this system user to run the SFTPGo service.
+
+You are done! Now you can connect to you SFTPGo instance using any compatible `sftp` client on port `2022`.
+
+You can mix S3 users with local users but please be aware that we are running the service as the unprivileged `sftpgo` system user so if you set storage as `local` for an SFTPGo virtual user then the home directory for this user must be owned by the `sftpgo` system user. If you don't specify an home directory the default will be `/srv/sftpgo/data/<username>` which should be appropriate.

docs/howto/rest-api-https-auth.md

@@ -0,0 +1,122 @@
+# Expose Web Admin and REST API over HTTPS and password protected
+
+This tutorial shows how to expose the SFTPGo web interface and REST API over HTTPS and password protect them.
+
+## Preliminary Note
+
+Before proceeding further you need to have a SFTPGo instance already configured and running.
+
+We assume:
+
+- you are running SFTPGo as service using the dedicated `sftpgo` system user
+- the SFTPGo configuration directory is `/etc/sftpgo`
+- you are running SFTPGo on Ubuntu 20.04, however this instructions can be easily adapted for other Linux variants.
+
+## Authentication Setup
+
+First install the `htpasswd` tool. We use this tool to create the users for the Web Admin/REST API.
+
+```shell
+sudo apt install apache2-utils
+```
+
+Create a user for web based authentication.
+
+```shell
+sudo htpasswd -B -c /etc/sftpgo/httpauth sftpgoweb
+```
+
+If you want to create additional users omit the `-c` option.
+
+```shell
+sudo htpasswd -B /etc/sftpgo/httpauth anotheruser
+```
+
+Next open the SFTPGo configuration.
+
+```shell
+sudo vi /etc/sftpgo/sftpgo.json
+```
+
+Search for the `httpd` section and change it as follow.
+
+```json
+  "httpd": {
+    "bind_port": 8080,
+    "bind_address": "",
+    "templates_path": "templates",
+    "static_files_path": "static",
+    "backups_path": "backups",
+    "auth_user_file": "/etc/sftpgo/httpauth",
+    "certificate_file": "",
+    "certificate_key_file": ""
+  }
+```
+
+Setting an empty `bind_address` means that the service will listen on all available network interfaces and so it will be exposed over the network.
+
+Now restart the SFTPGo service to apply the changes.
+
+```shell
+sudo systemctl restart sftpgo
+```
+
+You are done! Now login to the Web Admin interface using the username and password created above.
+
+## Creation of a Self-Signed Certificate
+
+For demostration purpose we use a self-signed certificate here. These certificates are easy to make and do not cost money. However, they do not provide all of the security properties that certificates signed by a Public Certificate Authority (CA) aim to provide, you are encouraged to use a certificate signed by a Public CA.
+
+When creating a new SSL certificate, one needs to specify the duration validity of the same by changing the value 365 (as appearing in the message below) to the preferred number of days. It is important to mention here that the certificate so created stands to auto-expire upon completion of one year.
+
+```shell
+sudo mkdir /etc/sftpgo/ssl
+sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/sftpgo/ssl/sftpgo.key -out /etc/sftpgo/ssl/sftpgo.crt
+```
+
+The above command is rather versatile, and lets you create both the self-signed SSL certificate and the server key to safeguard it, in addition to placing both of these into the `etc/sftpgo/ssl` directory. Answer to the questions to create the certificate and the key for HTTPS.
+
+Assign the proper permissions to the generated certificates.
+
+```shell
+sudo chown -R sftpgo:sftpgo /etc/sftpgo/ssl
+```
+
+## HTTPS Setup
+
+Open the SFTPGo configuration.
+
+```shell
+sudo vi /etc/sftpgo/sftpgo.json
+```
+
+Search for the `httpd` section and change it as follow.
+
+```json
+  "httpd": {
+    "bind_port": 8080,
+    "bind_address": "",
+    "templates_path": "templates",
+    "static_files_path": "static",
+    "backups_path": "backups",
+    "auth_user_file": "/etc/sftpgo/httpauth",
+    "certificate_file": "/etc/sftpgo/ssl/sftpgo.crt",
+    "certificate_key_file": "/etc/sftpgo/ssl/sftpgo.key"
+  }
+```
+
+Now restart the SFTPGo service to apply the changes.
+
+```shell
+sudo systemctl restart sftpgo
+```
+
+You are done! Now SFTPGo web admin and REST API are exposed over HTTPS and password protected.
+
+You can easily replace the self-signed certificate used here with a properly signed certificate.
+
+The certificate could frequently change if you use something like [let's encrypt](https://letsencrypt.org/). SFTPGo allows hot-certificate reloading using the following command.
+
+```shell
+sudo systemctl reload sftpgo
+```

docs/keyboard-interactive.md

@@ -9,6 +9,7 @@ To enable keyboard interactive authentication, you must set the absolute path of
 The external program can read the following environment variables to get info about the user trying to authenticate:
 
 - `SFTPGO_AUTHD_USERNAME`
+- `SFTPGO_AUTHD_IP`
 - `SFTPGO_AUTHD_PASSWORD`, this is the hashed password as stored inside the data provider
 
 Previous global environment variables aren't cleared when the script is called. The content of these variables is _not_ quoted. They may contain special characters.
@@ -77,13 +78,14 @@ The request body will contain a JSON struct with the following fields:
 
 - `request_id`, string. Unique request identifier
 - `username`, string
+- `ip`, string
 - `password`, string. This is the hashed password as stored inside the data provider
 - `answers`, list of string. It will be null for the first request
-- `questions`, list of string. It will contains the previous asked questions. It will be null for the first request
+- `questions`, list of string. It will contain the previously asked questions. It will be null for the first request
 
 The HTTP response code must be 200 and the body must contain the same JSON struct described for the program.
 
-Let's see a basic sample, the configured hook is `http://127.0.0.1:8000/keyIntHookPwd`, as soon as the user try to login, SFTPGo makes this HTTP POST request:
+Let's see a basic sample, the configured hook is `http://127.0.0.1:8000/keyIntHookPwd`, as soon as the user tries to login, SFTPGo makes this HTTP POST request:
 
 ```shell
 POST /keyIntHookPwd HTTP/1.1
@@ -93,7 +95,7 @@ Content-Length: 189
 Content-Type: application/json
 Accept-Encoding: gzip
 
-{"request_id":"bq1r5r7cdrpd2qtn25ng","username":"a","password":"$pbkdf2-sha512$150000$ClOPkLNujMTL$XktKy0xuJsOfMYBz+f2bIyPTdbvDTSnJ1q+7+zp/HPq5Qojwp6kcpSIiVHiwvbi8P6HFXI/D3UJv9BLcnQFqPA=="}
+{"request_id":"bq1r5r7cdrpd2qtn25ng","username":"a","ip":"127.0.0.1","password":"$pbkdf2-sha512$150000$ClOPkLNujMTL$XktKy0xuJsOfMYBz+f2bIyPTdbvDTSnJ1q+7+zp/HPq5Qojwp6kcpSIiVHiwvbi8P6HFXI/D3UJv9BLcnQFqPA=="}
 ```
 
 as you can see in this first requests `answers` and `questions` are null.
@@ -121,10 +123,10 @@ Content-Length: 233
 Content-Type: application/json
 Accept-Encoding: gzip
 
-{"request_id":"bq1r5r7cdrpd2qtn25ng","username":"a","password":"$pbkdf2-sha512$150000$ClOPkLNujMTL$XktKy0xuJsOfMYBz+f2bIyPTdbvDTSnJ1q+7+zp/HPq5Qojwp6kcpSIiVHiwvbi8P6HFXI/D3UJv9BLcnQFqPA==","answers":["OK"],"questions":["Password: "]}
+{"request_id":"bq1r5r7cdrpd2qtn25ng","username":"a","ip":"127.0.0.1","password":"$pbkdf2-sha512$150000$ClOPkLNujMTL$XktKy0xuJsOfMYBz+f2bIyPTdbvDTSnJ1q+7+zp/HPq5Qojwp6kcpSIiVHiwvbi8P6HFXI/D3UJv9BLcnQFqPA==","answers":["OK"],"questions":["Password: "]}
 ```
 
-Here is the HTTP response that istructs SFTPGo to ask for a new question:
+Here is the HTTP response that instructs SFTPGo to ask for a new question:
 
 ```shell
 HTTP/1.1 200 OK
@@ -147,7 +149,7 @@ Content-Length: 239
 Content-Type: application/json
 Accept-Encoding: gzip
 
-{"request_id":"bq1r5r7cdrpd2qtn25ng","username":"a","password":"$pbkdf2-sha512$150000$ClOPkLNujMTL$XktKy0xuJsOfMYBz+f2bIyPTdbvDTSnJ1q+7+zp/HPq5Qojwp6kcpSIiVHiwvbi8P6HFXI/D3UJv9BLcnQFqPA==","answers":["answer2"],"questions":["Question2: "]}
+{"request_id":"bq1r5r7cdrpd2qtn25ng","username":"a","ip":"127.0.0.1","password":"$pbkdf2-sha512$150000$ClOPkLNujMTL$XktKy0xuJsOfMYBz+f2bIyPTdbvDTSnJ1q+7+zp/HPq5Qojwp6kcpSIiVHiwvbi8P6HFXI/D3UJv9BLcnQFqPA==","answers":["answer2"],"questions":["Question2: "]}
 ```
 
 Here is the final HTTP response that allows the user login:
@@ -162,3 +164,5 @@ Content-Length: 18
 
 {"auth_result": 1}
 ```
+
+An example keyboard interactive program allowing to authenticate using [Twilio Authy 2FA](https://www.twilio.com/docs/authy) can be found inside the source tree [authy](../examples/OTP/authy) directory.

docs/logs.md

@@ -20,7 +20,7 @@ The logs can be divided into the following categories:
   - `connection_id` string. Unique connection identifier
   - `protocol` string. `SFTP` or `SCP`
 - **"command logs"**, SFTP/SCP command logs:
-  - `sender` string. `Rename`, `Rmdir`, `Mkdir`, `Symlink`, `Remove`, `Chmod`, `Chown`, `Chtimes`, `SSHCommand`
+  - `sender` string. `Rename`, `Rmdir`, `Mkdir`, `Symlink`, `Remove`, `Chmod`, `Chown`, `Chtimes`, `Truncate`, `SSHCommand`
   - `level` string
   - `username`, string
   - `file_path` string
@@ -30,6 +30,7 @@ The logs can be divided into the following categories:
   - `gid` integer. Valid for sender `Chown` otherwise -1
   - `access_time` datetime as YYYY-MM-DDTHH:MM:SS. Valid for sender `Chtimes` otherwise empty
   - `modification_time` datetime as YYYY-MM-DDTHH:MM:SS. Valid for sender `Chtimes` otherwise empty
+  - `size` int64. Valid for sender `Truncate` otherwise -1
   - `ssh_command`, string. Valid for sender `SSHCommand` otherwise empty
   - `connection_id` string. Unique connection identifier
   - `protocol` string. `SFTP`, `SCP` or `SSH`
@@ -50,5 +51,6 @@ The logs can be divided into the following categories:
   - `level` string
   - `username`, string. Can be empty if the connection is closed before an authentication attempt
   - `client_ip` string.
-  - `login_type` string. Can be `publickey`, `password`, `keyboard-interactive` or `no_auth_tryed`
+  - `protocol` string. Possible values are `SSH`, `FTP`, `DAV`
+  - `login_type` string. Can be `publickey`, `password`, `keyboard-interactive`, `publickey+password`, `publickey+keyboard-interactive` or `no_auth_tryed`
   - `error` string. Optional error description

docs/performance.md

@@ -32,7 +32,7 @@ Ethernet| Mellanox ConnectX-3 40GbE|
 ### Test configurations
 
 - `Baseline`: SFTPGo version 0.9.6.
-- `Devel`: SFTPGo commit b0ed1905918b9dcc22f9a20e89e354313f491734, compiled with Golang 1.14.2 .
+- `Devel`: SFTPGo commit b0ed1905918b9dcc22f9a20e89e354313f491734, compiled with Golang 1.14.2. This is basically the same as v1.0.0 as far as performance is concerned.
 - `Optimized`: Various [optimizations](#Optimizations-applied) applied on top of `Devel`.
 - `Balanced`: Two optimized instances, running on localhost, load balanced by HAProxy 2.1.3.
 - `OpenSSH`: OpenSSH_7.9p1 Debian-10+deb10u2, OpenSSL 1.1.1d  10 Sep 2019

docs/portable-mode.md

@@ -4,9 +4,10 @@ SFTPGo allows to share a single directory on demand using the `portable` subcomm
 
 ```console
 sftpgo portable --help
-To serve the current working directory with auto generated credentials simply use:
+To serve the current working directory with auto generated credentials simply
+use:
 
-sftpgo portable
+$ sftpgo portable
 
 Please take a look at the usage below to customize the serving parameters
 
@@ -14,40 +15,97 @@ Usage:
   sftpgo portable [flags]
 
 Flags:
-  -C, --advertise-credentials            If the SFTP service is advertised via multicast DNS, this flag allows to put username/password inside the advertised TXT record
-  -S, --advertise-service                Advertise SFTP service using multicast DNS
-      --allowed-extensions stringArray   Allowed file extensions case insensitive. The format is /dir::ext1,ext2. For example: "/somedir::.jpg,.png"
-      --denied-extensions stringArray    Denied file extensions case insensitive. The format is /dir::ext1,ext2. For example: "/somedir::.jpg,.png"
-  -d, --directory string                 Path to the directory to serve. This can be an absolute path or a path relative to the current directory (default ".")
-  -f, --fs-provider int                  0 means local filesystem, 1 Amazon S3 compatible, 2 Google Cloud Storage
-      --gcs-automatic-credentials int    0 means explicit credentials using a JSON credentials file, 1 automatic (default 1)
+  -C, --advertise-credentials           If the SFTP/FTP service is
+                                        advertised via multicast DNS, this
+                                        flag allows to put username/password
+                                        inside the advertised TXT record
+  -S, --advertise-service               Advertise SFTP/FTP service using
+                                        multicast DNS
+      --allowed-patterns stringArray    Allowed file patterns case insensitive.
+                                        The format is:
+                                        /dir::pattern1,pattern2.
+                                        For example: "/somedir::*.jpg,a*b?.png"
+      --az-access-tier string           Leave empty to use the default
+                                        container setting
+      --az-account-key string
+      --az-account-name string
+      --az-container string
+      --az-endpoint string              Leave empty to use the default:
+                                        "blob.core.windows.net"
+      --az-key-prefix string            Allows to restrict access to the
+                                        virtual folder identified by this
+                                        prefix and its contents
+      --az-sas-url string               Shared access signature URL
+      --az-upload-concurrency int       How many parts are uploaded in
+                                        parallel (default 2)
+      --az-upload-part-size int         The buffer size for multipart uploads
+                                        (MB) (default 4)
+      --az-use-emulator
+      --denied-patterns stringArray     Denied file patterns case insensitive.
+                                        The format is:
+                                        /dir::pattern1,pattern2.
+                                        For example: "/somedir::*.jpg,a*b?.png"
+  -d, --directory string                Path to the directory to serve.
+                                        This can be an absolute path or a path
+                                        relative to the current directory
+                                         (default ".")
+  -f, --fs-provider int                 0 => local filesystem
+                                        1 => AWS S3 compatible
+                                        2 => Google Cloud Storage
+                                        3 => Azure Blob Storage
+      --ftpd-cert string                Path to the certificate file for FTPS
+      --ftpd-key string                 Path to the key file for FTPS
+      --ftpd-port int                   0 means a random unprivileged port,
+                                        < 0 disabled (default -1)
+      --gcs-automatic-credentials int   0 means explicit credentials using
+                                        a JSON credentials file, 1 automatic
+                                         (default 1)
       --gcs-bucket string
-      --gcs-credentials-file string      Google Cloud Storage JSON credentials file
-      --gcs-key-prefix string            Allows to restrict access to the virtual folder identified by this prefix and its contents
+      --gcs-credentials-file string     Google Cloud Storage JSON credentials
+                                        file
+      --gcs-key-prefix string           Allows to restrict access to the
+                                        virtual folder identified by this
+                                        prefix and its contents
       --gcs-storage-class string
-  -h, --help                             help for portable
-  -l, --log-file-path string             Leave empty to disable logging
-  -v, --log-verbose                      Enable verbose logs
-  -p, --password string                  Leave empty to use an auto generated value
-  -g, --permissions strings              User's permissions. "*" means any permission (default [list,download])
+  -h, --help                            help for portable
+  -l, --log-file-path string            Leave empty to disable logging
+  -v, --log-verbose                     Enable verbose logs
+  -p, --password string                 Leave empty to use an auto generated
+                                        value
+  -g, --permissions strings             User's permissions. "*" means any
+                                        permission (default [list,download])
   -k, --public-key strings
       --s3-access-key string
       --s3-access-secret string
       --s3-bucket string
       --s3-endpoint string
-      --s3-key-prefix string             Allows to restrict access to the virtual folder identified by this prefix and its contents
+      --s3-key-prefix string            Allows to restrict access to the
+                                        virtual folder identified by this
+                                        prefix and its contents
       --s3-region string
       --s3-storage-class string
-      --s3-upload-concurrency int        How many parts are uploaded in parallel (default 2)
-      --s3-upload-part-size int          The buffer size for multipart uploads (MB) (default 5)
-  -s, --sftpd-port int                   0 means a random non privileged port
-  -c, --ssh-commands strings             SSH commands to enable. "*" means any supported SSH command including scp (default [md5sum,sha1sum,cd,pwd])
-  -u, --username string                  Leave empty to use an auto generated value
+      --s3-upload-concurrency int       How many parts are uploaded in
+                                        parallel (default 2)
+      --s3-upload-part-size int         The buffer size for multipart uploads
+                                        (MB) (default 5)
+  -s, --sftpd-port int                  0 means a random unprivileged port
+  -c, --ssh-commands strings            SSH commands to enable.
+                                        "*" means any supported SSH command
+                                        including scp
+                                         (default [md5sum,sha1sum,cd,pwd,scp])
+  -u, --username string                 Leave empty to use an auto generated
+                                        value
+      --webdav-cert string              Path to the certificate file for WebDAV
+                                        over HTTPS
+      --webdav-key string               Path to the key file for WebDAV over
+                                        HTTPS
+      --webdav-port int                 0 means a random unprivileged port,
+                                        < 0 disabled (default -1)
 ```
 
-In portable mode, SFTPGo can advertise the SFTP service and, optionally, the credentials via multicast DNS, so there is a standard way to discover the service and to automatically connect to it.
+In portable mode, SFTPGo can advertise the SFTP/FTP services and, optionally, the credentials via multicast DNS, so there is a standard way to discover the service and to automatically connect to it.
 
-Here is an example of the advertised service including credentials as seen using `avahi-browse`:
+Here is an example of the advertised SFTP service including credentials as seen using `avahi-browse`:
 
 ```console
 = enp0s31f6 IPv4 SFTPGo portable 53705                         SFTP File Transfer   local

docs/post-connect-hook.md

@@ -0,0 +1,26 @@
+# Post-connect hook
+
+This hook is executed as soon as a new connection is established. It notifies the connection's IP address and protocol. Based on the received response, the connection is accepted or rejected. Combining this hook with the [Post-login hook](./post-login-hook.md) you can implement your own (even for Protocol) blacklist/whitelist of IP addresses.
+
+Please keep in mind that you can easily configure specialized program such as [Fail2ban](http://www.fail2ban.org/) for brute force protection. Executing a hook for each connection can be heavy.
+
+The `post-connect-hook` can be defined as the absolute path of your program or an HTTP URL.
+
+If the hook defines an external program it can read the following environment variables:
+
+- `SFTPGO_CONNECTION_IP`
+- `SFTPGO_CONNECTION_PROTOCOL`
+
+If the external command completes with a zero exit status the connection will be accepted otherwise rejected.
+
+Previous global environment variables aren't cleared when the script is called.
+The program must finish within 20 seconds.
+
+If the hook defines an HTTP URL then this URL will be invoked as HTTP GET with the following query parameters:
+
+- `ip`
+- `protocol`
+
+The connection is accepted if the HTTP response code is `200` otherwise rejected.
+
+The HTTP request will use the global configuration for HTTP clients.

docs/post-login-hook.md

@@ -0,0 +1,36 @@
+# Post-login hook
+
+This hook is executed after a login or after closing a connection for authentication timeout. Defining an appropriate `post_login_scope` you can get notifications for failed logins, successful logins or both.
+
+Combining this hook with the [Post-connect hook](./post-connect-hook.md) you can implement your own (even for Protocol) blacklist/whitelist of IP addresses.
+
+Please keep in mind that you can easily configure specialized program such as [Fail2ban](http://www.fail2ban.org/) for brute force protection. Executing a hook after each login can be heavy.
+
+The `post-login-hook` can be defined as the absolute path of your program or an HTTP URL.
+
+If the hook defines an external program it can reads the following environment variables:
+
+- `SFTPGO_LOGIND_USER`, username, can be empty if the connection is closed for authentication timeout
+- `SFTPGO_LOGIND_IP`
+- `SFTPGO_LOGIND_METHOD`, possible values are `publickey`, `password`, `keyboard-interactive`, `publickey+password`, `publickey+keyboard-interactive` or `no_auth_tryed`
+- `SFTPGO_LOGIND_STATUS`, 1 means login OK, 0 login KO
+- `SFTPGO_LOGIND_PROTOCOL`, possible values are `SSH`, `FTP`, `DAV`
+
+Previous global environment variables aren't cleared when the script is called.
+The program must finish within 20 seconds.
+
+If the hook is an HTTP URL then it will be invoked as HTTP POST. The request body will contain a JSON serialized struct with the following fields:
+
+- `username`
+- `login_method`
+- `ip`
+- `protocol`
+- `status`
+
+The HTTP request will use the global configuration for HTTP clients.
+
+The `post_login_scope` supports the following configuration values:
+
+- `0` means notify both failed and successful logins
+- `1` means notify failed logins. Connections closed for authentication timeout are notified as failed connections. You will get an empty username in this case
+- `2` means notify successful logins

docs/s3.md

@@ -10,26 +10,26 @@ AWS SDK has different options for credentials. [More Detail](https://docs.aws.am
 
 So, you need to provide access keys to activate option 1, or leave them blank to use the other ways to specify credentials.
 
-Most S3 backends require HTTPS connections so if you are running SFTPGo as docker image please be sure to uncomment the line that install `ca-certificates`, inside your `Dockerfile`, to be able to properly verify certificate authorities.
-
 Specifying a different `key_prefix`, you can assign different "folders" of the same bucket to different users. This is similar to a chroot directory for local filesystem. Each SFTP/SCP user can only access the assigned folder and its contents. The folder identified by `key_prefix` does not need to be pre-created.
 
 SFTPGo uses multipart uploads and parallel downloads for storing and retrieving files from S3.
 
-For multipart uploads you can customize the parts size and the upload concurrency. Please note that if the upload bandwidth between the SFTP client and SFTPGo is greater than the upload bandwidth between SFTPGo and S3 then the SFTP client have to wait for the upload of the last parts to S3 after it ends the file upload to SFTPGo, and it may time out. Keep this in mind if you customize these parameters.
+For multipart uploads you can customize the parts size and the upload concurrency. Please note that if the upload bandwidth between the client and SFTPGo is greater than the upload bandwidth between SFTPGo and S3 then the client should wait for the last parts to be uploaded to S3 after finishing uploading the file to SFTPGo, and it may time out. Keep this in mind if you customize these parameters.
 
 The configured bucket must exist.
 
 Some SFTP commands don't work over S3:
 
-- `symlink` and `chtimes` will fail
-- `chown` and `chmod` are silently ignored
+- `chtimes`, `chown` and `chmod` will fail. If you want to silently ignore these method set `setstat_mode` to `1` or `2` in your configuration file
+- `truncate`, `symlink`, `readlink` are not supported
+- opening a file for both reading and writing at the same time is not supported
 - upload resume is not supported
 - upload mode `atomic` is ignored since S3 uploads are already atomic
 
 Other notes:
 
 - `rename` is a two step operation: server-side copy and then deletion. So, it is not atomic as for local filesystem.
-- We don't support renaming non empty directories since we should rename all the contents too and this could take a long time: think about directories with thousands of files; for each file we should do an AWS API call.
+- We don't support renaming non empty directories since we should rename all the contents too and this could take a long time: think about directories with thousands of files: for each file we should do an AWS API call.
 - For server side encryption, you have to configure the mapped bucket to automatically encrypt objects.
 - A local home directory is still required to store temporary files.
+- Clients that require advanced filesystem-like features such as `sshfs` are not supported.

docs/service.md

@@ -1,30 +1,58 @@
 # Running SFTPGo as a service
 
+Download a binary SFTPGo [release](https://github.com/drakkan/sftpgo/releases) or a build artifact for the [latest commit](https://github.com/drakkan/sftpgo/actions) or build SFTPGo yourself.
+
+Run the following instructions from the directory that contains the sftpgo binary and the accompanying files.
+
 ## Linux
 
-For Linux, a `systemd` sample [service](../init/sftpgo.service "systemd service") can be found inside the source tree.
+The easiest way to run SFTPGo as a service is to download and install the pre-compiled deb/rpm package or use one of the Arch Linux PKGBUILDs we maintain.
 
-Here are some basic instructions to run SFTPGo as service, please run the following commands from the directory where you downloaded SFTPGo:
+This section describes the procedure to use if you prefer to build SFTPGo yourself or if you want to download and configure a pre-built release as tar.
+
+A `systemd` sample [service](../init/sftpgo.service "systemd service") can be found inside the source tree.
+
+Here are some basic instructions to run SFTPGo as service using a dedicated `sftpgo` system account.
+
+Please run the following commands from the directory where you downloaded/compiled SFTPGo:
 
 ```bash
+# create the sftpgo user and group
+sudo groupadd --system sftpgo
+sudo useradd --system \
+  --gid sftpgo \
+  --no-create-home \
+  --home-dir /var/lib/sftpgo \
+  --shell /usr/sbin/nologin \
+  --comment "SFTPGo user" \
+  sftpgo
 # create the required directories
 sudo mkdir -p /etc/sftpgo \
-  /var/lib/sftpgo
+  /var/lib/sftpgo \
+  /usr/share/sftpgo
 
-# install sftpgo executable
+# install the sftpgo executable
 sudo install -Dm755 sftpgo /usr/bin/sftpgo
 # install the default configuration file, edit it if required
 sudo install -Dm644 sftpgo.json /etc/sftpgo/
 # override some configuration keys using environment variables
-sudo sh -c 'echo "SFTPGO_HTTPD__TEMPLATES_PATH=/var/lib/sftpgo/templates" > /etc/sftpgo/sftpgo.env'
-sudo sh -c 'echo "SFTPGO_HTTPD__STATIC_FILES_PATH=/var/lib/sftpgo/static" >> /etc/sftpgo/sftpgo.env'
+sudo sh -c 'echo "SFTPGO_HTTPD__TEMPLATES_PATH=/usr/share/sftpgo/templates" > /etc/sftpgo/sftpgo.env'
+sudo sh -c 'echo "SFTPGO_HTTPD__STATIC_FILES_PATH=/usr/share/sftpgo/static" >> /etc/sftpgo/sftpgo.env'
 sudo sh -c 'echo "SFTPGO_HTTPD__BACKUPS_PATH=/var/lib/sftpgo/backups" >> /etc/sftpgo/sftpgo.env'
 sudo sh -c 'echo "SFTPGO_DATA_PROVIDER__CREDENTIALS_PATH=/var/lib/sftpgo/credentials" >> /etc/sftpgo/sftpgo.env'
+# if you use a file based data provider such as sqlite or bolt consider to set the database path too, for example:
+#sudo sh -c 'echo "SFTPGO_DATA_PROVIDER__NAME=/var/lib/sftpgo/sftpgo.db" >> /etc/sftpgo/sftpgo.env'
+# also set the provider's PATH as env var to get initprovider to work with SQLite provider:
+#export SFTPGO_DATA_PROVIDER__NAME=/var/lib/sftpgo/sftpgo.db
 # install static files and templates for the web UI
-sudo cp -r static templates /var/lib/sftpgo/
+sudo cp -r static templates /usr/share/sftpgo/
+# set files and directory permissions
+sudo chown -R sftpgo:sftpgo /etc/sftpgo /var/lib/sftpgo
+sudo chmod 750 /etc/sftpgo /var/lib/sftpgo
+sudo chmod 640 /etc/sftpgo/sftpgo.json /etc/sftpgo/sftpgo.env
 # initialize the configured data provider
 # if you want to use MySQL or PostgreSQL you need to create the configured database before running the initprovider command
-sudo /usr/bin/sftpgo initprovider -c /etc/sftpgo/
+sudo -E su - sftpgo -m -s /bin/bash -c 'sftpgo initprovider -c /etc/sftpgo'
 # install the systemd service
 sudo install -Dm644 init/sftpgo.service /etc/systemd/system
 # start the service
@@ -34,7 +62,11 @@ sudo systemctl status sftpgo
 # automatically start sftpgo on boot
 sudo systemctl enable sftpgo
 # optional, install the REST API CLI. It requires python-requests to run
-sudo install -Dm755 examples/rest-api-cli/sftpgo_api_cli.py /usr/bin/sftpgo_api_cli
+sudo install -Dm755 examples/rest-api-cli/sftpgo_api_cli /usr/bin/sftpgo_api_cli
+# optional, create shell completion script, for example for bash
+sudo sh -c '/usr/bin/sftpgo gen completion bash > /usr/share/bash-completion/completions/sftpgo'
+# optional, create man pages
+sudo /usr/bin/sftpgo gen man -d /usr/share/man/man1
 ```
 
 ## macOS
@@ -47,6 +79,7 @@ Here are some basic instructions to run SFTPGo as service, please run the follow
 # create the required directories
 sudo mkdir -p /usr/local/opt/sftpgo/init \
   /usr/local/opt/sftpgo/var/lib \
+  /usr/local/opt/sftpgo/usr/share \
   /usr/local/opt/sftpgo/var/log \
   /usr/local/opt/sftpgo/etc \
   /usr/local/opt/sftpgo/bin
@@ -59,7 +92,7 @@ sudo chown root:wheel /usr/local/opt/sftpgo/init/com.github.drakkan.sftpgo.plist
 # install the default configuration file, edit it if required
 sudo cp sftpgo.json /usr/local/opt/sftpgo/etc/
 # install static files and templates for the web UI
-sudo cp -r static templates /usr/local/opt/sftpgo/var/lib/
+sudo cp -r static templates /usr/local/opt/sftpgo/usr/share/
 # initialize the configured data provider
 # if you want to use MySQL or PostgreSQL you need to create the configured database before running the initprovider command
 sudo /usr/local/opt/sftpgo/bin/sftpgo initprovider -c /usr/local/opt/sftpgo/etc/
@@ -70,7 +103,7 @@ sudo launchctl load -w /Library/LaunchDaemons/com.github.drakkan.sftpgo.plist
 # verify that the service is started
 sudo launchctl list com.github.drakkan.sftpgo
 # optional, install the REST API CLI. It requires python-requests to run, this python module is not installed by default
-sudo cp examples/rest-api-cli/sftpgo_api_cli.py /usr/local/opt/sftpgo/bin/
+sudo cp examples/rest-api-cli/sftpgo_api_cli /usr/local/opt/sftpgo/bin/
 ```
 
 ## Windows
@@ -79,7 +112,7 @@ On Windows, you can register SFTPGo as Windows Service. Take a look at the CLI u
 
 ```powershell
 PS> sftpgo.exe service --help
-Install, Uninstall, Start, Stop, Reload and retrieve status for SFTPGo Windows Service
+Manage SFTPGo Windows Service
 
 Usage:
   sftpgo service [command]
@@ -87,7 +120,7 @@ Usage:
 Available Commands:
   install     Install SFTPGo as Windows Service
   reload      Reload the SFTPGo Windows Service sending a "paramchange" request
-  rotatelogs  Signal to the running service to close the existing log file and immediately create a new one
+  rotatelogs  Signal to the running service to rotate the logs
   start       Start SFTPGo Windows Service
   status      Retrieve the status for the SFTPGo Windows Service
   stop        Stop SFTPGo Windows Service
@@ -107,4 +140,6 @@ After installing as a Windows Service, please remember to allow network access t
 PS> netsh advfirewall firewall add rule name="SFTPGo Service" dir=in action=allow program="C:\Program Files\SFTPGo\sftpgo.exe"
 ```
 
-(Or through the Windows Firewall GUI.)
+Or through the Windows Firewall GUI.
+
+The Windows installer will register the service and allow network access for it automatically.

docs/sftp-subsystem.md

@@ -0,0 +1,64 @@
+# SFTP subsystem mode
+
+In this mode SFTPGo speaks the server side of SFTP protocol to stdout and expects client requests from stdin.
+You can use SFTPGo as subsystem via the `startsubsys` command.
+This mode is not intended to be called directly, but from sshd using the `Subsystem` option.
+For example adding a line like this one in `/etc/ssh/sshd_config`:
+
+```shell
+Subsystem    sftp    sftpgo startsubsys
+```
+
+Command-line flags should be specified in the Subsystem declaration.
+
+```shell
+Usage:
+  sftpgo startsubsys [flags]
+
+Flags:
+  -d, --base-home-dir string   If the user does not exist specify an alternate
+                               starting directory. The home directory for a new
+                               user will be:
+
+                               <base-home-dir>/<username>
+
+                               base-home-dir must be an absolute path.
+  -c, --config-dir string      Location for SFTPGo config dir. This directory
+                               should contain the "sftpgo" configuration file
+                               or the configured config-file and it is used as
+                               the base for files with a relative path (eg. the
+                               private keys for the SFTP server, the SQLite
+                               database if you use SQLite as data provider).
+                               This flag can be set using SFTPGO_CONFIG_DIR
+                               env var too. (default ".")
+  -f, --config-file string     Name for SFTPGo configuration file. It must be
+                               the name of a file stored in config-dir not the
+                               absolute path to the configuration file. The
+                               specified file name must have no extension we
+                               automatically load JSON, YAML, TOML, HCL and
+                               Java properties. Therefore if you set "sftpgo"
+                               then "sftpgo.json", "sftpgo.yaml" and so on
+                               are searched.
+                               This flag can be set using SFTPGO_CONFIG_FILE
+                               env var too. (default "sftpgo")
+  -h, --help                   help for startsubsys
+  -j, --log-to-journald        Send logs to journald. Only available on Linux.
+                               Use:
+
+                               $ journalctl -o verbose -f
+
+                               To see full logs.
+                               If not set, the logs will be sent to the standard
+                               error
+  -v, --log-verbose            Enable verbose logs. This flag can be set
+                               using SFTPGO_LOG_VERBOSE env var too.
+                                (default true)
+  -p, --preserve-home          If the user already exists, the existing home
+                               directory will not be changed
+```
+
+In this mode `bolt` and `sqlite` providers are not usable as the same database file cannot be shared among multiple processes, if one of these provider is configured it will be automatically changed to `memory` provider.
+
+The username and home directory for the logged in user are determined using [user.Current()](https://golang.org/pkg/os/user/#Current).
+If the user who is logging is not found within the SFTPGo data provider, it is added automatically.
+You can pre-configure the users inside the SFTPGo data provider, this way you can use a different home directory, restrict permissions and such.

docs/ssh-commands.md

@@ -8,21 +8,34 @@ For system commands we have no direct control on file creation/deletion and so t
 - system commands work only on local filyestem
 - we cannot avoid to leak real filesystem paths
 - quota check is suboptimal
+- maximum size restriction on single file is not respected
 
- If quota is enabled and SFTPGO receives a system command, the used size and number of files are checked at the command start and not while new files are created/deleted. While the command is running the number of files is not checked, the remaining size is calculated as the difference between the max allowed quota and the used one, and it is checked against the bytes transferred via SSH. The command is aborted if it uploads more bytes than the remaining allowed size calculated at the command start. Anyway, we only see the bytes that the remote command sends to the local one via SSH. These bytes contain both protocol commands and files, and so the size of the files is different from the size trasferred via SSH: for example, a command can send compressed files, or a protocol command (few bytes) could delete a big file. To mitigate these issues, quotas are recalculated at the command end with a full scan of the directory specified for the system command. This could be heavy for big directories. If you need system commands and quotas you could consider disabling quota restrictions and periodically update quota usage yourself using the REST API.
+ If quota is enabled and SFTPGo receives a system command, the used size and number of files are checked at the command start and not while new files are created/deleted. While the command is running the number of files is not checked, the remaining size is calculated as the difference between the max allowed quota and the used one, and it is checked against the bytes transferred via SSH. The command is aborted if it uploads more bytes than the remaining allowed size calculated at the command start. Anyway, we only see the bytes that the remote command sends to the local one via SSH. These bytes contain both protocol commands and files, and so the size of the files is different from the size transferred via SSH: for example, a command can send compressed files, or a protocol command (few bytes) could delete a big file. To mitigate these issues, quotas are recalculated at the command end with a full scan of the directory specified for the system command. This could be heavy for big directories. If you need system commands and quotas you could consider disabling quota restrictions and periodically update quota usage yourself using the REST API.
 
- For these reasons we should limit system commands usage as much as possibile, we currently support the following system commands:
+ For these reasons we should limit system commands usage as much as possible, we currently support the following system commands:
 
 - `git-receive-pack`, `git-upload-pack`, `git-upload-archive`. These commands enable support for Git repositories over SSH. They need to be installed and in your system's `PATH`.
-- `rsync`. The `rsync` command needs to be installed and in your system's `PATH`. We cannot avoid that rsync creates symlinks, so if the user has the permission to create symlinks, we add the option `--safe-links` to the received rsync command if it is not already set. This should prevent creating symlinks that point outside the home dir. If the user cannot create symlinks, we add the option `--munge-links` if it is not already set. This should make symlinks unusable (but manually recoverable).
+- `rsync`. The `rsync` command needs to be installed and in your system's `PATH`.
 
-SFTPGo support the following built-in SSH commands:
+At least the following permissions are required to be able to run system commands:
+
+- `list`
+- `download`
+- `upload`
+- `create_dirs`
+- `overwrite`
+- `delete`
+
+For `rsync`  we cannot avoid that it creates symlinks so if the `create_symlinks` permission is granted we add the option `--safe-links`, if it is not already set, to the received `rsync` command. This should prevent to create symlinks that point outside the home directory.
+If the user cannot create symlinks we add the option `--munge-links`, if it is not already set, to the received `rsync` command. This should make symlinks unusable (but manually recoverable).
+
+SFTPGo supports the following built-in SSH commands:
 
 - `scp`, SFTPGo implements the SCP protocol so we can support it for cloud filesystems too and we can avoid the other system commands limitations. SCP between two remote hosts is supported using the `-3` scp option.
 - `md5sum`, `sha1sum`, `sha256sum`, `sha384sum`, `sha512sum`. Useful to check message digests for uploaded files.
 - `cd`, `pwd`. Some SFTP clients do not support the SFTP SSH_FXP_REALPATH packet type, so they use `cd` and `pwd` SSH commands to get the initial directory. Currently `cd` does nothing and `pwd` always returns the `/` path.
-- `sftpgo-copy`. This is a built-in copy implementation. It allows server side copy for files and directories. The first argument is the source file/directory and the second one is the destination file/directory, for example `sftpgo-copy <src> <dst>`. The command will fail if the destination exists. Copy for directories spanning virtual folders is not supported. Only local filesystem is supported: recursive copy for Cloud Storage filesystems requires a new request for every file in any case, so a real server side copy is not possibile.
-- `sftpgo-remove`. This is a built-in remove implementation. It allows to remove single files and to recursively remove directories. The first argument is the file/directory to remove, for example `sftpgo-remove <dst>`. Only local filesystem is supported: recursive remove for Cloud Storage filesystems requires a new request for every file in any case, so a server side remove is not possibile.
+- `sftpgo-copy`. This is a built-in copy implementation. It allows server side copy for files and directories. The first argument is the source file/directory and the second one is the destination file/directory, for example `sftpgo-copy <src> <dst>`. The command will fail if the destination exists. Copy for directories spanning virtual folders is not supported. Only local filesystem is supported: recursive copy for Cloud Storage filesystems requires a new request for every file in any case, so a real server side copy is not possible.
+- `sftpgo-remove`. This is a built-in remove implementation. It allows to remove single files and to recursively remove directories. The first argument is the file/directory to remove, for example `sftpgo-remove <dst>`. Only local filesystem is supported: recursive remove for Cloud Storage filesystems requires a new request for every file in any case, so a server side remove is not possible.
 
 The following SSH commands are enabled by default:
 

docs/virtual-folders.md

@@ -1,6 +1,6 @@
 # Virtual Folders
 
-A virtual folder is a mapping between a SFTP/SCP virtual path and a filesystem path outside the user home directory.
+A virtual folder is a mapping between an SFTP/SCP virtual path and a filesystem path outside the user home directory.
 The specified paths must be absolute and the virtual path cannot be "/", it must be a sub directory.
 The parent directory to the specified virtual path must exist. SFTPGo will try to automatically create any missing parent directory for the configured virtual folders at user login.
 
@@ -16,7 +16,7 @@ For example if you configure `/tmp/mapped` or `C:\mapped` as mapped path and `/v
 The same virtual folder, identified by the `mapped_path`, can be shared among users and different folder quota limits for each user are supported.
 Folder quota limits can also be included inside the user quota but in this case the folder is considered "private" and sharing it with other users will break user quota calculation.
 
-You don't need to create virtual folders, inside the data provider, to associate them to the users: any missing virtual folder will be automatically created when you add/update an user. You only have to create the folder on the filesystem.
+You don't need to create virtual folders, inside the data provider, to associate them to the users: any missing virtual folder will be automatically created when you add/update a user. You only have to create the folder on the filesystem.
 
 Using the REST API you can:
 

docs/webdav.md

@@ -0,0 +1,31 @@
+# WebDAV
+
+The experimental `WebDAV` support can be enabled by setting a `bind_port` inside the `webdavd` configuration section.
+
+Each user has his own path like `http/s://<SFTPGo ip>:<WevDAVPORT>/<username>` and it must authenticate using password credentials.
+
+WebDAV is quite a different protocol than SCP/FTP, there is no session concept, each command is a separate HTTP request and must be authenticated, to improve performance SFTPGo caches authenticated users. This way SFTPGo don't need to do a dataprovider query and a password check for each request.
+
+The user caching configuration allows to set:
+
+- `expiration_time` in minutes. If a user is cached for more than the specified minutes it will be removed from the cache and a new dataprovider query will be performed. Please note that the `last_login` field will not be updated and `external_auth_hook`, `pre_login_hook` and `check_password_hook` will not be executed if the user is obtained from the cache.
+- `max_size`. Maximum number of users to cache. When this limit is reached the user with the oldest expiration date will be removed from the cache. 0 means no limit however the cache size cannot exceed the number of users so if you have a small number of users you can set this value to 0.
+
+Users are automatically removed from the cache after an update/delete.
+
+WebDAV protocol requires the MIME type for each file. SFTPGo will first try to guess the MIME type by extension. If this fails it will send a `HEAD` request for Cloud backends and, as last resort, it will try to guess the MIME type reading the first 512 bytes of the file. This may slow down the directory listing, especially for Cloud based backends, if you have directories containing many files with unregistered extensions. To mitigate this problem, you can enable caching of MIME types so that the MIME type detection is done only once.
+
+The MIME types caching configurations allows to set the maximum number of MIME types to cache. Once the cache reaches the configured maximum size no new MIME types will be added. The MIME types cache  is a non-persistent in-memory cache. If you need a persistent cache add your MIME types to `/etc/mime.types` on Linux or inside the registry on Windows.
+
+WebDAV should work as expected for most use cases but there are some minor issues and some missing features.
+
+Know issues:
+
+- removing a directory tree on Cloud Storage backends could generate a `not found` error when removing the last (virtual) directory. This happens if the client cycles the directories tree itself and removes files and directories one by one instead of issuing a single remove command
+- the used [WebDAV library](https://pkg.go.dev/golang.org/x/net/webdav?tab=doc) asks to open a file to execute a `stat` and sometimes reads some bytes to find the content type. Stat calls are executed before and after a download too, so to be able to properly list a directory you need to grant both `list` and `download` permissions and to be able to upload files you need to gran both `list` and `upload` permissions
+- the used `WebDAV library` not always returns a proper error code/message, most of the times it simply returns `Method not Allowed`. I'll try to improve the library error codes in the future
+- if an object within a directory cannot be accessed, for example due to OS permissions issues or because is a missing mapped path for a virtual folder, the directory listing will fail. In SFTP/FTP the directory listing will succeed and you'll only get an error if you try to access to the problematic file/directory
+
+We plan to add [Dead Properties](https://tools.ietf.org/html/rfc4918#section-3) support in future releases. We need a design decision here, probably the best solution is to store dead properties inside the data provider but this could increase a lot its size. Alternately we could store them on disk for local filesystem and add as metadata for Cloud Storage, this means that we need to do a separate `HEAD` request to retrieve dead properties for an S3 file. For big folders will do a lot of requests to the Cloud Provider, I don't like this solution. Another option is to expose a hook and allow you to implement `dead properties` outside SFTPGo.
+
+If you find any other quirks or problems please let us know opening a GitHub issue, thank you!

examples/OTP/authy/README.md

@@ -0,0 +1,58 @@
+# Authy
+
+These example show how-to integrate [Twilio Authy API](https://www.twilio.com/docs/authy/api) for One-Time-Password logins.
+
+The examples assume that the user has the free [Authy app](https://authy.com/) installed and uses it to generate offline [TOTP](https://en.wikipedia.org/wiki/Time-based_One-time_Password_algorithm) codes (soft tokens).
+
+You first need to [create an Authy Application in the Twilio Console](https://twilio.com/console/authy/applications?_ga=2.205553366.451688189.1597667213-1526360003.1597667213), then you can create a new Authy user and store a reference to the matching SFTPGo account.
+
+Verify that your Authy application is successfully registered:
+
+```bash
+export AUTHY_API_KEY=<your api key here>
+curl 'https://api.authy.com/protected/json/app/details' -H "X-Authy-API-Key: $AUTHY_API_KEY"
+```
+
+now create an Authy user:
+
+```bash
+curl -XPOST "https://api.authy.com/protected/json/users/new" \
+-H "X-Authy-API-Key: $AUTHY_API_KEY" \
+--data-urlencode user[email]="user@domain.com" \
+--data-urlencode user[cellphone]="317-338-9302" \
+--data-urlencode user[country_code]="54"
+```
+
+The response is something like this:
+
+```json
+{"message":"User created successfully.","user":{"id":xxxxxxxx},"success":true}
+```
+
+Save the user id somewhere and add a reference to the matching SFTPGo account.
+
+After this step you can use the Authy app installed on your phone to generate TOTP codes.
+
+Now you can verify the token using an HTTP GET request:
+
+```bash
+export TOKEN=<TOTP you read from Authy app>
+export AUTHY_ID=<user id>
+curl -i "https://api.authy.com/protected/json/verify/${TOKEN}/${AUTHY_ID}" \
+    -H "X-Authy-API-Key: $AUTHY_API_KEY"
+```
+
+So inside your hook you need to check:
+
+- the HTTP response code for the verify request, it must be `200`
+- the JSON reponse body, it must contains the key `success` with the value `true` (as string)
+
+If these conditions are met the token is valid and you allow the user to login.
+
+We provide the following examples:
+
+- [Keyboard interactive authentication](./keyint/README.md) for 2FA using password + Authy one time token.
+- [External authentication](./extauth/README.md) using Authy one time tokens as passwords.
+- [Check password hook](./checkpwd/README.md) for 2FA using a password consisting of a fixed string and a One Time Token.
+
+Please note that these are sample programs not intended for production use, you should write your own hook based on them and you should prefer HTTP based hooks if performance is a concern.

examples/OTP/authy/checkpwd/README.md

@@ -0,0 +1,3 @@
+# Authy 2FA via check password hook
+
+This example shows how to use 2FA via the check password hook using a password consisting of a fixed part and an Authy TOTP token. The hook will check the TOTP token using the Authy API and SFTPGo will check the fixed part. Please read the [sample code](./main.go), it should be self explanatory.

examples/OTP/authy/checkpwd/go.mod

@@ -0,0 +1,3 @@
+module github.com/drakkan/sftpgo/authy/checkpwd
+
+go 1.15

examples/OTP/authy/checkpwd/main.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"time"
+)
+
+type userMapping struct {
+	SFTPGoUsername string
+	AuthyID        int64
+	AuthyAPIKey    string
+}
+
+type checkPasswordResponse struct {
+	// 0 KO, 1 OK, 2 partial success
+	Status int `json:"status"`
+	// for status == 2 this is the password that SFTPGo will check against the one stored
+	// inside the data provider
+	ToVerify string `json:"to_verify"`
+}
+
+var (
+	mapping []userMapping
+)
+
+func init() {
+	// this is for demo only, you probably want to get this mapping dynamically, for example using a database query
+	mapping = append(mapping, userMapping{
+		SFTPGoUsername: "<SFTPGo username>",
+		AuthyID:        1234567,
+		AuthyAPIKey:    "<your api key>",
+	})
+}
+
+func printResponse(status int, toVerify string) {
+	r := checkPasswordResponse{
+		Status:   status,
+		ToVerify: toVerify,
+	}
+	resp, _ := json.Marshal(r)
+	fmt.Printf("%v\n", string(resp))
+	if status > 0 {
+		os.Exit(0)
+	} else {
+		os.Exit(1)
+	}
+}
+
+func main() {
+	// get credentials from env vars
+	username := os.Getenv("SFTPGO_AUTHD_USERNAME")
+	password := os.Getenv("SFTPGO_AUTHD_PASSWORD")
+
+	for _, m := range mapping {
+		if m.SFTPGoUsername == username {
+			// Authy token len is 7, we assume that we have the password followed by the token
+			pwdLen := len(password)
+			if pwdLen <= 7 {
+				printResponse(0, "")
+			}
+			pwd := password[:pwdLen-7]
+			authyToken := password[pwdLen-7:]
+			// now verify the authy token and instruct SFTPGo to check the password if the token is OK
+			url := fmt.Sprintf("https://api.authy.com/protected/json/verify/%v/%v", authyToken, m.AuthyID)
+			req, err := http.NewRequest(http.MethodGet, url, nil)
+			if err != nil {
+				log.Fatal(err)
+			}
+			req.Header.Set("X-Authy-API-Key", m.AuthyAPIKey)
+			httpClient := &http.Client{
+				Timeout: 10 * time.Second,
+			}
+			resp, err := httpClient.Do(req)
+			if err != nil {
+				printResponse(0, "")
+			}
+			defer resp.Body.Close()
+			if resp.StatusCode != http.StatusOK {
+				// status code 200 is expected
+				printResponse(0, "")
+			}
+			var authyResponse map[string]interface{}
+			respBody, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				printResponse(0, "")
+			}
+			err = json.Unmarshal(respBody, &authyResponse)
+			if err != nil {
+				printResponse(0, "")
+			}
+			if authyResponse["success"].(string) == "true" {
+				printResponse(2, pwd)
+			}
+			printResponse(0, "")
+			break
+		}
+	}
+
+	// no mapping found
+	printResponse(0, "")
+}

examples/OTP/authy/extauth/README.md

@@ -0,0 +1,3 @@
+# Authy external authentication
+
+This example shows how to use Authy TOTP token as password for SFTPGo users. Please read the [sample code](./main.go), it should be self explanatory.

examples/OTP/authy/extauth/go.mod

@@ -0,0 +1,3 @@
+module github.com/drakkan/sftpgo/authy/extauth
+
+go 1.15

examples/OTP/authy/extauth/main.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+type userMapping struct {
+	SFTPGoUsername string
+	AuthyID        int64
+	AuthyAPIKey    string
+}
+
+// we assume that the SFTPGo already exists, we only check the one time token.
+// If you need to create the SFTPGo user more fields are needed here
+type minimalSFTPGoUser struct {
+	Status      int                 `json:"status,omitempty"`
+	Username    string              `json:"username"`
+	HomeDir     string              `json:"home_dir,omitempty"`
+	Permissions map[string][]string `json:"permissions"`
+}
+
+var (
+	mapping []userMapping
+)
+
+func init() {
+	// this is for demo only, you probably want to get this mapping dynamically, for example using a database query
+	mapping = append(mapping, userMapping{
+		SFTPGoUsername: "<SFTPGo username>",
+		AuthyID:        1234567,
+		AuthyAPIKey:    "<your api key>",
+	})
+}
+
+func printResponse(username string) {
+	u := minimalSFTPGoUser{
+		Username: username,
+		Status:   1,
+		HomeDir:  filepath.Join(os.TempDir(), username),
+	}
+	u.Permissions = make(map[string][]string)
+	u.Permissions["/"] = []string{"*"}
+	resp, _ := json.Marshal(u)
+	fmt.Printf("%v\n", string(resp))
+	if len(username) > 0 {
+		os.Exit(0)
+	} else {
+		os.Exit(1)
+	}
+}
+
+func main() {
+	// get credentials from env vars
+	username := os.Getenv("SFTPGO_AUTHD_USERNAME")
+	password := os.Getenv("SFTPGO_AUTHD_PASSWORD")
+	if len(password) == 0 {
+		// login method is not password
+		printResponse("")
+		return
+	}
+
+	for _, m := range mapping {
+		if m.SFTPGoUsername == username {
+			// mapping found we can now verify the token
+			url := fmt.Sprintf("https://api.authy.com/protected/json/verify/%v/%v", password, m.AuthyID)
+			req, err := http.NewRequest(http.MethodGet, url, nil)
+			if err != nil {
+				log.Fatal(err)
+			}
+			req.Header.Set("X-Authy-API-Key", m.AuthyAPIKey)
+			httpClient := &http.Client{
+				Timeout: 10 * time.Second,
+			}
+			resp, err := httpClient.Do(req)
+			if err != nil {
+				printResponse("")
+			}
+			defer resp.Body.Close()
+			if resp.StatusCode != http.StatusOK {
+				// status code 200 is expected
+				printResponse("")
+			}
+			var authyResponse map[string]interface{}
+			respBody, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				printResponse("")
+			}
+			err = json.Unmarshal(respBody, &authyResponse)
+			if err != nil {
+				printResponse("")
+			}
+			if authyResponse["success"].(string) == "true" {
+				printResponse(username)
+			}
+			printResponse("")
+			break
+		}
+	}
+
+	// no mapping found
+	printResponse("")
+}

examples/OTP/authy/keyint/README.md

@@ -0,0 +1,3 @@
+# Authy 2FA using keyboard interactive authentication
+
+This example shows how to authenticate SFTP users using 2FA (password + Authy token). Please read the [sample code](./main.go), it should be self explanatory.

examples/OTP/authy/keyint/go.mod

@@ -0,0 +1,3 @@
+module github.com/drakkan/sftpgo/authy/keyint
+
+go 1.15

examples/OTP/authy/keyint/main.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"time"
+)
+
+type userMapping struct {
+	SFTPGoUsername string
+	AuthyID        int64
+	AuthyAPIKey    string
+}
+
+type keyboardAuthHookResponse struct {
+	Instruction string   `json:"instruction,omitempty"`
+	Questions   []string `json:"questions,omitempty"`
+	Echos       []bool   `json:"echos,omitempty"`
+	AuthResult  int      `json:"auth_result"`
+	CheckPwd    int      `json:"check_password,omitempty"`
+}
+
+var (
+	mapping []userMapping
+)
+
+func init() {
+	// this is for demo only, you probably want to get this mapping dynamically, for example using a database query
+	mapping = append(mapping, userMapping{
+		SFTPGoUsername: "<SFTPGo username>",
+		AuthyID:        1234567,
+		AuthyAPIKey:    "<your api key>",
+	})
+}
+
+func printAuthResponse(result int) {
+	resp, _ := json.Marshal(keyboardAuthHookResponse{
+		AuthResult: result,
+	})
+	fmt.Printf("%v\n", string(resp))
+	if result == 1 {
+		os.Exit(0)
+	} else {
+		os.Exit(1)
+	}
+}
+
+func main() {
+	// get credentials from env vars
+	username := os.Getenv("SFTPGO_AUTHD_USERNAME")
+	var userMap userMapping
+	for _, m := range mapping {
+		if m.SFTPGoUsername == username {
+			userMap = m
+			break
+		}
+	}
+
+	if userMap.SFTPGoUsername != username {
+		// no mapping found
+		os.Exit(1)
+	}
+
+	checkPwdQuestion := keyboardAuthHookResponse{
+		Instruction: "This is a sample keyboard authentication program that ask for your password + Authy token",
+		Questions:   []string{"Your password: "},
+		Echos:       []bool{false},
+		CheckPwd:    1,
+		AuthResult:  0,
+	}
+
+	q, _ := json.Marshal(checkPwdQuestion)
+	fmt.Printf("%v\n", string(q))
+
+	// in a real world app you probably want to use a read timeout
+	scanner := bufio.NewScanner(os.Stdin)
+	scanner.Scan()
+	if scanner.Err() != nil {
+		printAuthResponse(-1)
+	}
+	response := scanner.Text()
+	if response != "OK" {
+		printAuthResponse(-1)
+	}
+
+	checkTokenQuestion := keyboardAuthHookResponse{
+		Instruction: "",
+		Questions:   []string{"Authy token: "},
+		Echos:       []bool{false},
+		CheckPwd:    0,
+		AuthResult:  0,
+	}
+
+	q, _ = json.Marshal(checkTokenQuestion)
+	fmt.Printf("%v\n", string(q))
+	scanner.Scan()
+	if scanner.Err() != nil {
+		printAuthResponse(-1)
+	}
+	authyToken := scanner.Text()
+
+	url := fmt.Sprintf("https://api.authy.com/protected/json/verify/%v/%v", authyToken, userMap.AuthyID)
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		printAuthResponse(-1)
+	}
+	req.Header.Set("X-Authy-API-Key", userMap.AuthyAPIKey)
+	httpClient := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		printAuthResponse(-1)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		// status code 200 is expected
+		printAuthResponse(-1)
+	}
+	var authyResponse map[string]interface{}
+	respBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		printAuthResponse(-1)
+	}
+	err = json.Unmarshal(respBody, &authyResponse)
+	if err != nil {
+		printAuthResponse(-1)
+	}
+	if authyResponse["success"].(string) == "true" {
+		printAuthResponse(1)
+	}
+	printAuthResponse(-1)
+}

examples/ldapauth/README.md

@@ -7,7 +7,7 @@ You need to change the LDAP connection parameters and the user search query to m
 You can build this example using the following command:
 
 ```console
-go build -i -ldflags "-s -w" -o ldapauth
+go build -ldflags "-s -w" -o ldapauth
 ```
 
 This program assumes that the 389ds schema was extended to add support for public keys using the following ldif file placed in `/etc/dirsrv/schema/98openssh-ldap.ldif`:

examples/ldapauth/go.mod

@@ -3,8 +3,7 @@ module github.com/drakkan/ldapauth
 go 1.14
 
 require (
-	github.com/go-asn1-ber/asn1-ber v1.4.1 // indirect
-	github.com/go-ldap/ldap/v3 v3.1.8
-	golang.org/x/crypto v0.0.0-20200406173513-056763e48d71
-	golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa // indirect
+	github.com/go-ldap/ldap/v3 v3.2.3
+	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
+	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect
 )

examples/ldapauth/go.sum

@@ -1,13 +1,15 @@
-github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-asn1-ber/asn1-ber v1.4.1 h1:qP/QDxOtmMoJVgXHCXNzDpA0+wkgYB2x5QoLMVOciyw=
-github.com/go-asn1-ber/asn1-ber v1.4.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-ldap/ldap/v3 v3.1.8 h1:5vU/2jOh9HqprwXp8aF915s9p6Z8wmbSEVF7/gdTFhM=
-github.com/go-ldap/ldap/v3 v3.1.8/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q=
+github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28=
+github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
+github.com/go-asn1-ber/asn1-ber v1.5.1 h1:pDbRAunXzIUXfx4CB2QJFv5IuPiuoW+sWvr/Us009o8=
+github.com/go-asn1-ber/asn1-ber v1.5.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-ldap/ldap/v3 v3.2.3 h1:FBt+5w3q/vPVPb4eYMQSn+pOiz4zewPamYhlGMmc7yM=
+github.com/go-ldap/ldap/v3 v3.2.3/go.mod h1:iYS1MdmrmceOJ1QOTnRXrIs7i3kloqtmGQjRvjKpyMg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200406173513-056763e48d71 h1:DOmugCavvUtnUD114C1Wh+UgTgQZ4pMLzXxi1pSt+/Y=
-golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

examples/ldapauth/main.go

@@ -35,8 +35,8 @@ func exitError() {
 	u := minimalSFTPGoUser{
 		Username: "",
 	}
-	json, _ := json.Marshal(u)
-	fmt.Printf("%v\n", string(json))
+	resp, _ := json.Marshal(u)
+	fmt.Printf("%v\n", string(resp))
 	os.Exit(1)
 }
 
@@ -52,8 +52,8 @@ func printSuccessResponse(username, homeDir string, uid, gid int) {
 	u.Permissions["/"] = []string{"*"}
 	// uncomment the next line to require publickey+password authentication
 	//u.Filters.DeniedLoginMethods = []string{"publickey", "password", "keyboard-interactive", "publickey+keyboard-interactive"}
-	json, _ := json.Marshal(u)
-	fmt.Printf("%v\n", string(json))
+	resp, _ := json.Marshal(u)
+	fmt.Printf("%v\n", string(resp))
 	os.Exit(0)
 }
 

examples/ldapauthserver/README.md

@@ -7,5 +7,5 @@ You can configure the server using the [ldapauth.toml](./ldapauth.toml) configur
 You can build this example using the following command:
 
 ```console
-go build -i -ldflags "-s -w" -o ldapauthserver
+go build -ldflags "-s -w" -o ldapauthserver
 ```

examples/ldapauthserver/cmd/root.go

@@ -77,17 +77,26 @@ func addConfigFlags(cmd *cobra.Command) {
 	viper.SetDefault(configDirKey, defaultConfigDir)
 	viper.BindEnv(configDirKey, "LDAPAUTH_CONFIG_DIR")
 	cmd.Flags().StringVarP(&configDir, configDirFlag, "c", viper.GetString(configDirKey),
-		"Location for the config dir. This directory should contain the \"ldapauth\" configuration file or the configured "+
-			"config-file. This flag can be set using LDAPAUTH_CONFIG_DIR env var too.")
+		`Location for the config dir. This directory
+should contain the "ldapauth" configuration
+file or the configured config-file. This flag
+can be set using LDAPAUTH_CONFIG_DIR env var too.
+`)
 	viper.BindPFlag(configDirKey, cmd.Flags().Lookup(configDirFlag))
 
 	viper.SetDefault(configFileKey, defaultConfigName)
 	viper.BindEnv(configFileKey, "LDAPAUTH_CONFIG_FILE")
 	cmd.Flags().StringVarP(&configFile, configFileFlag, "f", viper.GetString(configFileKey),
-		"Name for the configuration file. It must be the name of a file stored in config-dir not the absolute path to the "+
-			"configuration file. The specified file name must have no extension we automatically load JSON, YAML, TOML, HCL and "+
-			"Java properties. Therefore if you set \"ldapauth\" then \"ldapauth.toml\", \"ldapauth.yaml\" and so on are searched. "+
-			"This flag can be set using LDAPAUTH_CONFIG_FILE env var too.")
+		`Name for the configuration file. It must be
+the name of a file stored in config-dir not
+the absolute path to the configuration file.
+The specified file name must have no extension
+we automatically load JSON, YAML, TOML, HCL and
+Java properties. Therefore if you set \"ldapauth\"
+then \"ldapauth.toml\", \"ldapauth.yaml\" and
+so on are searched. This flag can be set using
+LDAPAUTH_CONFIG_FILE env var too.
+`)
 	viper.BindPFlag(configFileKey, cmd.Flags().Lookup(configFileFlag))
 }
 
@@ -97,41 +106,53 @@ func addServeFlags(cmd *cobra.Command) {
 	viper.SetDefault(logFilePathKey, defaultLogFile)
 	viper.BindEnv(logFilePathKey, "LDAPAUTH_LOG_FILE_PATH")
 	cmd.Flags().StringVarP(&logFilePath, logFilePathFlag, "l", viper.GetString(logFilePathKey),
-		"Location for the log file. Leave empty to write logs to the standard output. This flag can be set using LDAPAUTH_LOG_FILE_PATH "+
-			"env var too.")
+		`Location for the log file. Leave empty to write
+logs to the standard output. This flag can be
+set using LDAPAUTH_LOG_FILE_PATH env var too.
+`)
 	viper.BindPFlag(logFilePathKey, cmd.Flags().Lookup(logFilePathFlag))
 
 	viper.SetDefault(logMaxSizeKey, defaultLogMaxSize)
 	viper.BindEnv(logMaxSizeKey, "LDAPAUTH_LOG_MAX_SIZE")
 	cmd.Flags().IntVarP(&logMaxSize, logMaxSizeFlag, "s", viper.GetInt(logMaxSizeKey),
-		"Maximum size in megabytes of the log file before it gets rotated. This flag can be set using LDAPAUTH_LOG_MAX_SIZE "+
-			"env var too. It is unused if log-file-path is empty.")
+		`Maximum size in megabytes of the log file
+before it gets rotated. This flag can be set
+using LDAPAUTH_LOG_MAX_SIZE env var too. It
+is unused if log-file-path is empty.`)
 	viper.BindPFlag(logMaxSizeKey, cmd.Flags().Lookup(logMaxSizeFlag))
 
 	viper.SetDefault(logMaxBackupKey, defaultLogMaxBackup)
 	viper.BindEnv(logMaxBackupKey, "LDAPAUTH_LOG_MAX_BACKUPS")
 	cmd.Flags().IntVarP(&logMaxBackups, "log-max-backups", "b", viper.GetInt(logMaxBackupKey),
-		"Maximum number of old log files to retain. This flag can be set using LDAPAUTH_LOG_MAX_BACKUPS env var too. "+
-			"It is unused if log-file-path is empty.")
+		`Maximum number of old log files to retain.
+This flag can be set using LDAPAUTH_LOG_MAX_BACKUPS
+env var too. It is unused if log-file-path is
+empty.`)
 	viper.BindPFlag(logMaxBackupKey, cmd.Flags().Lookup(logMaxBackupFlag))
 
 	viper.SetDefault(logMaxAgeKey, defaultLogMaxAge)
 	viper.BindEnv(logMaxAgeKey, "LDAPAUTH_LOG_MAX_AGE")
 	cmd.Flags().IntVarP(&logMaxAge, "log-max-age", "a", viper.GetInt(logMaxAgeKey),
-		"Maximum number of days to retain old log files. This flag can be set using LDAPAUTH_LOG_MAX_AGE env var too. "+
-			"It is unused if log-file-path is empty.")
+		`Maximum number of days to retain old log files.
+This flag can be set using LDAPAUTH_LOG_MAX_AGE
+env var too. It is unused if log-file-path is
+empty.`)
 	viper.BindPFlag(logMaxAgeKey, cmd.Flags().Lookup(logMaxAgeFlag))
 
 	viper.SetDefault(logCompressKey, defaultLogCompress)
 	viper.BindEnv(logCompressKey, "LDAPAUTH_LOG_COMPRESS")
-	cmd.Flags().BoolVarP(&logCompress, logCompressFlag, "z", viper.GetBool(logCompressKey), "Determine if the rotated "+
-		"log files should be compressed using gzip. This flag can be set using LDAPAUTH_LOG_COMPRESS env var too. "+
-		"It is unused if log-file-path is empty.")
+	cmd.Flags().BoolVarP(&logCompress, logCompressFlag, "z", viper.GetBool(logCompressKey),
+		`Determine if the rotated log files
+should be compressed using gzip. This flag can
+be set using LDAPAUTH_LOG_COMPRESS env var too.
+It is unused if log-file-path is empty.`)
 	viper.BindPFlag(logCompressKey, cmd.Flags().Lookup(logCompressFlag))
 
 	viper.SetDefault(logVerboseKey, defaultLogVerbose)
 	viper.BindEnv(logVerboseKey, "LDAPAUTH_LOG_VERBOSE")
-	cmd.Flags().BoolVarP(&logVerbose, logVerboseFlag, "v", viper.GetBool(logVerboseKey), "Enable verbose logs. "+
-		"This flag can be set using LDAPAUTH_LOG_VERBOSE env var too.")
+	cmd.Flags().BoolVarP(&logVerbose, logVerboseFlag, "v", viper.GetBool(logVerboseKey),
+		`Enable verbose logs. This flag can be set
+using LDAPAUTH_LOG_VERBOSE env var too.
+`)
 	viper.BindPFlag(logVerboseKey, cmd.Flags().Lookup(logVerboseFlag))
 }

examples/ldapauthserver/go.mod

@@ -4,23 +4,24 @@ go 1.14
 
 require (
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.4.1 // indirect
-	github.com/go-chi/chi v4.1.1+incompatible
+	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/go-chi/render v1.0.1
-	github.com/go-ldap/ldap/v3 v3.1.8
-	github.com/mitchellh/mapstructure v1.2.2 // indirect
+	github.com/go-ldap/ldap/v3 v3.2.3
+	github.com/json-iterator/go v1.1.9 // indirect
+	github.com/mitchellh/mapstructure v1.3.2 // indirect
 	github.com/nathanaelle/password/v2 v2.0.1
-	github.com/pelletier/go-toml v1.7.0 // indirect
-	github.com/rs/zerolog v1.18.0
-	github.com/spf13/afero v1.2.2 // indirect
+	github.com/pelletier/go-toml v1.8.0 // indirect
+	github.com/rs/zerolog v1.19.0
+	github.com/spf13/afero v1.3.2 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.6.3
-	golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a
-	golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f // indirect
-	golang.org/x/text v0.3.2 // indirect
-	gopkg.in/ini.v1 v1.55.0 // indirect
+	github.com/spf13/viper v1.7.0
+	github.com/zenazn/goji v0.9.0 // indirect
+	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
+	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect
+	golang.org/x/text v0.3.3 // indirect
+	gopkg.in/ini.v1 v1.57.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )

examples/ldapauthserver/go.sum

@@ -1,43 +1,60 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28=
+github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-asn1-ber/asn1-ber v1.3.1 h1:gvPdv/Hr++TRFCl0UbPFHC54P9N9jgsRPnmnr419Uck=
-github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-asn1-ber/asn1-ber v1.4.1 h1:qP/QDxOtmMoJVgXHCXNzDpA0+wkgYB2x5QoLMVOciyw=
-github.com/go-asn1-ber/asn1-ber v1.4.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-chi/chi v4.1.1+incompatible h1:MmTgB0R8Bt/jccxp+t6S/1VGIKdJw5J74CK/c9tTfA4=
-github.com/go-chi/chi v4.1.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-asn1-ber/asn1-ber v1.5.1 h1:pDbRAunXzIUXfx4CB2QJFv5IuPiuoW+sWvr/Us009o8=
+github.com/go-asn1-ber/asn1-ber v1.5.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
+github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
 github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
 github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-ldap/ldap/v3 v3.1.8 h1:5vU/2jOh9HqprwXp8aF915s9p6Z8wmbSEVF7/gdTFhM=
-github.com/go-ldap/ldap/v3 v3.1.8/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q=
+github.com/go-ldap/ldap/v3 v3.2.3 h1:FBt+5w3q/vPVPb4eYMQSn+pOiz4zewPamYhlGMmc7yM=
+github.com/go-ldap/ldap/v3 v3.2.3/go.mod h1:iYS1MdmrmceOJ1QOTnRXrIs7i3kloqtmGQjRvjKpyMg=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -46,60 +63,98 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4=
-github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg=
+github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nathanaelle/password/v2 v2.0.1 h1:ItoCTdsuIWzilYmllQPa3DR3YoCXcpfxScWLqr8Ii2s=
 github.com/nathanaelle/password/v2 v2.0.1/go.mod h1:eaoT+ICQEPNtikBRIAatN8ThWwMhVG+r1jTw60BvPJk=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI=
-github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
+github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw=
+github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -110,44 +165,41 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
-github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8=
-github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I=
+github.com/rs/zerolog v1.19.0 h1:hYz4ZVdUgjXTBUmrkrw55j1nHx68LfOKIQk5IYtyScg=
+github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/afero v1.3.2 h1:GDarE4TJQI52kYSbSAmLiId1Elfj+xgSDqrUZxFhxlU=
+github.com/spf13/afero v1.3.2/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
-github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs=
-github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw=
+github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -156,73 +208,144 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a h1:y6sBfNd1b9Wy08a6K1Z1DZc4aXABUN5TKjkYhz7UKmo=
-golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8=
-golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ=
-gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
+gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

examples/ldapauthserver/httpd/auth.go

@@ -32,10 +32,10 @@ type httpAuthProvider interface {
 }
 
 type basicAuthProvider struct {
-	Path  string
+	Path string
+	sync.RWMutex
 	Info  os.FileInfo
 	Users map[string]string
-	lock  *sync.RWMutex
 }
 
 func newBasicAuthProvider(authUserFile string) (httpAuthProvider, error) {
@@ -43,7 +43,6 @@ func newBasicAuthProvider(authUserFile string) (httpAuthProvider, error) {
 		Path:  authUserFile,
 		Info:  nil,
 		Users: make(map[string]string),
-		lock:  new(sync.RWMutex),
 	}
 	return &basicAuthProvider, basicAuthProvider.loadUsers()
 }
@@ -53,8 +52,8 @@ func (p *basicAuthProvider) isEnabled() bool {
 }
 
 func (p *basicAuthProvider) isReloadNeeded(info os.FileInfo) bool {
-	p.lock.RLock()
-	defer p.lock.RUnlock()
+	p.RLock()
+	defer p.RUnlock()
 	return p.Info == nil || p.Info.ModTime() != info.ModTime() || p.Info.Size() != info.Size()
 }
 
@@ -83,8 +82,8 @@ func (p *basicAuthProvider) loadUsers() error {
 			logger.Debug(logSender, "", "unable to parse basic auth users file: %v", err)
 			return err
 		}
-		p.lock.Lock()
-		defer p.lock.Unlock()
+		p.Lock()
+		defer p.Unlock()
 		p.Users = make(map[string]string)
 		for _, record := range records {
 			if len(record) == 2 {
@@ -102,8 +101,8 @@ func (p *basicAuthProvider) getHashedPassword(username string) (string, bool) {
 	if err != nil {
 		return "", false
 	}
-	p.lock.RLock()
-	defer p.lock.RUnlock()
+	p.RLock()
+	defer p.RUnlock()
 	pwd, ok := p.Users[username]
 	return pwd, ok
 }

examples/ldapauthserver/httpd/httpd.go

@@ -86,6 +86,7 @@ func StartHTTPServer(configDir string, httpConfig config.HTTPDConfig) error {
 		}
 		config := &tls.Config{
 			GetCertificate: certMgr.GetCertificateFunc(),
+			MinVersion:     tls.VersionTLS12,
 		}
 		httpServer.TLSConfig = config
 		return httpServer.ListenAndServeTLS("", "")

examples/ldapauthserver/httpd/models.go

@@ -51,7 +51,7 @@ type GCSFsConfig struct {
 
 // SFTPGoFilesystem defines cloud storage filesystem details
 type SFTPGoFilesystem struct {
-	// 0 local filesystem, 1 Amazon S3 compatible, 2 Google Cloud Storage
+	// 0 local filesystem, 1 AWS S3 compatible, 2 Google Cloud Storage
 	Provider  int         `json:"provider"`
 	S3Config  S3FsConfig  `json:"s3config,omitempty"`
 	GCSConfig GCSFsConfig `json:"gcsconfig,omitempty"`

examples/ldapauthserver/httpd/tlsutils.go

@@ -8,10 +8,10 @@ import (
 )
 
 type certManager struct {
-	cert     *tls.Certificate
 	certPath string
 	keyPath  string
-	lock     *sync.RWMutex
+	sync.RWMutex
+	cert *tls.Certificate
 }
 
 func (m *certManager) loadCertificate() error {
@@ -21,16 +21,16 @@ func (m *certManager) loadCertificate() error {
 		return err
 	}
 	logger.Debug(logSender, "", "https certificate successfully loaded")
-	m.lock.Lock()
-	defer m.lock.Unlock()
+	m.Lock()
+	defer m.Unlock()
 	m.cert = &newCert
 	return nil
 }
 
 func (m *certManager) GetCertificateFunc() func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
 	return func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
-		m.lock.RLock()
-		defer m.lock.RUnlock()
+		m.RLock()
+		defer m.RUnlock()
 		return m.cert, nil
 	}
 }
@@ -40,7 +40,6 @@ func newCertManager(certificateFile, certificateKeyFile string) (*certManager, e
 		cert:     nil,
 		certPath: certificateFile,
 		keyPath:  certificateKeyFile,
-		lock:     new(sync.RWMutex),
 	}
 	err := manager.loadCertificate()
 	if err != nil {

examples/ldapauthserver/logger/logger.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
-	"sync"
 
 	"github.com/rs/zerolog"
 	lumberjack "gopkg.in/natefinch/lumberjack.v2"
@@ -38,9 +37,9 @@ func InitLogger(logFilePath string, logMaxSize, logMaxBackups, logMaxAge int, lo
 		})
 		EnableConsoleLogger(level)
 	} else {
-		logger = zerolog.New(logSyncWrapper{
+		logger = zerolog.New(&logSyncWrapper{
 			output: os.Stdout,
-			lock:   new(sync.Mutex)})
+		})
 		consoleLogger = zerolog.Nop()
 	}
 	logger.Level(level)

examples/ldapauthserver/logger/request_logger.go

@@ -58,7 +58,7 @@ func (l *StructuredLoggerEntry) Write(status, bytes int, header http.Header, ela
 		Int("resp_status", status).
 		Int("resp_size", bytes).
 		Int64("elapsed_ms", elapsed.Nanoseconds()/1000000).
-		Msg("")
+		Send()
 }
 
 // Panic logs panics
@@ -69,5 +69,5 @@ func (l *StructuredLoggerEntry) Panic(v interface{}, stack []byte) {
 		Fields(l.fields).
 		Str("stack", string(stack)).
 		Str("panic", fmt.Sprintf("%+v", v)).
-		Msg("")
+		Send()
 }

examples/ldapauthserver/logger/sync_wrapper.go

@@ -6,12 +6,12 @@ import (
 )
 
 type logSyncWrapper struct {
-	lock   *sync.Mutex
+	sync.Mutex
 	output *os.File
 }
 
-func (l logSyncWrapper) Write(b []byte) (n int, err error) {
-	l.lock.Lock()
-	defer l.lock.Unlock()
+func (l *logSyncWrapper) Write(b []byte) (n int, err error) {
+	l.Lock()
+	defer l.Unlock()
 	return l.output.Write(b)
 }