set version to 1.0.0

and log the real error, it could leak a filesystem path

.github/workflows/development.yml

@@ -0,0 +1,175 @@
+name: CI
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+
+jobs:
+  test-deploy:
+    name: Test and deploy
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        go: [1.14]
+        os: [ubuntu-latest, macos-latest]
+        upload-coverage: [true]
+        include:
+          - go: 1.13
+            os: ubuntu-latest
+            upload-coverage: false
+          - go: 1.14
+            os: windows-latest
+            upload-coverage: false
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: Build for Linux/macOS
+        if: startsWith(matrix.os, 'windows-') != true
+        run: go build -i -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
+
+      - name: Build for Windows
+        if: startsWith(matrix.os, 'windows-')
+        run: |
+          $GIT_COMMIT = (git describe --always --dirty) | Out-String
+          $DATE_TIME = ([datetime]::Now.ToUniversalTime().toString("yyyy-MM-ddTHH:mm:ssZ")) | Out-String
+          go build -i -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/version.date=$DATE_TIME" -o sftpgo.exe
+
+      - name: Initialize data provider
+        run: ./sftpgo initprovider
+        shell: bash
+
+      - name: Run test cases using SQLite provider
+        run: go test -v ./... -coverprofile=coverage.txt -covermode=atomic
+
+      - name: Upload coverage to Codecov
+        if: ${{ matrix.upload-coverage }}
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./coverage.txt
+          fail_ci_if_error: false
+
+      - name: Run test cases using bolt provider
+        run: |
+          go test -v ./config -covermode=atomic
+          go test -v ./httpd -covermode=atomic
+          go test -v ./sftpd -covermode=atomic
+        env:
+          SFTPGO_DATA_PROVIDER__DRIVER: bolt
+          SFTPGO_DATA_PROVIDER__NAME: 'sftpgo_bolt.db'
+
+      - name: Run test cases using memory provider
+        run: go test -v ./... -covermode=atomic
+        env:
+          SFTPGO_DATA_PROVIDER__DRIVER: memory
+          SFTPGO_DATA_PROVIDER__NAME: ''
+
+      - name: Prepare build artifact for Linux/macOS
+        if: startsWith(matrix.os, 'windows-') != true
+        run: |
+          mkdir output
+          cp sftpgo output/
+          cp sftpgo.json output/
+          cp -r templates output/
+          cp -r static output/
+
+      - name: Prepare build artifact for Windows
+        if: startsWith(matrix.os, 'windows-')
+        run: |
+          mkdir output
+          copy .\sftpgo.exe .\output
+          copy .\sftpgo.json .\output
+          mkdir output\templates
+          xcopy .\templates .\output\templates\ /E
+          mkdir output\static
+          xcopy .\static .\output\static\ /E
+
+      - name: Upload build artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: sftpgo-${{ matrix.os }}-go${{ matrix.go }}
+          path: output
+
+  test-postgresql-mysql:
+    name: Test with PostgreSQL/MySQL
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: sftpgo
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      mariadb:
+        image: mariadb:latest
+        env:
+          MYSQL_ROOT_PASSWORD: mysql
+          MYSQL_DATABASE: sftpgo
+          MYSQL_USER: sftpgo
+          MYSQL_PASSWORD: sftpgo
+        options: >-
+          --health-cmd "mysqladmin status -h 127.0.0.1 -P 3306 -u root -p$MYSQL_ROOT_PASSWORD"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 6
+        ports:
+          - 3307:3306
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14
+
+      - name: Build
+        run: go build -i -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
+
+      - name: Run tests using PostgreSQL provider
+        run: |
+          ./sftpgo initprovider
+          go test -v ./... -covermode=atomic
+        env:
+          SFTPGO_DATA_PROVIDER__DRIVER: postgresql
+          SFTPGO_DATA_PROVIDER__NAME: sftpgo
+          SFTPGO_DATA_PROVIDER__HOST: localhost
+          SFTPGO_DATA_PROVIDER__PORT: 5432
+          SFTPGO_DATA_PROVIDER__USERNAME: postgres
+          SFTPGO_DATA_PROVIDER__PASSWORD: postgres
+
+      - name: Run tests using MySQL provider
+        run: |
+          ./sftpgo initprovider
+          go test -v ./... -covermode=atomic
+        env:
+          SFTPGO_DATA_PROVIDER__DRIVER: mysql
+          SFTPGO_DATA_PROVIDER__NAME: sftpgo
+          SFTPGO_DATA_PROVIDER__HOST: localhost
+          SFTPGO_DATA_PROVIDER__PORT: 3307
+          SFTPGO_DATA_PROVIDER__USERNAME: sftpgo
+          SFTPGO_DATA_PROVIDER__PASSWORD: sftpgo
+
+  golangci-lint:
+    name: golangci-lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v1
+        with:
+          version: v1.27

.github/workflows/release.yml

@@ -0,0 +1,225 @@
+name: Release
+
+on:
+  push:
+    tags: 'v*'
+
+env:
+  GO_VERSION: 1.14
+
+jobs:
+  create-release:
+    name: Create
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Release ${{ github.ref }}
+          draft: false
+          prerelease: false
+
+      - name: Save release upload URL
+        run: echo "${{ steps.create_release.outputs.upload_url }}" > ./upload_url.txt
+        shell: bash
+
+      - name: Store release upload URL
+        uses: actions/upload-artifact@v2
+        with:
+          name: upload_url
+          path: ./upload_url.txt
+
+  release-sources-with-deps:
+    name: Publish sources
+    needs: create-release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Get SFTPGo version
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+
+      - name: Prepare release
+        run: |
+          go mod vendor
+          echo "${SFTPGO_VERSION}" > VERSION.txt
+          tar cJvf sftpgo_${SFTPGO_VERSION}_src_with_deps.tar.xz *
+        env:
+          SFTPGO_VERSION: ${{ steps.get_version.outputs.VERSION }}
+
+      - name: Download release upload URL
+        uses: actions/download-artifact@v2
+        with:
+          name: upload_url
+
+      - name: Get release upload URL
+        id: upload_url
+        run: |
+          URL=$(cat upload_url.txt)
+          echo "::set-output name=url::${URL}"
+        shell: bash
+
+      - name: Upload Release
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.upload_url.outputs.url }}
+          asset_path: ./sftpgo_${{ steps.get_version.outputs.VERSION }}_src_with_deps.tar.xz
+          asset_name: sftpgo_${{ steps.get_version.outputs.VERSION }}_src_with_deps.tar.xz
+          asset_content_type: application/x-xz
+
+  publish:
+    name: Publish binary
+    needs: create-release
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Set up Python
+        if: startsWith(matrix.os, 'windows-')
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.x
+
+      - name: Build for Linux/macOS
+        if: startsWith(matrix.os, 'windows-') != true
+        run: go build -i -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
+
+      - name: Build for Windows
+        if: startsWith(matrix.os, 'windows-')
+        run: |
+          $GIT_COMMIT = (git describe --always --dirty) | Out-String
+          $DATE_TIME = ([datetime]::Now.ToUniversalTime().toString("yyyy-MM-ddTHH:mm:ssZ")) | Out-String
+          go build -i -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/version.date=$DATE_TIME" -o sftpgo.exe
+
+      - name: Initialize data provider
+        run: ./sftpgo initprovider
+        shell: bash
+
+      - name: Get SFTPGo version
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+        shell: bash
+
+      - name: Get OS name
+        id: get_os_name
+        run: |
+          if [ $MATRIX_OS == 'ubuntu-latest' ]
+          then
+            echo ::set-output name=OS::linux
+          elif [ $MATRIX_OS == 'macos-latest' ]
+          then
+            echo ::set-output name=OS::macOS
+          else
+            echo ::set-output name=OS::windows
+          fi
+        shell: bash
+        env:
+          MATRIX_OS: ${{ matrix.os }}
+
+      - name: Build REST API CLI for Windows
+        if: startsWith(matrix.os, 'windows-')
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          pip install requests
+          pip install pygments
+          pip install pyinstaller
+          pyinstaller --hidden-import="pkg_resources.py2_warn" --noupx --onefile examples\rest-api-cli\sftpgo_api_cli.py
+
+      - name: Prepare Release for Linux/macOS
+        if: startsWith(matrix.os, 'windows-') != true
+        run: |
+          mkdir -p output/{init,examples/rest-api-cli,sqlite}
+          echo "For documentation please take a look here:" > output/README.txt
+          echo "" >> output/README.txt
+          echo "https://github.com/drakkan/sftpgo/blob/${SFTPGO_VERSION}/README.md" >> output/README.txt
+          cp LICENSE output/
+          cp sftpgo output/
+          cp sftpgo.json output/
+          cp sftpgo.db output/sqlite/
+          cp -r static output/
+          cp -r templates output/
+          if [ $OS == 'linux' ]
+          then
+            cp -r init/sftpgo.service output/init/
+          else
+            cp -r init/com.github.drakkan.sftpgo.plist output/init/
+          fi
+          cp examples/rest-api-cli/sftpgo_api_cli.py output/examples/rest-api-cli/
+          cd output
+          tar cJvf sftpgo_${SFTPGO_VERSION}_${OS}_x86_64.tar.xz *
+          cd ..
+        env:
+          SFTPGO_VERSION: ${{ steps.get_version.outputs.VERSION }}
+          OS: ${{ steps.get_os_name.outputs.OS }}
+
+      - name: Prepare Release for Windows
+        if: startsWith(matrix.os, 'windows-')
+        run: |
+          mkdir output
+          copy .\sftpgo.exe .\output
+          copy .\sftpgo.json .\output
+          copy .\sftpgo.db .\output
+          copy .\dist\sftpgo_api_cli.exe .\output
+          copy .\LICENSE .\output\LICENSE.txt
+          mkdir output\templates
+          xcopy .\templates .\output\templates\ /E
+          mkdir output\static
+          xcopy .\static .\output\static\ /E
+          iscc windows-installer\sftpgo.iss
+        env:
+          SFTPGO_ISS_VERSION: ${{ steps.get_version.outputs.VERSION }}
+          SFTPGO_ISS_DOC_URL: https://github.com/drakkan/sftpgo/blob/${{ steps.get_version.outputs.VERSION }}/README.md
+
+      - name: Download release upload URL
+        uses: actions/download-artifact@v2
+        with:
+          name: upload_url
+
+      - name: Get release upload URL
+        id: upload_url
+        run: |
+          URL=$(cat upload_url.txt)
+          echo "::set-output name=url::${URL}"
+        shell: bash
+
+      - name: Upload Linux/macOS Release
+        if: startsWith(matrix.os, 'windows-') != true
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.upload_url.outputs.url }}
+          asset_path: ./output/sftpgo_${{ steps.get_version.outputs.VERSION }}_${{ steps.get_os_name.outputs.OS }}_x86_64.tar.xz
+          asset_name: sftpgo_${{ steps.get_version.outputs.VERSION }}_${{ steps.get_os_name.outputs.OS }}_x86_64.tar.xz
+          asset_content_type: application/x-xz
+
+      - name: Upload Windows Release
+        if: startsWith(matrix.os, 'windows-')
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.upload_url.outputs.url }}
+          asset_path: ./sftpgo_windows_x86_64.exe
+          asset_name: sftpgo_${{ steps.get_version.outputs.VERSION }}_${{ steps.get_os_name.outputs.OS }}_x86_64.exe
+          asset_content_type: application/x-dosexec

.gitignore

@@ -0,0 +1,3 @@
+# compilation output
+sftpgo
+sftpgo.exe

.golangci.yml

@@ -0,0 +1,42 @@
+run:
+  timeout: 5m
+  issues-exit-code: 1
+  tests: true
+
+
+linters-settings:
+  dupl:
+    threshold: 150
+  errcheck:
+    check-type-assertions: false
+    check-blank: false
+  goconst:
+    min-len: 3
+    min-occurrences: 3
+  gocyclo:
+    min-complexity: 15
+  gofmt:
+    simplify: true
+  goimports:
+    local-prefixes: github.com/drakkan/sftpgo
+  maligned:
+    suggest-new: true
+
+linters:
+  enable:
+    - goconst
+    - errcheck
+    - gofmt
+    - goimports
+    - golint
+    - unconvert
+    - unparam
+    - bodyclose
+    - gocyclo
+    - misspell
+    - maligned
+    - whitespace
+    - dupl
+    - scopelint
+    - rowserrcheck
+    - dogsled

.travis.yml

@@ -1,24 +0,0 @@
-language: go
-
-os:
-  - linux
-  - osx
-
-go:
-  - "1.12.x"
-  - "1.13.x"
-
-env:
-  - GO111MODULE=on
-
-before_script:
-  - sqlite3 sftpgo.db 'CREATE TABLE "users" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE, "password" varchar(255) NULL, "public_keys" text NULL, "home_dir" varchar(255) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL);'
-
-install:
-  - go get -v -t ./...
-
-script:
-  - go test -v ./... -coverprofile=coverage.txt -covermode=atomic
-
-after_success:
-  - bash <(curl -s https://codecov.io/bash)

README.md

@@ -1,409 +1,186 @@
 # SFTPGo
-[![Build Status](https://travis-ci.org/drakkan/sftpgo.svg?branch=master)](https://travis-ci.org/drakkan/sftpgo) [![Code Coverage](https://codecov.io/gh/drakkan/sftpgo/branch/master/graph/badge.svg)](https://codecov.io/gh/drakkan/sftpgo/branch/master) [![Go Report Card](https://goreportcard.com/badge/github.com/drakkan/sftpgo)](https://goreportcard.com/report/github.com/drakkan/sftpgo) [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) [![Mentioned in Awesome Go](https://awesome.re/mentioned-badge.svg)](https://github.com/avelino/awesome-go)
 
-Full featured and highly configurable SFTP server
+![CI Status](https://github.com/drakkan/sftpgo/workflows/CI/badge.svg?branch=master&event=push)
+[![Code Coverage](https://codecov.io/gh/drakkan/sftpgo/branch/master/graph/badge.svg)](https://codecov.io/gh/drakkan/sftpgo/branch/master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/drakkan/sftpgo)](https://goreportcard.com/report/github.com/drakkan/sftpgo)
+[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
+[![Mentioned in Awesome Go](https://awesome.re/mentioned-badge.svg)](https://github.com/avelino/awesome-go)
+
+Fully featured and highly configurable SFTP server, written in Go
 
 ## Features
 
-- Each account is chrooted to his Home Dir.
+- Each account is chrooted to its home directory.
 - SFTP accounts are virtual accounts stored in a "data provider".
-- SQLite, MySQL, PostgreSQL and bbolt (key/value store in pure Go) data providers are supported.
+- SQLite, MySQL, PostgreSQL, bbolt (key/value store in pure Go) and in-memory data providers are supported.
 - Public key and password authentication. Multiple public keys per user are supported.
+- SSH user [certificate authentication](https://cvsweb.openbsd.org/src/usr.bin/ssh/PROTOCOL.certkeys?rev=1.8).
+- Keyboard interactive authentication. You can easily setup a customizable multi-factor authentication.
+- Partial authentication. You can configure multi-step authentication requiring, for example, the user password after successful public key authentication.
+- Per user authentication methods. You can configure the allowed authentication methods for each user.
+- Custom authentication via external programs/HTTP API is supported.
+- Dynamic user modification before login via external programs/HTTP API is supported.
 - Quota support: accounts can have individual quota expressed as max total size and/or max number of files.
 - Bandwidth throttling is supported, with distinct settings for upload and download.
 - Per user maximum concurrent sessions.
-- Per user permissions: list directories content, upload, overwrite, download, delete, rename, create directories, create symlinks can be enabled or disabled.
-- Per user files/folders ownership: you can map all the users to the system account that runs SFTPGo (all platforms are supported) or you can run SFTPGo as root user and map each user or group of users to a different system account (*NIX only).
-- Configurable custom commands and/or HTTP notifications on upload, download, delete or rename.
+- Per user and per directory permission management: list directory contents, upload, overwrite, download, delete, rename, create directories, create symlinks, change owner/group and mode, change access and modification times.
+- Per user files/folders ownership mapping: you can map all the users to the system account that runs SFTPGo (all platforms are supported) or you can run SFTPGo as root user and map each user or group of users to a different system account (\*NIX only).
+- Per user IP filters are supported: login can be restricted to specific ranges of IP addresses or to a specific IP address.
+- Per user and per directory file extensions filters are supported: files can be allowed or denied based on their extensions.
+- Virtual folders are supported: directories outside the user home directory can be exposed as virtual folders.
+- Configurable custom commands and/or HTTP notifications on file upload, download, pre-delete, delete, rename, on SSH commands and on user add, update and delete.
 - Automatically terminating idle connections.
 - Atomic uploads are configurable.
-- Optional SCP support.
-- REST API for users and quota management and real time reports for the active connections with possibility of forcibly closing a connection.
-- Prometheus metrics are exposed.
-- Configuration is a your choice: JSON, TOML, YAML, HCL, envfile are supported.
-- Log files are accurate and they are saved in the easily parsable JSON format.
+- Support for Git repositories over SSH.
+- SCP and rsync are supported.
+- Support for serving local filesystem, S3 Compatible Object Storage and Google Cloud Storage over SFTP/SCP.
+- [Prometheus metrics](./docs/metrics.md) are exposed.
+- Support for HAProxy PROXY protocol: you can proxy and/or load balance the SFTP/SCP service without losing the information about the client's address.
+- [REST API](./docs/rest-api.md) for users and folders management, backup, restore and real time reports of the active connections with possibility of forcibly closing a connection.
+- [Web based administration interface](./docs/web-admin.md) to easily manage users, folders and connections.
+- Easy [migration](./examples/rest-api-cli#convert-users-from-other-stores) from Linux system user accounts.
+- [Portable mode](./docs/portable-mode.md): a convenient way to share a single directory on demand.
+- Performance analysis using built-in [profiler](./docs/profiling.md).
+- Configuration format is at your choice: JSON, TOML, YAML, HCL, envfile are supported.
+- Log files are accurate and they are saved in the easily parsable JSON format ([more information](./docs/logs.md)).
 
 ## Platforms
 
-SFTPGo is developed and tested on Linux. After each commit the code is automatically built and tested on Linux and macOS using Travis CI.
-Regularly the test cases are manually executed and pass on Windows. Other UNIX variants such as *BSD should work too.
+SFTPGo is developed and tested on Linux. After each commit, the code is automatically built and tested on Linux, macOS and Windows using a [GitHub Action](./.github/workflows/development.yml). Other UNIX variants such as \*BSD should work too.
 
 ## Requirements
 
-- Go 1.12 or higher.
-- A suitable SQL server or key/value store to use as data provider: PostreSQL 9.4+ or MySQL 5.6+ or SQLite 3.x or bbolt 1.3.x
+- Go 1.13 or higher as build only dependency.
+- A suitable SQL server or key/value store to use as data provider: PostgreSQL 9.4+ or MySQL 5.6+ or SQLite 3.x or bbolt 1.3.x
 
 ## Installation
 
-Simple install the package to your [$GOPATH](https://github.com/golang/go/wiki/GOPATH "GOPATH") with the [go tool](https://golang.org/cmd/go/ "go command") from shell:
+Binary releases for Linux, macOS, and Windows are available. Please visit the [releases](https://github.com/drakkan/sftpgo/releases "releases") page.
 
-```
-$ go get -u github.com/drakkan/sftpgo
-```
+Sample Dockerfiles for [Debian](https://www.debian.org) and [Alpine](https://alpinelinux.org) are available inside the source tree [docker](./docker) directory.
 
-Make sure [Git is installed](https://git-scm.com/downloads) on your machine and in your system's `PATH`.
+Some Linux distro packages are available:
 
-SFTPGo depends on [go-sqlite3](https://github.com/mattn/go-sqlite3) that is a CGO package and so it requires a `C` compiler at build time.
-On Linux and macOS a compiler is easy to install or already installed, on Windows you need to download [MinGW-w64](https://sourceforge.net/projects/mingw-w64/files/) and build SFTPGo from its command prompt.
+- For Arch Linux via AUR:
+  - [sftpgo](https://aur.archlinux.org/packages/sftpgo/). This package follows stable releases. It requires `git`, `gcc` and `go` to build.
+  - [sftpgo-bin](https://aur.archlinux.org/packages/sftpgo-bin/). This package follows stable releases downloading the prebuilt linux binary from GitHub. It does not require `git`, `gcc` and `go` to build.
+  - [sftpgo-git](https://aur.archlinux.org/packages/sftpgo-git/). This package builds and installs the latest git master. It requires `git`, `gcc` and `go` to build.
 
-The compiler is a build time only dependency, it is not not required at runtime.
+You can easily test new features selecting a commit from the [Actions](https://github.com/drakkan/sftpgo/actions) page and downloading the matching build artifacts for Linux, macOS or Windows. GitHub stores artifacts for 90 days.
 
-If you don't need SQLite, you can also get/build SFTPGo setting the environment variable `GCO_ENABLED` to 0, this way SQLite support will be disabled but PostgreSQL, MySQL and bbolt will work and you don't need a `C` compiler for building.
-
-Version info, such as git commit and build date, can be embedded setting the following string variables at build time:
-
-- `github.com/drakkan/sftpgo/utils.commit`
-- `github.com/drakkan/sftpgo/utils.date`
-
-For example you can build using the following command:
-
-```
-go build -i -ldflags "-s -w -X github.com/drakkan/sftpgo/utils.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/utils.date=`date -u +%FT%TZ`" -o sftpgo
-```
-
-and you will get a version that includes git commit and build date like this one:
-
-```
-sftpgo -v
-SFTPGo version: 0.9.0-dev-90607d4-dirty-2019-08-08T19:28:36Z
-```
-
-For Linux, a `systemd` sample [service](https://github.com/drakkan/sftpgo/tree/master/init/sftpgo.service "systemd service") can be found inside the source tree.
-
-Alternately you can use distro packages:
-
-- Arch Linux PKGBUILD is available on [AUR](https://aur.archlinux.org/packages/sftpgo/ "SFTPGo")
-
-For macOS a `launchd` sample [service](https://github.com/drakkan/sftpgo/tree/master/init/com.github.drakkan.sftpgo.plist "launchd plist") can be found inside the source tree. The `launchd` plist assumes that `sftpgo` has `/usr/local/opt/sftpgo` as base directory.
-
-On Windows you can run `SFTPGo` as Windows Service, please read the "Configuration" section below for more details.
+Alternately, you can [build from source](./docs/build-from-source.md).
 
 ## Configuration
 
-The `sftpgo` executable can be used this way:
+A full explanation of all configuration methods can be found [here](./docs/full-configuration.md).
 
-```
-Usage:
-  sftpgo [command]
+Please make sure to [initialize the data provider](#data-provider-initialization) before running the daemon!
 
-Available Commands:
-  help        Help about any command
-  serve       Start the SFTP Server
+To start the SFTP server with default settings, simply run:
 
-Flags:
-  -h, --help      help for sftpgo
-  -v, --version
-
- Use "sftpgo [command] --help" for more information about a command 
-```
-
-The `serve` subcommand supports the following flags:
-
-- `--config-dir` string. Location of the config dir. This directory should contain the `sftpgo` configuration file and is used as the base for files with a relative path (eg. the private keys for the SFTP server, the SQLite or bblot database if you use SQLite or bbolt as data provider). The default value is "." or the value of `SFTPGO_CONFIG_DIR` environment variable.
-- `--config-file` string. Name of the configuration file. It must be the name of a file stored in config-dir not the absolute path to the configuration file. The specified file name must have no extension we automatically load JSON, YAML, TOML, HCL and Java properties. The default value is "sftpgo" (and therefore `sftpgo.json`, `sftpgo.yaml` and so on are searched) or the value of `SFTPGO_CONFIG_FILE` environment variable.
-- `--log-compress` boolean. Determine if the rotated log files should be compressed using gzip. Default `false` or the value of `SFTPGO_LOG_COMPRESS` environment variable (1 or `true`, 0 or `false`). It is unused if `log-file-path` is empty.
-- `--log-file-path` string. Location for the log file, default "sftpgo.log" or the value of `SFTPGO_LOG_FILE_PATH` environment variable. Leave empty to write logs to the standard error.
-- `--log-max-age` int. Maximum number of days to retain old log files. Default 28 or the value of `SFTPGO_LOG_MAX_AGE` environment variable. It is unused if `log-file-path` is empty.
-- `--log-max-backups` int. Maximum number of old log files to retain. Default 5 or the value of `SFTPGO_LOG_MAX_BACKUPS` environment variable. It is unused if `log-file-path` is empty.
-- `--log-max-size` int. Maximum size in megabytes of the log file before it gets rotated. Default 10 or the value of `SFTPGO_LOG_MAX_SIZE` environment variable. It is unused if `log-file-path` is empty.
-- `--log-verbose` boolean. Enable verbose logs. Default `true` or the value of `SFTPGO_LOG_VERBOSE` environment variable (1 or `true`, 0 or `false`).
-
-If you don't configure any private host keys, the daemon will use `id_rsa` in the configuration directory. If that file doesn't exist, the daemon will attempt to autogenerate it (if the user that executes SFTPGo has write access to the config-dir). The server supports any private key format supported by [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/keys.go#L32).
-
-Before starting `sftpgo` a dataprovider must be configured.
-
-Sample SQL scripts to create the required database structure can be found inside the source tree [sql](https://github.com/drakkan/sftpgo/tree/master/sql "sql") directory. The SQL scripts filename's is, by convention, the date as `YYYYMMDD` and the suffix `.sql`. You need to apply all the SQL scripts for your database ordered by name, for example `20190706.sql` must be applied before `20190728.sql` and so on.
-
-The `sftpgo` configuration file contains the following sections:
-
-- **"sftpd"**, the configuration for the SFTP server
-    - `bind_port`, integer. The port used for serving SFTP requests. Default: 2022
-    - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: ""
-    - `idle_timeout`, integer. Time in minutes after which an idle client will be disconnected. 0 menas disabled. Default: 15
-    - `max_auth_tries` integer. Maximum number of authentication attempts permitted per connection. If set to a negative number, the number of attempts are unlimited. If set to zero, the number of attempts are limited to 6.
-    - `umask`, string. Umask for the new files and directories. This setting has no effect on Windows. Default: "0022"
-    - `banner`, string. Identification string used by the server. Leave empty to use the default banner. Default "SFTPGo_version"
-    - `upload_mode` integer. 0 means standard, the files are uploaded directly to the requested path. 1 means atomic: files are uploaded to a temporary path and renamed to the requested path when the client ends the upload. Atomic mode avoids problems such as a web server that serves partial files when the files are being uploaded. In atomic mode if there is an upload error the temporary file is deleted and so the requested upload path will not contain a partial file.
-    - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions
-        - `execute_on`, list of strings. Valid values are `download`, `upload`, `delete`, `rename`. On folder deletion a `delete` notification will be sent for each deleted file. Actions will be not executed if an error is detected and so a partial file is uploaded or downloaded. Leave empty to disable actions. The `upload` condition includes both uploads to new files and overwrite existing files 
-        - `command`, string. Absolute path to the command to execute. Leave empty to disable. The command is invoked with the following arguments:
-            - `action`, any valid `execute_on` string
-            - `username`, user who did the action
-            - `path` to the affected file. For `rename` action this is the old file name
-            - `target_path`, non empty for `rename` action, this is the new file name
-        - `http_notification_url`, a valid URL. An HTTP GET request will be executed to this URL. Leave empty to disable. The query string will contain the following parameters that have the same meaning of the command's arguments:
-            - `action`
-            - `username`
-            - `path`
-            - `target_path`, added for `rename` action only
-    - `keys`, struct array. It contains the daemon's private keys. If empty or missing the daemon will search or try to generate `id_rsa` in the configuration directory.
-        - `private_key`, path to the private key file. It can be a path relative to the config dir or an absolute one.
-    - `enable_scp`, boolean. Default disabled. Set to `true` to enable SCP support. SCP is an experimental feature, we have our own SCP implementation since we can't rely on `scp` system command to proper handle permissions, quota and user's home dir restrictions. The SCP protocol is quite simple but there is no official docs about it, so we need more testing and feedbacks before enabling it by default. We may not handle some borderline cases or have sneaky bugs. Please do accurate tests yourself before enabling SCP and let us known if something does not work as expected for your use cases. SCP between two remote hosts is supported using the `-3` scp option.
-    - `kex_algorithms`, list of strings. Available KEX (Key Exchange) algorithms in preference order. Leave empty to use default values. The supported values can be found here: [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/common.go#L46 "Supported kex algos")
-    - `ciphers`, list of strings. Allowed ciphers. Leave empty to use default values. The supported values can be found here: [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/common.go#L28 "Supported ciphers")
-    - `macs`, list of strings. available MAC (message authentication code) algorithms in preference order. Leave empty to use default values. The supported values can be found here: [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/common.go#L84 "Supported MACs")
-    - `login_banner_file`, path to the login banner file. The contents of the specified file, if any, are sent to the remote user before authentication is allowed. It can be a path relative to the config dir or an absolute one. Leave empty to send no login banner
-- **"data_provider"**, the configuration for the data provider
-    - `driver`, string. Supported drivers are `sqlite`, `mysql`, `postgresql`, `bolt`
-    - `name`, string. Database name. For driver `sqlite` this can be the database name relative to the config dir or the absolute path to the SQLite database.
-    - `host`, string. Database host. Leave empty for driver `sqlite` and `bolt`
-    - `port`, integer. Database port. Leave empty for driver `sqlite` and `bolt`
-    - `username`, string. Database user. Leave empty for driver `sqlite` and `bolt`
-    - `password`, string. Database password. Leave empty for driver `sqlite` and `bolt`
-    - `sslmode`, integer. Used for drivers `mysql` and `postgresql`. 0 disable SSL/TLS connections, 1 require ssl, 2 set ssl mode to `verify-ca` for driver `postgresql` and `skip-verify` for driver `mysql`, 3 set ssl mode to `verify-full` for driver `postgresql` and `preferred` for driver `mysql`
-    - `connectionstring`, string. Provide a custom database connection string. If not empty this connection string will be used instead of build one using the previous parameters. Leave empty for driver `bolt`
-    - `users_table`, string. Database table for SFTP users
-    - `manage_users`, integer. Set to 0 to disable users management, 1 to enable
-    - `track_quota`, integer. Set the preferred way to track users quota between the following choices:
-        - 0, disable quota tracking. REST API to scan user dir and update quota will do nothing
-        - 1, quota is updated each time a user upload or delete a file even if the user has no quota restrictions
-        - 2, quota is updated each time a user upload or delete a file but only for users with quota restrictions. With this configuration the "quota scan" REST API can still be used to periodically update space usage for users without quota restrictions
-    - `pool_size`, integer. Sets the maximum number of open connections for `mysql` and `postgresql` driver. Default 0 (unlimited)
-- **"httpd"**, the configuration for the HTTP server used to serve REST API
-    - `bind_port`, integer. The port used for serving HTTP requests. Set to 0 to disable HTTP server. Default: 8080
-    - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "127.0.0.1"
-
-Here is a full example showing the default config in JSON format:
-
-```json
-{
-  "sftpd": {
-    "bind_port": 2022,
-    "bind_address": "",
-    "idle_timeout": 15,
-    "max_auth_tries": 0,
-    "umask": "0022",
-    "banner": "SFTPGo",
-    "actions": {
-      "execute_on": [],
-      "command": "",
-      "http_notification_url": ""
-    },
-    "keys": [],
-    "enable_scp": false
-  },
-  "data_provider": {
-    "driver": "sqlite",
-    "name": "sftpgo.db",
-    "host": "",
-    "port": 5432,
-    "username": "",
-    "password": "",
-    "sslmode": 0,
-    "connection_string": "",
-    "users_table": "users",
-    "manage_users": 1,
-    "track_quota": 2,
-    "pool_size": 0
-  },
-  "httpd": {
-    "bind_port": 8080,
-    "bind_address": "127.0.0.1"
-  }
-}
-```
-
-If you want to use a private key that use an algorithm different from RSA or more than one private key then replace the empty `keys` array with something like this:
-
-```json
-"keys": [
-  {
-    "private_key": "id_rsa"
-  },
-  {
-    "private_key": "id_ecdsa"
-  }
-]
-```
-
-The configuration can be read from JSON, TOML, YAML, HCL, envfile and Java properties config files, if your `config-file` flag is set to `sftpgo` (default value) you need to create a configuration file called `sftpgo.json` or `sftpgo.yaml` and so on inside `config-dir`.
-
-You can also override all the available configuration options using environment variables, sftpgo will check for environment variables with a name matching the key uppercased and prefixed with the `SFTPGO_`. You need to use `__` to traverse a struct.
-
-Let's see some examples:
-
-- To set sftpd `bind_port` you need to define the env var `SFTPGO_SFTPD__BIND_PORT`
-- To set the `execute_on` actions you need to define the env var `SFTPGO_SFTPD__ACTIONS__EXECUTE_ON` for example `SFTPGO_SFTPD__ACTIONS__EXECUTE_ON=upload,download`
-
-Please note that to override configuration options with environment variables a configuration file containing the options to override is required. You can, for example, deploy the default configuration file and then override the options you need to customize using environment variables.
-
-To start the SFTP Server with the default values for the command line flags simply use:
-
-```
+```bash
 sftpgo serve
 ```
 
-On Windows you can register `SFTPGo` as Windows Service, take a look at the CLI usage to learn how:
+Check out [this documentation](./docs/service.md) if you want to run SFTPGo as a service.
 
-```
-sftpgo.exe service --help
-Install, Uninstall, Start, Stop and retrieve status for SFTPGo Windows Service
+### Data provider initialization
 
-Usage:
-  sftpgo service [command]
+Before starting the SFTPGo server, please ensure that the configured data provider is properly initialized.
 
-Available Commands:
-  install     Install SFTPGo as Windows Service
-  start       Start SFTPGo Windows Service
-  status      Retrieve the status for the SFTPGo Windows Service
-  stop        Stop SFTPGo Windows Service
-  uninstall   Uninstall SFTPGo Windows Service
+SQL based data providers (SQLite, MySQL, PostgreSQL) require the creation of a database containing the required tables. Memory and bolt data providers do not require an initialization.
 
-Flags:
-  -h, --help   help for service
+After configuring the data provider using the configuration file, you can create the required database structure using the `initprovider` command.
+For SQLite provider, the `initprovider` command will auto create the database file, if missing, and the required tables.
+For PostgreSQL and MySQL providers, you need to create the configured database, and the `initprovider` command will create the required tables.
 
-Use "sftpgo service [command] --help" for more information about a command.
+For example, you can simply execute the following command from the configuration directory:
+
+```bash
+sftpgo initprovider
 ```
 
-`install` subcommand accepts the same flags valid for `serve`.
+Take a look at the CLI usage to learn how to specify a different configuration file:
 
-After installing as Windows Service please remember to allow network access to the SFTPGo executable using something like this:
-
-```
-netsh advfirewall firewall add rule name="SFTPGo Service" dir=in action=allow program="C:\Program Files\SFTPGo\sftpgo.exe"
+```bash
+sftpgo initprovider --help
 ```
 
-or through the Windows Firewall GUI.
+The `initprovider` command is enough for new installations. From now on, the database structure will be automatically checked and updated, if required, at startup.
+
+#### Upgrading
+
+If you are upgrading from version 0.9.5 or before, you have to manually execute the SQL scripts to create the required database structure. These scripts can be found inside the source tree [sql](./sql "sql") directory. The SQL scripts filename is, by convention, the date as `YYYYMMDD` and the suffix `.sql`. You need to apply all the SQL scripts for your database ordered by name. For example, `20190828.sql` must be applied before `20191112.sql`, and so on.
+Example for SQLite: `find sql/sqlite/ -type f -iname '*.sql' -print | sort -n | xargs cat | sqlite3 sftpgo.db`.
+After applying these scripts, your database structure is the same as the one obtained using `initprovider` for new installations, so from now on, you don't have to manually upgrade your database anymore.
+
+## Authentication options
+
+### External Authentication
+
+Custom authentication methods can easily be added. SFTPGo supports external authentication modules, and writing a new backend can be as simple as a few lines of shell script. More information can be found [here](./docs/external-auth.md).
+
+### Keyboard Interactive Authentication
+
+Keyboard interactive authentication is, in general, a series of questions asked by the server with responses provided by the client.
+This authentication method is typically used for multi-factor authentication.
+
+More information can be found [here](./docs/keyboard-interactive.md).
+
+## Dynamic user creation or modification
+
+A user can be created or modified by an external program just before the login. More information about this can be found [here](./docs/dynamic-user-mod.md).
+
+## Custom Actions
+
+SFTPGo allows to configure custom commands and/or HTTP notifications on file upload, download, delete, rename, on SSH commands and on user add, update and delete.
+
+More information about custom actions can be found [here](./docs/custom-actions.md).
+
+## Virtual folders
+
+Directories outside the user home directory can be exposed as virtual folders, more information [here](./docs/virtual-folders.md).
+
+## Storage backends
+
+### S3 Compabible Object Storage backends
+
+Each user can be mapped to whole bucket or to a bucket virtual folder. This way, the mapped bucket/virtual folder is exposed over SFTP/SCP. More information about S3 integration can be found [here](./docs/s3.md).
+
+### Google Cloud Storage backend
+
+Each user can be mapped with a Google Cloud Storage bucket or a bucket virtual folder. This way, the mapped bucket/virtual folder is exposed over SFTP/SCP. More information about Google Cloud Storage integration can be found [here](./docs/google-cloud-storage.md).
+
+### Other Storage backends
+
+Adding new storage backends is quite easy:
+
+- implement the [Fs interface](./vfs/vfs.go#L18 "interface for filesystem backends").
+- update the user method `GetFilesystem` to return the new backend
+- update the web interface and the REST API CLI
+- add the flags for the new storage backed to the `portable` mode
+
+Anyway, some backends require a pay per use account (or they offer free account for a limited time period only). To be able to add support for such backends or to review pull requests, please provide a test account. The test account must be available for enough time to be able to maintain the backend and do basic tests before each new release.
+
+## Brute force protection
+
+The [connection failed logs](./docs/logs.md) can be used for integration in tools such as [Fail2ban](http://www.fail2ban.org/). Example of [jails](./fail2ban/jails) and [filters](./fail2ban/filters) working with `systemd`/`journald` are available in fail2ban directory.
 
 ## Account's configuration properties
 
-For each account the following properties can be configured:
+Details information about account configuration properties can be found [here](./docs/account.md).
 
-- `username`
-- `password` used for password authentication. For users created using SFTPGo REST API if the password has no known hashing algo prefix it will be stored using argon2id. SFTPGo supports checking passwords stored with bcrypt, pbkdf2 and sha512crypt too. For pbkdf2 the supported format is `$<algo>$<iterations>$<salt>$<hashed pwd base64 encoded>`, where algo is `pbkdf2-sha1` or `pbkdf2-sha256` or `pbkdf2-sha512`. For example the `pbkdf2-sha256` of the word `password` using 150000 iterations and `E86a9YMX3zC7` as salt must be stored as `$pbkdf2-sha256$150000$E86a9YMX3zC7$R5J62hsSq+pYw00hLLPKBbcGXmq7fj5+/M0IFoYtZbo=`. For bcrypt the format must be the one supported by golang's [crypto/bcrypt](https://godoc.org/golang.org/x/crypto/bcrypt) package, for example the password `secret` with cost `14` must be stored as `$2a$14$ajq8Q7fbtFRQvXpdCq7Jcuy.Rx1h/L4J60Otx.gyNLbAYctGMJ9tK`. For sha512crypt we support the format used in `/etc/shadow` with the `$6$` prefix, this is useful if you are migrating from system user's accounts.  Using the REST API you can send a password hashed as bcrypt, pbkdf2 or sha512crypt and it will be stored as is.
-- `public_keys` array of public keys. At least one public key or the password is mandatory.
-- `home_dir` The user cannot upload or download files outside this directory. Must be an absolute path
-- `uid`, `gid`. If sftpgo runs as root system user then the created files and directories will be assigned to this system uid/gid. Ignored on windows and if sftpgo runs as non root user: in this case files and directories for all SFTP users will be owned by the system user that runs sftpgo.
-- `max_sessions` maximum concurrent sessions. 0 means unlimited
-- `quota_size` maximum size allowed as bytes. 0 means unlimited
-- `quota_files` maximum number of files allowed. 0 means unlimited
-- `permissions` the following permissions are supported:
-    - `*` all permissions are granted
-    - `list` list items is allowed
-    - `download` download files is allowed
-    - `upload` upload files is allowed
-    - `overwrite` overwrite an existing file, while uploading, is allowed. `upload` permission is required to allow file overwrite
-    - `delete` delete files or directories is allowed
-    - `rename` rename files or directories is allowed
-    - `create_dirs` create directories is allowed
-    - `create_symlinks` create symbolic links is allowed
-- `upload_bandwidth` maximum upload bandwidth as KB/s, 0 means unlimited
-- `download_bandwidth` maximum download bandwidth as KB/s, 0 means unlimited
+## Performance
 
-These properties are stored inside the data provider. If you want to use your existing accounts, you can create a database view. Since a view is read only, you have to disable user management and quota tracking so SFTPGo will never try to write to the view.
+SFTPGo can easily saturate a Gigabit connection on low end hardware with no special configuration, this is generally enough for most use cases.
 
-## REST API
-
-SFTPGo exposes REST API to manage users and quota and to get real time reports for the active connections with possibility of forcibly closing a connection.
-
-If quota tracking is enabled in `sftpgo` configuration file, then the used size and number of files are updated each time a file is added/removed. If files are added/removed not using SFTP or if you change `track_quota` from `2` to `1`, you can rescan the user home dir and update the used quota using the REST API.
-
-REST API is designed to run on localhost or on a trusted network, if you need HTTPS or authentication you can setup a reverse proxy using an HTTP Server such as Apache or NGNIX.
-
-For example you can keep SFTPGo listening on localhost and expose it externally configuring a reverse proxy using Apache HTTP Server this way:
-
-```
-ProxyPass /api/v1 http://127.0.0.1:8080/api/v1
-ProxyPassReverse /api/v1 http://127.0.0.1:8080/api/v1
-```
-
-and you can add authentication with something like this:
-
-```
-<Location /api/v1>
-	AuthType Digest
-	AuthName "Private"
-	AuthDigestDomain "/api/v1"
-	AuthDigestProvider file
-	AuthUserFile "/etc/httpd/conf/auth_digest"
-	Require valid-user
-</Location>
-```
-
-and, of course, you can configure the web server to use HTTPS.
-
-The OpenAPI 3 schema for the exposed API can be found inside the source tree: [openapi.yaml](https://github.com/drakkan/sftpgo/tree/master/api/schema/openapi.yaml "OpenAPI 3 specs").
-
-A sample CLI client for the REST API can be found inside the source tree [scripts](https://github.com/drakkan/sftpgo/tree/master/scripts "scripts") directory.
-
-You can also generate your own REST client, in your preferred programming language or even bash scripts, using an OpenAPI generator such as [swagger-codegen](https://github.com/swagger-api/swagger-codegen) or [OpenAPI Generator](https://openapi-generator.tech/)
-
-## Metrics
-
-SFTPGo exposes [Prometheus](https://prometheus.io/) metrics at the `/metrics` HTTP endpoint.
-Several counters and gauges are available, for example:
-
-- Total uploads and downloads
-- Total uploads and downloads size
-- Total uploads and downloads errors
-- Number of active connections
-- Data provider availability
-- Total successful and failed logins using a password or a public key
-- Total HTTP requests served and totals for response code
-- Go's runtime like details about GC, number of gouroutines and OS threads
-- Process information like CPU, memory, file descriptor usage and start time
-
-Please check the `/metrics` page for more details.
-
-## Logs
-
-Inside the log file each line is a JSON struct, each struct has a `sender` fields that identify the log type.
-
-The logs can be divided into the following categories:
-
-- **"app logs"**, internal logs used to debug `sftpgo`:
-    - `sender` string. This is generally the package name that emits the log
-    - `time` string. Date/time with millisecond precision
-    - `level` string
-    - `message` string
-- **"transfer logs"**, SFTP/SCP transfer logs:
-    - `sender` string. `Upload` or `Download`
-    - `time` string. Date/time with millisecond precision
-    - `level` string
-    - `elapsed_ms`, int64. Elapsed time, as milliseconds, for the upload/download
-    - `size_bytes`, int64. Size, as bytes, of the download/upload
-    - `username`, string
-    - `file_path` string
-    - `connection_id` string. Unique connection identifier
-    - `protocol` string. `SFTP` or `SCP`
-- **"command logs"**, SFTP/SCP command logs:
-    - `sender` string. `Rename`, `Rmdir`, `Mkdir`, `Symlink`, `Remove`
-    - `level` string
-    - `username`, string
-    - `file_path` string
-    - `target_path` string
-    - `connection_id` string. Unique connection identifier
-    - `protocol` string. `SFTP` or `SCP`
-- **"http logs"**, REST API logs:
-    - `sender` string. `httpd`
-    - `level` string
-    - `remote_addr` string. IP and port of the remote client
-    - `proto` string, for example `HTTP/1.1`
-    - `method` string. HTTP method (`GET`, `POST`, `PUT`, `DELETE` etc.)
-    - `user_agent` string
-    - `uri` string. Full uri
-    - `resp_status` integer. HTTP response status code
-    - `resp_size` integer. Size in bytes of the HTTP response
-    - `elapsed_ms` int64. Elapsed time, as milliseconds, to complete the request
-    - `request_id` string. Unique request identifier
+More in-depth analysis of performance can be found [here](./docs/performance.md).
 
 ## Acknowledgements
 
-- [pkg/sftp](https://github.com/pkg/sftp)
-- [go-chi](https://github.com/go-chi/chi)
-- [zerolog](https://github.com/rs/zerolog)
-- [lumberjack](https://gopkg.in/natefinch/lumberjack.v2)
-- [argon2id](https://github.com/alexedwards/argon2id)
-- [go-sqlite3](https://github.com/mattn/go-sqlite3)
-- [go-sql-driver/mysql](https://github.com/go-sql-driver/mysql)
-- [bbolt](https://github.com/etcd-io/bbolt)
-- [lib/pq](https://github.com/lib/pq)
-- [viper](https://github.com/spf13/viper)
-- [cobra](https://github.com/spf13/cobra)
-- [xid](https://github.com/rs/xid)
-- [nathanaelle/password](https://github.com/nathanaelle/password)
-
-Some code was initially taken from [Pterodactyl sftp server](https://github.com/pterodactyl/sftp-server)
+SFTPGo makes use of the third party libraries listed inside [go.mod](./go.mod).
+Some code was initially taken from [Pterodactyl SFTP Server](https://github.com/pterodactyl/sftp-server).
+We are very grateful to all the people who contributed with ideas and/or pull requests.
 
 ## License
 

api/api.go

@@ -1,78 +0,0 @@
-// Package api implements REST API for sftpgo.
-// REST API allows to manage users and quota and to get real time reports for the active connections
-// with possibility of forcibly closing a connection.
-// The OpenAPI 3 schema for the exposed API can be found inside the source tree:
-// https://github.com/drakkan/sftpgo/tree/master/api/schema/openapi.yaml
-package api
-
-import (
-	"net/http"
-
-	"github.com/drakkan/sftpgo/dataprovider"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/render"
-)
-
-const (
-	logSender             = "api"
-	activeConnectionsPath = "/api/v1/connection"
-	quotaScanPath         = "/api/v1/quota_scan"
-	userPath              = "/api/v1/user"
-	versionPath           = "/api/v1/version"
-	metricsPath           = "/metrics"
-)
-
-var (
-	router       *chi.Mux
-	dataProvider dataprovider.Provider
-)
-
-// HTTPDConf httpd daemon configuration
-type HTTPDConf struct {
-	// The port used for serving HTTP requests. 0 disable the HTTP server. Default: 8080
-	BindPort int `json:"bind_port" mapstructure:"bind_port"`
-	// The address to listen on. A blank value means listen on all available network interfaces. Default: "127.0.0.1"
-	BindAddress string `json:"bind_address" mapstructure:"bind_address"`
-}
-
-type apiResponse struct {
-	Error      string `json:"error"`
-	Message    string `json:"message"`
-	HTTPStatus int    `json:"status"`
-}
-
-func init() {
-	initializeRouter()
-}
-
-// SetDataProvider sets the data provider to use to fetch the data about users
-func SetDataProvider(provider dataprovider.Provider) {
-	dataProvider = provider
-}
-
-func sendAPIResponse(w http.ResponseWriter, r *http.Request, err error, message string, code int) {
-	var errorString string
-	if err != nil {
-		errorString = err.Error()
-	}
-	resp := apiResponse{
-		Error:      errorString,
-		Message:    message,
-		HTTPStatus: code,
-	}
-	if code != http.StatusOK {
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
-		w.WriteHeader(code)
-	}
-	render.JSON(w, r, resp)
-}
-
-func getRespStatus(err error) int {
-	if _, ok := err.(*dataprovider.ValidationError); ok {
-		return http.StatusBadRequest
-	}
-	if _, ok := err.(*dataprovider.MethodDisabledError); ok {
-		return http.StatusForbidden
-	}
-	return http.StatusInternalServerError
-}

api/api_test.go

@@ -1,762 +0,0 @@
-package api_test
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/go-chi/render"
-	_ "github.com/go-sql-driver/mysql"
-	_ "github.com/lib/pq"
-	_ "github.com/mattn/go-sqlite3"
-	"github.com/rs/zerolog"
-
-	"github.com/drakkan/sftpgo/api"
-	"github.com/drakkan/sftpgo/config"
-	"github.com/drakkan/sftpgo/dataprovider"
-	"github.com/drakkan/sftpgo/logger"
-	"github.com/drakkan/sftpgo/sftpd"
-)
-
-const (
-	defaultUsername       = "test_user"
-	defaultPassword       = "test_password"
-	testPubKey            = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC03jj0D+djk7pxIf/0OhrxrchJTRZklofJ1NoIu4752Sq02mdXmarMVsqJ1cAjV5LBVy3D1F5U6XW4rppkXeVtd04Pxb09ehtH0pRRPaoHHlALiJt8CoMpbKYMA8b3KXPPriGxgGomvtU2T2RMURSwOZbMtpsugfjYSWenyYX+VORYhylWnSXL961LTyC21ehd6d6QnW9G7E5hYMITMY9TuQZz3bROYzXiTsgN0+g6Hn7exFQp50p45StUMfV/SftCMdCxlxuyGny2CrN/vfjO7xxOo2uv7q1qm10Q46KPWJQv+pgZ/OfL+EDjy07n5QVSKHlbx+2nT4Q0EgOSQaCTYwn3YjtABfIxWwgAFdyj6YlPulCL22qU4MYhDcA6PSBwDdf8hvxBfvsiHdM+JcSHvv8/VeJhk6CmnZxGY0fxBupov27z3yEO8nAg8k+6PaUiW1MSUfuGMF/ktB8LOstXsEPXSszuyXiOv4DaryOXUiSn7bmRqKcEFlJusO6aZP0= nicola@p1"
-	logSender             = "APITesting"
-	userPath              = "/api/v1/user"
-	activeConnectionsPath = "/api/v1/connection"
-	quotaScanPath         = "/api/v1/quota_scan"
-	versionPath           = "/api/v1/version"
-	metricsPath           = "/metrics"
-)
-
-var (
-	defaultPerms = []string{dataprovider.PermAny}
-	homeBasePath string
-	testServer   *httptest.Server
-)
-
-func TestMain(m *testing.M) {
-	if runtime.GOOS == "windows" {
-		homeBasePath = "C:\\"
-	} else {
-		homeBasePath = "/tmp"
-	}
-	configDir := ".."
-	logfilePath := filepath.Join(configDir, "sftpgo_api_test.log")
-	logger.InitLogger(logfilePath, 5, 1, 28, false, zerolog.DebugLevel)
-	config.LoadConfig(configDir, "")
-	providerConf := config.GetProviderConf()
-
-	err := dataprovider.Initialize(providerConf, configDir)
-	if err != nil {
-		logger.Warn(logSender, "", "error initializing data provider: %v", err)
-		os.Exit(1)
-	}
-	dataProvider := dataprovider.GetProvider()
-	httpdConf := config.GetHTTPDConfig()
-	router := api.GetHTTPRouter()
-
-	httpdConf.BindPort = 8081
-	api.SetBaseURL("http://127.0.0.1:8081")
-
-	sftpd.SetDataProvider(dataProvider)
-	api.SetDataProvider(dataProvider)
-
-	go func() {
-		logger.Debug(logSender, "", "initializing HTTP server with config %+v", httpdConf)
-		s := &http.Server{
-			Addr:           fmt.Sprintf("%s:%d", httpdConf.BindAddress, httpdConf.BindPort),
-			Handler:        router,
-			ReadTimeout:    300 * time.Second,
-			WriteTimeout:   300 * time.Second,
-			MaxHeaderBytes: 1 << 20, // 1MB
-		}
-		if err := s.ListenAndServe(); err != nil {
-			logger.Error(logSender, "", "could not start HTTP server: %v", err)
-		}
-	}()
-
-	testServer = httptest.NewServer(api.GetHTTPRouter())
-	defer testServer.Close()
-
-	waitTCPListening(fmt.Sprintf("%s:%d", httpdConf.BindAddress, httpdConf.BindPort))
-
-	exitCode := m.Run()
-	os.Remove(logfilePath)
-	os.Exit(exitCode)
-}
-
-func TestBasicUserHandling(t *testing.T) {
-	user, _, err := api.AddUser(getTestUser(), http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to add user: %v", err)
-	}
-	user.MaxSessions = 10
-	user.QuotaSize = 4096
-	user.QuotaFiles = 2
-	user.UploadBandwidth = 128
-	user.DownloadBandwidth = 64
-	user, _, err = api.UpdateUser(user, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to update user: %v", err)
-	}
-	users, _, err := api.GetUsers(0, 0, defaultUsername, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to get users: %v", err)
-	}
-	if len(users) != 1 {
-		t.Errorf("number of users mismatch, expected: 1, actual: %v", len(users))
-	}
-	_, err = api.RemoveUser(user, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to remove: %v", err)
-	}
-}
-
-func TestAddUserNoCredentials(t *testing.T) {
-	u := getTestUser()
-	u.Password = ""
-	u.PublicKeys = []string{}
-	_, _, err := api.AddUser(u, http.StatusBadRequest)
-	if err != nil {
-		t.Errorf("unexpected error adding user with no credentials: %v", err)
-	}
-}
-
-func TestAddUserNoUsername(t *testing.T) {
-	u := getTestUser()
-	u.Username = ""
-	_, _, err := api.AddUser(u, http.StatusBadRequest)
-	if err != nil {
-		t.Errorf("unexpected error adding user with no home dir: %v", err)
-	}
-}
-
-func TestAddUserNoHomeDir(t *testing.T) {
-	u := getTestUser()
-	u.HomeDir = ""
-	_, _, err := api.AddUser(u, http.StatusBadRequest)
-	if err != nil {
-		t.Errorf("unexpected error adding user with no home dir: %v", err)
-	}
-}
-
-func TestAddUserInvalidHomeDir(t *testing.T) {
-	u := getTestUser()
-	u.HomeDir = "relative_path"
-	_, _, err := api.AddUser(u, http.StatusBadRequest)
-	if err != nil {
-		t.Errorf("unexpected error adding user with invalid home dir: %v", err)
-	}
-}
-
-func TestAddUserNoPerms(t *testing.T) {
-	u := getTestUser()
-	u.Permissions = []string{}
-	_, _, err := api.AddUser(u, http.StatusBadRequest)
-	if err != nil {
-		t.Errorf("unexpected error adding user with no perms: %v", err)
-	}
-}
-
-func TestAddUserInvalidPerms(t *testing.T) {
-	u := getTestUser()
-	u.Permissions = []string{"invalidPerm"}
-	_, _, err := api.AddUser(u, http.StatusBadRequest)
-	if err != nil {
-		t.Errorf("unexpected error adding user with no perms: %v", err)
-	}
-}
-
-func TestUserPublicKey(t *testing.T) {
-	u := getTestUser()
-	invalidPubKey := "invalid"
-	validPubKey := "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC03jj0D+djk7pxIf/0OhrxrchJTRZklofJ1NoIu4752Sq02mdXmarMVsqJ1cAjV5LBVy3D1F5U6XW4rppkXeVtd04Pxb09ehtH0pRRPaoHHlALiJt8CoMpbKYMA8b3KXPPriGxgGomvtU2T2RMURSwOZbMtpsugfjYSWenyYX+VORYhylWnSXL961LTyC21ehd6d6QnW9G7E5hYMITMY9TuQZz3bROYzXiTsgN0+g6Hn7exFQp50p45StUMfV/SftCMdCxlxuyGny2CrN/vfjO7xxOo2uv7q1qm10Q46KPWJQv+pgZ/OfL+EDjy07n5QVSKHlbx+2nT4Q0EgOSQaCTYwn3YjtABfIxWwgAFdyj6YlPulCL22qU4MYhDcA6PSBwDdf8hvxBfvsiHdM+JcSHvv8/VeJhk6CmnZxGY0fxBupov27z3yEO8nAg8k+6PaUiW1MSUfuGMF/ktB8LOstXsEPXSszuyXiOv4DaryOXUiSn7bmRqKcEFlJusO6aZP0= nicola@p1"
-	u.PublicKeys = []string{invalidPubKey}
-	_, _, err := api.AddUser(u, http.StatusBadRequest)
-	if err != nil {
-		t.Errorf("unexpected error adding user with invalid pub key: %v", err)
-	}
-	u.PublicKeys = []string{validPubKey}
-	user, _, err := api.AddUser(u, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to add user: %v", err)
-	}
-	user.PublicKeys = []string{validPubKey, invalidPubKey}
-	_, _, err = api.UpdateUser(user, http.StatusBadRequest)
-	if err != nil {
-		t.Errorf("update user with invalid public key must fail: %v", err)
-	}
-	user.PublicKeys = []string{validPubKey, validPubKey, validPubKey}
-	_, _, err = api.UpdateUser(user, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to update user: %v", err)
-	}
-	_, err = api.RemoveUser(user, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to remove: %v", err)
-	}
-}
-
-func TestUpdateUser(t *testing.T) {
-	user, _, err := api.AddUser(getTestUser(), http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to add user: %v", err)
-	}
-	user.HomeDir = filepath.Join(homeBasePath, "testmod")
-	user.UID = 33
-	user.GID = 101
-	user.MaxSessions = 10
-	user.QuotaSize = 4096
-	user.QuotaFiles = 2
-	user.Permissions = []string{dataprovider.PermCreateDirs, dataprovider.PermDelete, dataprovider.PermDownload}
-	user.UploadBandwidth = 1024
-	user.DownloadBandwidth = 512
-	user, _, err = api.UpdateUser(user, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to update user: %v", err)
-	}
-	_, err = api.RemoveUser(user, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to remove: %v", err)
-	}
-}
-
-func TestUpdateUserNoCredentials(t *testing.T) {
-	user, _, err := api.AddUser(getTestUser(), http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to add user: %v", err)
-	}
-	user.Password = ""
-	user.PublicKeys = []string{}
-	// password and public key will be omitted from json serialization if empty and so they will remain unchanged
-	// and no validation error will be raised
-	_, _, err = api.UpdateUser(user, http.StatusOK)
-	if err != nil {
-		t.Errorf("unexpected error updating user with no credentials: %v", err)
-	}
-	_, err = api.RemoveUser(user, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to remove: %v", err)
-	}
-}
-
-func TestUpdateUserEmptyHomeDir(t *testing.T) {
-	user, _, err := api.AddUser(getTestUser(), http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to add user: %v", err)
-	}
-	user.HomeDir = ""
-	_, _, err = api.UpdateUser(user, http.StatusBadRequest)
-	if err != nil {
-		t.Errorf("unexpected error updating user with empty home dir: %v", err)
-	}
-	_, err = api.RemoveUser(user, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to remove: %v", err)
-	}
-}
-
-func TestUpdateUserInvalidHomeDir(t *testing.T) {
-	user, _, err := api.AddUser(getTestUser(), http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to add user: %v", err)
-	}
-	user.HomeDir = "relative_path"
-	_, _, err = api.UpdateUser(user, http.StatusBadRequest)
-	if err != nil {
-		t.Errorf("unexpected error updating user with empty home dir: %v", err)
-	}
-	_, err = api.RemoveUser(user, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to remove: %v", err)
-	}
-}
-
-func TestUpdateNonExistentUser(t *testing.T) {
-	_, _, err := api.UpdateUser(getTestUser(), http.StatusNotFound)
-	if err != nil {
-		t.Errorf("unable to update user: %v", err)
-	}
-}
-
-func TestGetNonExistentUser(t *testing.T) {
-	_, _, err := api.GetUserByID(0, http.StatusNotFound)
-	if err != nil {
-		t.Errorf("unable to get user: %v", err)
-	}
-}
-
-func TestDeleteNonExistentUser(t *testing.T) {
-	_, err := api.RemoveUser(getTestUser(), http.StatusNotFound)
-	if err != nil {
-		t.Errorf("unable to remove user: %v", err)
-	}
-}
-
-func TestAddDuplicateUser(t *testing.T) {
-	user, _, err := api.AddUser(getTestUser(), http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to add user: %v", err)
-	}
-	_, _, err = api.AddUser(getTestUser(), http.StatusInternalServerError)
-	if err != nil {
-		t.Errorf("unable to add second user: %v", err)
-	}
-	_, _, err = api.AddUser(getTestUser(), http.StatusOK)
-	if err == nil {
-		t.Errorf("adding a duplicate user must fail")
-	}
-	_, err = api.RemoveUser(user, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to remove user: %v", err)
-	}
-}
-
-func TestGetUsers(t *testing.T) {
-	user1, _, err := api.AddUser(getTestUser(), http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to add user: %v", err)
-	}
-	u := getTestUser()
-	u.Username = defaultUsername + "1"
-	user2, _, err := api.AddUser(u, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to add second user: %v", err)
-	}
-	users, _, err := api.GetUsers(0, 0, "", http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to get users: %v", err)
-	}
-	if len(users) < 2 {
-		t.Errorf("at least 2 users are expected")
-	}
-	users, _, err = api.GetUsers(1, 0, "", http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to get users: %v", err)
-	}
-	if len(users) != 1 {
-		t.Errorf("1 user is expected")
-	}
-	users, _, err = api.GetUsers(1, 1, "", http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to get users: %v", err)
-	}
-	if len(users) != 1 {
-		t.Errorf("1 user is expected")
-	}
-	_, _, err = api.GetUsers(1, 1, "", http.StatusInternalServerError)
-	if err == nil {
-		t.Errorf("get users must succeed, we requested a fail for a good request")
-	}
-	_, err = api.RemoveUser(user1, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to remove user: %v", err)
-	}
-	_, err = api.RemoveUser(user2, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to remove user: %v", err)
-	}
-}
-
-func TestGetQuotaScans(t *testing.T) {
-	_, _, err := api.GetQuotaScans(http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to get quota scans: %v", err)
-	}
-	_, _, err = api.GetQuotaScans(http.StatusInternalServerError)
-	if err == nil {
-		t.Errorf("quota scan request must succeed, we requested to check a wrong status code")
-	}
-}
-
-func TestStartQuotaScan(t *testing.T) {
-	user, _, err := api.AddUser(getTestUser(), http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to add user: %v", err)
-	}
-	_, err = api.StartQuotaScan(user, http.StatusCreated)
-	if err != nil {
-		t.Errorf("unable to start quota scan: %v", err)
-	}
-	_, err = api.RemoveUser(user, http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to remove user: %v", err)
-	}
-}
-
-func TestGetVersion(t *testing.T) {
-	_, _, err := api.GetVersion(http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to get sftp version: %v", err)
-	}
-	_, _, err = api.GetVersion(http.StatusInternalServerError)
-	if err == nil {
-		t.Errorf("get version request must succeed, we requested to check a wrong status code")
-	}
-}
-
-func TestGetConnections(t *testing.T) {
-	_, _, err := api.GetConnections(http.StatusOK)
-	if err != nil {
-		t.Errorf("unable to get sftp connections: %v", err)
-	}
-	_, _, err = api.GetConnections(http.StatusInternalServerError)
-	if err == nil {
-		t.Errorf("get sftp connections request must succeed, we requested to check a wrong status code")
-	}
-}
-
-func TestCloseActiveConnection(t *testing.T) {
-	_, err := api.CloseConnection("non_existent_id", http.StatusNotFound)
-	if err != nil {
-		t.Errorf("unexpected error closing non existent sftp connection: %v", err)
-	}
-}
-
-// test using mock http server
-
-func TestBasicUserHandlingMock(t *testing.T) {
-	user := getTestUser()
-	userAsJSON := getUserAsJSON(t, user)
-	req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer(userAsJSON))
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-	err := render.DecodeJSON(rr.Body, &user)
-	if err != nil {
-		t.Errorf("Error get user: %v", err)
-	}
-	req, _ = http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer(userAsJSON))
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusInternalServerError, rr.Code)
-	user.MaxSessions = 10
-	user.UploadBandwidth = 128
-	userAsJSON = getUserAsJSON(t, user)
-	req, _ = http.NewRequest(http.MethodPut, userPath+"/"+strconv.FormatInt(user.ID, 10), bytes.NewBuffer(userAsJSON))
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-
-	req, _ = http.NewRequest(http.MethodGet, userPath+"/"+strconv.FormatInt(user.ID, 10), nil)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-
-	var updatedUser dataprovider.User
-	err = render.DecodeJSON(rr.Body, &updatedUser)
-	if err != nil {
-		t.Errorf("Error decoding updated user: %v", err)
-	}
-	if user.MaxSessions != updatedUser.MaxSessions || user.UploadBandwidth != updatedUser.UploadBandwidth {
-		t.Errorf("Error modifying user actual: %v, %v", updatedUser.MaxSessions, updatedUser.UploadBandwidth)
-	}
-	req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-}
-
-func TestGetUserByIdInvalidParamsMock(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodGet, userPath+"/0", nil)
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusNotFound, rr.Code)
-	req, _ = http.NewRequest(http.MethodGet, userPath+"/a", nil)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-}
-
-func TestAddUserNoUsernameMock(t *testing.T) {
-	user := getTestUser()
-	user.Username = ""
-	userAsJSON := getUserAsJSON(t, user)
-	req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer(userAsJSON))
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-}
-
-func TestAddUserInvalidHomeDirMock(t *testing.T) {
-	user := getTestUser()
-	user.HomeDir = "relative_path"
-	userAsJSON := getUserAsJSON(t, user)
-	req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer(userAsJSON))
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-}
-
-func TestAddUserInvalidPermsMock(t *testing.T) {
-	user := getTestUser()
-	user.Permissions = []string{}
-	userAsJSON := getUserAsJSON(t, user)
-	req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer(userAsJSON))
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-}
-
-func TestAddUserInvalidJsonMock(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer([]byte("invalid json")))
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-}
-
-func TestUpdateUserInvalidJsonMock(t *testing.T) {
-	user := getTestUser()
-	userAsJSON := getUserAsJSON(t, user)
-	req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer(userAsJSON))
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-	err := render.DecodeJSON(rr.Body, &user)
-	if err != nil {
-		t.Errorf("Error get user: %v", err)
-	}
-	req, _ = http.NewRequest(http.MethodPut, userPath+"/"+strconv.FormatInt(user.ID, 10), bytes.NewBuffer([]byte("Invalid json")))
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-	req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-}
-
-func TestUpdateUserInvalidParamsMock(t *testing.T) {
-	user := getTestUser()
-	userAsJSON := getUserAsJSON(t, user)
-	req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer(userAsJSON))
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-	err := render.DecodeJSON(rr.Body, &user)
-	if err != nil {
-		t.Errorf("Error get user: %v", err)
-	}
-	user.HomeDir = ""
-	userAsJSON = getUserAsJSON(t, user)
-	req, _ = http.NewRequest(http.MethodPut, userPath+"/"+strconv.FormatInt(user.ID, 10), bytes.NewBuffer(userAsJSON))
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-	userID := user.ID
-	user.ID = 0
-	userAsJSON = getUserAsJSON(t, user)
-	req, _ = http.NewRequest(http.MethodPut, userPath+"/"+strconv.FormatInt(userID, 10), bytes.NewBuffer(userAsJSON))
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-	user.ID = userID
-	req, _ = http.NewRequest(http.MethodPut, userPath+"/0", bytes.NewBuffer(userAsJSON))
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusNotFound, rr.Code)
-	req, _ = http.NewRequest(http.MethodPut, userPath+"/a", bytes.NewBuffer(userAsJSON))
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-	req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-}
-
-func TestGetUsersMock(t *testing.T) {
-	user := getTestUser()
-	userAsJSON := getUserAsJSON(t, user)
-	req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer(userAsJSON))
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-	err := render.DecodeJSON(rr.Body, &user)
-	if err != nil {
-		t.Errorf("Error get user: %v", err)
-	}
-	req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=510&offset=0&order=ASC&username="+defaultUsername, nil)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-	var users []dataprovider.User
-	err = render.DecodeJSON(rr.Body, &users)
-	if err != nil {
-		t.Errorf("Error decoding users: %v", err)
-	}
-	if len(users) != 1 {
-		t.Errorf("1 user is expected")
-	}
-	req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=a&offset=0&order=ASC", nil)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-	req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=1&offset=a&order=ASC", nil)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-	req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=1&offset=0&order=ASCa", nil)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-
-	req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-}
-
-func TestDeleteUserInvalidParamsMock(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodDelete, userPath+"/0", nil)
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusNotFound, rr.Code)
-	req, _ = http.NewRequest(http.MethodDelete, userPath+"/a", nil)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-}
-
-func TestGetQuotaScansMock(t *testing.T) {
-	req, err := http.NewRequest("GET", quotaScanPath, nil)
-	if err != nil {
-		t.Errorf("error get quota scan: %v", err)
-	}
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-}
-
-func TestStartQuotaScanMock(t *testing.T) {
-	user := getTestUser()
-	userAsJSON := getUserAsJSON(t, user)
-	req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer(userAsJSON))
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-	err := render.DecodeJSON(rr.Body, &user)
-	if err != nil {
-		t.Errorf("Error get user: %v", err)
-	}
-	_, err = os.Stat(user.HomeDir)
-	if err == nil {
-		os.Remove(user.HomeDir)
-	}
-	// simulate a duplicate quota scan
-	userAsJSON = getUserAsJSON(t, user)
-	sftpd.AddQuotaScan(user.Username)
-	req, _ = http.NewRequest(http.MethodPost, quotaScanPath, bytes.NewBuffer(userAsJSON))
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusConflict, rr.Code)
-	sftpd.RemoveQuotaScan(user.Username)
-
-	userAsJSON = getUserAsJSON(t, user)
-	req, _ = http.NewRequest(http.MethodPost, quotaScanPath, bytes.NewBuffer(userAsJSON))
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusCreated, rr.Code)
-
-	req, _ = http.NewRequest(http.MethodGet, quotaScanPath, nil)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-	var scans []sftpd.ActiveQuotaScan
-	err = render.DecodeJSON(rr.Body, &scans)
-	if err != nil {
-		t.Errorf("Error get active scans: %v", err)
-	}
-	for len(scans) > 0 {
-		req, _ = http.NewRequest(http.MethodGet, quotaScanPath, nil)
-		rr = executeRequest(req)
-		checkResponseCode(t, http.StatusOK, rr.Code)
-		err = render.DecodeJSON(rr.Body, &scans)
-		if err != nil {
-			t.Errorf("Error get active scans: %v", err)
-			break
-		}
-	}
-	_, err = os.Stat(user.HomeDir)
-	if err != nil && os.IsNotExist(err) {
-		os.MkdirAll(user.HomeDir, 0777)
-	}
-	req, _ = http.NewRequest(http.MethodPost, quotaScanPath, bytes.NewBuffer(userAsJSON))
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusCreated, rr.Code)
-	req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-}
-
-func TestStartQuotaScanBadUserMock(t *testing.T) {
-	user := getTestUser()
-	userAsJSON := getUserAsJSON(t, user)
-	req, _ := http.NewRequest(http.MethodPost, quotaScanPath, bytes.NewBuffer(userAsJSON))
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusNotFound, rr.Code)
-}
-
-func TestStartQuotaScanNonExistentUserMock(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodPost, quotaScanPath, bytes.NewBuffer([]byte("invalid json")))
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr.Code)
-}
-
-func TestGetVersionMock(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodGet, versionPath, nil)
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-}
-
-func TestGetConnectionsMock(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodGet, activeConnectionsPath, nil)
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-}
-
-func TestDeleteActiveConnectionMock(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodDelete, activeConnectionsPath+"/connectionID", nil)
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusNotFound, rr.Code)
-}
-
-func TestNotFoundMock(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodGet, "/non/existing/path", nil)
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusNotFound, rr.Code)
-}
-
-func TestMethodNotAllowedMock(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodPost, activeConnectionsPath, nil)
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusMethodNotAllowed, rr.Code)
-}
-
-func TestMetricsMock(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodGet, metricsPath, nil)
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-}
-
-func waitTCPListening(address string) {
-	for {
-		conn, err := net.Dial("tcp", address)
-		if err != nil {
-			logger.WarnToConsole("tcp server %v not listening: %v\n", address, err)
-			time.Sleep(100 * time.Millisecond)
-			continue
-		}
-		logger.InfoToConsole("tcp server %v now listening\n", address)
-		defer conn.Close()
-		break
-	}
-}
-
-func getTestUser() dataprovider.User {
-	return dataprovider.User{
-		Username:    defaultUsername,
-		Password:    defaultPassword,
-		HomeDir:     filepath.Join(homeBasePath, defaultUsername),
-		Permissions: defaultPerms,
-	}
-}
-
-func getUserAsJSON(t *testing.T, user dataprovider.User) []byte {
-	json, err := json.Marshal(user)
-	if err != nil {
-		t.Errorf("error get user as json: %v", err)
-		return []byte("{}")
-	}
-	return json
-}
-
-func executeRequest(req *http.Request) *httptest.ResponseRecorder {
-	rr := httptest.NewRecorder()
-	testServer.Config.Handler.ServeHTTP(rr, req)
-	return rr
-}
-
-func checkResponseCode(t *testing.T, expected, actual int) {
-	if expected != actual {
-		t.Errorf("Expected response code %d. Got %d", expected, actual)
-	}
-}

api/api_utils.go

@@ -1,330 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"path"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/drakkan/sftpgo/dataprovider"
-	"github.com/drakkan/sftpgo/sftpd"
-	"github.com/drakkan/sftpgo/utils"
-	"github.com/go-chi/render"
-)
-
-var (
-	httpBaseURL = "http://127.0.0.1:8080"
-)
-
-// SetBaseURL sets the base url to use for HTTP requests, default is "http://127.0.0.1:8080"
-func SetBaseURL(url string) {
-	httpBaseURL = url
-}
-
-// gets an HTTP Client with a timeout
-func getHTTPClient() *http.Client {
-	return &http.Client{
-		Timeout: 15 * time.Second,
-	}
-}
-
-func buildURLRelativeToBase(paths ...string) string {
-	// we need to use path.Join and not filepath.Join
-	// since filepath.Join will use backslash separator on Windows
-	p := path.Join(paths...)
-	return fmt.Sprintf("%s/%s", strings.TrimRight(httpBaseURL, "/"), strings.TrimLeft(p, "/"))
-}
-
-// AddUser adds a new user and checks the received HTTP Status code against expectedStatusCode.
-func AddUser(user dataprovider.User, expectedStatusCode int) (dataprovider.User, []byte, error) {
-	var newUser dataprovider.User
-	var body []byte
-	userAsJSON, err := json.Marshal(user)
-	if err != nil {
-		return newUser, body, err
-	}
-	resp, err := getHTTPClient().Post(buildURLRelativeToBase(userPath), "application/json", bytes.NewBuffer(userAsJSON))
-	if err != nil {
-		return newUser, body, err
-	}
-	defer resp.Body.Close()
-	err = checkResponse(resp.StatusCode, expectedStatusCode)
-	if expectedStatusCode != http.StatusOK {
-		body, _ = getResponseBody(resp)
-		return newUser, body, err
-	}
-	if err == nil {
-		err = render.DecodeJSON(resp.Body, &newUser)
-	} else {
-		body, _ = getResponseBody(resp)
-	}
-	if err == nil {
-		err = checkUser(user, newUser)
-	}
-	return newUser, body, err
-}
-
-// UpdateUser updates an existing user and checks the received HTTP Status code against expectedStatusCode.
-func UpdateUser(user dataprovider.User, expectedStatusCode int) (dataprovider.User, []byte, error) {
-	var newUser dataprovider.User
-	var body []byte
-	userAsJSON, err := json.Marshal(user)
-	if err != nil {
-		return user, body, err
-	}
-	req, err := http.NewRequest(http.MethodPut, buildURLRelativeToBase(userPath, strconv.FormatInt(user.ID, 10)),
-		bytes.NewBuffer(userAsJSON))
-	if err != nil {
-		return user, body, err
-	}
-	resp, err := getHTTPClient().Do(req)
-	if err != nil {
-		return user, body, err
-	}
-	defer resp.Body.Close()
-	body, _ = getResponseBody(resp)
-	err = checkResponse(resp.StatusCode, expectedStatusCode)
-	if expectedStatusCode != http.StatusOK {
-		return newUser, body, err
-	}
-	if err == nil {
-		newUser, body, err = GetUserByID(user.ID, expectedStatusCode)
-	}
-	if err == nil {
-		err = checkUser(user, newUser)
-	}
-	return newUser, body, err
-}
-
-// RemoveUser removes an existing user and checks the received HTTP Status code against expectedStatusCode.
-func RemoveUser(user dataprovider.User, expectedStatusCode int) ([]byte, error) {
-	var body []byte
-	req, err := http.NewRequest(http.MethodDelete, buildURLRelativeToBase(userPath, strconv.FormatInt(user.ID, 10)), nil)
-	if err != nil {
-		return body, err
-	}
-	resp, err := getHTTPClient().Do(req)
-	if err != nil {
-		return body, err
-	}
-	defer resp.Body.Close()
-	body, _ = getResponseBody(resp)
-	return body, checkResponse(resp.StatusCode, expectedStatusCode)
-}
-
-// GetUserByID gets an user by database id and checks the received HTTP Status code against expectedStatusCode.
-func GetUserByID(userID int64, expectedStatusCode int) (dataprovider.User, []byte, error) {
-	var user dataprovider.User
-	var body []byte
-	resp, err := getHTTPClient().Get(buildURLRelativeToBase(userPath, strconv.FormatInt(userID, 10)))
-	if err != nil {
-		return user, body, err
-	}
-	defer resp.Body.Close()
-	err = checkResponse(resp.StatusCode, expectedStatusCode)
-	if err == nil && expectedStatusCode == http.StatusOK {
-		err = render.DecodeJSON(resp.Body, &user)
-	} else {
-		body, _ = getResponseBody(resp)
-	}
-	return user, body, err
-}
-
-// GetUsers allows to get a list of users and checks the received HTTP Status code against expectedStatusCode.
-// The number of results can be limited specifying a limit.
-// Some results can be skipped specifying an offset.
-// The results can be filtered specifying an username, the username filter is an exact match
-func GetUsers(limit int64, offset int64, username string, expectedStatusCode int) ([]dataprovider.User, []byte, error) {
-	var users []dataprovider.User
-	var body []byte
-	url, err := url.Parse(buildURLRelativeToBase(userPath))
-	if err != nil {
-		return users, body, err
-	}
-	q := url.Query()
-	if limit > 0 {
-		q.Add("limit", strconv.FormatInt(limit, 10))
-	}
-	if offset > 0 {
-		q.Add("offset", strconv.FormatInt(offset, 10))
-	}
-	if len(username) > 0 {
-		q.Add("username", username)
-	}
-	url.RawQuery = q.Encode()
-	resp, err := getHTTPClient().Get(url.String())
-	if err != nil {
-		return users, body, err
-	}
-	defer resp.Body.Close()
-	err = checkResponse(resp.StatusCode, expectedStatusCode)
-	if err == nil && expectedStatusCode == http.StatusOK {
-		err = render.DecodeJSON(resp.Body, &users)
-	} else {
-		body, _ = getResponseBody(resp)
-	}
-	return users, body, err
-}
-
-// GetQuotaScans gets active quota scans and checks the received HTTP Status code against expectedStatusCode.
-func GetQuotaScans(expectedStatusCode int) ([]sftpd.ActiveQuotaScan, []byte, error) {
-	var quotaScans []sftpd.ActiveQuotaScan
-	var body []byte
-	resp, err := getHTTPClient().Get(buildURLRelativeToBase(quotaScanPath))
-	if err != nil {
-		return quotaScans, body, err
-	}
-	defer resp.Body.Close()
-	err = checkResponse(resp.StatusCode, expectedStatusCode)
-	if err == nil && expectedStatusCode == http.StatusOK {
-		err = render.DecodeJSON(resp.Body, &quotaScans)
-	} else {
-		body, _ = getResponseBody(resp)
-	}
-	return quotaScans, body, err
-}
-
-// StartQuotaScan start a new quota scan for the given user and checks the received HTTP Status code against expectedStatusCode.
-func StartQuotaScan(user dataprovider.User, expectedStatusCode int) ([]byte, error) {
-	var body []byte
-	userAsJSON, err := json.Marshal(user)
-	if err != nil {
-		return body, err
-	}
-	resp, err := getHTTPClient().Post(buildURLRelativeToBase(quotaScanPath), "application/json", bytes.NewBuffer(userAsJSON))
-	if err != nil {
-		return body, err
-	}
-	defer resp.Body.Close()
-	body, _ = getResponseBody(resp)
-	return body, checkResponse(resp.StatusCode, expectedStatusCode)
-}
-
-// GetConnections returns status and stats for active SFTP/SCP connections
-func GetConnections(expectedStatusCode int) ([]sftpd.ConnectionStatus, []byte, error) {
-	var connections []sftpd.ConnectionStatus
-	var body []byte
-	resp, err := getHTTPClient().Get(buildURLRelativeToBase(activeConnectionsPath))
-	if err != nil {
-		return connections, body, err
-	}
-	defer resp.Body.Close()
-	err = checkResponse(resp.StatusCode, expectedStatusCode)
-	if err == nil && expectedStatusCode == http.StatusOK {
-		err = render.DecodeJSON(resp.Body, &connections)
-	} else {
-		body, _ = getResponseBody(resp)
-	}
-	return connections, body, err
-}
-
-// CloseConnection closes an active  connection identified by connectionID
-func CloseConnection(connectionID string, expectedStatusCode int) ([]byte, error) {
-	var body []byte
-	req, err := http.NewRequest(http.MethodDelete, buildURLRelativeToBase(activeConnectionsPath, connectionID), nil)
-	if err != nil {
-		return body, err
-	}
-	resp, err := getHTTPClient().Do(req)
-	if err != nil {
-		return body, err
-	}
-	defer resp.Body.Close()
-	err = checkResponse(resp.StatusCode, expectedStatusCode)
-	body, _ = getResponseBody(resp)
-	return body, err
-}
-
-// GetVersion returns version details
-func GetVersion(expectedStatusCode int) (utils.VersionInfo, []byte, error) {
-	var version utils.VersionInfo
-	var body []byte
-	resp, err := getHTTPClient().Get(buildURLRelativeToBase(versionPath))
-	if err != nil {
-		return version, body, err
-	}
-	defer resp.Body.Close()
-	err = checkResponse(resp.StatusCode, expectedStatusCode)
-	if err == nil && expectedStatusCode == http.StatusOK {
-		err = render.DecodeJSON(resp.Body, &version)
-	} else {
-		body, _ = getResponseBody(resp)
-	}
-	return version, body, err
-}
-
-func checkResponse(actual int, expected int) error {
-	if expected != actual {
-		return fmt.Errorf("wrong status code: got %v want %v", actual, expected)
-	}
-	return nil
-}
-
-func getResponseBody(resp *http.Response) ([]byte, error) {
-	return ioutil.ReadAll(resp.Body)
-}
-
-func checkUser(expected dataprovider.User, actual dataprovider.User) error {
-	if len(actual.Password) > 0 {
-		return errors.New("User password must not be visible")
-	}
-	if len(actual.PublicKeys) > 0 {
-		return errors.New("User public keys must not be visible")
-	}
-	if expected.ID <= 0 {
-		if actual.ID <= 0 {
-			return errors.New("actual user ID must be > 0")
-		}
-	} else {
-		if actual.ID != expected.ID {
-			return errors.New("user ID mismatch")
-		}
-	}
-	for _, v := range expected.Permissions {
-		if !utils.IsStringInSlice(v, actual.Permissions) {
-			return errors.New("Permissions contents mismatch")
-		}
-	}
-	return compareEqualsUserFields(expected, actual)
-}
-
-func compareEqualsUserFields(expected dataprovider.User, actual dataprovider.User) error {
-	if expected.Username != actual.Username {
-		return errors.New("Username mismatch")
-	}
-	if expected.HomeDir != actual.HomeDir {
-		return errors.New("HomeDir mismatch")
-	}
-	if expected.UID != actual.UID {
-		return errors.New("UID mismatch")
-	}
-	if expected.GID != actual.GID {
-		return errors.New("GID mismatch")
-	}
-	if expected.MaxSessions != actual.MaxSessions {
-		return errors.New("MaxSessions mismatch")
-	}
-	if expected.QuotaSize != actual.QuotaSize {
-		return errors.New("QuotaSize mismatch")
-	}
-	if expected.QuotaFiles != actual.QuotaFiles {
-		return errors.New("QuotaFiles mismatch")
-	}
-	if len(expected.Permissions) != len(actual.Permissions) {
-		return errors.New("Permissions mismatch")
-	}
-	if expected.UploadBandwidth != actual.UploadBandwidth {
-		return errors.New("UploadBandwidth mismatch")
-	}
-	if expected.DownloadBandwidth != actual.DownloadBandwidth {
-		return errors.New("DownloadBandwidth mismatch")
-	}
-	return nil
-}

api/internal_test.go

@@ -1,228 +0,0 @@
-package api
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/drakkan/sftpgo/dataprovider"
-	"github.com/go-chi/chi"
-)
-
-const (
-	invalidURL  = "http://foo\x7f.com/"
-	inactiveURL = "http://127.0.0.1:12345"
-)
-
-func TestGetRespStatus(t *testing.T) {
-	var err error
-	err = &dataprovider.MethodDisabledError{}
-	respStatus := getRespStatus(err)
-	if respStatus != http.StatusForbidden {
-		t.Errorf("wrong resp status extected: %d got: %d", http.StatusForbidden, respStatus)
-	}
-	err = fmt.Errorf("generic error")
-	respStatus = getRespStatus(err)
-	if respStatus != http.StatusInternalServerError {
-		t.Errorf("wrong resp status extected: %d got: %d", http.StatusInternalServerError, respStatus)
-	}
-}
-
-func TestCheckResponse(t *testing.T) {
-	err := checkResponse(http.StatusOK, http.StatusCreated)
-	if err == nil {
-		t.Errorf("check must fail")
-	}
-	err = checkResponse(http.StatusBadRequest, http.StatusBadRequest)
-	if err != nil {
-		t.Errorf("test must succeed, error: %v", err)
-	}
-}
-
-func TestCheckUser(t *testing.T) {
-	expected := dataprovider.User{}
-	actual := dataprovider.User{}
-	actual.Password = "password"
-	err := checkUser(expected, actual)
-	if err == nil {
-		t.Errorf("actual password must be nil")
-	}
-	actual.Password = ""
-	actual.PublicKeys = []string{"pub key"}
-	err = checkUser(expected, actual)
-	if err == nil {
-		t.Errorf("actual public key must be nil")
-	}
-	actual.PublicKeys = []string{}
-	err = checkUser(expected, actual)
-	if err == nil {
-		t.Errorf("actual ID must be > 0")
-	}
-	expected.ID = 1
-	actual.ID = 2
-	err = checkUser(expected, actual)
-	if err == nil {
-		t.Errorf("actual ID must be equal to expected ID")
-	}
-	expected.ID = 2
-	actual.ID = 2
-	expected.Permissions = []string{dataprovider.PermCreateDirs, dataprovider.PermDelete, dataprovider.PermDownload}
-	actual.Permissions = []string{dataprovider.PermCreateDirs, dataprovider.PermCreateSymlinks}
-	err = checkUser(expected, actual)
-	if err == nil {
-		t.Errorf("Permissions are not equal")
-	}
-	expected.Permissions = append(expected.Permissions, dataprovider.PermRename)
-	err = checkUser(expected, actual)
-	if err == nil {
-		t.Errorf("Permissions are not equal")
-	}
-}
-
-func TestCompareUserFields(t *testing.T) {
-	expected := dataprovider.User{}
-	actual := dataprovider.User{}
-	expected.Username = "test"
-	err := compareEqualsUserFields(expected, actual)
-	if err == nil {
-		t.Errorf("Username does not match")
-	}
-	expected.Username = ""
-	expected.HomeDir = "homedir"
-	err = compareEqualsUserFields(expected, actual)
-	if err == nil {
-		t.Errorf("HomeDir does not match")
-	}
-	expected.HomeDir = ""
-	expected.UID = 1
-	err = compareEqualsUserFields(expected, actual)
-	if err == nil {
-		t.Errorf("UID does not match")
-	}
-	expected.UID = 0
-	expected.GID = 1
-	err = compareEqualsUserFields(expected, actual)
-	if err == nil {
-		t.Errorf("GID does not match")
-	}
-	expected.GID = 0
-	expected.MaxSessions = 2
-	err = compareEqualsUserFields(expected, actual)
-	if err == nil {
-		t.Errorf("MaxSessions do not match")
-	}
-	expected.MaxSessions = 0
-	expected.QuotaSize = 4096
-	err = compareEqualsUserFields(expected, actual)
-	if err == nil {
-		t.Errorf("QuotaSize does not match")
-	}
-	expected.QuotaSize = 0
-	expected.QuotaFiles = 2
-	err = compareEqualsUserFields(expected, actual)
-	if err == nil {
-		t.Errorf("QuotaFiles do not match")
-	}
-	expected.QuotaFiles = 0
-	expected.Permissions = []string{dataprovider.PermCreateDirs}
-	err = compareEqualsUserFields(expected, actual)
-	if err == nil {
-		t.Errorf("Permissions are not equal")
-	}
-	expected.Permissions = nil
-	expected.UploadBandwidth = 64
-	err = compareEqualsUserFields(expected, actual)
-	if err == nil {
-		t.Errorf("UploadBandwidth does not match")
-	}
-	expected.UploadBandwidth = 0
-	expected.DownloadBandwidth = 128
-	err = compareEqualsUserFields(expected, actual)
-	if err == nil {
-		t.Errorf("DownloadBandwidth does not match")
-	}
-}
-
-func TestApiCallsWithBadURL(t *testing.T) {
-	oldBaseURL := httpBaseURL
-	SetBaseURL(invalidURL)
-	u := dataprovider.User{}
-	_, _, err := UpdateUser(u, http.StatusBadRequest)
-	if err == nil {
-		t.Errorf("request with invalid URL must fail")
-	}
-	_, err = RemoveUser(u, http.StatusNotFound)
-	if err == nil {
-		t.Errorf("request with invalid URL must fail")
-	}
-	_, _, err = GetUsers(1, 0, "", http.StatusBadRequest)
-	if err == nil {
-		t.Errorf("request with invalid URL must fail")
-	}
-	_, err = CloseConnection("non_existent_id", http.StatusNotFound)
-	if err == nil {
-		t.Errorf("request with invalid URL must fail")
-	}
-	SetBaseURL(oldBaseURL)
-}
-
-func TestApiCallToNotListeningServer(t *testing.T) {
-	oldBaseURL := httpBaseURL
-	SetBaseURL(inactiveURL)
-	u := dataprovider.User{}
-	_, _, err := AddUser(u, http.StatusBadRequest)
-	if err == nil {
-		t.Errorf("request to an inactive URL must fail")
-	}
-	_, _, err = UpdateUser(u, http.StatusNotFound)
-	if err == nil {
-		t.Errorf("request to an inactive URL must fail")
-	}
-	_, err = RemoveUser(u, http.StatusNotFound)
-	if err == nil {
-		t.Errorf("request to an inactive URL must fail")
-	}
-	_, _, err = GetUserByID(-1, http.StatusNotFound)
-	if err == nil {
-		t.Errorf("request to an inactive URL must fail")
-	}
-	_, _, err = GetUsers(100, 0, "", http.StatusOK)
-	if err == nil {
-		t.Errorf("request to an inactive URL must fail")
-	}
-	_, _, err = GetQuotaScans(http.StatusOK)
-	if err == nil {
-		t.Errorf("request to an inactive URL must fail")
-	}
-	_, err = StartQuotaScan(u, http.StatusNotFound)
-	if err == nil {
-		t.Errorf("request to an inactive URL must fail")
-	}
-	_, _, err = GetConnections(http.StatusOK)
-	if err == nil {
-		t.Errorf("request to an inactive URL must fail")
-	}
-	_, err = CloseConnection("non_existent_id", http.StatusNotFound)
-	if err == nil {
-		t.Errorf("request to an inactive URL must fail")
-	}
-	_, _, err = GetVersion(http.StatusOK)
-	if err == nil {
-		t.Errorf("request to an inactive URL must fail")
-	}
-	SetBaseURL(oldBaseURL)
-}
-
-func TestCloseConnectionHandler(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodDelete, activeConnectionsPath+"/connectionID", nil)
-	rctx := chi.NewRouteContext()
-	rctx.URLParams.Add("connectionID", "")
-	req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
-	rr := httptest.NewRecorder()
-	handleCloseConnection(rr, req)
-	if rr.Code != http.StatusBadRequest {
-		t.Errorf("Expected response code 400. Got %d", rr.Code)
-	}
-}

api/quota.go

@@ -1,44 +0,0 @@
-package api
-
-import (
-	"net/http"
-
-	"github.com/drakkan/sftpgo/dataprovider"
-	"github.com/drakkan/sftpgo/logger"
-	"github.com/drakkan/sftpgo/sftpd"
-	"github.com/drakkan/sftpgo/utils"
-	"github.com/go-chi/render"
-)
-
-func getQuotaScans(w http.ResponseWriter, r *http.Request) {
-	render.JSON(w, r, sftpd.GetQuotaScans())
-}
-
-func startQuotaScan(w http.ResponseWriter, r *http.Request) {
-	var u dataprovider.User
-	err := render.DecodeJSON(r.Body, &u)
-	if err != nil {
-		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
-		return
-	}
-	user, err := dataprovider.UserExists(dataProvider, u.Username)
-	if err != nil {
-		sendAPIResponse(w, r, err, "", http.StatusNotFound)
-		return
-	}
-	if sftpd.AddQuotaScan(user.Username) {
-		sendAPIResponse(w, r, err, "Scan started", http.StatusCreated)
-		go func() {
-			numFiles, size, _, err := utils.ScanDirContents(user.HomeDir)
-			if err != nil {
-				logger.Warn(logSender, "", "error scanning user home dir %v: %v", user.HomeDir, err)
-			} else {
-				err := dataprovider.UpdateUserQuota(dataProvider, user, numFiles, size, true)
-				logger.Debug(logSender, "", "user dir scanned, user: %v, dir: %v, error: %v", user.Username, user.HomeDir, err)
-			}
-			sftpd.RemoveQuotaScan(user.Username)
-		}()
-	} else {
-		sendAPIResponse(w, r, err, "Another scan is already in progress", http.StatusConflict)
-	}
-}

api/router.go

@@ -1,89 +0,0 @@
-package api
-
-import (
-	"net/http"
-
-	"github.com/drakkan/sftpgo/logger"
-	"github.com/drakkan/sftpgo/sftpd"
-	"github.com/drakkan/sftpgo/utils"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/chi/middleware"
-	"github.com/go-chi/render"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-)
-
-// GetHTTPRouter returns the configured HTTP handler
-func GetHTTPRouter() http.Handler {
-	return router
-}
-
-func initializeRouter() {
-	router = chi.NewRouter()
-	router.Use(middleware.RequestID)
-	router.Use(middleware.RealIP)
-	router.Use(logger.NewStructuredLogger(logger.GetLogger()))
-	router.Use(middleware.Recoverer)
-
-	router.NotFound(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		sendAPIResponse(w, r, nil, "Not Found", http.StatusNotFound)
-	}))
-
-	router.MethodNotAllowed(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		sendAPIResponse(w, r, nil, "Method not allowed", http.StatusMethodNotAllowed)
-	}))
-
-	router.Handle(metricsPath, promhttp.Handler())
-
-	router.Get(versionPath, func(w http.ResponseWriter, r *http.Request) {
-		render.JSON(w, r, utils.GetAppVersion())
-	})
-
-	router.Get(activeConnectionsPath, func(w http.ResponseWriter, r *http.Request) {
-		render.JSON(w, r, sftpd.GetConnectionsStats())
-	})
-
-	router.Delete(activeConnectionsPath+"/{connectionID}", func(w http.ResponseWriter, r *http.Request) {
-		handleCloseConnection(w, r)
-	})
-
-	router.Get(quotaScanPath, func(w http.ResponseWriter, r *http.Request) {
-		getQuotaScans(w, r)
-	})
-
-	router.Post(quotaScanPath, func(w http.ResponseWriter, r *http.Request) {
-		startQuotaScan(w, r)
-	})
-
-	router.Get(userPath, func(w http.ResponseWriter, r *http.Request) {
-		getUsers(w, r)
-	})
-
-	router.Post(userPath, func(w http.ResponseWriter, r *http.Request) {
-		addUser(w, r)
-	})
-
-	router.Get(userPath+"/{userID}", func(w http.ResponseWriter, r *http.Request) {
-		getUserByID(w, r)
-	})
-
-	router.Put(userPath+"/{userID}", func(w http.ResponseWriter, r *http.Request) {
-		updateUser(w, r)
-	})
-
-	router.Delete(userPath+"/{userID}", func(w http.ResponseWriter, r *http.Request) {
-		deleteUser(w, r)
-	})
-}
-
-func handleCloseConnection(w http.ResponseWriter, r *http.Request) {
-	connectionID := chi.URLParam(r, "connectionID")
-	if connectionID == "" {
-		sendAPIResponse(w, r, nil, "connectionID is mandatory", http.StatusBadRequest)
-		return
-	}
-	if sftpd.CloseActiveConnection(connectionID) {
-		sendAPIResponse(w, r, nil, "Connection closed", http.StatusOK)
-	} else {
-		sendAPIResponse(w, r, nil, "Not Found", http.StatusNotFound)
-	}
-}

api/schema/openapi.yaml

@@ -1,691 +0,0 @@
-openapi: 3.0.1
-info:
-  title: SFTPGo
-  description: 'SFTPGo REST API'
-  version: 1.0.0
-  
-servers:
-- url: /api/v1
-paths:
-  /version:
-    get:
-      tags:
-      - version
-      summary: Get version details
-      operationId: get_version
-      responses:
-        200:
-          description: successful operation
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref : '#/components/schemas/VersionInfo'
-  /connection:
-    get:
-      tags:
-      - connections
-      summary: Get the active users and info about their uploads/downloads
-      operationId: get_connections
-      responses:
-        200:
-          description: successful operation
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref : '#/components/schemas/ConnectionStatus'
-  /connection/{connectionID}:
-    delete:
-      tags:
-      - connections
-      summary: Terminate an active connection
-      operationId: close_connection
-      parameters: 
-      - name: connectionID
-        in: path
-        description: ID of the connection to close
-        required: true
-        schema:
-          type: string
-      responses:
-        200:
-          description: successful operation
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 200
-                message: "Connection closed"
-                error: ""
-        400:
-          description: Bad request
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 400
-                message: ""
-                error: "Error description if any"
-        404:
-          description: Not Found
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 404
-                message: ""
-                error: "Error description if any"
-        500:
-          description: Internal Server Error
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 500
-                message: ""
-                error: "Error description if any"
-  /quota_scan:
-    get:
-      tags:
-      - quota
-      summary: Get the active quota scans
-      operationId: get_quota_scans
-      responses:
-        200:
-          description: successful operation
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref : '#/components/schemas/QuotaScan'
-    post:
-      tags:
-      - quota
-      summary: start a new quota scan
-      description: A quota scan update the number of files and their total size for the given user
-      operationId: start_quota_scan
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              $ref : '#/components/schemas/User'
-      responses:
-        201:
-          description: successful operation
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 201
-                message: "Scan started"
-                error: ""
-        400:
-          description: Bad request
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 400
-                message: ""
-                error: "Error description if any"
-        403:
-          description: Forbidden
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 403
-                message: ""
-                error: "Error description if any"
-        404:
-          description: Not Found
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 404
-                message: ""
-                error: "Error description if any"
-        409:
-          description: Another scan is already in progress for this user
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 409
-                message: "Another scan is already in progress"
-                error: "Error description if any" 
-        500:
-          description: Internal Server Error
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 500
-                message: ""
-                error: "Error description if any"
-  /user:
-    get:
-      tags:
-      - users
-      summary: Returns an array with one or more users
-      description: For security reasons password and public key are empty in the response
-      operationId: get_users
-      parameters:
-        - in: query
-          name: offset
-          schema:
-            type: integer
-            minimum: 0
-            default: 0
-          required: false
-        - in: query
-          name: limit
-          schema:
-            type: integer
-            minimum: 1
-            maximum: 500
-            default: 100
-          required: false
-          description: The maximum number of items to return. Max value is 500, default is 100
-        - in: query
-          name: order
-          required: false
-          description: Ordering users by username
-          schema:
-             type: string
-             enum:
-                - ASC
-                - DESC
-             example: ASC
-        - in: query
-          name: username
-          required: false
-          description: Filter by username, extact match case sensitive
-          schema:
-             type: string
-      responses:
-        200:
-          description: successful operation
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref : '#/components/schemas/User'
-        400:
-          description: Bad request
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 400
-                message: ""
-                error: "Error description if any"
-        403:
-          description: Forbidden
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 403
-                message: ""
-                error: "Error description if any"
-        500:
-          description: Internal Server Error
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 500
-                message: ""
-                error: "Error description if any"
-    post:
-      tags:
-      - users
-      summary: Adds a new SFTP/SCP user
-      operationId: add_user
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              $ref : '#/components/schemas/User'
-      responses:
-        200:
-          description: successful operation
-          content:
-            application/json:
-              schema:
-                $ref : '#/components/schemas/User'
-        400:
-          description: Bad request
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 400
-                message: ""
-                error: "Error description if any"
-        403:
-          description: Forbidden
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 403
-                message: ""
-                error: "Error description if any"
-        500:
-          description: Internal Server Error
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 500
-                message: ""
-                error: "Error description if any"
-  /user/{userID}:
-    get:
-      tags:
-      - users
-      summary: Find user by ID
-      description: For security reasons password and public key are empty in the response
-      operationId: get_user_by_id
-      parameters: 
-      - name: userID
-        in: path
-        description: ID of the user to retrieve
-        required: true
-        schema:
-          type: integer
-          format: int32
-      responses:
-        200:
-          description: successful operation
-          content:
-            application/json:
-              schema:
-                $ref : '#/components/schemas/User'
-        400:
-          description: Bad request
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 400
-                message: ""
-                error: "Error description if any"
-        403:
-          description: Forbidden
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 403
-                message: ""
-                error: "Error description if any"
-        404:
-          description: Not Found
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 404
-                message: ""
-                error: "Error description if any"
-        500:
-          description: Internal Server Error
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 500
-                message: ""
-                error: "Error description if any"
-    put:
-      tags:
-      - users
-      summary: Update an existing user
-      operationId: update_user
-      parameters: 
-      - name: userID
-        in: path
-        description: ID of the user to update
-        required: true
-        schema:
-          type: integer
-          format: int32
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              $ref : '#/components/schemas/User'
-      responses:
-        200:
-          description: successful operation
-          content:
-            application/json:
-              schema:
-                $ref : '#/components/schemas/ApiResponse'
-              example: 
-                status: 200
-                message: "User updated"
-                error: ""
-        400:
-          description: Bad request
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 400
-                message: ""
-                error: "Error description if any"
-        403:
-          description: Forbidden
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 403
-                message: ""
-                error: "Error description if any"
-        404:
-          description: Not Found
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 404
-                message: ""
-                error: "Error description if any"
-        500:
-          description: Internal Server Error
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 500
-                message: ""
-                error: "Error description if any"
-    delete:
-      tags:
-      - users
-      summary: Delete an existing user
-      operationId: delete_user
-      parameters: 
-      - name: userID
-        in: path
-        description: ID of the user to delete
-        required: true
-        schema:
-          type: integer
-          format: int32
-      responses:
-        200:
-          description: successful operation
-          content:
-            application/json:
-              schema:
-                $ref : '#/components/schemas/ApiResponse'
-              example: 
-                status: 200
-                message: "User deleted"
-                error: ""
-        400:
-          description: Bad request
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 400
-                message: ""
-                error: "Error description if any"
-        403:
-          description: Forbidden
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 403
-                message: ""
-                error: "Error description if any"
-        404:
-          description: Not Found
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 404
-                message: ""
-                error: "Error description if any"
-        500:
-          description: Internal Server Error
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example: 
-                status: 500
-                message: ""
-                error: "Error description if any"
-components:
-  schemas:
-    Permission:
-      type: string
-      enum:
-        - '*'
-        - list
-        - download
-        - upload
-        - overwrite
-        - delete
-        - rename
-        - create_dirs
-        - create_symlinks
-      description: >
-        Permissions:
-          * `*` - all permissions are granted
-          * `list` - list items is allowed
-          * `download` - download files is allowed
-          * `upload` - upload files is allowed
-          * `overwrite` - overwrite an existing file, while uploading, is allowed. upload permission is required to allow file overwrite
-          * `delete` - delete files or directories is allowed
-          * `rename` - rename files or directories is allowed
-          * `create_dirs` - create directories is allowed
-          * `create_symlinks` - create links is allowed
-    User:
-      type: object
-      properties:
-        id:
-          type: integer
-          format: int32
-          minimum: 1
-        username:
-          type: string
-        password:
-          type: string
-          nullable: true
-          description: password or public key are mandatory. If the password has no known hashing algo prefix it will be stored using argon2id. You can send a password hashed as bcrypt or pbkdf2 and it will be stored as is. For security reasons this field is omitted when you search/get users
-        public_keys:
-          type: array
-          items:
-            type: string
-          nullable: true
-          description: a password or at least one public key are mandatory. For security reasons this field is omitted when you search/get users.
-        home_dir:
-          type: string
-          description: path to the user home directory. The user cannot upload or download files outside this directory. SFTPGo tries to automatically create this folder if missing. Must be an absolute path
-        uid:
-          type: integer
-          format: int32
-          minimum: 0
-          maximum: 65535
-          description: if you run sftpgo as root user the created files and directories will be assigned to this uid. 0 means no change, the owner will be the user that runs sftpgo. Ignored on windows
-        gid:
-          type: integer
-          format: int32
-          minimum: 0
-          maximum: 65535
-          description: if you run sftpgo as root user the created files and directories will be assigned to this gid. 0 means no change, the group will be the one of the user that runs sftpgo. Ignored on windows
-        max_sessions:
-          type: integer
-          format: int32
-          description: limit the sessions that an user can open. 0 means unlimited
-        quota_size:
-          type: integer
-          format: int64
-          description: quota as size. 0 menas unlimited. Please note that quota is updated if files are added/removed via SFTP/SCP otherwise a quota scan is needed
-        quota_files:
-          type: integer
-          format: int32
-          description: quota as number of files. 0 menas unlimited. Please note that quota is updated if files are added/removed via SFTP/SCP otherwise a quota scan is needed
-        permissions:
-          type: array
-          items:
-            $ref: '#/components/schemas/Permission'
-          minItems: 1
-        used_quota_size:
-          type: integer
-          format: int64
-        used_quota_file:
-          type: integer
-          format: int32
-        last_quota_update:
-          type: integer
-          format: int64
-          description: last quota update as unix timestamp in milliseconds
-        upload_bandwidth:
-          type: integer
-          format: int32
-          description: Maximum upload bandwidth as KB/s, 0 means unlimited
-        download_bandwidth:
-          type: integer
-          format: int32
-          description: Maximum download bandwidth as KB/s, 0 means unlimited
-    Transfer:
-      type: object
-      properties:
-        operation_type:
-          type: string
-          enum: 
-            - upload
-            - download
-        path:
-          type: string
-          description: SFTP/SCP file path for the upload/download
-        start_time:
-          type: integer
-          format: int64
-          description: start time as unix timestamp in milliseconds
-        size:
-          type: integer
-          format: int64
-          description: bytes transferred
-        last_activity:
-          type: integer
-          format: int64
-          description: last transfer activity as unix timestamp in milliseconds
-    ConnectionStatus:
-      type: object
-      properties:
-        username:
-          type: string
-          description: connected username
-        connection_id:
-          type: string
-          description: unique connection identifier
-        client_version:
-          type: string
-          description: SFTP/SCP client version
-        remote_address:
-          type: string
-          description: Remote address for the connected SFTP/SCP client
-        connection_time:
-          type: integer
-          format: int64
-          description: connection time as unix timestamp in milliseconds
-        last_activity:
-          type: integer
-          format: int64
-          description: last client activity as unix timestamp in milliseconds
-        protocol:
-          type: string
-          enum:
-            - SFTP
-            - SCP
-        active_transfers:
-          type: array
-          items:
-            $ref : '#/components/schemas/Transfer'
-    QuotaScan:
-      type: object
-      properties:
-        username:
-          type: string
-          description: username with an active scan
-        start_time:
-          type: integer
-          format: int64
-          description: scan start time as unix timestamp in milliseconds
-    ApiResponse:
-      type: object
-      properties:
-        status:
-          type: integer
-          format: int32
-          minimum: 200
-          maximum: 500
-          example: 200
-          description: HTTP Status code, for example 200 OK, 400 Bad request and so on
-        message:
-          type: string
-          nullable: true
-          description: additional message if any
-        error:
-          type: string
-          nullable: true
-          description: error description if any
-    VersionInfo:
-      type: object
-      properties:
-        version:
-          type: string
-        build_date:
-          type: string
-        commit_hash:
-          type: string
-        

cmd/initprovider.go

@@ -0,0 +1,58 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/rs/zerolog"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/drakkan/sftpgo/config"
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/utils"
+)
+
+var (
+	initProviderCmd = &cobra.Command{
+		Use:   "initprovider",
+		Short: "Initializes the configured data provider",
+		Long: `This command reads the data provider connection details from the specified configuration file and creates the initial structure.
+
+Some data providers such as bolt and memory does not require an initialization.
+
+For SQLite provider the database file will be auto created if missing.
+
+For PostgreSQL and MySQL providers you need to create the configured database, this command will create the required tables.
+
+To initialize the data provider from the configuration directory simply use:
+
+sftpgo initprovider
+
+Please take a look at the usage below to customize the options.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			logger.DisableLogger()
+			logger.EnableConsoleLogger(zerolog.DebugLevel)
+			configDir = utils.CleanDirInput(configDir)
+			err := config.LoadConfig(configDir, configFile)
+			if err != nil {
+				logger.WarnToConsole("Unable to initialize data provider, config load error: %v", err)
+				return
+			}
+			providerConf := config.GetProviderConf()
+			logger.DebugToConsole("Initializing provider: %#v config file: %#v", providerConf.Driver, viper.ConfigFileUsed())
+			err = dataprovider.InitializeDatabase(providerConf, configDir)
+			if err == nil {
+				logger.DebugToConsole("Data provider successfully initialized")
+			} else {
+				logger.WarnToConsole("Unable to initialize data provider: %v", err)
+				os.Exit(1)
+			}
+		},
+	}
+)
+
+func init() {
+	rootCmd.AddCommand(initProviderCmd)
+	addConfigFlags(initProviderCmd)
+}

cmd/install_windows.go

@@ -2,9 +2,13 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+	"strconv"
+
+	"github.com/spf13/cobra"
 
 	"github.com/drakkan/sftpgo/service"
-	"github.com/spf13/cobra"
+	"github.com/drakkan/sftpgo/utils"
 )
 
 var (
@@ -18,7 +22,7 @@ sftpgo service install
 Please take a look at the usage below to customize the startup options`,
 		Run: func(cmd *cobra.Command, args []string) {
 			s := service.Service{
-				ConfigDir:     configDir,
+				ConfigDir:     utils.CleanDirInput(configDir),
 				ConfigFile:    configFile,
 				LogFilePath:   logFilePath,
 				LogMaxSize:    logMaxSize,
@@ -39,6 +43,7 @@ Please take a look at the usage below to customize the startup options`,
 			err := winService.Install(serviceArgs...)
 			if err != nil {
 				fmt.Printf("Error installing service: %v\r\n", err)
+				os.Exit(1)
 			} else {
 				fmt.Printf("Service installed!\r\n")
 			}
@@ -50,3 +55,42 @@ func init() {
 	serviceCmd.AddCommand(installCmd)
 	addServeFlags(installCmd)
 }
+
+func getCustomServeFlags() []string {
+	result := []string{}
+	if configDir != defaultConfigDir {
+		configDir = utils.CleanDirInput(configDir)
+		result = append(result, "--"+configDirFlag)
+		result = append(result, configDir)
+	}
+	if configFile != defaultConfigName {
+		result = append(result, "--"+configFileFlag)
+		result = append(result, configFile)
+	}
+	if logFilePath != defaultLogFile {
+		result = append(result, "--"+logFilePathFlag)
+		result = append(result, logFilePath)
+	}
+	if logMaxSize != defaultLogMaxSize {
+		result = append(result, "--"+logMaxSizeFlag)
+		result = append(result, strconv.Itoa(logMaxSize))
+	}
+	if logMaxBackups != defaultLogMaxBackup {
+		result = append(result, "--"+logMaxBackupFlag)
+		result = append(result, strconv.Itoa(logMaxBackups))
+	}
+	if logMaxAge != defaultLogMaxAge {
+		result = append(result, "--"+logMaxAgeFlag)
+		result = append(result, strconv.Itoa(logMaxAge))
+	}
+	if logVerbose != defaultLogVerbose {
+		result = append(result, "--"+logVerboseFlag+"=false")
+	}
+	if logCompress != defaultLogCompress {
+		result = append(result, "--"+logCompressFlag+"=true")
+	}
+	if profiler != defaultProfiler {
+		result = append(result, "--"+profilerFlag+"=true")
+	}
+	return result
+}

cmd/portable.go

@@ -0,0 +1,243 @@
+// +build !noportable
+
+package cmd
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/service"
+	"github.com/drakkan/sftpgo/sftpd"
+	"github.com/drakkan/sftpgo/version"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+var (
+	directoryToServe             string
+	portableSFTPDPort            int
+	portableAdvertiseService     bool
+	portableAdvertiseCredentials bool
+	portableUsername             string
+	portablePassword             string
+	portableLogFile              string
+	portableLogVerbose           bool
+	portablePublicKeys           []string
+	portablePermissions          []string
+	portableSSHCommands          []string
+	portableAllowedExtensions    []string
+	portableDeniedExtensions     []string
+	portableFsProvider           int
+	portableS3Bucket             string
+	portableS3Region             string
+	portableS3AccessKey          string
+	portableS3AccessSecret       string
+	portableS3Endpoint           string
+	portableS3StorageClass       string
+	portableS3KeyPrefix          string
+	portableS3ULPartSize         int
+	portableS3ULConcurrency      int
+	portableGCSBucket            string
+	portableGCSCredentialsFile   string
+	portableGCSAutoCredentials   int
+	portableGCSStorageClass      string
+	portableGCSKeyPrefix         string
+	portableCmd                  = &cobra.Command{
+		Use:   "portable",
+		Short: "Serve a single directory",
+		Long: `To serve the current working directory with auto generated credentials simply use:
+
+sftpgo portable
+
+Please take a look at the usage below to customize the serving parameters`,
+		Run: func(cmd *cobra.Command, args []string) {
+			portableDir := directoryToServe
+			if !filepath.IsAbs(portableDir) {
+				if portableFsProvider == 0 {
+					portableDir, _ = filepath.Abs(portableDir)
+				} else {
+					portableDir = os.TempDir()
+				}
+			}
+			permissions := make(map[string][]string)
+			permissions["/"] = portablePermissions
+			portableGCSCredentials := ""
+			if portableFsProvider == 2 && len(portableGCSCredentialsFile) > 0 {
+				fi, err := os.Stat(portableGCSCredentialsFile)
+				if err != nil {
+					fmt.Printf("Invalid GCS credentials file: %v\n", err)
+					os.Exit(1)
+				}
+				if fi.Size() > 1048576 {
+					fmt.Printf("Invalid GCS credentials file: %#v is too big %v/1048576 bytes\n", portableGCSCredentialsFile,
+						fi.Size())
+					os.Exit(1)
+				}
+				creds, err := ioutil.ReadFile(portableGCSCredentialsFile)
+				if err != nil {
+					fmt.Printf("Unable to read credentials file: %v\n", err)
+				}
+				portableGCSCredentials = base64.StdEncoding.EncodeToString(creds)
+				portableGCSAutoCredentials = 0
+			}
+			service := service.Service{
+				ConfigDir:     filepath.Clean(defaultConfigDir),
+				ConfigFile:    defaultConfigName,
+				LogFilePath:   portableLogFile,
+				LogMaxSize:    defaultLogMaxSize,
+				LogMaxBackups: defaultLogMaxBackup,
+				LogMaxAge:     defaultLogMaxAge,
+				LogCompress:   defaultLogCompress,
+				LogVerbose:    portableLogVerbose,
+				Profiler:      defaultProfiler,
+				Shutdown:      make(chan bool),
+				PortableMode:  1,
+				PortableUser: dataprovider.User{
+					Username:    portableUsername,
+					Password:    portablePassword,
+					PublicKeys:  portablePublicKeys,
+					Permissions: permissions,
+					HomeDir:     portableDir,
+					Status:      1,
+					FsConfig: dataprovider.Filesystem{
+						Provider: portableFsProvider,
+						S3Config: vfs.S3FsConfig{
+							Bucket:            portableS3Bucket,
+							Region:            portableS3Region,
+							AccessKey:         portableS3AccessKey,
+							AccessSecret:      portableS3AccessSecret,
+							Endpoint:          portableS3Endpoint,
+							StorageClass:      portableS3StorageClass,
+							KeyPrefix:         portableS3KeyPrefix,
+							UploadPartSize:    int64(portableS3ULPartSize),
+							UploadConcurrency: portableS3ULConcurrency,
+						},
+						GCSConfig: vfs.GCSFsConfig{
+							Bucket:               portableGCSBucket,
+							Credentials:          portableGCSCredentials,
+							AutomaticCredentials: portableGCSAutoCredentials,
+							StorageClass:         portableGCSStorageClass,
+							KeyPrefix:            portableGCSKeyPrefix,
+						},
+					},
+					Filters: dataprovider.UserFilters{
+						FileExtensions: parseFileExtensionsFilters(),
+					},
+				},
+			}
+			if err := service.StartPortableMode(portableSFTPDPort, portableSSHCommands, portableAdvertiseService,
+				portableAdvertiseCredentials); err == nil {
+				service.Wait()
+				os.Exit(0)
+			}
+			os.Exit(1)
+		},
+	}
+)
+
+func init() {
+	version.AddFeature("+portable")
+
+	portableCmd.Flags().StringVarP(&directoryToServe, "directory", "d", ".",
+		"Path to the directory to serve. This can be an absolute path or a path relative to the current directory")
+	portableCmd.Flags().IntVarP(&portableSFTPDPort, "sftpd-port", "s", 0, "0 means a random non privileged port")
+	portableCmd.Flags().StringSliceVarP(&portableSSHCommands, "ssh-commands", "c", sftpd.GetDefaultSSHCommands(),
+		"SSH commands to enable. \"*\" means any supported SSH command including scp")
+	portableCmd.Flags().StringVarP(&portableUsername, "username", "u", "", "Leave empty to use an auto generated value")
+	portableCmd.Flags().StringVarP(&portablePassword, "password", "p", "", "Leave empty to use an auto generated value")
+	portableCmd.Flags().StringVarP(&portableLogFile, logFilePathFlag, "l", "", "Leave empty to disable logging")
+	portableCmd.Flags().BoolVarP(&portableLogVerbose, logVerboseFlag, "v", false, "Enable verbose logs")
+	portableCmd.Flags().StringSliceVarP(&portablePublicKeys, "public-key", "k", []string{}, "")
+	portableCmd.Flags().StringSliceVarP(&portablePermissions, "permissions", "g", []string{"list", "download"},
+		"User's permissions. \"*\" means any permission")
+	portableCmd.Flags().StringArrayVar(&portableAllowedExtensions, "allowed-extensions", []string{},
+		"Allowed file extensions case insensitive. The format is /dir::ext1,ext2. For example: \"/somedir::.jpg,.png\"")
+	portableCmd.Flags().StringArrayVar(&portableDeniedExtensions, "denied-extensions", []string{},
+		"Denied file extensions case insensitive. The format is /dir::ext1,ext2. For example: \"/somedir::.jpg,.png\"")
+	portableCmd.Flags().BoolVarP(&portableAdvertiseService, "advertise-service", "S", false,
+		"Advertise SFTP service using multicast DNS")
+	portableCmd.Flags().BoolVarP(&portableAdvertiseCredentials, "advertise-credentials", "C", false,
+		"If the SFTP service is advertised via multicast DNS, this flag allows to put username/password inside the advertised TXT record")
+	portableCmd.Flags().IntVarP(&portableFsProvider, "fs-provider", "f", 0, "0 means local filesystem, 1 Amazon S3 compatible, "+
+		"2 Google Cloud Storage")
+	portableCmd.Flags().StringVar(&portableS3Bucket, "s3-bucket", "", "")
+	portableCmd.Flags().StringVar(&portableS3Region, "s3-region", "", "")
+	portableCmd.Flags().StringVar(&portableS3AccessKey, "s3-access-key", "", "")
+	portableCmd.Flags().StringVar(&portableS3AccessSecret, "s3-access-secret", "", "")
+	portableCmd.Flags().StringVar(&portableS3Endpoint, "s3-endpoint", "", "")
+	portableCmd.Flags().StringVar(&portableS3StorageClass, "s3-storage-class", "", "")
+	portableCmd.Flags().StringVar(&portableS3KeyPrefix, "s3-key-prefix", "", "Allows to restrict access to the virtual folder "+
+		"identified by this prefix and its contents")
+	portableCmd.Flags().IntVar(&portableS3ULPartSize, "s3-upload-part-size", 5, "The buffer size for multipart uploads (MB)")
+	portableCmd.Flags().IntVar(&portableS3ULConcurrency, "s3-upload-concurrency", 2, "How many parts are uploaded in parallel")
+	portableCmd.Flags().StringVar(&portableGCSBucket, "gcs-bucket", "", "")
+	portableCmd.Flags().StringVar(&portableGCSStorageClass, "gcs-storage-class", "", "")
+	portableCmd.Flags().StringVar(&portableGCSKeyPrefix, "gcs-key-prefix", "", "Allows to restrict access to the virtual folder "+
+		"identified by this prefix and its contents")
+	portableCmd.Flags().StringVar(&portableGCSCredentialsFile, "gcs-credentials-file", "", "Google Cloud Storage JSON credentials file")
+	portableCmd.Flags().IntVar(&portableGCSAutoCredentials, "gcs-automatic-credentials", 1, "0 means explicit credentials using a JSON "+
+		"credentials file, 1 automatic")
+	rootCmd.AddCommand(portableCmd)
+}
+
+func parseFileExtensionsFilters() []dataprovider.ExtensionsFilter {
+	var extensions []dataprovider.ExtensionsFilter
+	for _, val := range portableAllowedExtensions {
+		p, exts := getExtensionsFilterValues(strings.TrimSpace(val))
+		if len(p) > 0 {
+			extensions = append(extensions, dataprovider.ExtensionsFilter{
+				Path:              path.Clean(p),
+				AllowedExtensions: exts,
+				DeniedExtensions:  []string{},
+			})
+		}
+	}
+	for _, val := range portableDeniedExtensions {
+		p, exts := getExtensionsFilterValues(strings.TrimSpace(val))
+		if len(p) > 0 {
+			found := false
+			for index, e := range extensions {
+				if path.Clean(e.Path) == path.Clean(p) {
+					extensions[index].DeniedExtensions = append(extensions[index].DeniedExtensions, exts...)
+					found = true
+					break
+				}
+			}
+			if !found {
+				extensions = append(extensions, dataprovider.ExtensionsFilter{
+					Path:              path.Clean(p),
+					AllowedExtensions: []string{},
+					DeniedExtensions:  exts,
+				})
+			}
+		}
+	}
+	return extensions
+}
+
+func getExtensionsFilterValues(value string) (string, []string) {
+	if strings.Contains(value, "::") {
+		dirExts := strings.Split(value, "::")
+		if len(dirExts) > 1 {
+			dir := strings.TrimSpace(dirExts[0])
+			exts := []string{}
+			for _, e := range strings.Split(dirExts[1], ",") {
+				cleanedExt := strings.TrimSpace(e)
+				if len(cleanedExt) > 0 {
+					exts = append(exts, cleanedExt)
+				}
+			}
+			if len(dir) > 0 && len(exts) > 0 {
+				return dir, exts
+			}
+		}
+	}
+	return "", nil
+}

cmd/portable_disabled.go

@@ -0,0 +1,9 @@
+// +build noportable
+
+package cmd
+
+import "github.com/drakkan/sftpgo/version"
+
+func init() {
+	version.AddFeature("-portable")
+}

cmd/reload_windows.go

@@ -0,0 +1,35 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/drakkan/sftpgo/service"
+)
+
+var (
+	reloadCmd = &cobra.Command{
+		Use:   "reload",
+		Short: "Reload the SFTPGo Windows Service sending a \"paramchange\" request",
+		Run: func(cmd *cobra.Command, args []string) {
+			s := service.WindowsService{
+				Service: service.Service{
+					Shutdown: make(chan bool),
+				},
+			}
+			err := s.Reload()
+			if err != nil {
+				fmt.Printf("Error sending reload signal: %v\r\n", err)
+				os.Exit(1)
+			} else {
+				fmt.Printf("Reload signal sent!\r\n")
+			}
+		},
+	}
+)
+
+func init() {
+	serviceCmd.AddCommand(reloadCmd)
+}

cmd/root.go

@@ -1,18 +1,18 @@
+// Package cmd provides Command Line Interface support
 package cmd
 
 import (
 	"fmt"
 	"os"
-	"strconv"
 
-	"github.com/drakkan/sftpgo/config"
-	"github.com/drakkan/sftpgo/utils"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
+	"github.com/drakkan/sftpgo/config"
+	"github.com/drakkan/sftpgo/version"
 )
 
 const (
-	logSender           = "cmd"
 	configDirFlag       = "config-dir"
 	configDirKey        = "config_dir"
 	configFileFlag      = "config-file"
@@ -29,6 +29,8 @@ const (
 	logCompressKey      = "log_compress"
 	logVerboseFlag      = "log-verbose"
 	logVerboseKey       = "log_verbose"
+	profilerFlag        = "profiler"
+	profilerKey         = "profiler"
 	defaultConfigDir    = "."
 	defaultConfigName   = config.DefaultConfigName
 	defaultLogFile      = "sftpgo.log"
@@ -37,6 +39,7 @@ const (
 	defaultLogMaxAge    = 28
 	defaultLogCompress  = false
 	defaultLogVerbose   = true
+	defaultProfiler     = false
 )
 
 var (
@@ -48,6 +51,7 @@ var (
 	logMaxAge     int
 	logCompress   bool
 	logVerbose    bool
+	profiler      bool
 
 	rootCmd = &cobra.Command{
 		Use:   "sftpgo",
@@ -56,10 +60,9 @@ var (
 )
 
 func init() {
-	version := utils.GetAppVersion()
 	rootCmd.Flags().BoolP("version", "v", false, "")
-	rootCmd.Version = version.GetVersionAsString()
-	rootCmd.SetVersionTemplate(`{{printf "SFTPGo version: "}}{{printf "%s" .Version}}
+	rootCmd.Version = version.GetAsString()
+	rootCmd.SetVersionTemplate(`{{printf "SFTPGo "}}{{printf "%s" .Version}}
 `)
 }
 
@@ -72,97 +75,73 @@ func Execute() {
 	}
 }
 
-func addServeFlags(cmd *cobra.Command) {
+func addConfigFlags(cmd *cobra.Command) {
 	viper.SetDefault(configDirKey, defaultConfigDir)
-	viper.BindEnv(configDirKey, "SFTPGO_CONFIG_DIR")
+	viper.BindEnv(configDirKey, "SFTPGO_CONFIG_DIR") //nolint:errcheck // err is not nil only if the key to bind is missing
 	cmd.Flags().StringVarP(&configDir, configDirFlag, "c", viper.GetString(configDirKey),
 		"Location for SFTPGo config dir. This directory should contain the \"sftpgo\" configuration file or the configured "+
 			"config-file and it is used as the base for files with a relative path (eg. the private keys for the SFTP server, "+
 			"the SQLite database if you use SQLite as data provider). This flag can be set using SFTPGO_CONFIG_DIR env var too.")
-	viper.BindPFlag(configDirKey, cmd.Flags().Lookup(configDirFlag))
+	viper.BindPFlag(configDirKey, cmd.Flags().Lookup(configDirFlag)) //nolint:errcheck
 
 	viper.SetDefault(configFileKey, defaultConfigName)
-	viper.BindEnv(configFileKey, "SFTPGO_CONFIG_FILE")
+	viper.BindEnv(configFileKey, "SFTPGO_CONFIG_FILE") //nolint:errcheck
 	cmd.Flags().StringVarP(&configFile, configFileFlag, "f", viper.GetString(configFileKey),
 		"Name for SFTPGo configuration file. It must be the name of a file stored in config-dir not the absolute path to the "+
 			"configuration file. The specified file name must have no extension we automatically load JSON, YAML, TOML, HCL and "+
 			"Java properties. Therefore if you set \"sftpgo\" then \"sftpgo.json\", \"sftpgo.yaml\" and so on are searched. "+
 			"This flag can be set using SFTPGO_CONFIG_FILE env var too.")
-	viper.BindPFlag(configFileKey, cmd.Flags().Lookup(configFileFlag))
+	viper.BindPFlag(configFileKey, cmd.Flags().Lookup(configFileFlag)) //nolint:errcheck
+}
+
+func addServeFlags(cmd *cobra.Command) {
+	addConfigFlags(cmd)
 
 	viper.SetDefault(logFilePathKey, defaultLogFile)
-	viper.BindEnv(logFilePathKey, "SFTPGO_LOG_FILE_PATH")
+	viper.BindEnv(logFilePathKey, "SFTPGO_LOG_FILE_PATH") //nolint:errcheck
 	cmd.Flags().StringVarP(&logFilePath, logFilePathFlag, "l", viper.GetString(logFilePathKey),
 		"Location for the log file. Leave empty to write logs to the standard output. This flag can be set using SFTPGO_LOG_FILE_PATH "+
 			"env var too.")
-	viper.BindPFlag(logFilePathKey, cmd.Flags().Lookup(logFilePathFlag))
+	viper.BindPFlag(logFilePathKey, cmd.Flags().Lookup(logFilePathFlag)) //nolint:errcheck
 
 	viper.SetDefault(logMaxSizeKey, defaultLogMaxSize)
-	viper.BindEnv(logMaxSizeKey, "SFTPGO_LOG_MAX_SIZE")
+	viper.BindEnv(logMaxSizeKey, "SFTPGO_LOG_MAX_SIZE") //nolint:errcheck
 	cmd.Flags().IntVarP(&logMaxSize, logMaxSizeFlag, "s", viper.GetInt(logMaxSizeKey),
 		"Maximum size in megabytes of the log file before it gets rotated. This flag can be set using SFTPGO_LOG_MAX_SIZE "+
 			"env var too. It is unused if log-file-path is empty.")
-	viper.BindPFlag(logMaxSizeKey, cmd.Flags().Lookup(logMaxSizeFlag))
+	viper.BindPFlag(logMaxSizeKey, cmd.Flags().Lookup(logMaxSizeFlag)) //nolint:errcheck
 
 	viper.SetDefault(logMaxBackupKey, defaultLogMaxBackup)
-	viper.BindEnv(logMaxBackupKey, "SFTPGO_LOG_MAX_BACKUPS")
+	viper.BindEnv(logMaxBackupKey, "SFTPGO_LOG_MAX_BACKUPS") //nolint:errcheck
 	cmd.Flags().IntVarP(&logMaxBackups, "log-max-backups", "b", viper.GetInt(logMaxBackupKey),
 		"Maximum number of old log files to retain. This flag can be set using SFTPGO_LOG_MAX_BACKUPS env var too. "+
 			"It is unused if log-file-path is empty.")
-	viper.BindPFlag(logMaxBackupKey, cmd.Flags().Lookup(logMaxBackupFlag))
+	viper.BindPFlag(logMaxBackupKey, cmd.Flags().Lookup(logMaxBackupFlag)) //nolint:errcheck
 
 	viper.SetDefault(logMaxAgeKey, defaultLogMaxAge)
-	viper.BindEnv(logMaxAgeKey, "SFTPGO_LOG_MAX_AGE")
+	viper.BindEnv(logMaxAgeKey, "SFTPGO_LOG_MAX_AGE") //nolint:errcheck
 	cmd.Flags().IntVarP(&logMaxAge, "log-max-age", "a", viper.GetInt(logMaxAgeKey),
 		"Maximum number of days to retain old log files. This flag can be set using SFTPGO_LOG_MAX_AGE env var too. "+
 			"It is unused if log-file-path is empty.")
-	viper.BindPFlag(logMaxAgeKey, cmd.Flags().Lookup(logMaxAgeFlag))
+	viper.BindPFlag(logMaxAgeKey, cmd.Flags().Lookup(logMaxAgeFlag)) //nolint:errcheck
 
 	viper.SetDefault(logCompressKey, defaultLogCompress)
-	viper.BindEnv(logCompressKey, "SFTPGO_LOG_COMPRESS")
+	viper.BindEnv(logCompressKey, "SFTPGO_LOG_COMPRESS") //nolint:errcheck
 	cmd.Flags().BoolVarP(&logCompress, logCompressFlag, "z", viper.GetBool(logCompressKey), "Determine if the rotated "+
 		"log files should be compressed using gzip. This flag can be set using SFTPGO_LOG_COMPRESS env var too. "+
 		"It is unused if log-file-path is empty.")
-	viper.BindPFlag(logCompressKey, cmd.Flags().Lookup(logCompressFlag))
+	viper.BindPFlag(logCompressKey, cmd.Flags().Lookup(logCompressFlag)) //nolint:errcheck
 
 	viper.SetDefault(logVerboseKey, defaultLogVerbose)
-	viper.BindEnv(logVerboseKey, "SFTPGO_LOG_VERBOSE")
+	viper.BindEnv(logVerboseKey, "SFTPGO_LOG_VERBOSE") //nolint:errcheck
 	cmd.Flags().BoolVarP(&logVerbose, logVerboseFlag, "v", viper.GetBool(logVerboseKey), "Enable verbose logs. "+
 		"This flag can be set using SFTPGO_LOG_VERBOSE env var too.")
-	viper.BindPFlag(logVerboseKey, cmd.Flags().Lookup(logVerboseFlag))
-}
+	viper.BindPFlag(logVerboseKey, cmd.Flags().Lookup(logVerboseFlag)) //nolint:errcheck
 
-func getCustomServeFlags() []string {
-	result := []string{}
-	if configDir != defaultConfigDir {
-		result = append(result, "--"+configDirFlag)
-		result = append(result, configDir)
-	}
-	if configFile != defaultConfigName {
-		result = append(result, "--"+configFileFlag)
-		result = append(result, configFile)
-	}
-	if logFilePath != defaultLogFile {
-		result = append(result, "--"+logFilePathFlag)
-		result = append(result, logFilePath)
-	}
-	if logMaxSize != defaultLogMaxSize {
-		result = append(result, "--"+logMaxSizeFlag)
-		result = append(result, strconv.Itoa(logMaxSize))
-	}
-	if logMaxBackups != defaultLogMaxBackup {
-		result = append(result, "--"+logMaxBackupFlag)
-		result = append(result, strconv.Itoa(logMaxBackups))
-	}
-	if logMaxAge != defaultLogMaxAge {
-		result = append(result, "--"+logMaxAgeFlag)
-		result = append(result, strconv.Itoa(logMaxAge))
-	}
-	if logVerbose != defaultLogVerbose {
-		result = append(result, "--"+logVerboseFlag+"=false")
-	}
-	if logCompress != defaultLogCompress {
-		result = append(result, "--"+logCompressFlag+"=true")
-	}
-	return result
+	viper.SetDefault(profilerKey, defaultProfiler)
+	viper.BindEnv(profilerKey, "SFTPGO_PROFILER") //nolint:errcheck
+	cmd.Flags().BoolVarP(&profiler, profilerFlag, "p", viper.GetBool(profilerKey), "Enable the built-in profiler. "+
+		"The profiler will be accessible via HTTP/HTTPS using the base URL \"/debug/pprof/\". "+
+		"This flag can be set using SFTPGO_PROFILER env var too.")
+	viper.BindPFlag(profilerKey, cmd.Flags().Lookup(profilerFlag)) //nolint:errcheck
 }

cmd/rotatelogs_windows.go

@@ -0,0 +1,35 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/drakkan/sftpgo/service"
+)
+
+var (
+	rotateLogCmd = &cobra.Command{
+		Use:   "rotatelogs",
+		Short: "Signal to the running service to close the existing log file and immediately create a new one",
+		Run: func(cmd *cobra.Command, args []string) {
+			s := service.WindowsService{
+				Service: service.Service{
+					Shutdown: make(chan bool),
+				},
+			}
+			err := s.RotateLogFile()
+			if err != nil {
+				fmt.Printf("Error sending rotate log file signal to the service: %v\r\n", err)
+				os.Exit(1)
+			} else {
+				fmt.Printf("Rotate log file signal sent!\r\n")
+			}
+		},
+	}
+)
+
+func init() {
+	serviceCmd.AddCommand(rotateLogCmd)
+}

cmd/serve.go

@@ -1,8 +1,12 @@
 package cmd
 
 import (
-	"github.com/drakkan/sftpgo/service"
+	"os"
+
 	"github.com/spf13/cobra"
+
+	"github.com/drakkan/sftpgo/service"
+	"github.com/drakkan/sftpgo/utils"
 )
 
 var (
@@ -16,7 +20,7 @@ sftpgo serve
 Please take a look at the usage below to customize the startup options`,
 		Run: func(cmd *cobra.Command, args []string) {
 			service := service.Service{
-				ConfigDir:     configDir,
+				ConfigDir:     utils.CleanDirInput(configDir),
 				ConfigFile:    configFile,
 				LogFilePath:   logFilePath,
 				LogMaxSize:    logMaxSize,
@@ -24,11 +28,14 @@ Please take a look at the usage below to customize the startup options`,
 				LogMaxAge:     logMaxAge,
 				LogCompress:   logCompress,
 				LogVerbose:    logVerbose,
+				Profiler:      profiler,
 				Shutdown:      make(chan bool),
 			}
 			if err := service.Start(); err == nil {
 				service.Wait()
+				os.Exit(0)
 			}
+			os.Exit(1)
 		},
 	}
 )

cmd/service_windows.go

@@ -7,7 +7,7 @@ import (
 var (
 	serviceCmd = &cobra.Command{
 		Use:   "service",
-		Short: "Install, Uninstall, Start, Stop and retrieve status for SFTPGo Windows Service",
+		Short: "Install, Uninstall, Start, Stop, Reload and retrieve status for SFTPGo Windows Service",
 	}
 )
 

cmd/start_windows.go

@@ -2,9 +2,13 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
 
 	"github.com/drakkan/sftpgo/service"
-	"github.com/spf13/cobra"
+	"github.com/drakkan/sftpgo/utils"
 )
 
 var (
@@ -12,6 +16,10 @@ var (
 		Use:   "start",
 		Short: "Start SFTPGo Windows Service",
 		Run: func(cmd *cobra.Command, args []string) {
+			configDir = utils.CleanDirInput(configDir)
+			if !filepath.IsAbs(logFilePath) && utils.IsFileInputValid(logFilePath) {
+				logFilePath = filepath.Join(configDir, logFilePath)
+			}
 			s := service.Service{
 				ConfigDir:     configDir,
 				ConfigFile:    configFile,
@@ -21,6 +29,7 @@ var (
 				LogMaxAge:     logMaxAge,
 				LogCompress:   logCompress,
 				LogVerbose:    logVerbose,
+				Profiler:      profiler,
 				Shutdown:      make(chan bool),
 			}
 			winService := service.WindowsService{
@@ -29,6 +38,7 @@ var (
 			err := winService.RunService()
 			if err != nil {
 				fmt.Printf("Error starting service: %v\r\n", err)
+				os.Exit(1)
 			} else {
 				fmt.Printf("Service started!\r\n")
 			}

cmd/status_windows.go

@@ -2,9 +2,11 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
 
 	"github.com/drakkan/sftpgo/service"
-	"github.com/spf13/cobra"
 )
 
 var (
@@ -20,6 +22,7 @@ var (
 			status, err := s.Status()
 			if err != nil {
 				fmt.Printf("Error querying service status: %v\r\n", err)
+				os.Exit(1)
 			} else {
 				fmt.Printf("Service status: %#v\r\n", status.String())
 			}

cmd/stop_windows.go

@@ -2,9 +2,11 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
 
 	"github.com/drakkan/sftpgo/service"
-	"github.com/spf13/cobra"
 )
 
 var (
@@ -20,6 +22,7 @@ var (
 			err := s.Stop()
 			if err != nil {
 				fmt.Printf("Error stopping service: %v\r\n", err)
+				os.Exit(1)
 			} else {
 				fmt.Printf("Service stopped!\r\n")
 			}

cmd/uninstall_windows.go

@@ -2,9 +2,11 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
 
 	"github.com/drakkan/sftpgo/service"
-	"github.com/spf13/cobra"
 )
 
 var (
@@ -20,6 +22,7 @@ var (
 			err := s.Uninstall()
 			if err != nil {
 				fmt.Printf("Error removing service: %v\r\n", err)
+				os.Exit(1)
 			} else {
 				fmt.Printf("Service uninstalled\r\n")
 			}

config/config.go

@@ -9,12 +9,15 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/drakkan/sftpgo/api"
+	"github.com/spf13/viper"
+
 	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/httpclient"
+	"github.com/drakkan/sftpgo/httpd"
 	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/sftpd"
 	"github.com/drakkan/sftpgo/utils"
-	"github.com/spf13/viper"
+	"github.com/drakkan/sftpgo/version"
 )
 
 const (
@@ -29,13 +32,14 @@ const (
 
 var (
 	globalConf    globalConfig
-	defaultBanner = fmt.Sprintf("SFTPGo_%v", utils.GetAppVersion().Version)
+	defaultBanner = fmt.Sprintf("SFTPGo_%v", version.Get().Version)
 )
 
 type globalConfig struct {
 	SFTPD        sftpd.Configuration `json:"sftpd" mapstructure:"sftpd"`
 	ProviderConf dataprovider.Config `json:"data_provider" mapstructure:"data_provider"`
-	HTTPDConfig  api.HTTPDConf       `json:"httpd" mapstructure:"httpd"`
+	HTTPDConfig  httpd.Conf          `json:"httpd" mapstructure:"httpd"`
+	HTTPConfig   httpclient.Config   `json:"http" mapstructure:"http"`
 }
 
 func init() {
@@ -50,16 +54,19 @@ func init() {
 			Umask:        "0022",
 			UploadMode:   0,
 			Actions: sftpd.Actions{
-				ExecuteOn:           []string{},
-				Command:             "",
-				HTTPNotificationURL: "",
+				ExecuteOn: []string{},
+				Hook:      "",
 			},
-			Keys:            []sftpd.Key{},
-			IsSCPEnabled:    false,
-			KexAlgorithms:   []string{},
-			Ciphers:         []string{},
-			MACs:            []string{},
-			LoginBannerFile: "",
+			HostKeys:                []string{},
+			KexAlgorithms:           []string{},
+			Ciphers:                 []string{},
+			MACs:                    []string{},
+			TrustedUserCAKeys:       []string{},
+			LoginBannerFile:         "",
+			EnabledSSHCommands:      sftpd.GetDefaultSSHCommands(),
+			KeyboardInteractiveHook: "",
+			ProxyProtocol:           0,
+			ProxyAllowed:            []string{},
 		},
 		ProviderConf: dataprovider.Config{
 			Driver:           "sqlite",
@@ -69,15 +76,35 @@ func init() {
 			Username:         "",
 			Password:         "",
 			ConnectionString: "",
-			UsersTable:       "users",
+			SQLTablesPrefix:  "",
 			ManageUsers:      1,
 			SSLMode:          0,
 			TrackQuota:       1,
 			PoolSize:         0,
+			UsersBaseDir:     "",
+			Actions: dataprovider.Actions{
+				ExecuteOn: []string{},
+				Hook:      "",
+			},
+			ExternalAuthHook:  "",
+			ExternalAuthScope: 0,
+			CredentialsPath:   "credentials",
+			PreLoginHook:      "",
 		},
-		HTTPDConfig: api.HTTPDConf{
-			BindPort:    8080,
-			BindAddress: "127.0.0.1",
+		HTTPDConfig: httpd.Conf{
+			BindPort:           8080,
+			BindAddress:        "127.0.0.1",
+			TemplatesPath:      "templates",
+			StaticFilesPath:    "static",
+			BackupsPath:        "backups",
+			AuthUserFile:       "",
+			CertificateFile:    "",
+			CertificateKeyFile: "",
+		},
+		HTTPConfig: httpclient.Config{
+			Timeout:        20,
+			CACertificates: nil,
+			SkipTLSVerify:  false,
 		},
 	}
 
@@ -94,16 +121,36 @@ func GetSFTPDConfig() sftpd.Configuration {
 	return globalConf.SFTPD
 }
 
+// SetSFTPDConfig sets the configuration for the SFTP server
+func SetSFTPDConfig(config sftpd.Configuration) {
+	globalConf.SFTPD = config
+}
+
 // GetHTTPDConfig returns the configuration for the HTTP server
-func GetHTTPDConfig() api.HTTPDConf {
+func GetHTTPDConfig() httpd.Conf {
 	return globalConf.HTTPDConfig
 }
 
+// SetHTTPDConfig sets the configuration for the HTTP server
+func SetHTTPDConfig(config httpd.Conf) {
+	globalConf.HTTPDConfig = config
+}
+
 //GetProviderConf returns the configuration for the data provider
 func GetProviderConf() dataprovider.Config {
 	return globalConf.ProviderConf
 }
 
+//SetProviderConf sets the configuration for the data provider
+func SetProviderConf(config dataprovider.Config) {
+	globalConf.ProviderConf = config
+}
+
+// GetHTTPConfig returns the configuration for HTTP clients
+func GetHTTPConfig() httpclient.Config {
+	return globalConf.HTTPConfig
+}
+
 func getRedactedGlobalConf() globalConfig {
 	conf := globalConf
 	conf.ProviderConf.Password = "[redacted]"
@@ -137,13 +184,92 @@ func LoadConfig(configDir, configName string) error {
 	if strings.TrimSpace(globalConf.SFTPD.Banner) == "" {
 		globalConf.SFTPD.Banner = defaultBanner
 	}
-	if globalConf.SFTPD.UploadMode < 0 || globalConf.SFTPD.UploadMode > 1 {
-		err = fmt.Errorf("Invalid upload_mode 0 and 1 are supported, configured: %v reset upload_mode to 0",
+	if len(globalConf.ProviderConf.UsersBaseDir) > 0 && !utils.IsFileInputValid(globalConf.ProviderConf.UsersBaseDir) {
+		err = fmt.Errorf("invalid users base dir %#v will be ignored", globalConf.ProviderConf.UsersBaseDir)
+		globalConf.ProviderConf.UsersBaseDir = ""
+		logger.Warn(logSender, "", "Configuration error: %v", err)
+		logger.WarnToConsole("Configuration error: %v", err)
+	}
+	if globalConf.SFTPD.UploadMode < 0 || globalConf.SFTPD.UploadMode > 2 {
+		err = fmt.Errorf("invalid upload_mode 0, 1 and 2 are supported, configured: %v reset upload_mode to 0",
 			globalConf.SFTPD.UploadMode)
 		globalConf.SFTPD.UploadMode = 0
 		logger.Warn(logSender, "", "Configuration error: %v", err)
 		logger.WarnToConsole("Configuration error: %v", err)
 	}
-	logger.Debug(logSender, "", "config file used: '%v', config loaded: %+v", viper.ConfigFileUsed(), getRedactedGlobalConf())
+	if globalConf.SFTPD.ProxyProtocol < 0 || globalConf.SFTPD.ProxyProtocol > 2 {
+		err = fmt.Errorf("invalid proxy_protocol 0, 1 and 2 are supported, configured: %v reset proxy_protocol to 0",
+			globalConf.SFTPD.ProxyProtocol)
+		globalConf.SFTPD.ProxyProtocol = 0
+		logger.Warn(logSender, "", "Configuration error: %v", err)
+		logger.WarnToConsole("Configuration error: %v", err)
+	}
+	if globalConf.ProviderConf.ExternalAuthScope < 0 || globalConf.ProviderConf.ExternalAuthScope > 7 {
+		err = fmt.Errorf("invalid external_auth_scope: %v reset to 0", globalConf.ProviderConf.ExternalAuthScope)
+		globalConf.ProviderConf.ExternalAuthScope = 0
+		logger.Warn(logSender, "", "Configuration error: %v", err)
+		logger.WarnToConsole("Configuration error: %v", err)
+	}
+	if len(globalConf.ProviderConf.CredentialsPath) == 0 {
+		err = fmt.Errorf("invalid credentials path, reset to \"credentials\"")
+		globalConf.ProviderConf.CredentialsPath = "credentials"
+		logger.Warn(logSender, "", "Configuration error: %v", err)
+		logger.WarnToConsole("Configuration error: %v", err)
+	}
+	checkHooksCompatibility()
+	checkHostKeyCompatibility()
+	logger.Debug(logSender, "", "config file used: '%#v', config loaded: %+v", viper.ConfigFileUsed(), getRedactedGlobalConf())
 	return err
 }
+
+func checkHooksCompatibility() {
+	// we copy deprecated fields to new ones to keep backward compatibility so lint is disabled
+	if len(globalConf.ProviderConf.ExternalAuthProgram) > 0 && len(globalConf.ProviderConf.ExternalAuthHook) == 0 { //nolint:staticcheck
+		logger.Warn(logSender, "", "external_auth_program is deprecated, please use external_auth_hook")
+		logger.WarnToConsole("external_auth_program is deprecated, please use external_auth_hook")
+		globalConf.ProviderConf.ExternalAuthHook = globalConf.ProviderConf.ExternalAuthProgram //nolint:staticcheck
+	}
+	if len(globalConf.ProviderConf.PreLoginProgram) > 0 && len(globalConf.ProviderConf.PreLoginHook) == 0 { //nolint:staticcheck
+		logger.Warn(logSender, "", "pre_login_program is deprecated, please use pre_login_hook")
+		logger.WarnToConsole("pre_login_program is deprecated, please use pre_login_hook")
+		globalConf.ProviderConf.PreLoginHook = globalConf.ProviderConf.PreLoginProgram //nolint:staticcheck
+	}
+	if len(globalConf.SFTPD.KeyboardInteractiveProgram) > 0 && len(globalConf.SFTPD.KeyboardInteractiveHook) == 0 { //nolint:staticcheck
+		logger.Warn(logSender, "", "keyboard_interactive_auth_program is deprecated, please use keyboard_interactive_auth_hook")
+		logger.WarnToConsole("keyboard_interactive_auth_program is deprecated, please use keyboard_interactive_auth_hook")
+		globalConf.SFTPD.KeyboardInteractiveHook = globalConf.SFTPD.KeyboardInteractiveProgram //nolint:staticcheck
+	}
+	if len(globalConf.SFTPD.Actions.Hook) == 0 {
+		if len(globalConf.SFTPD.Actions.HTTPNotificationURL) > 0 { //nolint:staticcheck
+			logger.Warn(logSender, "", "http_notification_url is deprecated, please use hook")
+			logger.WarnToConsole("http_notification_url is deprecated, please use hook")
+			globalConf.SFTPD.Actions.Hook = globalConf.SFTPD.Actions.HTTPNotificationURL //nolint:staticcheck
+		} else if len(globalConf.SFTPD.Actions.Command) > 0 { //nolint:staticcheck
+			logger.Warn(logSender, "", "command is deprecated, please use hook")
+			logger.WarnToConsole("command is deprecated, please use hook")
+			globalConf.SFTPD.Actions.Hook = globalConf.SFTPD.Actions.Command //nolint:staticcheck
+		}
+	}
+	if len(globalConf.ProviderConf.Actions.Hook) == 0 {
+		if len(globalConf.ProviderConf.Actions.HTTPNotificationURL) > 0 { //nolint:staticcheck
+			logger.Warn(logSender, "", "http_notification_url is deprecated, please use hook")
+			logger.WarnToConsole("http_notification_url is deprecated, please use hook")
+			globalConf.ProviderConf.Actions.Hook = globalConf.ProviderConf.Actions.HTTPNotificationURL //nolint:staticcheck
+		} else if len(globalConf.ProviderConf.Actions.Command) > 0 { //nolint:staticcheck
+			logger.Warn(logSender, "", "command is deprecated, please use hook")
+			logger.WarnToConsole("command is deprecated, please use hook")
+			globalConf.ProviderConf.Actions.Hook = globalConf.ProviderConf.Actions.Command //nolint:staticcheck
+		}
+	}
+}
+
+func checkHostKeyCompatibility() {
+	// we copy deprecated fields to new ones to keep backward compatibility so lint is disabled
+	if len(globalConf.SFTPD.Keys) > 0 && len(globalConf.SFTPD.HostKeys) == 0 { //nolint:staticcheck
+		logger.Warn(logSender, "", "keys is deprecated, please use host_keys")
+		logger.WarnToConsole("keys is deprecated, please use host_keys")
+		for _, k := range globalConf.SFTPD.Keys { //nolint:staticcheck
+			globalConf.SFTPD.HostKeys = append(globalConf.SFTPD.HostKeys, k.PrivateKey)
+		}
+	}
+}

config/config_test.go

@@ -8,92 +8,278 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/drakkan/sftpgo/api"
+	"github.com/stretchr/testify/assert"
+
 	"github.com/drakkan/sftpgo/config"
 	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/httpclient"
+	"github.com/drakkan/sftpgo/httpd"
 	"github.com/drakkan/sftpgo/sftpd"
+	"github.com/drakkan/sftpgo/utils"
 )
 
 const (
 	tempConfigName = "temp"
+	configName     = "sftpgo"
 )
 
 func TestLoadConfigTest(t *testing.T) {
 	configDir := ".."
-	err := config.LoadConfig(configDir, "")
-	if err != nil {
-		t.Errorf("error loading config")
-	}
-	emptyHTTPDConf := api.HTTPDConf{}
-	if config.GetHTTPDConfig() == emptyHTTPDConf {
-		t.Errorf("error loading httpd conf")
-	}
-	emptyProviderConf := dataprovider.Config{}
-	if config.GetProviderConf() == emptyProviderConf {
-		t.Errorf("error loading provider conf")
-	}
-	emptySFTPDConf := sftpd.Configuration{}
-	if config.GetSFTPDConfig().BindPort == emptySFTPDConf.BindPort {
-		t.Errorf("error loading SFTPD conf")
-	}
+	err := config.LoadConfig(configDir, configName)
+	assert.NoError(t, err)
+	assert.NotEqual(t, httpd.Conf{}, config.GetHTTPConfig())
+	assert.NotEqual(t, dataprovider.Config{}, config.GetProviderConf())
+	assert.NotEqual(t, sftpd.Configuration{}, config.GetSFTPDConfig())
+	assert.NotEqual(t, httpclient.Config{}, config.GetHTTPConfig())
 	confName := tempConfigName + ".json"
 	configFilePath := filepath.Join(configDir, confName)
 	err = config.LoadConfig(configDir, tempConfigName)
-	if err == nil {
-		t.Errorf("loading a non existent config file must fail")
-	}
-	ioutil.WriteFile(configFilePath, []byte("{invalid json}"), 0666)
+	assert.NotNil(t, err)
+	err = ioutil.WriteFile(configFilePath, []byte("{invalid json}"), 0666)
+	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
-	if err == nil {
-		t.Errorf("loading an invalid config file must fail")
-	}
-	ioutil.WriteFile(configFilePath, []byte("{\"sftpd\": {\"bind_port\": \"a\"}}"), 0666)
+	assert.NotNil(t, err)
+	err = ioutil.WriteFile(configFilePath, []byte("{\"sftpd\": {\"bind_port\": \"a\"}}"), 0666)
+	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
-	if err == nil {
-		t.Errorf("loading a config with an invalid bond_port must fail")
-	}
-	os.Remove(configFilePath)
+	assert.NotNil(t, err)
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
 }
 
 func TestEmptyBanner(t *testing.T) {
 	configDir := ".."
 	confName := tempConfigName + ".json"
 	configFilePath := filepath.Join(configDir, confName)
-	config.LoadConfig(configDir, "")
+	err := config.LoadConfig(configDir, configName)
+	assert.NoError(t, err)
 	sftpdConf := config.GetSFTPDConfig()
 	sftpdConf.Banner = " "
 	c := make(map[string]sftpd.Configuration)
 	c["sftpd"] = sftpdConf
 	jsonConf, _ := json.Marshal(c)
-	err := ioutil.WriteFile(configFilePath, jsonConf, 0666)
-	if err != nil {
-		t.Errorf("error saving temporary configuration")
-	}
-	config.LoadConfig(configDir, tempConfigName)
+	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, tempConfigName)
+	assert.NoError(t, err)
 	sftpdConf = config.GetSFTPDConfig()
-	if strings.TrimSpace(sftpdConf.Banner) == "" {
-		t.Errorf("SFTPD banner cannot be empty")
-	}
-	os.Remove(configFilePath)
+	assert.NotEmpty(t, strings.TrimSpace(sftpdConf.Banner))
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
 }
 
 func TestInvalidUploadMode(t *testing.T) {
 	configDir := ".."
 	confName := tempConfigName + ".json"
 	configFilePath := filepath.Join(configDir, confName)
-	config.LoadConfig(configDir, "")
+	err := config.LoadConfig(configDir, configName)
+	assert.NoError(t, err)
 	sftpdConf := config.GetSFTPDConfig()
 	sftpdConf.UploadMode = 10
 	c := make(map[string]sftpd.Configuration)
 	c["sftpd"] = sftpdConf
-	jsonConf, _ := json.Marshal(c)
-	err := ioutil.WriteFile(configFilePath, jsonConf, 0666)
-	if err != nil {
-		t.Errorf("error saving temporary configuration")
-	}
+	jsonConf, err := json.Marshal(c)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
-	if err == nil {
-		t.Errorf("Loading configuration with invalid upload_mode must fail")
-	}
-	os.Remove(configFilePath)
+	assert.NotNil(t, err)
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
+}
+
+func TestInvalidExternalAuthScope(t *testing.T) {
+	configDir := ".."
+	confName := tempConfigName + ".json"
+	configFilePath := filepath.Join(configDir, confName)
+	err := config.LoadConfig(configDir, configName)
+	assert.NoError(t, err)
+	providerConf := config.GetProviderConf()
+	providerConf.ExternalAuthScope = 10
+	c := make(map[string]dataprovider.Config)
+	c["data_provider"] = providerConf
+	jsonConf, err := json.Marshal(c)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, tempConfigName)
+	assert.NotNil(t, err)
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
+}
+
+func TestInvalidCredentialsPath(t *testing.T) {
+	configDir := ".."
+	confName := tempConfigName + ".json"
+	configFilePath := filepath.Join(configDir, confName)
+	err := config.LoadConfig(configDir, configName)
+	assert.NoError(t, err)
+	providerConf := config.GetProviderConf()
+	providerConf.CredentialsPath = ""
+	c := make(map[string]dataprovider.Config)
+	c["data_provider"] = providerConf
+	jsonConf, err := json.Marshal(c)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, tempConfigName)
+	assert.NotNil(t, err)
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
+}
+
+func TestInvalidProxyProtocol(t *testing.T) {
+	configDir := ".."
+	confName := tempConfigName + ".json"
+	configFilePath := filepath.Join(configDir, confName)
+	err := config.LoadConfig(configDir, configName)
+	assert.NoError(t, err)
+	sftpdConf := config.GetSFTPDConfig()
+	sftpdConf.ProxyProtocol = 10
+	c := make(map[string]sftpd.Configuration)
+	c["sftpd"] = sftpdConf
+	jsonConf, err := json.Marshal(c)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, tempConfigName)
+	assert.NotNil(t, err)
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
+}
+
+func TestInvalidUsersBaseDir(t *testing.T) {
+	configDir := ".."
+	confName := tempConfigName + ".json"
+	configFilePath := filepath.Join(configDir, confName)
+	err := config.LoadConfig(configDir, configName)
+	assert.NoError(t, err)
+	providerConf := config.GetProviderConf()
+	providerConf.UsersBaseDir = "."
+	c := make(map[string]dataprovider.Config)
+	c["data_provider"] = providerConf
+	jsonConf, err := json.Marshal(c)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, tempConfigName)
+	assert.NotNil(t, err)
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
+}
+
+func TestHookCompatibity(t *testing.T) {
+	configDir := ".."
+	confName := tempConfigName + ".json"
+	configFilePath := filepath.Join(configDir, confName)
+	err := config.LoadConfig(configDir, configName)
+	assert.NoError(t, err)
+	providerConf := config.GetProviderConf()
+	providerConf.ExternalAuthProgram = "ext_auth_program" //nolint:staticcheck
+	providerConf.PreLoginProgram = "pre_login_program"    //nolint:staticcheck
+	providerConf.Actions.Command = "/tmp/test_cmd"        //nolint:staticcheck
+	c := make(map[string]dataprovider.Config)
+	c["data_provider"] = providerConf
+	jsonConf, err := json.Marshal(c)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, tempConfigName)
+	assert.NoError(t, err)
+	providerConf = config.GetProviderConf()
+	assert.Equal(t, "ext_auth_program", providerConf.ExternalAuthHook)
+	assert.Equal(t, "pre_login_program", providerConf.PreLoginHook)
+	assert.Equal(t, "/tmp/test_cmd", providerConf.Actions.Hook)
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
+	providerConf.Actions.Hook = ""
+	providerConf.Actions.HTTPNotificationURL = "http://example.com/notify" //nolint:staticcheck
+	c = make(map[string]dataprovider.Config)
+	c["data_provider"] = providerConf
+	jsonConf, err = json.Marshal(c)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, tempConfigName)
+	assert.NoError(t, err)
+	providerConf = config.GetProviderConf()
+	assert.Equal(t, "http://example.com/notify", providerConf.Actions.Hook)
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
+	sftpdConf := config.GetSFTPDConfig()
+	sftpdConf.KeyboardInteractiveProgram = "key_int_program" //nolint:staticcheck
+	sftpdConf.Actions.Command = "/tmp/sftp_cmd"              //nolint:staticcheck
+	cnf := make(map[string]sftpd.Configuration)
+	cnf["sftpd"] = sftpdConf
+	jsonConf, err = json.Marshal(cnf)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, tempConfigName)
+	assert.NoError(t, err)
+	sftpdConf = config.GetSFTPDConfig()
+	assert.Equal(t, "key_int_program", sftpdConf.KeyboardInteractiveHook)
+	assert.Equal(t, "/tmp/sftp_cmd", sftpdConf.Actions.Hook)
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
+	sftpdConf.Actions.Hook = ""
+	sftpdConf.Actions.HTTPNotificationURL = "http://example.com/sftp" //nolint:staticcheck
+	cnf = make(map[string]sftpd.Configuration)
+	cnf["sftpd"] = sftpdConf
+	jsonConf, err = json.Marshal(cnf)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, tempConfigName)
+	assert.NoError(t, err)
+	sftpdConf = config.GetSFTPDConfig()
+	assert.Equal(t, "http://example.com/sftp", sftpdConf.Actions.Hook)
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
+}
+
+func TestHostKeyCompatibility(t *testing.T) {
+	configDir := ".."
+	confName := tempConfigName + ".json"
+	configFilePath := filepath.Join(configDir, confName)
+	err := config.LoadConfig(configDir, configName)
+	assert.NoError(t, err)
+	sftpdConf := config.GetSFTPDConfig()
+	sftpdConf.Keys = []sftpd.Key{ //nolint:staticcheck
+		{
+			PrivateKey: "rsa",
+		},
+		{
+			PrivateKey: "ecdsa",
+		},
+	}
+	c := make(map[string]sftpd.Configuration)
+	c["sftpd"] = sftpdConf
+	jsonConf, err := json.Marshal(c)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, tempConfigName)
+	assert.NoError(t, err)
+	sftpdConf = config.GetSFTPDConfig()
+	assert.Equal(t, 2, len(sftpdConf.HostKeys))
+	assert.True(t, utils.IsStringInSlice("rsa", sftpdConf.HostKeys))
+	assert.True(t, utils.IsStringInSlice("ecdsa", sftpdConf.HostKeys))
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
+}
+
+func TestSetGetConfig(t *testing.T) {
+	sftpdConf := config.GetSFTPDConfig()
+	sftpdConf.IdleTimeout = 3
+	config.SetSFTPDConfig(sftpdConf)
+	assert.Equal(t, sftpdConf.IdleTimeout, config.GetSFTPDConfig().IdleTimeout)
+	dataProviderConf := config.GetProviderConf()
+	dataProviderConf.Host = "test host"
+	config.SetProviderConf(dataProviderConf)
+	assert.Equal(t, dataProviderConf.Host, config.GetProviderConf().Host)
+	httpdConf := config.GetHTTPDConfig()
+	httpdConf.BindAddress = "0.0.0.0"
+	config.SetHTTPDConfig(httpdConf)
+	assert.Equal(t, httpdConf.BindAddress, config.GetHTTPDConfig().BindAddress)
 }

dataprovider/bolt.go

@@ -1,3 +1,5 @@
+// +build !nobolt
+
 package dataprovider
 
 import (
@@ -8,14 +10,24 @@ import (
 	"path/filepath"
 	"time"
 
+	bolt "go.etcd.io/bbolt"
+
 	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/utils"
-	bolt "go.etcd.io/bbolt"
+	"github.com/drakkan/sftpgo/version"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+const (
+	boltDatabaseVersion = 4
 )
 
 var (
 	usersBucket      = []byte("users")
 	usersIDIdxBucket = []byte("users_id_idx")
+	foldersBucket    = []byte("folders")
+	dbVersionBucket  = []byte("db_version")
+	dbVersionKey     = []byte("version")
 )
 
 // BoltProvider auth provider for bolt key/value store
@@ -23,10 +35,39 @@ type BoltProvider struct {
 	dbHandle *bolt.DB
 }
 
+type compatUserV2 struct {
+	ID                int64    `json:"id"`
+	Username          string   `json:"username"`
+	Password          string   `json:"password,omitempty"`
+	PublicKeys        []string `json:"public_keys,omitempty"`
+	HomeDir           string   `json:"home_dir"`
+	UID               int      `json:"uid"`
+	GID               int      `json:"gid"`
+	MaxSessions       int      `json:"max_sessions"`
+	QuotaSize         int64    `json:"quota_size"`
+	QuotaFiles        int      `json:"quota_files"`
+	Permissions       []string `json:"permissions"`
+	UsedQuotaSize     int64    `json:"used_quota_size"`
+	UsedQuotaFiles    int      `json:"used_quota_files"`
+	LastQuotaUpdate   int64    `json:"last_quota_update"`
+	UploadBandwidth   int64    `json:"upload_bandwidth"`
+	DownloadBandwidth int64    `json:"download_bandwidth"`
+	ExpirationDate    int64    `json:"expiration_date"`
+	LastLogin         int64    `json:"last_login"`
+	Status            int      `json:"status"`
+}
+
+func init() {
+	version.AddFeature("+bolt")
+}
+
 func initializeBoltProvider(basePath string) error {
 	var err error
-	logSender = BoltDataProviderName
+	logSender = fmt.Sprintf("dataprovider_%v", BoltDataProviderName)
 	dbPath := config.Name
+	if !utils.IsFileInputValid(dbPath) {
+		return fmt.Errorf("Invalid database path: %#v", dbPath)
+	}
 	if !filepath.IsAbs(dbPath) {
 		dbPath = filepath.Join(basePath, dbPath)
 	}
@@ -52,6 +93,22 @@ func initializeBoltProvider(basePath string) error {
 			providerLog(logger.LevelWarn, "error creating username idx bucket: %v", err)
 			return err
 		}
+		err = dbHandle.Update(func(tx *bolt.Tx) error {
+			_, e := tx.CreateBucketIfNotExists(foldersBucket)
+			return e
+		})
+		if err != nil {
+			providerLog(logger.LevelWarn, "error creating username idx bucket: %v", err)
+			return err
+		}
+		err = dbHandle.Update(func(tx *bolt.Tx) error {
+			_, e := tx.CreateBucketIfNotExists(dbVersionBucket)
+			return e
+		})
+		if err != nil {
+			providerLog(logger.LevelWarn, "error creating database version bucket: %v", err)
+			return err
+		}
 		provider = BoltProvider{dbHandle: dbHandle}
 	} else {
 		providerLog(logger.LevelWarn, "error creating bolt key/value store handler: %v", err)
@@ -60,7 +117,7 @@ func initializeBoltProvider(basePath string) error {
 }
 
 func (p BoltProvider) checkAvailability() error {
-	_, err := p.getUsers(1, 0, "ASC", "")
+	_, err := getBoltDatabaseVersion(p.dbHandle)
 	return err
 }
 
@@ -77,7 +134,7 @@ func (p BoltProvider) validateUserAndPass(username string, password string) (Use
 	return checkUserAndPass(user, password)
 }
 
-func (p BoltProvider) validateUserAndPubKey(username string, pubKey string) (User, string, error) {
+func (p BoltProvider) validateUserAndPubKey(username string, pubKey []byte) (User, string, error) {
 	var user User
 	if len(pubKey) == 0 {
 		return user, "", errors.New("Credentials cannot be null or empty")
@@ -104,14 +161,49 @@ func (p BoltProvider) getUserByID(ID int64) (User, error) {
 		}
 		u := bucket.Get(username)
 		if u == nil {
-			return &RecordNotFoundError{err: fmt.Sprintf("username %v and ID: %v does not exist", string(username), ID)}
+			return &RecordNotFoundError{err: fmt.Sprintf("username %#v and ID: %v does not exist", string(username), ID)}
 		}
-		return json.Unmarshal(u, &user)
+		folderBucket, err := getFolderBucket(tx)
+		if err != nil {
+			return err
+		}
+		user, err = joinUserAndFolders(u, folderBucket)
+		return err
 	})
 
 	return user, err
 }
 
+func (p BoltProvider) updateLastLogin(username string) error {
+	return p.dbHandle.Update(func(tx *bolt.Tx) error {
+		bucket, _, err := getBuckets(tx)
+		if err != nil {
+			return err
+		}
+		var u []byte
+		if u = bucket.Get([]byte(username)); u == nil {
+			return &RecordNotFoundError{err: fmt.Sprintf("username %#v does not exist, unable to update last login", username)}
+		}
+		var user User
+		err = json.Unmarshal(u, &user)
+		if err != nil {
+			return err
+		}
+		user.LastLogin = utils.GetTimeAsMsSinceEpoch(time.Now())
+		buf, err := json.Marshal(user)
+		if err != nil {
+			return err
+		}
+		err = bucket.Put([]byte(username), buf)
+		if err == nil {
+			providerLog(logger.LevelDebug, "last login updated for user %#v", username)
+		} else {
+			providerLog(logger.LevelWarn, "error updating last login for user %#v: %v", username, err)
+		}
+		return err
+	})
+}
+
 func (p BoltProvider) updateQuota(username string, filesAdd int, sizeAdd int64, reset bool) error {
 	return p.dbHandle.Update(func(tx *bolt.Tx) error {
 		bucket, _, err := getBuckets(tx)
@@ -120,7 +212,7 @@ func (p BoltProvider) updateQuota(username string, filesAdd int, sizeAdd int64,
 		}
 		var u []byte
 		if u = bucket.Get([]byte(username)); u == nil {
-			return &RecordNotFoundError{err: fmt.Sprintf("username %v does not exist, unable to update quota", username)}
+			return &RecordNotFoundError{err: fmt.Sprintf("username %#v does not exist, unable to update quota", username)}
 		}
 		var user User
 		err = json.Unmarshal(u, &user)
@@ -139,7 +231,10 @@ func (p BoltProvider) updateQuota(username string, filesAdd int, sizeAdd int64,
 		if err != nil {
 			return err
 		}
-		return bucket.Put([]byte(username), buf)
+		err = bucket.Put([]byte(username), buf)
+		providerLog(logger.LevelDebug, "quota updated for user %#v, files increment: %v size increment: %v is reset? %v",
+			username, filesAdd, sizeAdd, reset)
+		return err
 	})
 }
 
@@ -161,9 +256,14 @@ func (p BoltProvider) userExists(username string) (User, error) {
 		}
 		u := bucket.Get([]byte(username))
 		if u == nil {
-			return &RecordNotFoundError{err: fmt.Sprintf("username %v does not exist", user.Username)}
+			return &RecordNotFoundError{err: fmt.Sprintf("username %v does not exist", username)}
 		}
-		return json.Unmarshal(u, &user)
+		folderBucket, err := getFolderBucket(tx)
+		if err != nil {
+			return err
+		}
+		user, err = joinUserAndFolders(u, folderBucket)
+		return err
 	})
 	return user, err
 }
@@ -178,6 +278,10 @@ func (p BoltProvider) addUser(user User) error {
 		if err != nil {
 			return err
 		}
+		folderBucket, err := getFolderBucket(tx)
+		if err != nil {
+			return err
+		}
 		if u := bucket.Get([]byte(user.Username)); u != nil {
 			return fmt.Errorf("username %v already exists", user.Username)
 		}
@@ -186,15 +290,25 @@ func (p BoltProvider) addUser(user User) error {
 			return err
 		}
 		user.ID = int64(id)
+		user.LastQuotaUpdate = 0
+		user.UsedQuotaSize = 0
+		user.UsedQuotaFiles = 0
+		user.LastLogin = 0
+		for _, folder := range user.VirtualFolders {
+			err = addUserToFolderMapping(folder, user, folderBucket)
+			if err != nil {
+				return err
+			}
+		}
 		buf, err := json.Marshal(user)
 		if err != nil {
 			return err
 		}
-		userIDAsBytes := itob(user.ID)
 		err = bucket.Put([]byte(user.Username), buf)
 		if err != nil {
 			return err
 		}
+		userIDAsBytes := itob(user.ID)
 		return idxBucket.Put(userIDAsBytes, []byte(user.Username))
 	})
 }
@@ -209,9 +323,35 @@ func (p BoltProvider) updateUser(user User) error {
 		if err != nil {
 			return err
 		}
-		if u := bucket.Get([]byte(user.Username)); u == nil {
+		folderBucket, err := getFolderBucket(tx)
+		if err != nil {
+			return err
+		}
+		var u []byte
+		if u = bucket.Get([]byte(user.Username)); u == nil {
 			return &RecordNotFoundError{err: fmt.Sprintf("username %v does not exist", user.Username)}
 		}
+		var oldUser User
+		err = json.Unmarshal(u, &oldUser)
+		if err != nil {
+			return err
+		}
+		for _, folder := range oldUser.VirtualFolders {
+			err = removeUserFromFolderMapping(folder, oldUser, folderBucket)
+			if err != nil {
+				return err
+			}
+		}
+		for _, folder := range user.VirtualFolders {
+			err = addUserToFolderMapping(folder, user, folderBucket)
+			if err != nil {
+				return err
+			}
+		}
+		user.LastQuotaUpdate = oldUser.LastQuotaUpdate
+		user.UsedQuotaSize = oldUser.UsedQuotaSize
+		user.UsedQuotaFiles = oldUser.UsedQuotaFiles
+		user.LastLogin = oldUser.LastLogin
 		buf, err := json.Marshal(user)
 		if err != nil {
 			return err
@@ -226,6 +366,18 @@ func (p BoltProvider) deleteUser(user User) error {
 		if err != nil {
 			return err
 		}
+		if len(user.VirtualFolders) > 0 {
+			folderBucket, err := getFolderBucket(tx)
+			if err != nil {
+				return err
+			}
+			for _, folder := range user.VirtualFolders {
+				err = removeUserFromFolderMapping(folder, user, folderBucket)
+				if err != nil {
+					return err
+				}
+			}
+		}
 		userIDAsBytes := itob(user.ID)
 		userName := idxBucket.Get(userIDAsBytes)
 		if userName == nil {
@@ -239,38 +391,80 @@ func (p BoltProvider) deleteUser(user User) error {
 	})
 }
 
-func (p BoltProvider) getUsers(limit int, offset int, order string, username string) ([]User, error) {
+func (p BoltProvider) dumpUsers() ([]User, error) {
+	users := make([]User, 0, 100)
+	err := p.dbHandle.View(func(tx *bolt.Tx) error {
+		bucket, _, err := getBuckets(tx)
+		if err != nil {
+			return err
+		}
+		folderBucket, err := getFolderBucket(tx)
+		if err != nil {
+			return err
+		}
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			user, err := joinUserAndFolders(v, folderBucket)
+			if err != nil {
+				return err
+			}
+			err = addCredentialsToUser(&user)
+			if err != nil {
+				return err
+			}
+			users = append(users, user)
+		}
+		return err
+	})
+	return users, err
+}
+
+func (p BoltProvider) getUserWithUsername(username string) ([]User, error) {
 	users := []User{}
+	var user User
+	user, err := p.userExists(username)
+	if err == nil {
+		users = append(users, HideUserSensitiveData(&user))
+		return users, nil
+	}
+	if _, ok := err.(*RecordNotFoundError); ok {
+		err = nil
+	}
+	return users, err
+}
+
+func (p BoltProvider) getUsers(limit int, offset int, order string, username string) ([]User, error) {
+	users := make([]User, 0, limit)
 	var err error
+	if limit <= 0 {
+		return users, err
+	}
 	if len(username) > 0 {
 		if offset == 0 {
-			user, err := p.userExists(username)
-			if err == nil {
-				users = append(users, getUserNoCredentials(&user))
-			}
+			return p.getUserWithUsername(username)
 		}
 		return users, err
 	}
 	err = p.dbHandle.View(func(tx *bolt.Tx) error {
-		if limit <= 0 {
-			return nil
-		}
 		bucket, _, err := getBuckets(tx)
 		if err != nil {
 			return err
 		}
+		folderBucket, err := getFolderBucket(tx)
+		if err != nil {
+			return err
+		}
 		cursor := bucket.Cursor()
 		itNum := 0
-		if order == "ASC" {
+		if order == OrderASC {
 			for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
 				itNum++
 				if itNum <= offset {
 					continue
 				}
-				var user User
-				err = json.Unmarshal(v, &user)
+				user, err := joinUserAndFolders(v, folderBucket)
 				if err == nil {
-					users = append(users, getUserNoCredentials(&user))
+					users = append(users, HideUserSensitiveData(&user))
 				}
 				if len(users) >= limit {
 					break
@@ -282,10 +476,9 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str
 				if itNum <= offset {
 					continue
 				}
-				var user User
-				err = json.Unmarshal(v, &user)
+				user, err := joinUserAndFolders(v, folderBucket)
 				if err == nil {
-					users = append(users, getUserNoCredentials(&user))
+					users = append(users, HideUserSensitiveData(&user))
 				}
 				if len(users) >= limit {
 					break
@@ -297,10 +490,253 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str
 	return users, err
 }
 
-func getUserNoCredentials(user *User) User {
-	user.Password = ""
-	user.PublicKeys = []string{}
-	return *user
+func (p BoltProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
+	folders := make([]vfs.BaseVirtualFolder, 0, 50)
+	err := p.dbHandle.View(func(tx *bolt.Tx) error {
+		bucket, err := getFolderBucket(tx)
+		if err != nil {
+			return err
+		}
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var folder vfs.BaseVirtualFolder
+			err = json.Unmarshal(v, &folder)
+			if err != nil {
+				return err
+			}
+			folders = append(folders, folder)
+		}
+		return err
+	})
+	return folders, err
+}
+
+func (p BoltProvider) getFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualFolder, error) {
+	folders := make([]vfs.BaseVirtualFolder, 0, limit)
+	var err error
+	if limit <= 0 {
+		return folders, err
+	}
+	if len(folderPath) > 0 {
+		if offset == 0 {
+			var folder vfs.BaseVirtualFolder
+			folder, err = p.getFolderByPath(folderPath)
+			if err == nil {
+				folders = append(folders, folder)
+			}
+		}
+		return folders, err
+	}
+	err = p.dbHandle.View(func(tx *bolt.Tx) error {
+		bucket, err := getFolderBucket(tx)
+		if err != nil {
+			return err
+		}
+		cursor := bucket.Cursor()
+		itNum := 0
+		if order == OrderASC {
+			for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+				itNum++
+				if itNum <= offset {
+					continue
+				}
+				var folder vfs.BaseVirtualFolder
+				err = json.Unmarshal(v, &folder)
+				if err != nil {
+					return err
+				}
+				folders = append(folders, folder)
+				if len(folders) >= limit {
+					break
+				}
+			}
+		} else {
+			for k, v := cursor.Last(); k != nil; k, v = cursor.Prev() {
+				itNum++
+				if itNum <= offset {
+					continue
+				}
+				var folder vfs.BaseVirtualFolder
+				err = json.Unmarshal(v, &folder)
+				if err != nil {
+					return err
+				}
+				folders = append(folders, folder)
+				if len(folders) >= limit {
+					break
+				}
+			}
+		}
+		return err
+	})
+	return folders, err
+}
+
+func (p BoltProvider) getFolderByPath(name string) (vfs.BaseVirtualFolder, error) {
+	var folder vfs.BaseVirtualFolder
+	err := p.dbHandle.View(func(tx *bolt.Tx) error {
+		bucket, err := getFolderBucket(tx)
+		if err != nil {
+			return err
+		}
+		folder, err = folderExistsInternal(name, bucket)
+		return err
+	})
+	return folder, err
+}
+
+func (p BoltProvider) addFolder(folder vfs.BaseVirtualFolder) error {
+	err := validateFolder(&folder)
+	if err != nil {
+		return err
+	}
+	return p.dbHandle.Update(func(tx *bolt.Tx) error {
+		bucket, err := getFolderBucket(tx)
+		if err != nil {
+			return err
+		}
+		if f := bucket.Get([]byte(folder.MappedPath)); f != nil {
+			return fmt.Errorf("folder %v already exists", folder.MappedPath)
+		}
+		_, err = addFolderInternal(folder, bucket)
+		return err
+	})
+}
+
+func (p BoltProvider) deleteFolder(folder vfs.BaseVirtualFolder) error {
+	return p.dbHandle.Update(func(tx *bolt.Tx) error {
+		bucket, err := getFolderBucket(tx)
+		if err != nil {
+			return err
+		}
+		usersBucket, _, err := getBuckets(tx)
+		if err != nil {
+			return err
+		}
+		var f []byte
+		if f = bucket.Get([]byte(folder.MappedPath)); f == nil {
+			return &RecordNotFoundError{err: fmt.Sprintf("folder %v does not exist", folder.MappedPath)}
+		}
+		var folder vfs.BaseVirtualFolder
+		err = json.Unmarshal(f, &folder)
+		if err != nil {
+			return err
+		}
+		for _, username := range folder.Users {
+			var u []byte
+			if u = usersBucket.Get([]byte(username)); u == nil {
+				continue
+			}
+			var user User
+			err = json.Unmarshal(u, &user)
+			if err != nil {
+				return err
+			}
+			var folders []vfs.VirtualFolder
+			for _, userFolder := range user.VirtualFolders {
+				if folder.MappedPath != userFolder.MappedPath {
+					folders = append(folders, userFolder)
+				}
+			}
+			user.VirtualFolders = folders
+			buf, err := json.Marshal(user)
+			if err != nil {
+				return err
+			}
+			err = usersBucket.Put([]byte(user.Username), buf)
+			if err != nil {
+				return err
+			}
+		}
+
+		return bucket.Delete([]byte(folder.MappedPath))
+	})
+}
+
+func (p BoltProvider) updateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool) error {
+	return p.dbHandle.Update(func(tx *bolt.Tx) error {
+		bucket, err := getFolderBucket(tx)
+		if err != nil {
+			return err
+		}
+		var f []byte
+		if f = bucket.Get([]byte(mappedPath)); f == nil {
+			return &RecordNotFoundError{err: fmt.Sprintf("folder %v does not exist, unable to update quota", mappedPath)}
+		}
+		var folder vfs.BaseVirtualFolder
+		err = json.Unmarshal(f, &folder)
+		if err != nil {
+			return err
+		}
+		if reset {
+			folder.UsedQuotaSize = sizeAdd
+			folder.UsedQuotaFiles = filesAdd
+		} else {
+			folder.UsedQuotaSize += sizeAdd
+			folder.UsedQuotaFiles += filesAdd
+		}
+		folder.LastQuotaUpdate = utils.GetTimeAsMsSinceEpoch(time.Now())
+		buf, err := json.Marshal(folder)
+		if err != nil {
+			return err
+		}
+		return bucket.Put([]byte(folder.MappedPath), buf)
+	})
+}
+
+func (p BoltProvider) getUsedFolderQuota(mappedPath string) (int, int64, error) {
+	folder, err := p.getFolderByPath(mappedPath)
+	if err != nil {
+		providerLog(logger.LevelWarn, "unable to get quota for folder %#v error: %v", mappedPath, err)
+		return 0, 0, err
+	}
+	return folder.UsedQuotaFiles, folder.UsedQuotaSize, err
+}
+
+func (p BoltProvider) close() error {
+	return p.dbHandle.Close()
+}
+
+func (p BoltProvider) reloadConfig() error {
+	return nil
+}
+
+// initializeDatabase does nothing, no initilization is needed for bolt provider
+func (p BoltProvider) initializeDatabase() error {
+	return errNoInitRequired
+}
+
+func (p BoltProvider) migrateDatabase() error {
+	dbVersion, err := getBoltDatabaseVersion(p.dbHandle)
+	if err != nil {
+		return err
+	}
+	if dbVersion.Version == boltDatabaseVersion {
+		providerLog(logger.LevelDebug, "bolt database is updated, current version: %v", dbVersion.Version)
+		return nil
+	}
+	switch dbVersion.Version {
+	case 1:
+		err = updateDatabaseFrom1To2(p.dbHandle)
+		if err != nil {
+			return err
+		}
+		err = updateDatabaseFrom2To3(p.dbHandle)
+		if err != nil {
+			return err
+		}
+		return updateDatabaseFrom3To4(p.dbHandle)
+	case 2:
+		err = updateDatabaseFrom2To3(p.dbHandle)
+		if err != nil {
+			return err
+		}
+		return updateDatabaseFrom3To4(p.dbHandle)
+	case 3:
+		return updateDatabaseFrom3To4(p.dbHandle)
+	default:
+		return fmt.Errorf("Database version not handled: %v", dbVersion.Version)
+	}
 }
 
 // itob returns an 8-byte big endian representation of v.
@@ -310,12 +746,313 @@ func itob(v int64) []byte {
 	return b
 }
 
+func joinUserAndFolders(u []byte, foldersBucket *bolt.Bucket) (User, error) {
+	var user User
+	err := json.Unmarshal(u, &user)
+	if len(user.VirtualFolders) > 0 {
+		var folders []vfs.VirtualFolder
+		for _, folder := range user.VirtualFolders {
+			baseFolder, err := folderExistsInternal(folder.MappedPath, foldersBucket)
+			if err != nil {
+				continue
+			}
+			folder.UsedQuotaFiles = baseFolder.UsedQuotaFiles
+			folder.UsedQuotaSize = baseFolder.UsedQuotaSize
+			folder.LastQuotaUpdate = baseFolder.LastQuotaUpdate
+			folder.ID = baseFolder.ID
+			folders = append(folders, folder)
+		}
+		user.VirtualFolders = folders
+	}
+	return user, err
+}
+
+func folderExistsInternal(name string, bucket *bolt.Bucket) (vfs.BaseVirtualFolder, error) {
+	var folder vfs.BaseVirtualFolder
+	f := bucket.Get([]byte(name))
+	if f == nil {
+		err := &RecordNotFoundError{err: fmt.Sprintf("folder %v does not exist", name)}
+		return folder, err
+	}
+	err := json.Unmarshal(f, &folder)
+	return folder, err
+}
+
+func addFolderInternal(folder vfs.BaseVirtualFolder, bucket *bolt.Bucket) (vfs.BaseVirtualFolder, error) {
+	id, err := bucket.NextSequence()
+	if err != nil {
+		return folder, err
+	}
+	folder.ID = int64(id)
+	buf, err := json.Marshal(folder)
+	if err != nil {
+		return folder, err
+	}
+	err = bucket.Put([]byte(folder.MappedPath), buf)
+	return folder, err
+}
+
+func addUserToFolderMapping(folder vfs.VirtualFolder, user User, bucket *bolt.Bucket) error {
+	var baseFolder vfs.BaseVirtualFolder
+	var err error
+	if f := bucket.Get([]byte(folder.MappedPath)); f == nil {
+		// folder does not exists, try to create
+		baseFolder, err = addFolderInternal(folder.BaseVirtualFolder, bucket)
+	} else {
+		err = json.Unmarshal(f, &baseFolder)
+	}
+	if err != nil {
+		return err
+	}
+	if !utils.IsStringInSlice(user.Username, baseFolder.Users) {
+		baseFolder.Users = append(baseFolder.Users, user.Username)
+		buf, err := json.Marshal(baseFolder)
+		if err != nil {
+			return err
+		}
+		err = bucket.Put([]byte(folder.MappedPath), buf)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
+func removeUserFromFolderMapping(folder vfs.VirtualFolder, user User, bucket *bolt.Bucket) error {
+	var f []byte
+	if f = bucket.Get([]byte(folder.MappedPath)); f == nil {
+		// the folder does not exists so there is no associated user
+		return nil
+	}
+	var baseFolder vfs.BaseVirtualFolder
+	err := json.Unmarshal(f, &baseFolder)
+	if err != nil {
+		return err
+	}
+	if utils.IsStringInSlice(user.Username, baseFolder.Users) {
+		var newUserMapping []string
+		for _, u := range baseFolder.Users {
+			if u != user.Username {
+				newUserMapping = append(newUserMapping, u)
+			}
+		}
+		baseFolder.Users = newUserMapping
+		buf, err := json.Marshal(baseFolder)
+		if err != nil {
+			return err
+		}
+		return bucket.Put([]byte(folder.MappedPath), buf)
+	}
+	return err
+}
+
 func getBuckets(tx *bolt.Tx) (*bolt.Bucket, *bolt.Bucket, error) {
 	var err error
 	bucket := tx.Bucket(usersBucket)
 	idxBucket := tx.Bucket(usersIDIdxBucket)
 	if bucket == nil || idxBucket == nil {
-		err = fmt.Errorf("Unable to find required buckets, bolt database structure not correcly defined")
+		err = fmt.Errorf("unable to find required buckets, bolt database structure not correcly defined")
 	}
 	return bucket, idxBucket, err
 }
+
+func getFolderBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
+	var err error
+	bucket := tx.Bucket(foldersBucket)
+	if bucket == nil {
+		err = fmt.Errorf("unable to find required buckets, bolt database structure not correcly defined")
+	}
+	return bucket, err
+}
+
+func updateDatabaseFrom1To2(dbHandle *bolt.DB) error {
+	providerLog(logger.LevelInfo, "updating bolt database version: 1 -> 2")
+	usernames, err := getBoltAvailableUsernames(dbHandle)
+	if err != nil {
+		return err
+	}
+	for _, u := range usernames {
+		user, err := provider.userExists(u)
+		if err != nil {
+			return err
+		}
+		user.Status = 1
+		err = provider.updateUser(user)
+		if err != nil {
+			return err
+		}
+		providerLog(logger.LevelInfo, "user %#v updated, \"status\" setted to 1", user.Username)
+	}
+	return updateBoltDatabaseVersion(dbHandle, 2)
+}
+
+func updateDatabaseFrom2To3(dbHandle *bolt.DB) error {
+	providerLog(logger.LevelInfo, "updating bolt database version: 2 -> 3")
+	users := []User{}
+	err := dbHandle.View(func(tx *bolt.Tx) error {
+		bucket, _, err := getBuckets(tx)
+		if err != nil {
+			return err
+		}
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var compatUser compatUserV2
+			err = json.Unmarshal(v, &compatUser)
+			if err == nil {
+				user := User{}
+				user.ID = compatUser.ID
+				user.Username = compatUser.Username
+				user.Password = compatUser.Password
+				user.PublicKeys = compatUser.PublicKeys
+				user.HomeDir = compatUser.HomeDir
+				user.UID = compatUser.UID
+				user.GID = compatUser.GID
+				user.MaxSessions = compatUser.MaxSessions
+				user.QuotaSize = compatUser.QuotaSize
+				user.QuotaFiles = compatUser.QuotaFiles
+				user.Permissions = make(map[string][]string)
+				user.Permissions["/"] = compatUser.Permissions
+				user.UsedQuotaSize = compatUser.UsedQuotaSize
+				user.UsedQuotaFiles = compatUser.UsedQuotaFiles
+				user.LastQuotaUpdate = compatUser.LastQuotaUpdate
+				user.UploadBandwidth = compatUser.UploadBandwidth
+				user.DownloadBandwidth = compatUser.DownloadBandwidth
+				user.ExpirationDate = compatUser.ExpirationDate
+				user.LastLogin = compatUser.LastLogin
+				user.Status = compatUser.Status
+				users = append(users, user)
+			}
+		}
+		return err
+	})
+	if err != nil {
+		return err
+	}
+
+	for _, user := range users {
+		err = provider.updateUser(user)
+		if err != nil {
+			return err
+		}
+		providerLog(logger.LevelInfo, "user %#v updated, \"permissions\" setted to %+v", user.Username, user.Permissions)
+	}
+
+	return updateBoltDatabaseVersion(dbHandle, 3)
+}
+
+func updateDatabaseFrom3To4(dbHandle *bolt.DB) error {
+	providerLog(logger.LevelInfo, "updating bolt database version: 3 -> 4")
+	foldersToScan := []string{}
+	users := []userCompactVFolders{}
+	err := dbHandle.View(func(tx *bolt.Tx) error {
+		bucket, _, err := getBuckets(tx)
+		if err != nil {
+			return err
+		}
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var compatUser userCompactVFolders
+			err = json.Unmarshal(v, &compatUser)
+			if err == nil && len(compatUser.VirtualFolders) > 0 {
+				users = append(users, compatUser)
+			}
+		}
+		return err
+	})
+	if err != nil {
+		return err
+	}
+	for _, u := range users {
+		user, err := provider.userExists(u.Username)
+		if err != nil {
+			return err
+		}
+		var folders []vfs.VirtualFolder
+		for _, f := range u.VirtualFolders {
+			providerLog(logger.LevelInfo, "restoring virtual folder: %+v for user %#v", f, user.Username)
+			quotaSize := int64(-1)
+			quotaFiles := -1
+			if f.ExcludeFromQuota {
+				quotaSize = 0
+				quotaFiles = 0
+			}
+			folder := vfs.VirtualFolder{
+				QuotaSize:   quotaSize,
+				QuotaFiles:  quotaFiles,
+				VirtualPath: f.VirtualPath,
+			}
+			folder.MappedPath = f.MappedPath
+			folders = append(folders, folder)
+			if !utils.IsStringInSlice(folder.MappedPath, foldersToScan) {
+				foldersToScan = append(foldersToScan, folder.MappedPath)
+			}
+		}
+		user.VirtualFolders = folders
+		err = provider.updateUser(user)
+		providerLog(logger.LevelInfo, "number of virtual folders to restore %v, user %#v, error: %v", len(user.VirtualFolders),
+			user.Username, err)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = updateBoltDatabaseVersion(dbHandle, 4)
+	if err == nil {
+		go updateVFoldersQuotaAfterRestore(foldersToScan)
+	}
+	return err
+}
+
+func getBoltAvailableUsernames(dbHandle *bolt.DB) ([]string, error) {
+	usernames := []string{}
+	err := dbHandle.View(func(tx *bolt.Tx) error {
+		_, idxBucket, err := getBuckets(tx)
+		if err != nil {
+			return err
+		}
+		cursor := idxBucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			usernames = append(usernames, string(v))
+		}
+		return nil
+	})
+
+	return usernames, err
+}
+
+func getBoltDatabaseVersion(dbHandle *bolt.DB) (schemaVersion, error) {
+	var dbVersion schemaVersion
+	err := dbHandle.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(dbVersionBucket)
+		if bucket == nil {
+			return fmt.Errorf("unable to find database version bucket")
+		}
+		v := bucket.Get(dbVersionKey)
+		if v == nil {
+			dbVersion = schemaVersion{
+				Version: 1,
+			}
+			return nil
+		}
+		return json.Unmarshal(v, &dbVersion)
+	})
+	return dbVersion, err
+}
+
+func updateBoltDatabaseVersion(dbHandle *bolt.DB, version int) error {
+	err := dbHandle.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(dbVersionBucket)
+		if bucket == nil {
+			return fmt.Errorf("unable to find database version bucket")
+		}
+		newDbVersion := schemaVersion{
+			Version: version,
+		}
+		buf, err := json.Marshal(newDbVersion)
+		if err != nil {
+			return err
+		}
+		return bucket.Put(dbVersionKey, buf)
+	})
+	return err
+}

dataprovider/bolt_disabled.go

@@ -0,0 +1,17 @@
+// +build nobolt
+
+package dataprovider
+
+import (
+	"errors"
+
+	"github.com/drakkan/sftpgo/version"
+)
+
+func init() {
+	version.AddFeature("-bolt")
+}
+
+func initializeBoltProvider(basePath string) error {
+	return errors.New("bolt disabled at build time")
+}

dataprovider/memory.go

@@ -0,0 +1,676 @@
+package dataprovider
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/utils"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+var (
+	errMemoryProviderClosed = errors.New("memory provider is closed")
+)
+
+type memoryProviderHandle struct {
+	isClosed bool
+	// slice with ordered usernames
+	usernames []string
+	// mapping between ID and username
+	usersIdx map[int64]string
+	// map for users, username is the key
+	users map[string]User
+	// map for virtual folders, MappedPath is the key
+	vfolders map[string]vfs.BaseVirtualFolder
+	// slice with ordered folders mapped path
+	vfoldersPaths []string
+	// configuration file to use for loading users
+	configFile string
+	lock       *sync.Mutex
+}
+
+// MemoryProvider auth provider for a memory store
+type MemoryProvider struct {
+	dbHandle *memoryProviderHandle
+}
+
+func initializeMemoryProvider(basePath string) error {
+	logSender = fmt.Sprintf("dataprovider_%v", MemoryDataProviderName)
+	configFile := ""
+	if utils.IsFileInputValid(config.Name) {
+		configFile = config.Name
+		if !filepath.IsAbs(configFile) {
+			configFile = filepath.Join(basePath, configFile)
+		}
+	}
+	provider = MemoryProvider{
+		dbHandle: &memoryProviderHandle{
+			isClosed:      false,
+			usernames:     []string{},
+			usersIdx:      make(map[int64]string),
+			users:         make(map[string]User),
+			vfolders:      make(map[string]vfs.BaseVirtualFolder),
+			vfoldersPaths: []string{},
+			configFile:    configFile,
+			lock:          new(sync.Mutex),
+		},
+	}
+	return provider.reloadConfig()
+}
+
+func (p MemoryProvider) checkAvailability() error {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return errMemoryProviderClosed
+	}
+	return nil
+}
+
+func (p MemoryProvider) close() error {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return errMemoryProviderClosed
+	}
+	p.dbHandle.isClosed = true
+	return nil
+}
+
+func (p MemoryProvider) validateUserAndPass(username string, password string) (User, error) {
+	var user User
+	if len(password) == 0 {
+		return user, errors.New("Credentials cannot be null or empty")
+	}
+	user, err := p.userExists(username)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error authenticating user %#v, error: %v", username, err)
+		return user, err
+	}
+	return checkUserAndPass(user, password)
+}
+
+func (p MemoryProvider) validateUserAndPubKey(username string, pubKey []byte) (User, string, error) {
+	var user User
+	if len(pubKey) == 0 {
+		return user, "", errors.New("Credentials cannot be null or empty")
+	}
+	user, err := p.userExists(username)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error authenticating user %#v, error: %v", username, err)
+		return user, "", err
+	}
+	return checkUserAndPubKey(user, pubKey)
+}
+
+func (p MemoryProvider) getUserByID(ID int64) (User, error) {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return User{}, errMemoryProviderClosed
+	}
+	if val, ok := p.dbHandle.usersIdx[ID]; ok {
+		return p.userExistsInternal(val)
+	}
+	return User{}, &RecordNotFoundError{err: fmt.Sprintf("user with ID %v does not exist", ID)}
+}
+
+func (p MemoryProvider) updateLastLogin(username string) error {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return errMemoryProviderClosed
+	}
+	user, err := p.userExistsInternal(username)
+	if err != nil {
+		return err
+	}
+	user.LastLogin = utils.GetTimeAsMsSinceEpoch(time.Now())
+	p.dbHandle.users[user.Username] = user
+	return nil
+}
+
+func (p MemoryProvider) updateQuota(username string, filesAdd int, sizeAdd int64, reset bool) error {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return errMemoryProviderClosed
+	}
+	user, err := p.userExistsInternal(username)
+	if err != nil {
+		providerLog(logger.LevelWarn, "unable to update quota for user %#v error: %v", username, err)
+		return err
+	}
+	if reset {
+		user.UsedQuotaSize = sizeAdd
+		user.UsedQuotaFiles = filesAdd
+	} else {
+		user.UsedQuotaSize += sizeAdd
+		user.UsedQuotaFiles += filesAdd
+	}
+	user.LastQuotaUpdate = utils.GetTimeAsMsSinceEpoch(time.Now())
+	providerLog(logger.LevelDebug, "quota updated for user %#v, files increment: %v size increment: %v is reset? %v",
+		username, filesAdd, sizeAdd, reset)
+	p.dbHandle.users[user.Username] = user
+	return nil
+}
+
+func (p MemoryProvider) getUsedQuota(username string) (int, int64, error) {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return 0, 0, errMemoryProviderClosed
+	}
+	user, err := p.userExistsInternal(username)
+	if err != nil {
+		providerLog(logger.LevelWarn, "unable to get quota for user %#v error: %v", username, err)
+		return 0, 0, err
+	}
+	return user.UsedQuotaFiles, user.UsedQuotaSize, err
+}
+
+func (p MemoryProvider) addUser(user User) error {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return errMemoryProviderClosed
+	}
+	err := validateUser(&user)
+	if err != nil {
+		return err
+	}
+	_, err = p.userExistsInternal(user.Username)
+	if err == nil {
+		return fmt.Errorf("username %#v already exists", user.Username)
+	}
+	user.ID = p.getNextID()
+	user.LastQuotaUpdate = 0
+	user.UsedQuotaSize = 0
+	user.UsedQuotaFiles = 0
+	user.LastLogin = 0
+	user.VirtualFolders = p.joinVirtualFoldersFields(user)
+	p.dbHandle.users[user.Username] = user
+	p.dbHandle.usersIdx[user.ID] = user.Username
+	p.dbHandle.usernames = append(p.dbHandle.usernames, user.Username)
+	sort.Strings(p.dbHandle.usernames)
+	return nil
+}
+
+func (p MemoryProvider) updateUser(user User) error {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return errMemoryProviderClosed
+	}
+	err := validateUser(&user)
+	if err != nil {
+		return err
+	}
+	u, err := p.userExistsInternal(user.Username)
+	if err != nil {
+		return err
+	}
+	for _, oldFolder := range u.VirtualFolders {
+		p.removeUserFromFolderMapping(oldFolder.MappedPath, u.Username)
+	}
+	user.VirtualFolders = p.joinVirtualFoldersFields(user)
+	user.LastQuotaUpdate = u.LastQuotaUpdate
+	user.UsedQuotaSize = u.UsedQuotaSize
+	user.UsedQuotaFiles = u.UsedQuotaFiles
+	user.LastLogin = u.LastLogin
+	p.dbHandle.users[user.Username] = user
+	return nil
+}
+
+func (p MemoryProvider) deleteUser(user User) error {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return errMemoryProviderClosed
+	}
+	u, err := p.userExistsInternal(user.Username)
+	if err != nil {
+		return err
+	}
+	for _, oldFolder := range u.VirtualFolders {
+		p.removeUserFromFolderMapping(oldFolder.MappedPath, u.Username)
+	}
+	delete(p.dbHandle.users, user.Username)
+	delete(p.dbHandle.usersIdx, user.ID)
+	// this could be more efficient
+	p.dbHandle.usernames = []string{}
+	for username := range p.dbHandle.users {
+		p.dbHandle.usernames = append(p.dbHandle.usernames, username)
+	}
+	sort.Strings(p.dbHandle.usernames)
+	return nil
+}
+
+func (p MemoryProvider) dumpUsers() ([]User, error) {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	users := make([]User, 0, len(p.dbHandle.usernames))
+	var err error
+	if p.dbHandle.isClosed {
+		return users, errMemoryProviderClosed
+	}
+	for _, username := range p.dbHandle.usernames {
+		user := p.dbHandle.users[username]
+		err = addCredentialsToUser(&user)
+		if err != nil {
+			return users, err
+		}
+		users = append(users, user)
+	}
+	return users, err
+}
+
+func (p MemoryProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	folders := make([]vfs.BaseVirtualFolder, 0, len(p.dbHandle.vfoldersPaths))
+	if p.dbHandle.isClosed {
+		return folders, errMemoryProviderClosed
+	}
+	for _, f := range p.dbHandle.vfolders {
+		folders = append(folders, f)
+	}
+	return folders, nil
+}
+
+func (p MemoryProvider) getUsers(limit int, offset int, order string, username string) ([]User, error) {
+	users := make([]User, 0, limit)
+	var err error
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return users, errMemoryProviderClosed
+	}
+	if limit <= 0 {
+		return users, err
+	}
+	if len(username) > 0 {
+		if offset == 0 {
+			user, err := p.userExistsInternal(username)
+			if err == nil {
+				users = append(users, HideUserSensitiveData(&user))
+			}
+		}
+		return users, err
+	}
+	itNum := 0
+	if order == OrderASC {
+		for _, username := range p.dbHandle.usernames {
+			itNum++
+			if itNum <= offset {
+				continue
+			}
+			user := p.dbHandle.users[username]
+			users = append(users, HideUserSensitiveData(&user))
+			if len(users) >= limit {
+				break
+			}
+		}
+	} else {
+		for i := len(p.dbHandle.usernames) - 1; i >= 0; i-- {
+			itNum++
+			if itNum <= offset {
+				continue
+			}
+			username := p.dbHandle.usernames[i]
+			user := p.dbHandle.users[username]
+			users = append(users, HideUserSensitiveData(&user))
+			if len(users) >= limit {
+				break
+			}
+		}
+	}
+	return users, err
+}
+
+func (p MemoryProvider) userExists(username string) (User, error) {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return User{}, errMemoryProviderClosed
+	}
+	return p.userExistsInternal(username)
+}
+
+func (p MemoryProvider) userExistsInternal(username string) (User, error) {
+	if val, ok := p.dbHandle.users[username]; ok {
+		return val.getACopy(), nil
+	}
+	return User{}, &RecordNotFoundError{err: fmt.Sprintf("username %#v does not exist", username)}
+}
+
+func (p MemoryProvider) updateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool) error {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return errMemoryProviderClosed
+	}
+	folder, err := p.folderExistsInternal(mappedPath)
+	if err != nil {
+		providerLog(logger.LevelWarn, "unable to update quota for folder %#v error: %v", mappedPath, err)
+		return err
+	}
+	if reset {
+		folder.UsedQuotaSize = sizeAdd
+		folder.UsedQuotaFiles = filesAdd
+	} else {
+		folder.UsedQuotaSize += sizeAdd
+		folder.UsedQuotaFiles += filesAdd
+	}
+	folder.LastQuotaUpdate = utils.GetTimeAsMsSinceEpoch(time.Now())
+	p.dbHandle.vfolders[mappedPath] = folder
+	return nil
+}
+
+func (p MemoryProvider) getUsedFolderQuota(mappedPath string) (int, int64, error) {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return 0, 0, errMemoryProviderClosed
+	}
+	folder, err := p.folderExistsInternal(mappedPath)
+	if err != nil {
+		providerLog(logger.LevelWarn, "unable to get quota for folder %#v error: %v", mappedPath, err)
+		return 0, 0, err
+	}
+	return folder.UsedQuotaFiles, folder.UsedQuotaSize, err
+}
+
+func (p MemoryProvider) joinVirtualFoldersFields(user User) []vfs.VirtualFolder {
+	var folders []vfs.VirtualFolder
+	for _, folder := range user.VirtualFolders {
+		f, err := p.addOrGetFolderInternal(folder.MappedPath, user.Username, folder.UsedQuotaSize, folder.UsedQuotaFiles,
+			folder.LastQuotaUpdate)
+		if err == nil {
+			folder.UsedQuotaFiles = f.UsedQuotaFiles
+			folder.UsedQuotaSize = f.UsedQuotaSize
+			folder.LastQuotaUpdate = f.LastQuotaUpdate
+			folder.ID = f.ID
+			folders = append(folders, folder)
+		}
+	}
+	return folders
+}
+
+func (p MemoryProvider) removeUserFromFolderMapping(mappedPath, username string) {
+	folder, err := p.folderExistsInternal(mappedPath)
+	if err == nil {
+		var usernames []string
+		for _, user := range folder.Users {
+			if user != username {
+				usernames = append(usernames, user)
+			}
+		}
+		folder.Users = usernames
+		p.dbHandle.vfolders[folder.MappedPath] = folder
+	}
+}
+
+func (p MemoryProvider) updateFoldersMappingInternal(folder vfs.BaseVirtualFolder) {
+	p.dbHandle.vfolders[folder.MappedPath] = folder
+	if !utils.IsStringInSlice(folder.MappedPath, p.dbHandle.vfoldersPaths) {
+		p.dbHandle.vfoldersPaths = append(p.dbHandle.vfoldersPaths, folder.MappedPath)
+		sort.Strings(p.dbHandle.vfoldersPaths)
+	}
+}
+
+func (p MemoryProvider) addOrGetFolderInternal(mappedPath, username string, usedQuotaSize int64, usedQuotaFiles int, lastQuotaUpdate int64) (vfs.BaseVirtualFolder, error) {
+	folder, err := p.folderExistsInternal(mappedPath)
+	if _, ok := err.(*RecordNotFoundError); ok {
+		folder := vfs.BaseVirtualFolder{
+			ID:              p.getNextFolderID(),
+			MappedPath:      mappedPath,
+			UsedQuotaSize:   usedQuotaSize,
+			UsedQuotaFiles:  usedQuotaFiles,
+			LastQuotaUpdate: lastQuotaUpdate,
+			Users:           []string{username},
+		}
+		p.updateFoldersMappingInternal(folder)
+		return folder, nil
+	}
+	if err == nil && !utils.IsStringInSlice(username, folder.Users) {
+		folder.Users = append(folder.Users, username)
+		p.updateFoldersMappingInternal(folder)
+	}
+	return folder, err
+}
+
+func (p MemoryProvider) folderExistsInternal(mappedPath string) (vfs.BaseVirtualFolder, error) {
+	if val, ok := p.dbHandle.vfolders[mappedPath]; ok {
+		return val, nil
+	}
+	return vfs.BaseVirtualFolder{}, &RecordNotFoundError{err: fmt.Sprintf("folder %#v does not exist", mappedPath)}
+}
+
+func (p MemoryProvider) getFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualFolder, error) {
+	folders := make([]vfs.BaseVirtualFolder, 0, limit)
+	var err error
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return folders, errMemoryProviderClosed
+	}
+	if limit <= 0 {
+		return folders, err
+	}
+	if len(folderPath) > 0 {
+		if offset == 0 {
+			var folder vfs.BaseVirtualFolder
+			folder, err = p.folderExistsInternal(folderPath)
+			if err == nil {
+				folders = append(folders, folder)
+			}
+		}
+		return folders, err
+	}
+	itNum := 0
+	if order == OrderASC {
+		for _, mappedPath := range p.dbHandle.vfoldersPaths {
+			itNum++
+			if itNum <= offset {
+				continue
+			}
+			folder := p.dbHandle.vfolders[mappedPath]
+			folders = append(folders, folder)
+			if len(folders) >= limit {
+				break
+			}
+		}
+	} else {
+		for i := len(p.dbHandle.vfoldersPaths) - 1; i >= 0; i-- {
+			itNum++
+			if itNum <= offset {
+				continue
+			}
+			mappedPath := p.dbHandle.vfoldersPaths[i]
+			folder := p.dbHandle.vfolders[mappedPath]
+			folders = append(folders, folder)
+			if len(folders) >= limit {
+				break
+			}
+		}
+	}
+	return folders, err
+}
+
+func (p MemoryProvider) getFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error) {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return vfs.BaseVirtualFolder{}, errMemoryProviderClosed
+	}
+	return p.folderExistsInternal(mappedPath)
+}
+
+func (p MemoryProvider) addFolder(folder vfs.BaseVirtualFolder) error {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return errMemoryProviderClosed
+	}
+	err := validateFolder(&folder)
+	if err != nil {
+		return err
+	}
+	_, err = p.folderExistsInternal(folder.MappedPath)
+	if err == nil {
+		return fmt.Errorf("folder %#v already exists", folder.MappedPath)
+	}
+	folder.ID = p.getNextFolderID()
+	p.dbHandle.vfolders[folder.MappedPath] = folder
+	p.dbHandle.vfoldersPaths = append(p.dbHandle.vfoldersPaths, folder.MappedPath)
+	sort.Strings(p.dbHandle.vfoldersPaths)
+	return nil
+}
+
+func (p MemoryProvider) deleteFolder(folder vfs.BaseVirtualFolder) error {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	if p.dbHandle.isClosed {
+		return errMemoryProviderClosed
+	}
+
+	_, err := p.folderExistsInternal(folder.MappedPath)
+	if err != nil {
+		return err
+	}
+	for _, username := range folder.Users {
+		user, err := p.userExistsInternal(username)
+		if err == nil {
+			var folders []vfs.VirtualFolder
+			for _, userFolder := range user.VirtualFolders {
+				if folder.MappedPath != userFolder.MappedPath {
+					folders = append(folders, userFolder)
+				}
+			}
+			user.VirtualFolders = folders
+			p.dbHandle.users[user.Username] = user
+		}
+	}
+	delete(p.dbHandle.vfolders, folder.MappedPath)
+	p.dbHandle.vfoldersPaths = []string{}
+	for mappedPath := range p.dbHandle.vfolders {
+		p.dbHandle.vfoldersPaths = append(p.dbHandle.vfoldersPaths, mappedPath)
+	}
+	sort.Strings(p.dbHandle.vfoldersPaths)
+	return nil
+}
+
+func (p MemoryProvider) getNextID() int64 {
+	nextID := int64(1)
+	for id := range p.dbHandle.usersIdx {
+		if id >= nextID {
+			nextID = id + 1
+		}
+	}
+	return nextID
+}
+
+func (p MemoryProvider) getNextFolderID() int64 {
+	nextID := int64(1)
+	for _, v := range p.dbHandle.vfolders {
+		if v.ID >= nextID {
+			nextID = v.ID + 1
+		}
+	}
+	return nextID
+}
+
+func (p MemoryProvider) clear() {
+	p.dbHandle.lock.Lock()
+	defer p.dbHandle.lock.Unlock()
+	p.dbHandle.usernames = []string{}
+	p.dbHandle.usersIdx = make(map[int64]string)
+	p.dbHandle.users = make(map[string]User)
+	p.dbHandle.vfoldersPaths = []string{}
+	p.dbHandle.vfolders = make(map[string]vfs.BaseVirtualFolder)
+}
+
+func (p MemoryProvider) reloadConfig() error {
+	if len(p.dbHandle.configFile) == 0 {
+		providerLog(logger.LevelDebug, "no users configuration file defined")
+		return nil
+	}
+	providerLog(logger.LevelDebug, "loading users from file: %#v", p.dbHandle.configFile)
+	fi, err := os.Stat(p.dbHandle.configFile)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error loading users: %v", err)
+		return err
+	}
+	if fi.Size() == 0 {
+		err = errors.New("users configuration file is invalid, its size must be > 0")
+		providerLog(logger.LevelWarn, "error loading users: %v", err)
+		return err
+	}
+	if fi.Size() > 10485760 {
+		err = errors.New("users configuration file is invalid, its size must be <= 10485760 bytes")
+		providerLog(logger.LevelWarn, "error loading users: %v", err)
+		return err
+	}
+	content, err := ioutil.ReadFile(p.dbHandle.configFile)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error loading users: %v", err)
+		return err
+	}
+	var dump BackupData
+	err = json.Unmarshal(content, &dump)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error loading users: %v", err)
+		return err
+	}
+	p.clear()
+	for _, folder := range dump.Folders {
+		_, err := p.getFolderByPath(folder.MappedPath)
+		if err == nil {
+			logger.Debug(logSender, "", "folder %#v already exists, restore not needed", folder.MappedPath)
+			continue
+		}
+		folder.Users = nil
+		err = p.addFolder(folder)
+		if err != nil {
+			providerLog(logger.LevelWarn, "error adding folder %#v: %v", folder.MappedPath, err)
+			return err
+		}
+	}
+	for _, user := range dump.Users {
+		u, err := p.userExists(user.Username)
+		if err == nil {
+			user.ID = u.ID
+			err = p.updateUser(user)
+			if err != nil {
+				providerLog(logger.LevelWarn, "error updating user %#v: %v", user.Username, err)
+				return err
+			}
+		} else {
+			err = p.addUser(user)
+			if err != nil {
+				providerLog(logger.LevelWarn, "error adding user %#v: %v", user.Username, err)
+				return err
+			}
+		}
+	}
+	providerLog(logger.LevelDebug, "user and folders loaded from file: %#v", p.dbHandle.configFile)
+	return nil
+}
+
+// initializeDatabase does nothing, no initilization is needed for memory provider
+func (p MemoryProvider) initializeDatabase() error {
+	return errNoInitRequired
+}
+
+func (p MemoryProvider) migrateDatabase() error {
+	return nil
+}

dataprovider/mysql.go

@@ -1,11 +1,42 @@
+// +build !nomysql
+
 package dataprovider
 
 import (
 	"database/sql"
 	"fmt"
+	"strings"
 	"time"
 
+	// we import go-sql-driver/mysql here to be able to disable MySQL support using a build tag
+	_ "github.com/go-sql-driver/mysql"
+
 	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/version"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+const (
+	mysqlUsersTableSQL = "CREATE TABLE `{{users}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, " +
+		"`username` varchar(255) NOT NULL UNIQUE, `password` varchar(255) NULL, `public_keys` longtext NULL, " +
+		"`home_dir` varchar(255) NOT NULL, `uid` integer NOT NULL, `gid` integer NOT NULL, `max_sessions` integer NOT NULL, " +
+		" `quota_size` bigint NOT NULL, `quota_files` integer NOT NULL, `permissions` longtext NOT NULL, " +
+		"`used_quota_size` bigint NOT NULL, `used_quota_files` integer NOT NULL, `last_quota_update` bigint NOT NULL, " +
+		"`upload_bandwidth` integer NOT NULL, `download_bandwidth` integer NOT NULL, `expiration_date` bigint(20) NOT NULL, " +
+		"`last_login` bigint(20) NOT NULL, `status` int(11) NOT NULL, `filters` longtext DEFAULT NULL, " +
+		"`filesystem` longtext DEFAULT NULL);"
+	mysqlSchemaTableSQL = "CREATE TABLE `{{schema_version}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, `version` integer NOT NULL);"
+	mysqlV2SQL          = "ALTER TABLE `{{users}}` ADD COLUMN `virtual_folders` longtext NULL;"
+	mysqlV3SQL          = "ALTER TABLE `{{users}}` MODIFY `password` longtext NULL;"
+	mysqlV4SQL          = "CREATE TABLE `{{folders}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, `path` varchar(512) NOT NULL UNIQUE," +
+		"`used_quota_size` bigint NOT NULL, `used_quota_files` integer NOT NULL, `last_quota_update` bigint NOT NULL);" +
+		"ALTER TABLE `{{users}}` MODIFY `home_dir` varchar(512) NOT NULL;" +
+		"ALTER TABLE `{{users}}` DROP COLUMN `virtual_folders`;" +
+		"CREATE TABLE `{{folders_mapping}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, `virtual_path` varchar(512) NOT NULL, " +
+		"`quota_size` bigint NOT NULL, `quota_files` integer NOT NULL, `folder_id` integer NOT NULL, `user_id` integer NOT NULL);" +
+		"ALTER TABLE `{{folders_mapping}}` ADD CONSTRAINT `unique_mapping` UNIQUE (`user_id`, `folder_id`);" +
+		"ALTER TABLE `{{folders_mapping}}` ADD CONSTRAINT `folders_mapping_folder_id_fk_folders_id` FOREIGN KEY (`folder_id`) REFERENCES `{{folders}}` (`id`) ON DELETE CASCADE;" +
+		"ALTER TABLE `{{folders_mapping}}` ADD CONSTRAINT `folders_mapping_user_id_fk_users_id` FOREIGN KEY (`user_id`) REFERENCES `{{users}}` (`id`) ON DELETE CASCADE;"
 )
 
 // MySQLProvider auth provider for MySQL/MariaDB database
@@ -13,9 +44,13 @@ type MySQLProvider struct {
 	dbHandle *sql.DB
 }
 
+func init() {
+	version.AddFeature("+mysql")
+}
+
 func initializeMySQLProvider() error {
 	var err error
-	logSender = MySQLDataProviderName
+	logSender = fmt.Sprintf("dataprovider_%v", MySQLDataProviderName)
 	dbHandle, err := sql.Open("mysql", getMySQLConnectionString(false))
 	if err == nil {
 		providerLog(logger.LevelDebug, "mysql database handle created, connection string: %#v, pool size: %v",
@@ -52,7 +87,7 @@ func (p MySQLProvider) validateUserAndPass(username string, password string) (Us
 	return sqlCommonValidateUserAndPass(username, password, p.dbHandle)
 }
 
-func (p MySQLProvider) validateUserAndPubKey(username string, publicKey string) (User, string, error) {
+func (p MySQLProvider) validateUserAndPubKey(username string, publicKey []byte) (User, string, error) {
 	return sqlCommonValidateUserAndPubKey(username, publicKey, p.dbHandle)
 }
 
@@ -68,6 +103,10 @@ func (p MySQLProvider) getUsedQuota(username string) (int, int64, error) {
 	return sqlCommonGetUsedQuota(username, p.dbHandle)
 }
 
+func (p MySQLProvider) updateLastLogin(username string) error {
+	return sqlCommonUpdateLastLogin(username, p.dbHandle)
+}
+
 func (p MySQLProvider) userExists(username string) (User, error) {
 	return sqlCommonCheckUserExists(username, p.dbHandle)
 }
@@ -84,6 +123,120 @@ func (p MySQLProvider) deleteUser(user User) error {
 	return sqlCommonDeleteUser(user, p.dbHandle)
 }
 
+func (p MySQLProvider) dumpUsers() ([]User, error) {
+	return sqlCommonDumpUsers(p.dbHandle)
+}
+
 func (p MySQLProvider) getUsers(limit int, offset int, order string, username string) ([]User, error) {
 	return sqlCommonGetUsers(limit, offset, order, username, p.dbHandle)
 }
+
+func (p MySQLProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
+	return sqlCommonDumpFolders(p.dbHandle)
+}
+
+func (p MySQLProvider) getFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualFolder, error) {
+	return sqlCommonGetFolders(limit, offset, order, folderPath, p.dbHandle)
+}
+
+func (p MySQLProvider) getFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error) {
+	return sqlCommonCheckFolderExists(mappedPath, p.dbHandle)
+}
+
+func (p MySQLProvider) addFolder(folder vfs.BaseVirtualFolder) error {
+	return sqlCommonAddFolder(folder, p.dbHandle)
+}
+
+func (p MySQLProvider) deleteFolder(folder vfs.BaseVirtualFolder) error {
+	return sqlCommonDeleteFolder(folder, p.dbHandle)
+}
+
+func (p MySQLProvider) updateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool) error {
+	return sqlCommonUpdateFolderQuota(mappedPath, filesAdd, sizeAdd, reset, p.dbHandle)
+}
+
+func (p MySQLProvider) getUsedFolderQuota(mappedPath string) (int, int64, error) {
+	return sqlCommonGetFolderUsedQuota(mappedPath, p.dbHandle)
+}
+
+func (p MySQLProvider) close() error {
+	return p.dbHandle.Close()
+}
+
+func (p MySQLProvider) reloadConfig() error {
+	return nil
+}
+
+// initializeDatabase creates the initial database structure
+func (p MySQLProvider) initializeDatabase() error {
+	sqlUsers := strings.Replace(mysqlUsersTableSQL, "{{users}}", sqlTableUsers, 1)
+	tx, err := p.dbHandle.Begin()
+	if err != nil {
+		return err
+	}
+	_, err = tx.Exec(sqlUsers)
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	_, err = tx.Exec(strings.Replace(mysqlSchemaTableSQL, "{{schema_version}}", sqlTableSchemaVersion, 1))
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	_, err = tx.Exec(strings.Replace(initialDBVersionSQL, "{{schema_version}}", sqlTableSchemaVersion, 1))
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	return tx.Commit()
+}
+
+func (p MySQLProvider) migrateDatabase() error {
+	dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle)
+	if err != nil {
+		return err
+	}
+	if dbVersion.Version == sqlDatabaseVersion {
+		providerLog(logger.LevelDebug, "sql database is updated, current version: %v", dbVersion.Version)
+		return nil
+	}
+	switch dbVersion.Version {
+	case 1:
+		err = updateMySQLDatabaseFrom1To2(p.dbHandle)
+		if err != nil {
+			return err
+		}
+		err = updateMySQLDatabaseFrom2To3(p.dbHandle)
+		if err != nil {
+			return err
+		}
+		return updateMySQLDatabaseFrom3To4(p.dbHandle)
+	case 2:
+		err = updateMySQLDatabaseFrom2To3(p.dbHandle)
+		if err != nil {
+			return err
+		}
+		return updateMySQLDatabaseFrom3To4(p.dbHandle)
+	case 3:
+		return updateMySQLDatabaseFrom3To4(p.dbHandle)
+	default:
+		return fmt.Errorf("Database version not handled: %v", dbVersion.Version)
+	}
+}
+
+func updateMySQLDatabaseFrom1To2(dbHandle *sql.DB) error {
+	providerLog(logger.LevelInfo, "updating database version: 1 -> 2")
+	sql := strings.Replace(mysqlV2SQL, "{{users}}", sqlTableUsers, 1)
+	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 2)
+}
+
+func updateMySQLDatabaseFrom2To3(dbHandle *sql.DB) error {
+	providerLog(logger.LevelInfo, "updating database version: 2 -> 3")
+	sql := strings.Replace(mysqlV3SQL, "{{users}}", sqlTableUsers, 1)
+	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 3)
+}
+
+func updateMySQLDatabaseFrom3To4(dbHandle *sql.DB) error {
+	return sqlCommonUpdateDatabaseFrom3To4(mysqlV4SQL, dbHandle)
+}

dataprovider/mysql_disabled.go

@@ -0,0 +1,17 @@
+// +build nomysql
+
+package dataprovider
+
+import (
+	"errors"
+
+	"github.com/drakkan/sftpgo/version"
+)
+
+func init() {
+	version.AddFeature("-mysql")
+}
+
+func initializeMySQLProvider() error {
+	return errors.New("MySQL disabled at build time")
+}

dataprovider/pgsql.go

@@ -1,10 +1,41 @@
+// +build !nopgsql
+
 package dataprovider
 
 import (
 	"database/sql"
 	"fmt"
+	"strings"
+
+	// we import lib/pq here to be able to disable PostgreSQL support using a build tag
+	_ "github.com/lib/pq"
 
 	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/version"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+const (
+	pgsqlUsersTableSQL = `CREATE TABLE "{{users}}" ("id" serial NOT NULL PRIMARY KEY, "username" varchar(255) NOT NULL UNIQUE,
+"password" varchar(255) NULL, "public_keys" text NULL, "home_dir" varchar(255) NOT NULL, "uid" integer NOT NULL,
+"gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL,
+"permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL,
+"last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL,
+"expiration_date" bigint NOT NULL, "last_login" bigint NOT NULL, "status" integer NOT NULL, "filters" text NULL,
+"filesystem" text NULL);`
+	pgsqlSchemaTableSQL = `CREATE TABLE "{{schema_version}}" ("id" serial NOT NULL PRIMARY KEY, "version" integer NOT NULL);`
+	pgsqlV2SQL          = `ALTER TABLE "{{users}}" ADD COLUMN "virtual_folders" text NULL;`
+	pgsqlV3SQL          = `ALTER TABLE "{{users}}" ALTER COLUMN "password" TYPE text USING "password"::text;`
+	pgsqlV4SQL          = `CREATE TABLE "{{folders}}" ("id" serial NOT NULL PRIMARY KEY, "path" varchar(512) NOT NULL UNIQUE, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL);
+ALTER TABLE "{{users}}" ALTER COLUMN "home_dir" TYPE varchar(512) USING "home_dir"::varchar(512);
+ALTER TABLE "{{users}}" DROP COLUMN "virtual_folders" CASCADE;
+CREATE TABLE "{{folders_mapping}}" ("id" serial NOT NULL PRIMARY KEY, "virtual_path" varchar(512) NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "folder_id" integer NOT NULL, "user_id" integer NOT NULL);
+ALTER TABLE "{{folders_mapping}}" ADD CONSTRAINT "unique_mapping" UNIQUE ("user_id", "folder_id");
+ALTER TABLE "{{folders_mapping}}" ADD CONSTRAINT "folders_mapping_folder_id_fk_folders_id" FOREIGN KEY ("folder_id") REFERENCES "{{folders}}" ("id") MATCH SIMPLE ON UPDATE NO ACTION ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE "{{folders_mapping}}" ADD CONSTRAINT "folders_mapping_user_id_fk_users_id" FOREIGN KEY ("user_id") REFERENCES "{{users}}" ("id") MATCH SIMPLE ON UPDATE NO ACTION ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+CREATE INDEX "folders_mapping_folder_id_idx" ON "{{folders_mapping}}" ("folder_id");
+CREATE INDEX "folders_mapping_user_id_idx" ON "{{folders_mapping}}" ("user_id");
+`
 )
 
 // PGSQLProvider auth provider for PostgreSQL database
@@ -12,9 +43,13 @@ type PGSQLProvider struct {
 	dbHandle *sql.DB
 }
 
+func init() {
+	version.AddFeature("+pgsql")
+}
+
 func initializePGSQLProvider() error {
 	var err error
-	logSender = PGSQLDataProviderName
+	logSender = fmt.Sprintf("dataprovider_%v", PGSQLDataProviderName)
 	dbHandle, err := sql.Open("postgres", getPGSQLConnectionString(false))
 	if err == nil {
 		providerLog(logger.LevelDebug, "postgres database handle created, connection string: %#v, pool size: %v",
@@ -51,7 +86,7 @@ func (p PGSQLProvider) validateUserAndPass(username string, password string) (Us
 	return sqlCommonValidateUserAndPass(username, password, p.dbHandle)
 }
 
-func (p PGSQLProvider) validateUserAndPubKey(username string, publicKey string) (User, string, error) {
+func (p PGSQLProvider) validateUserAndPubKey(username string, publicKey []byte) (User, string, error) {
 	return sqlCommonValidateUserAndPubKey(username, publicKey, p.dbHandle)
 }
 
@@ -67,6 +102,10 @@ func (p PGSQLProvider) getUsedQuota(username string) (int, int64, error) {
 	return sqlCommonGetUsedQuota(username, p.dbHandle)
 }
 
+func (p PGSQLProvider) updateLastLogin(username string) error {
+	return sqlCommonUpdateLastLogin(username, p.dbHandle)
+}
+
 func (p PGSQLProvider) userExists(username string) (User, error) {
 	return sqlCommonCheckUserExists(username, p.dbHandle)
 }
@@ -83,6 +122,120 @@ func (p PGSQLProvider) deleteUser(user User) error {
 	return sqlCommonDeleteUser(user, p.dbHandle)
 }
 
+func (p PGSQLProvider) dumpUsers() ([]User, error) {
+	return sqlCommonDumpUsers(p.dbHandle)
+}
+
 func (p PGSQLProvider) getUsers(limit int, offset int, order string, username string) ([]User, error) {
 	return sqlCommonGetUsers(limit, offset, order, username, p.dbHandle)
 }
+
+func (p PGSQLProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
+	return sqlCommonDumpFolders(p.dbHandle)
+}
+
+func (p PGSQLProvider) getFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualFolder, error) {
+	return sqlCommonGetFolders(limit, offset, order, folderPath, p.dbHandle)
+}
+
+func (p PGSQLProvider) getFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error) {
+	return sqlCommonCheckFolderExists(mappedPath, p.dbHandle)
+}
+
+func (p PGSQLProvider) addFolder(folder vfs.BaseVirtualFolder) error {
+	return sqlCommonAddFolder(folder, p.dbHandle)
+}
+
+func (p PGSQLProvider) deleteFolder(folder vfs.BaseVirtualFolder) error {
+	return sqlCommonDeleteFolder(folder, p.dbHandle)
+}
+
+func (p PGSQLProvider) updateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool) error {
+	return sqlCommonUpdateFolderQuota(mappedPath, filesAdd, sizeAdd, reset, p.dbHandle)
+}
+
+func (p PGSQLProvider) getUsedFolderQuota(mappedPath string) (int, int64, error) {
+	return sqlCommonGetFolderUsedQuota(mappedPath, p.dbHandle)
+}
+
+func (p PGSQLProvider) close() error {
+	return p.dbHandle.Close()
+}
+
+func (p PGSQLProvider) reloadConfig() error {
+	return nil
+}
+
+// initializeDatabase creates the initial database structure
+func (p PGSQLProvider) initializeDatabase() error {
+	sqlUsers := strings.Replace(pgsqlUsersTableSQL, "{{users}}", sqlTableUsers, 1)
+	tx, err := p.dbHandle.Begin()
+	if err != nil {
+		return err
+	}
+	_, err = tx.Exec(sqlUsers)
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	_, err = tx.Exec(strings.Replace(pgsqlSchemaTableSQL, "{{schema_version}}", sqlTableSchemaVersion, 1))
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	_, err = tx.Exec(strings.Replace(initialDBVersionSQL, "{{schema_version}}", sqlTableSchemaVersion, 1))
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	return tx.Commit()
+}
+
+func (p PGSQLProvider) migrateDatabase() error {
+	dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle)
+	if err != nil {
+		return err
+	}
+	if dbVersion.Version == sqlDatabaseVersion {
+		providerLog(logger.LevelDebug, "sql database is updated, current version: %v", dbVersion.Version)
+		return nil
+	}
+	switch dbVersion.Version {
+	case 1:
+		err = updatePGSQLDatabaseFrom1To2(p.dbHandle)
+		if err != nil {
+			return err
+		}
+		err = updatePGSQLDatabaseFrom2To3(p.dbHandle)
+		if err != nil {
+			return err
+		}
+		return updatePGSQLDatabaseFrom3To4(p.dbHandle)
+	case 2:
+		err = updatePGSQLDatabaseFrom2To3(p.dbHandle)
+		if err != nil {
+			return err
+		}
+		return updatePGSQLDatabaseFrom3To4(p.dbHandle)
+	case 3:
+		return updatePGSQLDatabaseFrom3To4(p.dbHandle)
+	default:
+		return fmt.Errorf("Database version not handled: %v", dbVersion.Version)
+	}
+}
+
+func updatePGSQLDatabaseFrom1To2(dbHandle *sql.DB) error {
+	providerLog(logger.LevelInfo, "updating database version: 1 -> 2")
+	sql := strings.Replace(pgsqlV2SQL, "{{users}}", sqlTableUsers, 1)
+	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 2)
+}
+
+func updatePGSQLDatabaseFrom2To3(dbHandle *sql.DB) error {
+	providerLog(logger.LevelInfo, "updating database version: 2 -> 3")
+	sql := strings.Replace(pgsqlV3SQL, "{{users}}", sqlTableUsers, 1)
+	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 3)
+}
+
+func updatePGSQLDatabaseFrom3To4(dbHandle *sql.DB) error {
+	return sqlCommonUpdateDatabaseFrom3To4(pgsqlV4SQL, dbHandle)
+}

dataprovider/pgsql_disabled.go

@@ -0,0 +1,17 @@
+// +build nopgsql
+
+package dataprovider
+
+import (
+	"errors"
+
+	"github.com/drakkan/sftpgo/version"
+)
+
+func init() {
+	version.AddFeature("-pgsql")
+}
+
+func initializePGSQLProvider() error {
+	return errors.New("PostgreSQL disabled at build time")
+}

dataprovider/sqlcommon.go

@@ -5,13 +5,26 @@ import (
 	"database/sql"
 	"encoding/json"
 	"errors"
+	"strings"
 	"time"
 
 	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/utils"
+	"github.com/drakkan/sftpgo/vfs"
 )
 
-func getUserByUsername(username string, dbHandle *sql.DB) (User, error) {
+const (
+	sqlDatabaseVersion  = 4
+	initialDBVersionSQL = "INSERT INTO {{schema_version}} (version) VALUES (1);"
+)
+
+var errSQLFoldersAssosaction = errors.New("unable to associate virtual folders to user")
+
+type sqlQuerier interface {
+	Prepare(query string) (*sql.Stmt, error)
+}
+
+func getUserByUsername(username string, dbHandle sqlQuerier) (User, error) {
 	var user User
 	q := getUserByUsernameQuery()
 	stmt, err := dbHandle.Prepare(q)
@@ -22,7 +35,11 @@ func getUserByUsername(username string, dbHandle *sql.DB) (User, error) {
 	defer stmt.Close()
 
 	row := stmt.QueryRow(username)
-	return getUserFromDbRow(row, nil)
+	user, err = getUserFromDbRow(row, nil)
+	if err != nil {
+		return user, err
+	}
+	return getUserWithVirtualFolders(user, dbHandle)
 }
 
 func sqlCommonValidateUserAndPass(username string, password string, dbHandle *sql.DB) (User, error) {
@@ -38,7 +55,7 @@ func sqlCommonValidateUserAndPass(username string, password string, dbHandle *sq
 	return checkUserAndPass(user, password)
 }
 
-func sqlCommonValidateUserAndPubKey(username string, pubKey string, dbHandle *sql.DB) (User, string, error) {
+func sqlCommonValidateUserAndPubKey(username string, pubKey []byte, dbHandle *sql.DB) (User, string, error) {
 	var user User
 	if len(pubKey) == 0 {
 		return user, "", errors.New("Credentials cannot be null or empty")
@@ -68,7 +85,11 @@ func sqlCommonGetUserByID(ID int64, dbHandle *sql.DB) (User, error) {
 	defer stmt.Close()
 
 	row := stmt.QueryRow(ID)
-	return getUserFromDbRow(row, nil)
+	user, err = getUserFromDbRow(row, nil)
+	if err != nil {
+		return user, err
+	}
+	return getUserWithVirtualFolders(user, dbHandle)
 }
 
 func sqlCommonUpdateQuota(username string, filesAdd int, sizeAdd int64, reset bool, dbHandle *sql.DB) error {
@@ -81,10 +102,10 @@ func sqlCommonUpdateQuota(username string, filesAdd int, sizeAdd int64, reset bo
 	defer stmt.Close()
 	_, err = stmt.Exec(sizeAdd, filesAdd, utils.GetTimeAsMsSinceEpoch(time.Now()), username)
 	if err == nil {
-		providerLog(logger.LevelDebug, "quota updated for user %v, files increment: %v size increment: %v is reset? %v",
+		providerLog(logger.LevelDebug, "quota updated for user %#v, files increment: %v size increment: %v is reset? %v",
 			username, filesAdd, sizeAdd, reset)
 	} else {
-		providerLog(logger.LevelWarn, "error updating quota for username %v: %v", username, err)
+		providerLog(logger.LevelWarn, "error updating quota for user %#v: %v", username, err)
 	}
 	return err
 }
@@ -108,6 +129,23 @@ func sqlCommonGetUsedQuota(username string, dbHandle *sql.DB) (int, int64, error
 	return usedFiles, usedSize, err
 }
 
+func sqlCommonUpdateLastLogin(username string, dbHandle *sql.DB) error {
+	q := getUpdateLastLoginQuery()
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return err
+	}
+	defer stmt.Close()
+	_, err = stmt.Exec(utils.GetTimeAsMsSinceEpoch(time.Now()), username)
+	if err == nil {
+		providerLog(logger.LevelDebug, "last login updated for user %#v", username)
+	} else {
+		providerLog(logger.LevelWarn, "error updating last login for user %#v: %v", username, err)
+	}
+	return err
+}
+
 func sqlCommonCheckUserExists(username string, dbHandle *sql.DB) (User, error) {
 	var user User
 	q := getUserByUsernameQuery()
@@ -118,7 +156,11 @@ func sqlCommonCheckUserExists(username string, dbHandle *sql.DB) (User, error) {
 	}
 	defer stmt.Close()
 	row := stmt.QueryRow(username)
-	return getUserFromDbRow(row, nil)
+	user, err = getUserFromDbRow(row, nil)
+	if err != nil {
+		return user, err
+	}
+	return getUserWithVirtualFolders(user, dbHandle)
 }
 
 func sqlCommonAddUser(user User, dbHandle *sql.DB) error {
@@ -126,24 +168,51 @@ func sqlCommonAddUser(user User, dbHandle *sql.DB) error {
 	if err != nil {
 		return err
 	}
+	tx, err := dbHandle.Begin()
+	if err != nil {
+		return err
+	}
 	q := getAddUserQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := tx.Prepare(q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		sqlCommonRollbackTransaction(tx)
 		return err
 	}
 	defer stmt.Close()
 	permissions, err := user.GetPermissionsAsJSON()
 	if err != nil {
+		sqlCommonRollbackTransaction(tx)
 		return err
 	}
 	publicKeys, err := user.GetPublicKeysAsJSON()
 	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	filters, err := user.GetFiltersAsJSON()
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	fsConfig, err := user.GetFsConfigAsJSON()
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
 		return err
 	}
 	_, err = stmt.Exec(user.Username, user.Password, string(publicKeys), user.HomeDir, user.UID, user.GID, user.MaxSessions, user.QuotaSize,
-		user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth)
-	return err
+		user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.Status, user.ExpirationDate, string(filters),
+		string(fsConfig))
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	err = generateVirtualFoldersMapping(user, tx)
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	return tx.Commit()
 }
 
 func sqlCommonUpdateUser(user User, dbHandle *sql.DB) error {
@@ -151,24 +220,51 @@ func sqlCommonUpdateUser(user User, dbHandle *sql.DB) error {
 	if err != nil {
 		return err
 	}
+	tx, err := dbHandle.Begin()
+	if err != nil {
+		return err
+	}
 	q := getUpdateUserQuery()
-	stmt, err := dbHandle.Prepare(q)
+	stmt, err := tx.Prepare(q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		sqlCommonRollbackTransaction(tx)
 		return err
 	}
 	defer stmt.Close()
 	permissions, err := user.GetPermissionsAsJSON()
 	if err != nil {
+		sqlCommonRollbackTransaction(tx)
 		return err
 	}
 	publicKeys, err := user.GetPublicKeysAsJSON()
 	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	filters, err := user.GetFiltersAsJSON()
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	fsConfig, err := user.GetFsConfigAsJSON()
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
 		return err
 	}
 	_, err = stmt.Exec(user.Password, string(publicKeys), user.HomeDir, user.UID, user.GID, user.MaxSessions, user.QuotaSize,
-		user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.ID)
-	return err
+		user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.Status, user.ExpirationDate,
+		string(filters), string(fsConfig), user.ID)
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	err = generateVirtualFoldersMapping(user, tx)
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	return tx.Commit()
 }
 
 func sqlCommonDeleteUser(user User, dbHandle *sql.DB) error {
@@ -183,8 +279,37 @@ func sqlCommonDeleteUser(user User, dbHandle *sql.DB) error {
 	return err
 }
 
-func sqlCommonGetUsers(limit int, offset int, order string, username string, dbHandle *sql.DB) ([]User, error) {
-	users := []User{}
+func sqlCommonDumpUsers(dbHandle sqlQuerier) ([]User, error) {
+	users := make([]User, 0, 100)
+	q := getDumpUsersQuery()
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return nil, err
+	}
+	defer stmt.Close()
+	rows, err := stmt.Query()
+	if err != nil {
+		return users, err
+	}
+
+	defer rows.Close()
+	for rows.Next() {
+		u, err := getUserFromDbRow(nil, rows)
+		if err != nil {
+			return users, err
+		}
+		err = addCredentialsToUser(&u)
+		if err != nil {
+			return users, err
+		}
+		users = append(users, u)
+	}
+	return getUsersWithVirtualFolders(users, dbHandle)
+}
+
+func sqlCommonGetUsers(limit int, offset int, order string, username string, dbHandle sqlQuerier) ([]User, error) {
+	users := make([]User, 0, limit)
 	q := getUsersQuery(order, username)
 	stmt, err := dbHandle.Prepare(q)
 	if err != nil {
@@ -194,26 +319,44 @@ func sqlCommonGetUsers(limit int, offset int, order string, username string, dbH
 	defer stmt.Close()
 	var rows *sql.Rows
 	if len(username) > 0 {
-		rows, err = stmt.Query(username, limit, offset)
+		rows, err = stmt.Query(username, limit, offset) //nolint:rowserrcheck // rows.Err() is checked
 	} else {
-		rows, err = stmt.Query(limit, offset)
+		rows, err = stmt.Query(limit, offset) //nolint:rowserrcheck // rows.Err() is checked
 	}
 	if err == nil {
 		defer rows.Close()
 		for rows.Next() {
 			u, err := getUserFromDbRow(nil, rows)
-			// hide password and public key
-			if err == nil {
-				u.Password = ""
-				u.PublicKeys = []string{}
-				users = append(users, u)
-			} else {
-				break
+			if err != nil {
+				return users, err
 			}
+			users = append(users, HideUserSensitiveData(&u))
 		}
 	}
+	err = rows.Err()
+	if err != nil {
+		return users, err
+	}
+	return getUsersWithVirtualFolders(users, dbHandle)
+}
 
-	return users, err
+func updateUserPermissionsFromDb(user *User, permissions string) error {
+	var err error
+	perms := make(map[string][]string)
+	err = json.Unmarshal([]byte(permissions), &perms)
+	if err == nil {
+		user.Permissions = perms
+	} else {
+		// compatibility layer: until version 0.9.4 permissions were a string list
+		var list []string
+		err = json.Unmarshal([]byte(permissions), &list)
+		if err != nil {
+			return err
+		}
+		perms["/"] = list
+		user.Permissions = perms
+	}
+	return err
 }
 
 func getUserFromDbRow(row *sql.Row, rows *sql.Rows) (User, error) {
@@ -221,16 +364,17 @@ func getUserFromDbRow(row *sql.Row, rows *sql.Rows) (User, error) {
 	var permissions sql.NullString
 	var password sql.NullString
 	var publicKey sql.NullString
+	var filters sql.NullString
+	var fsConfig sql.NullString
 	var err error
 	if row != nil {
 		err = row.Scan(&user.ID, &user.Username, &password, &publicKey, &user.HomeDir, &user.UID, &user.GID, &user.MaxSessions,
 			&user.QuotaSize, &user.QuotaFiles, &permissions, &user.UsedQuotaSize, &user.UsedQuotaFiles, &user.LastQuotaUpdate,
-			&user.UploadBandwidth, &user.DownloadBandwidth)
-
+			&user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters, &fsConfig)
 	} else {
 		err = rows.Scan(&user.ID, &user.Username, &password, &publicKey, &user.HomeDir, &user.UID, &user.GID, &user.MaxSessions,
 			&user.QuotaSize, &user.QuotaFiles, &permissions, &user.UsedQuotaSize, &user.UsedQuotaFiles, &user.LastQuotaUpdate,
-			&user.UploadBandwidth, &user.DownloadBandwidth)
+			&user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters, &fsConfig)
 	}
 	if err != nil {
 		if err == sql.ErrNoRows {
@@ -241,6 +385,9 @@ func getUserFromDbRow(row *sql.Row, rows *sql.Rows) (User, error) {
 	if password.Valid {
 		user.Password = password.String
 	}
+	// we can have a empty string or an invalid json in null string
+	// so we do a relaxed test if the field is optional, for example we
+	// populate public keys only if unmarshal does not return an error
 	if publicKey.Valid {
 		var list []string
 		err = json.Unmarshal([]byte(publicKey.String), &list)
@@ -249,11 +396,495 @@ func getUserFromDbRow(row *sql.Row, rows *sql.Rows) (User, error) {
 		}
 	}
 	if permissions.Valid {
-		var list []string
-		err = json.Unmarshal([]byte(permissions.String), &list)
+		err = updateUserPermissionsFromDb(&user, permissions.String)
+		if err != nil {
+			return user, err
+		}
+	}
+	if filters.Valid {
+		var userFilters UserFilters
+		err = json.Unmarshal([]byte(filters.String), &userFilters)
 		if err == nil {
-			user.Permissions = list
+			user.Filters = userFilters
+		}
+	}
+	if fsConfig.Valid {
+		var fs Filesystem
+		err = json.Unmarshal([]byte(fsConfig.String), &fs)
+		if err == nil {
+			user.FsConfig = fs
 		}
 	}
 	return user, err
 }
+
+func sqlCommonCheckFolderExists(name string, dbHandle sqlQuerier) (vfs.BaseVirtualFolder, error) {
+	var folder vfs.BaseVirtualFolder
+	q := getFolderByPathQuery()
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return folder, err
+	}
+	defer stmt.Close()
+	row := stmt.QueryRow(name)
+	err = row.Scan(&folder.ID, &folder.MappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles, &folder.LastQuotaUpdate)
+	if err == sql.ErrNoRows {
+		return folder, &RecordNotFoundError{err: err.Error()}
+	}
+	return folder, err
+}
+
+func sqlCommonAddOrGetFolder(name string, usedQuotaSize int64, usedQuotaFiles int, lastQuotaUpdate int64, dbHandle sqlQuerier) (vfs.BaseVirtualFolder, error) {
+	folder, err := sqlCommonCheckFolderExists(name, dbHandle)
+	if _, ok := err.(*RecordNotFoundError); ok {
+		f := vfs.BaseVirtualFolder{
+			MappedPath:      name,
+			UsedQuotaSize:   usedQuotaSize,
+			UsedQuotaFiles:  usedQuotaFiles,
+			LastQuotaUpdate: lastQuotaUpdate,
+		}
+		err = sqlCommonAddFolder(f, dbHandle)
+		if err != nil {
+			return folder, err
+		}
+		return sqlCommonCheckFolderExists(name, dbHandle)
+	}
+	return folder, err
+}
+
+func sqlCommonAddFolder(folder vfs.BaseVirtualFolder, dbHandle sqlQuerier) error {
+	err := validateFolder(&folder)
+	if err != nil {
+		return err
+	}
+	q := getAddFolderQuery()
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return err
+	}
+	defer stmt.Close()
+	_, err = stmt.Exec(folder.MappedPath, folder.UsedQuotaSize, folder.UsedQuotaFiles, folder.LastQuotaUpdate)
+	return err
+}
+
+func sqlCommonDeleteFolder(folder vfs.BaseVirtualFolder, dbHandle sqlQuerier) error {
+	q := getDeleteFolderQuery()
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return err
+	}
+	defer stmt.Close()
+	_, err = stmt.Exec(folder.ID)
+	return err
+}
+
+func sqlCommonDumpFolders(dbHandle sqlQuerier) ([]vfs.BaseVirtualFolder, error) {
+	folders := make([]vfs.BaseVirtualFolder, 0, 50)
+	q := getDumpFoldersQuery()
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return nil, err
+	}
+	defer stmt.Close()
+	rows, err := stmt.Query()
+	if err != nil {
+		return folders, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var folder vfs.BaseVirtualFolder
+		err = rows.Scan(&folder.ID, &folder.MappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles, &folder.LastQuotaUpdate)
+		if err != nil {
+			return folders, err
+		}
+		folders = append(folders, folder)
+	}
+	err = rows.Err()
+	if err != nil {
+		return folders, err
+	}
+	return getVirtualFoldersWithUsers(folders, dbHandle)
+}
+
+func sqlCommonGetFolders(limit, offset int, order, folderPath string, dbHandle sqlQuerier) ([]vfs.BaseVirtualFolder, error) {
+	folders := make([]vfs.BaseVirtualFolder, 0, limit)
+	q := getFoldersQuery(order, folderPath)
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return nil, err
+	}
+	defer stmt.Close()
+	var rows *sql.Rows
+	if len(folderPath) > 0 {
+		rows, err = stmt.Query(folderPath, limit, offset) //nolint:rowserrcheck // rows.Err() is checked
+	} else {
+		rows, err = stmt.Query(limit, offset) //nolint:rowserrcheck // rows.Err() is checked
+	}
+	if err != nil {
+		return folders, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var folder vfs.BaseVirtualFolder
+		err = rows.Scan(&folder.ID, &folder.MappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles, &folder.LastQuotaUpdate)
+		if err != nil {
+			return folders, err
+		}
+		folders = append(folders, folder)
+	}
+
+	err = rows.Err()
+	if err != nil {
+		return folders, err
+	}
+	return getVirtualFoldersWithUsers(folders, dbHandle)
+}
+
+func sqlCommonClearFolderMapping(user User, dbHandle sqlQuerier) error {
+	q := getClearFolderMappingQuery()
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return err
+	}
+	defer stmt.Close()
+	_, err = stmt.Exec(user.Username)
+	return err
+}
+
+func sqlCommonAddFolderMapping(user User, folder vfs.VirtualFolder, dbHandle sqlQuerier) error {
+	q := getAddFolderMappingQuery()
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return err
+	}
+	defer stmt.Close()
+	_, err = stmt.Exec(folder.VirtualPath, folder.QuotaSize, folder.QuotaFiles, folder.ID, user.Username)
+	return err
+}
+
+func generateVirtualFoldersMapping(user User, dbHandle sqlQuerier) error {
+	err := sqlCommonClearFolderMapping(user, dbHandle)
+	if err != nil {
+		return err
+	}
+	for _, vfolder := range user.VirtualFolders {
+		f, err := sqlCommonAddOrGetFolder(vfolder.MappedPath, 0, 0, 0, dbHandle)
+		if err != nil {
+			return err
+		}
+		vfolder.BaseVirtualFolder = f
+		err = sqlCommonAddFolderMapping(user, vfolder, dbHandle)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
+func getUserWithVirtualFolders(user User, dbHandle sqlQuerier) (User, error) {
+	users, err := getUsersWithVirtualFolders([]User{user}, dbHandle)
+	if err != nil {
+		return user, err
+	}
+	if len(users) == 0 {
+		return user, errSQLFoldersAssosaction
+	}
+	return users[0], err
+}
+
+func getUsersWithVirtualFolders(users []User, dbHandle sqlQuerier) ([]User, error) {
+	var err error
+	usersVirtualFolders := make(map[int64][]vfs.VirtualFolder)
+	if len(users) == 0 {
+		return users, err
+	}
+	q := getRelatedFoldersForUsersQuery(users)
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return nil, err
+	}
+	defer stmt.Close()
+	rows, err := stmt.Query()
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var folder vfs.VirtualFolder
+		var userID int64
+		err = rows.Scan(&folder.ID, &folder.MappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles,
+			&folder.LastQuotaUpdate, &folder.VirtualPath, &folder.QuotaSize, &folder.QuotaFiles, &userID)
+		if err != nil {
+			return users, err
+		}
+		usersVirtualFolders[userID] = append(usersVirtualFolders[userID], folder)
+	}
+	err = rows.Err()
+	if err != nil {
+		return users, err
+	}
+	if len(usersVirtualFolders) == 0 {
+		return users, err
+	}
+	for idx := range users {
+		ref := &users[idx]
+		ref.VirtualFolders = usersVirtualFolders[ref.ID]
+	}
+	return users, err
+}
+
+func getVirtualFoldersWithUsers(folders []vfs.BaseVirtualFolder, dbHandle sqlQuerier) ([]vfs.BaseVirtualFolder, error) {
+	var err error
+	vFoldersUsers := make(map[int64][]string)
+	if len(folders) == 0 {
+		return folders, err
+	}
+	q := getRelatedUsersForFoldersQuery(folders)
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return nil, err
+	}
+	defer stmt.Close()
+	rows, err := stmt.Query()
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var username string
+		var folderID int64
+		err = rows.Scan(&folderID, &username)
+		if err != nil {
+			return folders, err
+		}
+		vFoldersUsers[folderID] = append(vFoldersUsers[folderID], username)
+	}
+	err = rows.Err()
+	if err != nil {
+		return folders, err
+	}
+	if len(vFoldersUsers) == 0 {
+		return folders, err
+	}
+	for idx := range folders {
+		ref := &folders[idx]
+		ref.Users = vFoldersUsers[ref.ID]
+	}
+	return folders, err
+}
+
+func sqlCommonUpdateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool, dbHandle *sql.DB) error {
+	q := getUpdateFolderQuotaQuery(reset)
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return err
+	}
+	defer stmt.Close()
+	_, err = stmt.Exec(sizeAdd, filesAdd, utils.GetTimeAsMsSinceEpoch(time.Now()), mappedPath)
+	if err == nil {
+		providerLog(logger.LevelDebug, "quota updated for folder %#v, files increment: %v size increment: %v is reset? %v",
+			mappedPath, filesAdd, sizeAdd, reset)
+	} else {
+		providerLog(logger.LevelWarn, "error updating quota for folder %#v: %v", mappedPath, err)
+	}
+	return err
+}
+
+func sqlCommonGetFolderUsedQuota(mappedPath string, dbHandle *sql.DB) (int, int64, error) {
+	q := getQuotaFolderQuery()
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return 0, 0, err
+	}
+	defer stmt.Close()
+
+	var usedFiles int
+	var usedSize int64
+	err = stmt.QueryRow(mappedPath).Scan(&usedSize, &usedFiles)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error getting quota for folder: %v, error: %v", mappedPath, err)
+		return 0, 0, err
+	}
+	return usedFiles, usedSize, err
+}
+
+func sqlCommonRollbackTransaction(tx *sql.Tx) {
+	err := tx.Rollback()
+	if err != nil {
+		providerLog(logger.LevelWarn, "error rolling back transaction: %v", err)
+	}
+}
+
+func sqlCommonGetDatabaseVersion(dbHandle *sql.DB) (schemaVersion, error) {
+	var result schemaVersion
+	q := getDatabaseVersionQuery()
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return result, err
+	}
+	defer stmt.Close()
+	row := stmt.QueryRow()
+	err = row.Scan(&result.Version)
+	return result, err
+}
+
+func sqlCommonUpdateDatabaseVersion(dbHandle sqlQuerier, version int) error {
+	q := getUpdateDBVersionQuery()
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return err
+	}
+	defer stmt.Close()
+	_, err = stmt.Exec(version)
+	return err
+}
+
+func sqlCommonExecSQLAndUpdateDBVersion(dbHandle *sql.DB, sql []string, newVersion int) error {
+	tx, err := dbHandle.Begin()
+	if err != nil {
+		return err
+	}
+	for _, q := range sql {
+		if len(strings.TrimSpace(q)) == 0 {
+			continue
+		}
+		_, err = tx.Exec(q)
+		if err != nil {
+			sqlCommonRollbackTransaction(tx)
+			return err
+		}
+	}
+	err = sqlCommonUpdateDatabaseVersion(tx, newVersion)
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	return tx.Commit()
+}
+
+func sqlCommonGetCompatVirtualFolders(dbHandle *sql.DB) ([]userCompactVFolders, error) {
+	users := []userCompactVFolders{}
+	q := getCompatVirtualFoldersQuery()
+	stmt, err := dbHandle.Prepare(q)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
+		return nil, err
+	}
+	defer stmt.Close()
+	rows, err := stmt.Query()
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var user userCompactVFolders
+		var virtualFolders sql.NullString
+		err = rows.Scan(&user.ID, &user.Username, &virtualFolders)
+		if err != nil {
+			return nil, err
+		}
+		if virtualFolders.Valid {
+			var list []virtualFoldersCompact
+			err = json.Unmarshal([]byte(virtualFolders.String), &list)
+			if err == nil && len(list) > 0 {
+				user.VirtualFolders = list
+				users = append(users, user)
+			}
+		}
+	}
+	return users, rows.Err()
+}
+
+func sqlCommonRestoreCompatVirtualFolders(users []userCompactVFolders, dbHandle sqlQuerier) ([]string, error) {
+	foldersToScan := []string{}
+	for _, user := range users {
+		for _, vfolder := range user.VirtualFolders {
+			providerLog(logger.LevelInfo, "restoring virtual folder: %+v for user %#v", vfolder, user.Username)
+			// -1 means included in user quota, 0 means unlimited
+			quotaSize := int64(-1)
+			quotaFiles := -1
+			if vfolder.ExcludeFromQuota {
+				quotaFiles = 0
+				quotaSize = 0
+			}
+			b, err := sqlCommonAddOrGetFolder(vfolder.MappedPath, 0, 0, 0, dbHandle)
+			if err != nil {
+				providerLog(logger.LevelWarn, "error restoring virtual folder for user %#v: %v", user.Username, err)
+				return foldersToScan, err
+			}
+			u := User{
+				ID:       user.ID,
+				Username: user.Username,
+			}
+			f := vfs.VirtualFolder{
+				BaseVirtualFolder: b,
+				VirtualPath:       vfolder.VirtualPath,
+				QuotaSize:         quotaSize,
+				QuotaFiles:        quotaFiles,
+			}
+			err = sqlCommonAddFolderMapping(u, f, dbHandle)
+			if err != nil {
+				providerLog(logger.LevelWarn, "error adding virtual folder mapping for user %#v: %v", user.Username, err)
+				return foldersToScan, err
+			}
+			if !utils.IsStringInSlice(vfolder.MappedPath, foldersToScan) {
+				foldersToScan = append(foldersToScan, vfolder.MappedPath)
+			}
+			providerLog(logger.LevelInfo, "virtual folder: %+v for user %#v successfully restored", vfolder, user.Username)
+		}
+	}
+	return foldersToScan, nil
+}
+
+func sqlCommonUpdateDatabaseFrom3To4(sqlV4 string, dbHandle *sql.DB) error {
+	providerLog(logger.LevelInfo, "updating database version: 3 -> 4")
+	users, err := sqlCommonGetCompatVirtualFolders(dbHandle)
+	if err != nil {
+		return err
+	}
+	sql := strings.ReplaceAll(sqlV4, "{{users}}", sqlTableUsers)
+	sql = strings.ReplaceAll(sql, "{{folders}}", sqlTableFolders)
+	sql = strings.ReplaceAll(sql, "{{folders_mapping}}", sqlTableFoldersMapping)
+	tx, err := dbHandle.Begin()
+	if err != nil {
+		return err
+	}
+	for _, q := range strings.Split(sql, ";") {
+		if len(strings.TrimSpace(q)) == 0 {
+			continue
+		}
+		_, err = tx.Exec(q)
+		if err != nil {
+			sqlCommonRollbackTransaction(tx)
+			return err
+		}
+	}
+	foldersToScan, err := sqlCommonRestoreCompatVirtualFolders(users, tx)
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	err = sqlCommonUpdateDatabaseVersion(tx, 4)
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	err = tx.Commit()
+	if err == nil {
+		go updateVFoldersQuotaAfterRestore(foldersToScan)
+	}
+	return err
+}

dataprovider/sqlite.go

@@ -1,13 +1,67 @@
+// +build !nosqlite
+
 package dataprovider
 
 import (
 	"database/sql"
-	"errors"
 	"fmt"
-	"os"
 	"path/filepath"
+	"strings"
+
+	// we import go-sqlite3 here to be able to disable SQLite support using a build tag
+	_ "github.com/mattn/go-sqlite3"
 
 	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/utils"
+	"github.com/drakkan/sftpgo/version"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+const (
+	sqliteUsersTableSQL = `CREATE TABLE "{{users}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255)
+NOT NULL UNIQUE, "password" varchar(255) NULL, "public_keys" text NULL, "home_dir" varchar(255) NOT NULL, "uid" integer NOT NULL,
+"gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL,
+"permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL,
+"last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL,
+"expiration_date" bigint NOT NULL, "last_login" bigint NOT NULL, "status" integer NOT NULL, "filters" text NULL,
+"filesystem" text NULL);`
+	sqliteSchemaTableSQL = `CREATE TABLE "{{schema_version}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "version" integer NOT NULL);`
+	sqliteV2SQL          = `ALTER TABLE "{{users}}" ADD COLUMN "virtual_folders" text NULL;`
+	sqliteV3SQL          = `CREATE TABLE "new__users" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE,
+	"password" text NULL, "public_keys" text NULL, "home_dir" varchar(255) NOT NULL, "uid" integer NOT NULL,
+"gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL,
+"permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL,
+"upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL, "expiration_date" bigint NOT NULL, "last_login" bigint NOT NULL,
+"status" integer NOT NULL, "filters" text NULL, "filesystem" text NULL, "virtual_folders" text NULL);
+INSERT INTO "new__users" ("id", "username", "public_keys", "home_dir", "uid", "gid", "max_sessions", "quota_size", "quota_files",
+"permissions", "used_quota_size", "used_quota_files", "last_quota_update", "upload_bandwidth", "download_bandwidth", "expiration_date",
+"last_login", "status", "filters", "filesystem", "virtual_folders", "password") SELECT "id", "username", "public_keys", "home_dir",
+"uid", "gid", "max_sessions", "quota_size", "quota_files", "permissions", "used_quota_size", "used_quota_files", "last_quota_update",
+"upload_bandwidth", "download_bandwidth", "expiration_date", "last_login", "status", "filters", "filesystem", "virtual_folders",
+"password" FROM "{{users}}";
+DROP TABLE "{{users}}";
+ALTER TABLE "new__users" RENAME TO "{{users}}";`
+	sqliteV4SQL = `CREATE TABLE "{{folders}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "path" varchar(512) NOT NULL UNIQUE,
+"used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL);
+CREATE TABLE "{{folders_mapping}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "virtual_path" varchar(512) NOT NULL,
+"quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "folder_id" integer NOT NULL REFERENCES "{{folders}}" ("id")
+ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, "user_id" integer NOT NULL REFERENCES "{{users}}" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+CONSTRAINT "unique_mapping" UNIQUE ("user_id", "folder_id"));
+CREATE TABLE "new__users" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE, "password" text NULL,
+"public_keys" text NULL, "home_dir" varchar(512) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL,
+"quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL,
+"used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL,
+"expiration_date" bigint NOT NULL, "last_login" bigint NOT NULL, "status" integer NOT NULL, "filters" text NULL, "filesystem" text NULL);
+INSERT INTO "new__users" ("id", "username", "password", "public_keys", "home_dir", "uid", "gid", "max_sessions", "quota_size", "quota_files",
+"permissions", "used_quota_size", "used_quota_files", "last_quota_update", "upload_bandwidth", "download_bandwidth", "expiration_date",
+"last_login", "status", "filters", "filesystem") SELECT "id", "username", "password", "public_keys", "home_dir", "uid", "gid", "max_sessions",
+"quota_size", "quota_files", "permissions", "used_quota_size", "used_quota_files", "last_quota_update", "upload_bandwidth", "download_bandwidth",
+"expiration_date", "last_login", "status", "filters", "filesystem" FROM "{{users}}";
+DROP TABLE "{{users}}";
+ALTER TABLE "new__users" RENAME TO "{{users}}";
+CREATE INDEX "folders_mapping_folder_id_idx" ON "{{folders_mapping}}" ("folder_id");
+CREATE INDEX "folders_mapping_user_id_idx" ON "{{folders_mapping}}" ("user_id");
+`
 )
 
 // SQLiteProvider auth provider for SQLite database
@@ -15,26 +69,23 @@ type SQLiteProvider struct {
 	dbHandle *sql.DB
 }
 
+func init() {
+	version.AddFeature("+sqlite")
+}
+
 func initializeSQLiteProvider(basePath string) error {
 	var err error
 	var connectionString string
-	logSender = SQLiteDataProviderName
+	logSender = fmt.Sprintf("dataprovider_%v", SQLiteDataProviderName)
 	if len(config.ConnectionString) == 0 {
 		dbPath := config.Name
+		if !utils.IsFileInputValid(dbPath) {
+			return fmt.Errorf("Invalid database path: %#v", dbPath)
+		}
 		if !filepath.IsAbs(dbPath) {
 			dbPath = filepath.Join(basePath, dbPath)
 		}
-		fi, err := os.Stat(dbPath)
-		if err != nil {
-			providerLog(logger.LevelWarn, "sqlite database file does not exists, please be sure to create and initialize"+
-				" a database before starting sftpgo")
-			return err
-		}
-		if fi.Size() == 0 {
-			return errors.New("sqlite database file is invalid, please be sure to create and initialize" +
-				" a database before starting sftpgo")
-		}
-		connectionString = fmt.Sprintf("file:%v?cache=shared", dbPath)
+		connectionString = fmt.Sprintf("file:%v?cache=shared&_foreign_keys=1", dbPath)
 	} else {
 		connectionString = config.ConnectionString
 	}
@@ -58,7 +109,7 @@ func (p SQLiteProvider) validateUserAndPass(username string, password string) (U
 	return sqlCommonValidateUserAndPass(username, password, p.dbHandle)
 }
 
-func (p SQLiteProvider) validateUserAndPubKey(username string, publicKey string) (User, string, error) {
+func (p SQLiteProvider) validateUserAndPubKey(username string, publicKey []byte) (User, string, error) {
 	return sqlCommonValidateUserAndPubKey(username, publicKey, p.dbHandle)
 }
 
@@ -74,6 +125,10 @@ func (p SQLiteProvider) getUsedQuota(username string) (int, int64, error) {
 	return sqlCommonGetUsedQuota(username, p.dbHandle)
 }
 
+func (p SQLiteProvider) updateLastLogin(username string) error {
+	return sqlCommonUpdateLastLogin(username, p.dbHandle)
+}
+
 func (p SQLiteProvider) userExists(username string) (User, error) {
 	return sqlCommonCheckUserExists(username, p.dbHandle)
 }
@@ -90,6 +145,120 @@ func (p SQLiteProvider) deleteUser(user User) error {
 	return sqlCommonDeleteUser(user, p.dbHandle)
 }
 
+func (p SQLiteProvider) dumpUsers() ([]User, error) {
+	return sqlCommonDumpUsers(p.dbHandle)
+}
+
 func (p SQLiteProvider) getUsers(limit int, offset int, order string, username string) ([]User, error) {
 	return sqlCommonGetUsers(limit, offset, order, username, p.dbHandle)
 }
+
+func (p SQLiteProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
+	return sqlCommonDumpFolders(p.dbHandle)
+}
+
+func (p SQLiteProvider) getFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualFolder, error) {
+	return sqlCommonGetFolders(limit, offset, order, folderPath, p.dbHandle)
+}
+
+func (p SQLiteProvider) getFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error) {
+	return sqlCommonCheckFolderExists(mappedPath, p.dbHandle)
+}
+
+func (p SQLiteProvider) addFolder(folder vfs.BaseVirtualFolder) error {
+	return sqlCommonAddFolder(folder, p.dbHandle)
+}
+
+func (p SQLiteProvider) deleteFolder(folder vfs.BaseVirtualFolder) error {
+	return sqlCommonDeleteFolder(folder, p.dbHandle)
+}
+
+func (p SQLiteProvider) updateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool) error {
+	return sqlCommonUpdateFolderQuota(mappedPath, filesAdd, sizeAdd, reset, p.dbHandle)
+}
+
+func (p SQLiteProvider) getUsedFolderQuota(mappedPath string) (int, int64, error) {
+	return sqlCommonGetFolderUsedQuota(mappedPath, p.dbHandle)
+}
+
+func (p SQLiteProvider) close() error {
+	return p.dbHandle.Close()
+}
+
+func (p SQLiteProvider) reloadConfig() error {
+	return nil
+}
+
+// initializeDatabase creates the initial database structure
+func (p SQLiteProvider) initializeDatabase() error {
+	sqlUsers := strings.Replace(sqliteUsersTableSQL, "{{users}}", sqlTableUsers, 1)
+	tx, err := p.dbHandle.Begin()
+	if err != nil {
+		return err
+	}
+	_, err = tx.Exec(sqlUsers)
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	_, err = tx.Exec(strings.Replace(sqliteSchemaTableSQL, "{{schema_version}}", sqlTableSchemaVersion, 1))
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	_, err = tx.Exec(strings.Replace(initialDBVersionSQL, "{{schema_version}}", sqlTableSchemaVersion, 1))
+	if err != nil {
+		sqlCommonRollbackTransaction(tx)
+		return err
+	}
+	return tx.Commit()
+}
+
+func (p SQLiteProvider) migrateDatabase() error {
+	dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle)
+	if err != nil {
+		return err
+	}
+	if dbVersion.Version == sqlDatabaseVersion {
+		providerLog(logger.LevelDebug, "sql database is updated, current version: %v", dbVersion.Version)
+		return nil
+	}
+	switch dbVersion.Version {
+	case 1:
+		err = updateSQLiteDatabaseFrom1To2(p.dbHandle)
+		if err != nil {
+			return err
+		}
+		err = updateSQLiteDatabaseFrom2To3(p.dbHandle)
+		if err != nil {
+			return err
+		}
+		return updateSQLiteDatabaseFrom3To4(p.dbHandle)
+	case 2:
+		err = updateSQLiteDatabaseFrom2To3(p.dbHandle)
+		if err != nil {
+			return err
+		}
+		return updateSQLiteDatabaseFrom3To4(p.dbHandle)
+	case 3:
+		return updateSQLiteDatabaseFrom3To4(p.dbHandle)
+	default:
+		return fmt.Errorf("Database version not handled: %v", dbVersion.Version)
+	}
+}
+
+func updateSQLiteDatabaseFrom1To2(dbHandle *sql.DB) error {
+	providerLog(logger.LevelInfo, "updating database version: 1 -> 2")
+	sql := strings.Replace(sqliteV2SQL, "{{users}}", sqlTableUsers, 1)
+	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 2)
+}
+
+func updateSQLiteDatabaseFrom2To3(dbHandle *sql.DB) error {
+	providerLog(logger.LevelInfo, "updating database version: 2 -> 3")
+	sql := strings.ReplaceAll(sqliteV3SQL, "{{users}}", sqlTableUsers)
+	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 3)
+}
+
+func updateSQLiteDatabaseFrom3To4(dbHandle *sql.DB) error {
+	return sqlCommonUpdateDatabaseFrom3To4(sqliteV4SQL, dbHandle)
+}

dataprovider/sqlite_disabled.go

@@ -0,0 +1,17 @@
+// +build nosqlite
+
+package dataprovider
+
+import (
+	"errors"
+
+	"github.com/drakkan/sftpgo/version"
+)
+
+func init() {
+	version.AddFeature("-sqlite")
+}
+
+func initializeSQLiteProvider(basePath string) error {
+	return errors.New("SQLite disabled at build time")
+}

dataprovider/sqlqueries.go

@@ -1,10 +1,17 @@
 package dataprovider
 
-import "fmt"
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/drakkan/sftpgo/vfs"
+)
 
 const (
-	selectUserFields = "id,username,password,public_keys,home_dir,uid,gid,max_sessions,quota_size,quota_files,permissions," +
-		"used_quota_size,used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth"
+	selectUserFields = "id,username,password,public_keys,home_dir,uid,gid,max_sessions,quota_size,quota_files,permissions,used_quota_size," +
+		"used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,expiration_date,last_login,status,filters,filesystem"
+	selectFolderFields = "id,path,used_quota_size,used_quota_files,last_quota_update"
 )
 
 func getSQLPlaceholders() []string {
@@ -20,51 +27,160 @@ func getSQLPlaceholders() []string {
 }
 
 func getUserByUsernameQuery() string {
-	return fmt.Sprintf(`SELECT %v FROM %v WHERE username = %v`, selectUserFields, config.UsersTable, sqlPlaceholders[0])
+	return fmt.Sprintf(`SELECT %v FROM %v WHERE username = %v`, selectUserFields, sqlTableUsers, sqlPlaceholders[0])
 }
 
 func getUserByIDQuery() string {
-	return fmt.Sprintf(`SELECT %v FROM %v WHERE id = %v`, selectUserFields, config.UsersTable, sqlPlaceholders[0])
+	return fmt.Sprintf(`SELECT %v FROM %v WHERE id = %v`, selectUserFields, sqlTableUsers, sqlPlaceholders[0])
 }
 
 func getUsersQuery(order string, username string) string {
 	if len(username) > 0 {
 		return fmt.Sprintf(`SELECT %v FROM %v WHERE username = %v ORDER BY username %v LIMIT %v OFFSET %v`,
-			selectUserFields, config.UsersTable, sqlPlaceholders[0], order, sqlPlaceholders[1], sqlPlaceholders[2])
+			selectUserFields, sqlTableUsers, sqlPlaceholders[0], order, sqlPlaceholders[1], sqlPlaceholders[2])
 	}
-	return fmt.Sprintf(`SELECT %v FROM %v ORDER BY username %v LIMIT %v OFFSET %v`, selectUserFields, config.UsersTable,
+	return fmt.Sprintf(`SELECT %v FROM %v ORDER BY username %v LIMIT %v OFFSET %v`, selectUserFields, sqlTableUsers,
 		order, sqlPlaceholders[0], sqlPlaceholders[1])
 }
 
+func getDumpUsersQuery() string {
+	return fmt.Sprintf(`SELECT %v FROM %v`, selectUserFields, sqlTableUsers)
+}
+
+func getDumpFoldersQuery() string {
+	return fmt.Sprintf(`SELECT %v FROM %v`, selectFolderFields, sqlTableFolders)
+}
+
 func getUpdateQuotaQuery(reset bool) string {
 	if reset {
-		return fmt.Sprintf(`UPDATE %v SET used_quota_size = %v,used_quota_files = %v,last_quota_update = %v 
-			WHERE username = %v`, config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
+		return fmt.Sprintf(`UPDATE %v SET used_quota_size = %v,used_quota_files = %v,last_quota_update = %v
+			WHERE username = %v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
 	}
-	return fmt.Sprintf(`UPDATE %v SET used_quota_size = used_quota_size + %v,used_quota_files = used_quota_files + %v,last_quota_update = %v 
-		WHERE username = %v`, config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
+	return fmt.Sprintf(`UPDATE %v SET used_quota_size = used_quota_size + %v,used_quota_files = used_quota_files + %v,last_quota_update = %v
+		WHERE username = %v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
+}
+
+func getUpdateLastLoginQuery() string {
+	return fmt.Sprintf(`UPDATE %v SET last_login = %v WHERE username = %v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1])
 }
 
 func getQuotaQuery() string {
-	return fmt.Sprintf(`SELECT used_quota_size,used_quota_files FROM %v WHERE username = %v`, config.UsersTable,
+	return fmt.Sprintf(`SELECT used_quota_size,used_quota_files FROM %v WHERE username = %v`, sqlTableUsers,
 		sqlPlaceholders[0])
 }
 
 func getAddUserQuery() string {
 	return fmt.Sprintf(`INSERT INTO %v (username,password,public_keys,home_dir,uid,gid,max_sessions,quota_size,quota_files,permissions,
-		used_quota_size,used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth) 
-		VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0,0,0,%v,%v)`, config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1],
+		used_quota_size,used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,status,last_login,expiration_date,filters,
+		filesystem)
+		VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0,0,0,%v,%v,%v,0,%v,%v,%v)`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1],
 		sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7],
-		sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10], sqlPlaceholders[11])
+		sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13],
+		sqlPlaceholders[14], sqlPlaceholders[15])
 }
 
 func getUpdateUserQuery() string {
 	return fmt.Sprintf(`UPDATE %v SET password=%v,public_keys=%v,home_dir=%v,uid=%v,gid=%v,max_sessions=%v,quota_size=%v,
-		quota_files=%v,permissions=%v,upload_bandwidth=%v,download_bandwidth=%v WHERE id = %v`, config.UsersTable,
-		sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5],
-		sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10], sqlPlaceholders[11])
+		quota_files=%v,permissions=%v,upload_bandwidth=%v,download_bandwidth=%v,status=%v,expiration_date=%v,filters=%v,filesystem=%v
+		WHERE id = %v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3],
+		sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9],
+		sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13], sqlPlaceholders[14], sqlPlaceholders[15])
 }
 
 func getDeleteUserQuery() string {
-	return fmt.Sprintf(`DELETE FROM %v WHERE id = %v`, config.UsersTable, sqlPlaceholders[0])
+	return fmt.Sprintf(`DELETE FROM %v WHERE id = %v`, sqlTableUsers, sqlPlaceholders[0])
+}
+
+func getFolderByPathQuery() string {
+	return fmt.Sprintf(`SELECT %v FROM %v WHERE path = %v`, selectFolderFields, sqlTableFolders, sqlPlaceholders[0])
+}
+
+func getAddFolderQuery() string {
+	return fmt.Sprintf(`INSERT INTO %v (path,used_quota_size,used_quota_files,last_quota_update) VALUES (%v,%v,%v,%v)`,
+		sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
+}
+
+func getDeleteFolderQuery() string {
+	return fmt.Sprintf(`DELETE FROM %v WHERE id = %v`, sqlTableFolders, sqlPlaceholders[0])
+}
+
+func getClearFolderMappingQuery() string {
+	return fmt.Sprintf(`DELETE FROM %v WHERE user_id = (SELECT id FROM %v WHERE username = %v)`, sqlTableFoldersMapping,
+		sqlTableUsers, sqlPlaceholders[0])
+}
+
+func getAddFolderMappingQuery() string {
+	return fmt.Sprintf(`INSERT INTO %v (virtual_path,quota_size,quota_files,folder_id,user_id)
+		VALUES (%v,%v,%v,%v,(SELECT id FROM %v WHERE username = %v))`, sqlTableFoldersMapping, sqlPlaceholders[0],
+		sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlTableUsers, sqlPlaceholders[4])
+}
+
+func getFoldersQuery(order, folderPath string) string {
+	if len(folderPath) > 0 {
+		return fmt.Sprintf(`SELECT %v FROM %v WHERE path = %v ORDER BY path %v LIMIT %v OFFSET %v`,
+			selectFolderFields, sqlTableFolders, sqlPlaceholders[0], order, sqlPlaceholders[1], sqlPlaceholders[2])
+	}
+	return fmt.Sprintf(`SELECT %v FROM %v ORDER BY path %v LIMIT %v OFFSET %v`, selectFolderFields, sqlTableFolders,
+		order, sqlPlaceholders[0], sqlPlaceholders[1])
+}
+
+func getUpdateFolderQuotaQuery(reset bool) string {
+	if reset {
+		return fmt.Sprintf(`UPDATE %v SET used_quota_size = %v,used_quota_files = %v,last_quota_update = %v
+			WHERE path = %v`, sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
+	}
+	return fmt.Sprintf(`UPDATE %v SET used_quota_size = used_quota_size + %v,used_quota_files = used_quota_files + %v,last_quota_update = %v
+		WHERE path = %v`, sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
+}
+
+func getQuotaFolderQuery() string {
+	return fmt.Sprintf(`SELECT used_quota_size,used_quota_files FROM %v WHERE path = %v`, sqlTableFolders,
+		sqlPlaceholders[0])
+}
+
+func getRelatedFoldersForUsersQuery(users []User) string {
+	var sb strings.Builder
+	for _, u := range users {
+		if sb.Len() == 0 {
+			sb.WriteString("(")
+		} else {
+			sb.WriteString(",")
+		}
+		sb.WriteString(strconv.FormatInt(u.ID, 10))
+	}
+	if sb.Len() > 0 {
+		sb.WriteString(")")
+	}
+	return fmt.Sprintf(`SELECT f.id,f.path,f.used_quota_size,f.used_quota_files,f.last_quota_update,fm.virtual_path,fm.quota_size,fm.quota_files,fm.user_id
+		FROM %v f INNER JOIN %v fm ON f.id = fm.folder_id WHERE fm.user_id IN %v ORDER BY fm.user_id`, sqlTableFolders,
+		sqlTableFoldersMapping, sb.String())
+}
+
+func getRelatedUsersForFoldersQuery(folders []vfs.BaseVirtualFolder) string {
+	var sb strings.Builder
+	for _, f := range folders {
+		if sb.Len() == 0 {
+			sb.WriteString("(")
+		} else {
+			sb.WriteString(",")
+		}
+		sb.WriteString(strconv.FormatInt(f.ID, 10))
+	}
+	if sb.Len() > 0 {
+		sb.WriteString(")")
+	}
+	return fmt.Sprintf(`SELECT fm.folder_id,u.username FROM %v fm INNER JOIN %v u ON fm.user_id = u.id
+		WHERE fm.folder_id IN %v ORDER BY fm.folder_id`, sqlTableFoldersMapping, sqlTableUsers, sb.String())
+}
+
+func getDatabaseVersionQuery() string {
+	return fmt.Sprintf("SELECT version from %v LIMIT 1", sqlTableSchemaVersion)
+}
+
+func getUpdateDBVersionQuery() string {
+	return fmt.Sprintf(`UPDATE %v SET version=%v`, sqlTableSchemaVersion, sqlPlaceholders[0])
+}
+
+func getCompatVirtualFoldersQuery() string {
+	return fmt.Sprintf(`SELECT id,username,virtual_folders FROM %v`, sqlTableUsers)
 }

dataprovider/user.go

@@ -2,9 +2,19 @@ package dataprovider
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"path"
 	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
 
+	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/utils"
+	"github.com/drakkan/sftpgo/vfs"
 )
 
 // Available permissions for SFTP users
@@ -28,24 +38,96 @@ const (
 	PermCreateDirs = "create_dirs"
 	// create symbolic links is allowed
 	PermCreateSymlinks = "create_symlinks"
+	// changing file or directory permissions is allowed
+	PermChmod = "chmod"
+	// changing file or directory owner and group is allowed
+	PermChown = "chown"
+	// changing file or directory access and modification time is allowed
+	PermChtimes = "chtimes"
 )
 
+// Available SSH login methods
+const (
+	SSHLoginMethodPublicKey           = "publickey"
+	SSHLoginMethodPassword            = "password"
+	SSHLoginMethodKeyboardInteractive = "keyboard-interactive"
+	SSHLoginMethodKeyAndPassword      = "publickey+password"
+	SSHLoginMethodKeyAndKeyboardInt   = "publickey+keyboard-interactive"
+)
+
+var (
+	errNoMatchingVirtualFolder = errors.New("no matching virtual folder found")
+)
+
+// ExtensionsFilter defines filters based on file extensions.
+// These restrictions do not apply to files listing for performance reasons, so
+// a denied file cannot be downloaded/overwritten/renamed but will still be
+// it will still be listed in the list of files.
+// System commands such as Git and rsync interacts with the filesystem directly
+// and they are not aware about these restrictions so they are not allowed
+// inside paths with extensions filters
+type ExtensionsFilter struct {
+	// SFTP/SCP path, if no other specific filter is defined, the filter apply for
+	// sub directories too.
+	// For example if filters are defined for the paths "/" and "/sub" then the
+	// filters for "/" are applied for any file outside the "/sub" directory
+	Path string `json:"path"`
+	// only files with these, case insensitive, extensions are allowed.
+	// Shell like expansion is not supported so you have to specify ".jpg" and
+	// not "*.jpg"
+	AllowedExtensions []string `json:"allowed_extensions,omitempty"`
+	// files with these, case insensitive, extensions are not allowed.
+	// Denied file extensions are evaluated before the allowed ones
+	DeniedExtensions []string `json:"denied_extensions,omitempty"`
+}
+
+// UserFilters defines additional restrictions for a user
+type UserFilters struct {
+	// only clients connecting from these IP/Mask are allowed.
+	// IP/Mask must be in CIDR notation as defined in RFC 4632 and RFC 4291
+	// for example "192.0.2.0/24" or "2001:db8::/32"
+	AllowedIP []string `json:"allowed_ip,omitempty"`
+	// clients connecting from these IP/Mask are not allowed.
+	// Denied rules will be evaluated before allowed ones
+	DeniedIP []string `json:"denied_ip,omitempty"`
+	// these login methods are not allowed.
+	// If null or empty any available login method is allowed
+	DeniedLoginMethods []string `json:"denied_login_methods,omitempty"`
+	// filters based on file extensions.
+	// Please note that these restrictions can be easily bypassed.
+	FileExtensions []ExtensionsFilter `json:"file_extensions,omitempty"`
+}
+
+// Filesystem defines cloud storage filesystem details
+type Filesystem struct {
+	// 0 local filesystem, 1 Amazon S3 compatible, 2 Google Cloud Storage
+	Provider  int             `json:"provider"`
+	S3Config  vfs.S3FsConfig  `json:"s3config,omitempty"`
+	GCSConfig vfs.GCSFsConfig `json:"gcsconfig,omitempty"`
+}
+
 // User defines an SFTP user
 type User struct {
 	// Database unique identifier
 	ID int64 `json:"id"`
+	// 1 enabled, 0 disabled (login is not allowed)
+	Status int `json:"status"`
 	// Username
 	Username string `json:"username"`
+	// Account expiration date as unix timestamp in milliseconds. An expired account cannot login.
+	// 0 means no expiration
+	ExpirationDate int64 `json:"expiration_date"`
 	// Password used for password authentication.
 	// For users created using SFTPGo REST API the password is be stored using argon2id hashing algo.
-	// Checking passwords stored with bcrypt is supported too.
-	// Currently, as fallback, there is a clear text password checking but you should not store passwords
-	// as clear text and this support could be removed at any time, so please don't depend on it.
+	// Checking passwords stored with bcrypt, pbkdf2, md5crypt and sha512crypt is supported too.
 	Password string `json:"password,omitempty"`
 	// PublicKeys used for public key authentication. At least one between password and a public key is mandatory
 	PublicKeys []string `json:"public_keys,omitempty"`
 	// The user cannot upload or download files outside this directory. Must be an absolute path
 	HomeDir string `json:"home_dir"`
+	// Mapping between virtual paths and filesystem paths outside the home directory.
+	// Supported for local filesystem only
+	VirtualFolders []vfs.VirtualFolder `json:"virtual_folders,omitempty"`
 	// If sftpgo runs as root system user then the created files and directories will be assigned to this system UID
 	UID int `json:"uid"`
 	// If sftpgo runs as root system user then the created files and directories will be assigned to this system GID
@@ -57,7 +139,7 @@ type User struct {
 	// Maximum number of files allowed. 0 means unlimited
 	QuotaFiles int `json:"quota_files"`
 	// List of the granted permissions
-	Permissions []string `json:"permissions"`
+	Permissions map[string][]string `json:"permissions"`
 	// Used quota as bytes
 	UsedQuotaSize int64 `json:"used_quota_size"`
 	// Used quota as number of files
@@ -68,14 +150,330 @@ type User struct {
 	UploadBandwidth int64 `json:"upload_bandwidth"`
 	// Maximum download bandwidth as KB/s, 0 means unlimited
 	DownloadBandwidth int64 `json:"download_bandwidth"`
+	// Last login as unix timestamp in milliseconds
+	LastLogin int64 `json:"last_login"`
+	// Additional restrictions
+	Filters UserFilters `json:"filters"`
+	// Filesystem configuration details
+	FsConfig Filesystem `json:"filesystem"`
+}
+
+// GetFilesystem returns the filesystem for this user
+func (u *User) GetFilesystem(connectionID string) (vfs.Fs, error) {
+	if u.FsConfig.Provider == 1 {
+		return vfs.NewS3Fs(connectionID, u.GetHomeDir(), u.FsConfig.S3Config)
+	} else if u.FsConfig.Provider == 2 {
+		config := u.FsConfig.GCSConfig
+		config.CredentialFile = u.getGCSCredentialsFilePath()
+		return vfs.NewGCSFs(connectionID, u.GetHomeDir(), config)
+	}
+	return vfs.NewOsFs(connectionID, u.GetHomeDir(), u.VirtualFolders), nil
+}
+
+// GetPermissionsForPath returns the permissions for the given path.
+// The path must be an SFTP path
+func (u *User) GetPermissionsForPath(p string) []string {
+	permissions := []string{}
+	if perms, ok := u.Permissions["/"]; ok {
+		// if only root permissions are defined returns them unconditionally
+		if len(u.Permissions) == 1 {
+			return perms
+		}
+		// fallback permissions
+		permissions = perms
+	}
+	dirsForPath := utils.GetDirsForSFTPPath(p)
+	// dirsForPath contains all the dirs for a given path in reverse order
+	// for example if the path is: /1/2/3/4 it contains:
+	// [ "/1/2/3/4", "/1/2/3", "/1/2", "/1", "/" ]
+	// so the first match is the one we are interested to
+	for _, val := range dirsForPath {
+		if perms, ok := u.Permissions[val]; ok {
+			permissions = perms
+			break
+		}
+	}
+	return permissions
+}
+
+// GetVirtualFolderForPath returns the virtual folder containing the specified sftp path.
+// If the path is not inside a virtual folder an error is returned
+func (u *User) GetVirtualFolderForPath(sftpPath string) (vfs.VirtualFolder, error) {
+	var folder vfs.VirtualFolder
+	if len(u.VirtualFolders) == 0 || u.FsConfig.Provider != 0 {
+		return folder, errNoMatchingVirtualFolder
+	}
+	dirsForPath := utils.GetDirsForSFTPPath(sftpPath)
+	for _, val := range dirsForPath {
+		for _, v := range u.VirtualFolders {
+			if v.VirtualPath == val {
+				return v, nil
+			}
+		}
+	}
+	return folder, errNoMatchingVirtualFolder
+}
+
+// AddVirtualDirs adds virtual folders, if defined, to the given files list
+func (u *User) AddVirtualDirs(list []os.FileInfo, sftpPath string) []os.FileInfo {
+	if len(u.VirtualFolders) == 0 {
+		return list
+	}
+	for _, v := range u.VirtualFolders {
+		if path.Dir(v.VirtualPath) == sftpPath {
+			fi := vfs.NewFileInfo(path.Base(v.VirtualPath), true, 0, time.Time{})
+			found := false
+			for index, f := range list {
+				if f.Name() == fi.Name() {
+					list[index] = fi
+					found = true
+					break
+				}
+			}
+			if !found {
+				list = append(list, fi)
+			}
+		}
+	}
+	return list
+}
+
+// IsMappedPath returns true if the specified filesystem path has a virtual folder mapping.
+// The filesystem path must be cleaned before calling this method
+func (u *User) IsMappedPath(fsPath string) bool {
+	for _, v := range u.VirtualFolders {
+		if fsPath == v.MappedPath {
+			return true
+		}
+	}
+	return false
+}
+
+// IsVirtualFolder returns true if the specified sftp path is a virtual folder
+func (u *User) IsVirtualFolder(sftpPath string) bool {
+	for _, v := range u.VirtualFolders {
+		if sftpPath == v.VirtualPath {
+			return true
+		}
+	}
+	return false
+}
+
+// HasVirtualFoldersInside returns true if there are virtual folders inside the
+// specified SFTP path. We assume that path are cleaned
+func (u *User) HasVirtualFoldersInside(sftpPath string) bool {
+	if sftpPath == "/" && len(u.VirtualFolders) > 0 {
+		return true
+	}
+	for _, v := range u.VirtualFolders {
+		if len(v.VirtualPath) > len(sftpPath) {
+			if strings.HasPrefix(v.VirtualPath, sftpPath+"/") {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// HasPermissionsInside returns true if the specified sftpPath has no permissions itself and
+// no subdirs with defined permissions
+func (u *User) HasPermissionsInside(sftpPath string) bool {
+	for dir := range u.Permissions {
+		if dir == sftpPath {
+			return true
+		} else if len(dir) > len(sftpPath) {
+			if strings.HasPrefix(dir, sftpPath+"/") {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// HasOverlappedMappedPaths returns true if this user has virtual folders with overlapped mapped paths
+func (u *User) HasOverlappedMappedPaths() bool {
+	if len(u.VirtualFolders) <= 1 {
+		return false
+	}
+	for _, v1 := range u.VirtualFolders {
+		for _, v2 := range u.VirtualFolders {
+			if v1.VirtualPath == v2.VirtualPath {
+				continue
+			}
+			if isMappedDirOverlapped(v1.MappedPath, v2.MappedPath) {
+				return true
+			}
+		}
+	}
+	return false
 }
 
 // HasPerm returns true if the user has the given permission or any permission
-func (u *User) HasPerm(permission string) bool {
-	if utils.IsStringInSlice(PermAny, u.Permissions) {
+func (u *User) HasPerm(permission, path string) bool {
+	perms := u.GetPermissionsForPath(path)
+	if utils.IsStringInSlice(PermAny, perms) {
 		return true
 	}
-	return utils.IsStringInSlice(permission, u.Permissions)
+	return utils.IsStringInSlice(permission, perms)
+}
+
+// HasPerms return true if the user has all the given permissions
+func (u *User) HasPerms(permissions []string, path string) bool {
+	perms := u.GetPermissionsForPath(path)
+	if utils.IsStringInSlice(PermAny, perms) {
+		return true
+	}
+	for _, permission := range permissions {
+		if !utils.IsStringInSlice(permission, perms) {
+			return false
+		}
+	}
+	return true
+}
+
+// HasNoQuotaRestrictions returns true if no quota restrictions need to be applyed
+func (u *User) HasNoQuotaRestrictions(checkFiles bool) bool {
+	if u.QuotaSize == 0 && (!checkFiles || u.QuotaFiles == 0) {
+		return true
+	}
+	return false
+}
+
+// IsLoginMethodAllowed returns true if the specified login method is allowed
+func (u *User) IsLoginMethodAllowed(loginMethod string, partialSuccessMethods []string) bool {
+	if len(u.Filters.DeniedLoginMethods) == 0 {
+		return true
+	}
+	if len(partialSuccessMethods) == 1 {
+		for _, method := range u.GetNextAuthMethods(partialSuccessMethods) {
+			if method == loginMethod {
+				return true
+			}
+		}
+	}
+	if utils.IsStringInSlice(loginMethod, u.Filters.DeniedLoginMethods) {
+		return false
+	}
+	return true
+}
+
+// GetNextAuthMethods returns the list of authentications methods that
+// can continue for multi-step authentication
+func (u *User) GetNextAuthMethods(partialSuccessMethods []string) []string {
+	var methods []string
+	if len(partialSuccessMethods) != 1 {
+		return methods
+	}
+	if partialSuccessMethods[0] != SSHLoginMethodPublicKey {
+		return methods
+	}
+	for _, method := range u.GetAllowedLoginMethods() {
+		if method == SSHLoginMethodKeyAndPassword {
+			methods = append(methods, SSHLoginMethodPassword)
+		}
+		if method == SSHLoginMethodKeyAndKeyboardInt {
+			methods = append(methods, SSHLoginMethodKeyboardInteractive)
+		}
+	}
+	return methods
+}
+
+// IsPartialAuth returns true if the specified login method is a step for
+// a multi-step Authentication.
+// We support publickey+password and publickey+keyboard-interactive, so
+// only publickey can returns partial success.
+// We can have partial success if only multi-step Auth methods are enabled
+func (u *User) IsPartialAuth(loginMethod string) bool {
+	if loginMethod != SSHLoginMethodPublicKey {
+		return false
+	}
+	for _, method := range u.GetAllowedLoginMethods() {
+		if !utils.IsStringInSlice(method, SSHMultiStepsLoginMethods) {
+			return false
+		}
+	}
+	return true
+}
+
+// GetAllowedLoginMethods returns the allowed login methods
+func (u *User) GetAllowedLoginMethods() []string {
+	var allowedMethods []string
+	for _, method := range ValidSSHLoginMethods {
+		if !utils.IsStringInSlice(method, u.Filters.DeniedLoginMethods) {
+			allowedMethods = append(allowedMethods, method)
+		}
+	}
+	return allowedMethods
+}
+
+// IsFileAllowed returns true if the specified file is allowed by the file restrictions filters
+func (u *User) IsFileAllowed(sftpPath string) bool {
+	if len(u.Filters.FileExtensions) == 0 {
+		return true
+	}
+	dirsForPath := utils.GetDirsForSFTPPath(path.Dir(sftpPath))
+	var filter ExtensionsFilter
+	for _, dir := range dirsForPath {
+		for _, f := range u.Filters.FileExtensions {
+			if f.Path == dir {
+				filter = f
+				break
+			}
+		}
+		if len(filter.Path) > 0 {
+			break
+		}
+	}
+	if len(filter.Path) > 0 {
+		toMatch := strings.ToLower(sftpPath)
+		for _, denied := range filter.DeniedExtensions {
+			if strings.HasSuffix(toMatch, denied) {
+				return false
+			}
+		}
+		for _, allowed := range filter.AllowedExtensions {
+			if strings.HasSuffix(toMatch, allowed) {
+				return true
+			}
+		}
+		return len(filter.AllowedExtensions) == 0
+	}
+	return true
+}
+
+// IsLoginFromAddrAllowed returns true if the login is allowed from the specified remoteAddr.
+// If AllowedIP is defined only the specified IP/Mask can login.
+// If DeniedIP is defined the specified IP/Mask cannot login.
+// If an IP is both allowed and denied then login will be denied
+func (u *User) IsLoginFromAddrAllowed(remoteAddr string) bool {
+	if len(u.Filters.AllowedIP) == 0 && len(u.Filters.DeniedIP) == 0 {
+		return true
+	}
+	remoteIP := net.ParseIP(utils.GetIPFromRemoteAddress(remoteAddr))
+	// if remoteIP is invalid we allow login, this should never happen
+	if remoteIP == nil {
+		logger.Warn(logSender, "", "login allowed for invalid IP. remote address: %#v", remoteAddr)
+		return true
+	}
+	for _, IPMask := range u.Filters.DeniedIP {
+		_, IPNet, err := net.ParseCIDR(IPMask)
+		if err != nil {
+			return false
+		}
+		if IPNet.Contains(remoteIP) {
+			return false
+		}
+	}
+	for _, IPMask := range u.Filters.AllowedIP {
+		_, IPNet, err := net.ParseCIDR(IPMask)
+		if err != nil {
+			return false
+		}
+		if IPNet.Contains(remoteIP) {
+			return true
+		}
+	}
+	return len(u.Filters.AllowedIP) == 0
 }
 
 // GetPermissionsAsJSON returns the permissions as json byte array
@@ -88,6 +486,16 @@ func (u *User) GetPublicKeysAsJSON() ([]byte, error) {
 	return json.Marshal(u.PublicKeys)
 }
 
+// GetFiltersAsJSON returns the filters as json byte array
+func (u *User) GetFiltersAsJSON() ([]byte, error) {
+	return json.Marshal(u.Filters)
+}
+
+// GetFsConfigAsJSON returns the filesystem config as json byte array
+func (u *User) GetFsConfigAsJSON() ([]byte, error) {
+	return json.Marshal(u.FsConfig)
+}
+
 // GetUID returns a validate uid, suitable for use with os.Chown
 func (u *User) GetUID() int {
 	if u.UID <= 0 || u.UID > 65535 {
@@ -114,12 +522,232 @@ func (u *User) HasQuotaRestrictions() bool {
 	return u.QuotaFiles > 0 || u.QuotaSize > 0
 }
 
-// GetRelativePath returns the path for a file relative to the user's home dir.
-// This is the path as seen by SFTP users
-func (u *User) GetRelativePath(path string) string {
-	rel, err := filepath.Rel(u.GetHomeDir(), path)
-	if err != nil {
-		return ""
+// GetQuotaSummary returns used quota and limits if defined
+func (u *User) GetQuotaSummary() string {
+	var result string
+	result = "Files: " + strconv.Itoa(u.UsedQuotaFiles)
+	if u.QuotaFiles > 0 {
+		result += "/" + strconv.Itoa(u.QuotaFiles)
 	}
-	return "/" + filepath.ToSlash(rel)
+	if u.UsedQuotaSize > 0 || u.QuotaSize > 0 {
+		result += ". Size: " + utils.ByteCountSI(u.UsedQuotaSize)
+		if u.QuotaSize > 0 {
+			result += "/" + utils.ByteCountSI(u.QuotaSize)
+		}
+	}
+	return result
+}
+
+// GetPermissionsAsString returns the user's permissions as comma separated string
+func (u *User) GetPermissionsAsString() string {
+	result := ""
+	for dir, perms := range u.Permissions {
+		var dirPerms string
+		for _, p := range perms {
+			if len(dirPerms) > 0 {
+				dirPerms += ", "
+			}
+			dirPerms += p
+		}
+		dp := fmt.Sprintf("%#v: %#v", dir, dirPerms)
+		if dir == "/" {
+			if len(result) > 0 {
+				result = dp + ", " + result
+			} else {
+				result = dp
+			}
+		} else {
+			if len(result) > 0 {
+				result += ", "
+			}
+			result += dp
+		}
+	}
+	return result
+}
+
+// GetBandwidthAsString returns bandwidth limits if defines
+func (u *User) GetBandwidthAsString() string {
+	result := "Download: "
+	if u.DownloadBandwidth > 0 {
+		result += utils.ByteCountSI(u.DownloadBandwidth*1000) + "/s."
+	} else {
+		result += "unlimited."
+	}
+	result += " Upload: "
+	if u.UploadBandwidth > 0 {
+		result += utils.ByteCountSI(u.UploadBandwidth*1000) + "/s."
+	} else {
+		result += "unlimited."
+	}
+	return result
+}
+
+// GetInfoString returns user's info as string.
+// Storage provider, number of public keys, max sessions, uid,
+// gid, denied and allowed IP/Mask are returned
+func (u *User) GetInfoString() string {
+	var result string
+	if u.LastLogin > 0 {
+		t := utils.GetTimeFromMsecSinceEpoch(u.LastLogin)
+		result += fmt.Sprintf("Last login: %v ", t.Format("2006-01-02 15:04:05")) // YYYY-MM-DD HH:MM:SS
+	}
+	if u.FsConfig.Provider == 1 {
+		result += "Storage: S3 "
+	} else if u.FsConfig.Provider == 2 {
+		result += "Storage: GCS "
+	}
+	if len(u.PublicKeys) > 0 {
+		result += fmt.Sprintf("Public keys: %v ", len(u.PublicKeys))
+	}
+	if u.MaxSessions > 0 {
+		result += fmt.Sprintf("Max sessions: %v ", u.MaxSessions)
+	}
+	if u.UID > 0 {
+		result += fmt.Sprintf("UID: %v ", u.UID)
+	}
+	if u.GID > 0 {
+		result += fmt.Sprintf("GID: %v ", u.GID)
+	}
+	if len(u.Filters.DeniedIP) > 0 {
+		result += fmt.Sprintf("Denied IP/Mask: %v ", len(u.Filters.DeniedIP))
+	}
+	if len(u.Filters.AllowedIP) > 0 {
+		result += fmt.Sprintf("Allowed IP/Mask: %v ", len(u.Filters.AllowedIP))
+	}
+	return result
+}
+
+// GetExpirationDateAsString returns expiration date formatted as YYYY-MM-DD
+func (u *User) GetExpirationDateAsString() string {
+	if u.ExpirationDate > 0 {
+		t := utils.GetTimeFromMsecSinceEpoch(u.ExpirationDate)
+		return t.Format("2006-01-02")
+	}
+	return ""
+}
+
+// GetAllowedIPAsString returns the allowed IP as comma separated string
+func (u User) GetAllowedIPAsString() string {
+	result := ""
+	for _, IPMask := range u.Filters.AllowedIP {
+		if len(result) > 0 {
+			result += ","
+		}
+		result += IPMask
+	}
+	return result
+}
+
+// GetDeniedIPAsString returns the denied IP as comma separated string
+func (u User) GetDeniedIPAsString() string {
+	result := ""
+	for _, IPMask := range u.Filters.DeniedIP {
+		if len(result) > 0 {
+			result += ","
+		}
+		result += IPMask
+	}
+	return result
+}
+
+func (u *User) getACopy() User {
+	pubKeys := make([]string, len(u.PublicKeys))
+	copy(pubKeys, u.PublicKeys)
+	virtualFolders := make([]vfs.VirtualFolder, len(u.VirtualFolders))
+	copy(virtualFolders, u.VirtualFolders)
+	permissions := make(map[string][]string)
+	for k, v := range u.Permissions {
+		perms := make([]string, len(v))
+		copy(perms, v)
+		permissions[k] = perms
+	}
+	filters := UserFilters{}
+	filters.AllowedIP = make([]string, len(u.Filters.AllowedIP))
+	copy(filters.AllowedIP, u.Filters.AllowedIP)
+	filters.DeniedIP = make([]string, len(u.Filters.DeniedIP))
+	copy(filters.DeniedIP, u.Filters.DeniedIP)
+	filters.DeniedLoginMethods = make([]string, len(u.Filters.DeniedLoginMethods))
+	copy(filters.DeniedLoginMethods, u.Filters.DeniedLoginMethods)
+	filters.FileExtensions = make([]ExtensionsFilter, len(u.Filters.FileExtensions))
+	copy(filters.FileExtensions, u.Filters.FileExtensions)
+	fsConfig := Filesystem{
+		Provider: u.FsConfig.Provider,
+		S3Config: vfs.S3FsConfig{
+			Bucket:            u.FsConfig.S3Config.Bucket,
+			Region:            u.FsConfig.S3Config.Region,
+			AccessKey:         u.FsConfig.S3Config.AccessKey,
+			AccessSecret:      u.FsConfig.S3Config.AccessSecret,
+			Endpoint:          u.FsConfig.S3Config.Endpoint,
+			StorageClass:      u.FsConfig.S3Config.StorageClass,
+			KeyPrefix:         u.FsConfig.S3Config.KeyPrefix,
+			UploadPartSize:    u.FsConfig.S3Config.UploadPartSize,
+			UploadConcurrency: u.FsConfig.S3Config.UploadConcurrency,
+		},
+		GCSConfig: vfs.GCSFsConfig{
+			Bucket:               u.FsConfig.GCSConfig.Bucket,
+			CredentialFile:       u.FsConfig.GCSConfig.CredentialFile,
+			AutomaticCredentials: u.FsConfig.GCSConfig.AutomaticCredentials,
+			StorageClass:         u.FsConfig.GCSConfig.StorageClass,
+			KeyPrefix:            u.FsConfig.GCSConfig.KeyPrefix,
+		},
+	}
+
+	return User{
+		ID:                u.ID,
+		Username:          u.Username,
+		Password:          u.Password,
+		PublicKeys:        pubKeys,
+		HomeDir:           u.HomeDir,
+		VirtualFolders:    virtualFolders,
+		UID:               u.UID,
+		GID:               u.GID,
+		MaxSessions:       u.MaxSessions,
+		QuotaSize:         u.QuotaSize,
+		QuotaFiles:        u.QuotaFiles,
+		Permissions:       permissions,
+		UsedQuotaSize:     u.UsedQuotaSize,
+		UsedQuotaFiles:    u.UsedQuotaFiles,
+		LastQuotaUpdate:   u.LastQuotaUpdate,
+		UploadBandwidth:   u.UploadBandwidth,
+		DownloadBandwidth: u.DownloadBandwidth,
+		Status:            u.Status,
+		ExpirationDate:    u.ExpirationDate,
+		LastLogin:         u.LastLogin,
+		Filters:           filters,
+		FsConfig:          fsConfig,
+	}
+}
+
+func (u *User) getNotificationFieldsAsSlice(action string) []string {
+	return []string{action, u.Username,
+		strconv.FormatInt(u.ID, 10),
+		strconv.FormatInt(int64(u.Status), 10),
+		strconv.FormatInt(u.ExpirationDate, 10),
+		u.HomeDir,
+		strconv.FormatInt(int64(u.UID), 10),
+		strconv.FormatInt(int64(u.GID), 10),
+	}
+}
+
+func (u *User) getNotificationFieldsAsEnvVars(action string) []string {
+	return []string{fmt.Sprintf("SFTPGO_USER_ACTION=%v", action),
+		fmt.Sprintf("SFTPGO_USER_USERNAME=%v", u.Username),
+		fmt.Sprintf("SFTPGO_USER_PASSWORD=%v", u.Password),
+		fmt.Sprintf("SFTPGO_USER_ID=%v", u.ID),
+		fmt.Sprintf("SFTPGO_USER_STATUS=%v", u.Status),
+		fmt.Sprintf("SFTPGO_USER_EXPIRATION_DATE=%v", u.ExpirationDate),
+		fmt.Sprintf("SFTPGO_USER_HOME_DIR=%v", u.HomeDir),
+		fmt.Sprintf("SFTPGO_USER_UID=%v", u.UID),
+		fmt.Sprintf("SFTPGO_USER_GID=%v", u.GID),
+		fmt.Sprintf("SFTPGO_USER_QUOTA_FILES=%v", u.QuotaFiles),
+		fmt.Sprintf("SFTPGO_USER_QUOTA_SIZE=%v", u.QuotaSize),
+		fmt.Sprintf("SFTPGO_USER_UPLOAD_BANDWIDTH=%v", u.UploadBandwidth),
+		fmt.Sprintf("SFTPGO_USER_DOWNLOAD_BANDWIDTH=%v", u.DownloadBandwidth),
+		fmt.Sprintf("SFTPGO_USER_MAX_SESSIONS=%v", u.MaxSessions),
+		fmt.Sprintf("SFTPGO_USER_FS_PROVIDER=%v", u.FsConfig.Provider)}
+}
+
+func (u *User) getGCSCredentialsFilePath() string {
+	return filepath.Join(credentialsDirPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username))
 }

docker/README.md

@@ -1,5 +1,5 @@
-## Dockerfile examples
+# Dockerfile examples
 
 Sample Dockerfiles for `sftpgo` daemon and the REST API CLI.
 
-We don't want to add a `Dockerfile` for each single `sftpgo` configuration options or data provider. You can use the docker configurations here as starting point that you can customize to run `sftpgo` with [Docker](http://www.docker.io "Docker").
+We don't want to add a `Dockerfile` for each single `sftpgo` configuration options or data provider. You can use the docker configurations here as starting point that you can customize to run `sftpgo` with [Docker](http://www.docker.io "Docker").

docker/rest-api-cli/Dockerfile

@@ -2,7 +2,7 @@ FROM debian:latest
 LABEL maintainer="nicola.murino@gmail.com"
 RUN apt-get update && apt-get install -y curl python3-requests python3-pygments
 
-RUN curl https://raw.githubusercontent.com/drakkan/sftpgo/master/scripts/sftpgo_api_cli.py --output /usr/bin/sftpgo_api_cli.py
+RUN curl https://raw.githubusercontent.com/drakkan/sftpgo/master/examples/rest-api-cli/sftpgo_api_cli.py --output /usr/bin/sftpgo_api_cli.py
 
 ENTRYPOINT ["python3", "/usr/bin/sftpgo_api_cli.py" ]
 CMD []

docker/sftpgo/alpine/Dockerfile

@@ -3,22 +3,29 @@ FROM golang:alpine as builder
 RUN apk add --no-cache git gcc g++ ca-certificates \
   && go get -d github.com/drakkan/sftpgo
 WORKDIR /go/src/github.com/drakkan/sftpgo
-# uncomment the next line to get the latest stable version instead of the latest git
-#RUN git checkout `git rev-list --tags --max-count=1`
-RUN go build -i -ldflags "-s -w -X github.com/drakkan/sftpgo/utils.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/utils.date=`date -u +%FT%TZ`" -o /go/bin/sftpgo
+ARG TAG
+ARG FEATURES
+# Use --build-arg TAG=LATEST for latest tag. Use e.g. --build-arg TAG=0.9.6 for a specific tag/commit. Otherwise HEAD (master) is built.
+RUN git checkout $(if [ "${TAG}" = LATEST ]; then echo `git rev-list --tags --max-count=1`; elif [ -n "${TAG}" ]; then echo "${TAG}"; else echo HEAD; fi)
+RUN go build -i $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o /go/bin/sftpgo
 
 FROM alpine:latest
 
-RUN  apk add --no-cache ca-certificates su-exec \
-  && mkdir -p /data /etc/sftpgo /srv/sftpgo/config
+RUN apk add --no-cache ca-certificates su-exec \
+  && mkdir -p /data /etc/sftpgo /srv/sftpgo/config /srv/sftpgo/web /srv/sftpgo/backups
+
+# git and rsync are optional, uncomment the next line to add support for them if needed.
+#RUN apk add --no-cache git rsync
 
 COPY --from=builder /go/bin/sftpgo /bin/
 COPY --from=builder /go/src/github.com/drakkan/sftpgo/sftpgo.json /etc/sftpgo/sftpgo.json
+COPY --from=builder /go/src/github.com/drakkan/sftpgo/templates /srv/sftpgo/web/templates
+COPY --from=builder /go/src/github.com/drakkan/sftpgo/static /srv/sftpgo/web/static
 COPY docker-entrypoint.sh /bin/entrypoint.sh
 RUN chmod +x /bin/entrypoint.sh
 
-VOLUME [ "/data", "/srv/sftpgo/config" ]
+VOLUME [ "/data", "/srv/sftpgo/config", "/srv/sftpgo/backups" ]
 EXPOSE 2022 8080
 
 ENTRYPOINT ["/bin/entrypoint.sh"]
-CMD []
+CMD ["serve"]

docker/sftpgo/alpine/README.md

@@ -2,38 +2,56 @@
 
 This DockerFile is made to build image to host multiple instances of SFTPGo started with different users.
 
-### Example
+## Example
+
 > 1003 is a custom uid:gid for this instance of SFTPGo
+
 ```bash
 # Prereq on docker host
 sudo groupadd -g 1003 sftpgrp && \
   sudo useradd -u 1003 -g 1003 sftpuser -d /home/sftpuser/ && \
   sudo -u sftpuser mkdir /home/sftpuser/{conf,data} && \
-  curl https://raw.githubusercontent.com/drakkan/sftpgo/master/sql/sqlite/20190828.sql | sqlite3 /home/sftpuser/conf/sftpgo.db && \
   curl https://raw.githubusercontent.com/drakkan/sftpgo/master/sftpgo.json -o /home/sftpuser/conf/sftpgo.json
 
-# Get and build SFTPGo image
+# Edit sftpgo.json as you need
+
+# Get and build SFTPGo image.
+# Add --build-arg TAG=LATEST to build the latest tag or e.g. TAG=0.9.6 for a specific tag/commit.
+# Add --build-arg FEATURES=<build features comma separated> to specify the features to build.
 git clone https://github.com/drakkan/sftpgo.git && \
   cd sftpgo && \
   sudo docker build -t sftpgo docker/sftpgo/alpine/
 
-# Starting image
+# Initialize the configured provider. For PostgreSQL and MySQL providers you need to create the configured database and the "initprovider" command will create the required tables.
 sudo docker run --name sftpgo \
+  -e PUID=1003 \
+  -e GUID=1003 \
+  -v /home/sftpuser/conf/:/srv/sftpgo/config \
+  sftpgo initprovider -c /srv/sftpgo/config
+
+# Start the image
+sudo docker rm sftpgo && sudo docker run --name sftpgo \
   -e SFTPGO_LOG_FILE_PATH= \
   -e SFTPGO_CONFIG_DIR=/srv/sftpgo/config \
+  -e SFTPGO_HTTPD__TEMPLATES_PATH=/srv/sftpgo/web/templates \
+  -e SFTPGO_HTTPD__STATIC_FILES_PATH=/srv/sftpgo/web/static \
+  -e SFTPGO_HTTPD__BACKUPS_PATH=/srv/sftpgo/backups \
   -p 8080:8080 \
   -p 2022:2022 \
   -e PUID=1003 \
   -e GUID=1003 \
   -v /home/sftpuser/conf/:/srv/sftpgo/config \
   -v /home/sftpuser/data:/data \
+  -v /home/sftpuser/backups:/srv/sftpgo/backups \
   sftpgo
 ```
-The script `entrypoint.sh` makes sure to correct the permissions of directories and start the process with the right user
+
+The script `entrypoint.sh` makes sure to correct the permissions of directories and start the process with the right user.
 
 Several images can be run with different parameters.
 
-### Custom systemd script
+## Custom systemd script
+
 An example of systemd script is present [here](sftpgo.service), with `Environment` parameter to set `PUID` and `GUID`
 
-`WorkingDirectory` parameter must be exist with one file in this directory like `sftpgo-${PUID}.env` corresponding to the variable file for SFTPGo instance.
+`WorkingDirectory` parameter must be exist with one file in this directory like `sftpgo-${PUID}.env` corresponding to the variable file for SFTPGo instance.

docker/sftpgo/alpine/docker-entrypoint.sh

@@ -2,6 +2,6 @@
 
 set -eu
 
-chown -R "${PUID}:${GUID}" /data /etc/sftpgo /srv/sftpgo/config \
+chown -R "${PUID}:${GUID}" /data /etc/sftpgo /srv/sftpgo/config /srv/sftpgo/backups \
 	&& exec su-exec "${PUID}:${GUID}" \
-  /bin/sftpgo serve "$@"
+  /bin/sftpgo "$@"

docker/sftpgo/alpine/sftpgo.service

@@ -8,13 +8,15 @@ Group=root
 WorkingDirectory=/etc/sftpgo
 Environment=PUID=1003
 Environment=GUID=1003
-EnvironmentFile=-/etc/sysconfig/sftpgo.conf
+EnvironmentFile=-/etc/sysconfig/sftpgo.env
 ExecStartPre=-docker kill sftpgo
 ExecStartPre=-docker rm sftpgo
 ExecStart=docker run --name sftpgo \
   --env-file sftpgo-${PUID}.env \
   -e PUID=${PUID} \
   -e GUID=${GUID} \
+  -e SFTPGO_HTTPD__TEMPLATES_PATH=/srv/sftpgo/web/templates \
+  -e SFTPGO_HTTPD__STATIC_FILES_PATH=/srv/sftpgo/web/static \
   -p 8080:8080 \
   -p 2022:2022 \
   -v /home/sftpuser/conf/:/srv/sftpgo/config \

docker/sftpgo/debian/Dockerfile

@@ -3,20 +3,30 @@ FROM golang:latest as buildenv
 LABEL maintainer="nicola.murino@gmail.com"
 RUN go get -d github.com/drakkan/sftpgo
 WORKDIR /go/src/github.com/drakkan/sftpgo
-# uncomment the next line to get the latest stable version instead of the latest git
-#RUN git checkout `git rev-list --tags --max-count=1`
-RUN go build -i -ldflags "-s -w -X github.com/drakkan/sftpgo/utils.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/utils.date=`date -u +%FT%TZ`" -o sftpgo
+ARG TAG
+ARG FEATURES
+# Use --build-arg TAG=LATEST for latest tag. Use e.g. --build-arg TAG=0.9.6 for a specific tag/commit. Otherwise HEAD (master) is built.
+RUN git checkout $(if [ "${TAG}" = LATEST ]; then echo `git rev-list --tags --max-count=1`; elif [ -n "${TAG}" ]; then echo "${TAG}"; else echo HEAD; fi)
+RUN go build -i $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
 
 # now define the run environment
 FROM debian:latest
 
+# ca-certificates is needed for Cloud Storage Support and to expose the REST API over HTTPS.
+RUN apt-get update && apt-get install -y ca-certificates
+
+# git and rsync are optional, uncomment the next line to add support for them if needed.
+#RUN apt-get update && apt-get install -y git rsync
+
 ARG BASE_DIR=/app
 ARG DATA_REL_DIR=data
 ARG CONFIG_REL_DIR=config
+ARG BACKUP_REL_DIR=backups
 ARG USERNAME=sftpgo
 ARG GROUPNAME=sftpgo
 ARG UID=515
 ARG GID=515
+ARG WEB_REL_PATH=web
 
 # HOME_DIR for sftpgo itself
 ENV HOME_DIR=${BASE_DIR}/${USERNAME}
@@ -24,8 +34,11 @@ ENV HOME_DIR=${BASE_DIR}/${USERNAME}
 ENV DATA_DIR=${BASE_DIR}/${DATA_REL_DIR}
 # CONFIG_DIR, this is a volume to persist the daemon private keys, configuration file ecc..
 ENV CONFIG_DIR=${BASE_DIR}/${CONFIG_REL_DIR}
+# BACKUPS_DIR, this is a volume to store backups done using "dumpdata" REST API
+ENV BACKUPS_DIR=${BASE_DIR}/${BACKUP_REL_DIR}
+ENV WEB_DIR=${BASE_DIR}/${WEB_REL_PATH}
 
-RUN mkdir -p ${DATA_DIR} ${CONFIG_DIR}
+RUN mkdir -p ${DATA_DIR} ${CONFIG_DIR} ${WEB_DIR} ${BACKUPS_DIR}
 RUN groupadd --system -g ${GID} ${GROUPNAME}
 RUN useradd --system --create-home --no-log-init --home-dir ${HOME_DIR} --comment "SFTPGo user" --shell /bin/false --gid ${GID} --uid ${UID} ${USERNAME}
 
@@ -36,7 +49,9 @@ COPY --from=buildenv /go/src/github.com/drakkan/sftpgo/sftpgo bin/sftpgo
 # default config file to use if no config file is found inside the CONFIG_DIR volume.
 # You can override each configuration options via env vars too
 COPY --from=buildenv /go/src/github.com/drakkan/sftpgo/sftpgo.json .config/sftpgo/
-RUN chown -R ${UID}:${GID} ${DATA_DIR}
+COPY --from=buildenv /go/src/github.com/drakkan/sftpgo/templates ${WEB_DIR}/templates
+COPY --from=buildenv /go/src/github.com/drakkan/sftpgo/static ${WEB_DIR}/static
+RUN chown -R ${UID}:${GID} ${DATA_DIR} ${BACKUPS_DIR}
 
 # run as non root user
 USER ${USERNAME}
@@ -44,13 +59,17 @@ USER ${USERNAME}
 EXPOSE 2022 8080
 
 # the defined volumes must have write access for the UID and GID defined above
-VOLUME [ "$DATA_DIR", "$CONFIG_DIR" ]
+VOLUME [ "$DATA_DIR", "$CONFIG_DIR", "$BACKUPS_DIR" ]
 
 # override some default configuration options using env vars
 ENV SFTPGO_CONFIG_DIR=${CONFIG_DIR}
 # setting SFTPGO_LOG_FILE_PATH to an empty string will log to stdout
-ENV SFTPGO_LOG_FILE_PATH=${CONFIG_DIR}/sftpgo.log
+ENV SFTPGO_LOG_FILE_PATH=""
 ENV SFTPGO_HTTPD__BIND_ADDRESS=""
+ENV SFTPGO_HTTPD__TEMPLATES_PATH=${WEB_DIR}/templates
+ENV SFTPGO_HTTPD__STATIC_FILES_PATH=${WEB_DIR}/static
+ENV SFTPGO_DATA_PROVIDER__USERS_BASE_DIR=${DATA_DIR}
+ENV SFTPGO_HTTPD__BACKUPS_PATH=${BACKUPS_DIR}
 
 ENTRYPOINT ["sftpgo"]
-CMD ["serve"]
+CMD ["serve"]

docker/sftpgo/debian/README.md

@@ -1,4 +1,4 @@
-## Dockerfile based on Debian stable
+# Dockerfile based on Debian stable
 
 Please read the comments inside the `Dockerfile` to learn how to customize things for your setup.
 
@@ -8,15 +8,48 @@ You can build the container image using `docker build`, for example:
 docker build -t="drakkan/sftpgo" .
 ```
 
-and you can run the Dockerfile using something like this:
+This will build master of github.com/drakkan/sftpgo.
+
+To build the latest tag you can add `--build-arg TAG=LATEST` and to build a specific tag/commit you can use for example `TAG=0.9.6`, like this:
 
 ```bash
-docker run --name sftpgo -p 8080:8080 -p 2022:2022 --mount type=bind,source=/srv/sftpgo/data,target=/app/data --mount type=bind,source=/srv/sftpgo/config,target=/app/config drakkan/sftpgo
+docker build -t="drakkan/sftpgo" --build-arg TAG=0.9.6 .
 ```
 
-where  `/srv/sftpgo/data` and `/srv/sftpgo/config` are two folders on the host system with write access for UID/GID defined inside the `Dockerfile`. You can choose to create a new user, on the host system, with a matching UID/GID pair or simply do something like:
-
+To specify the features to build you can add `--build-arg FEATURES=<build features comma separated>`. For example you can disable SQLite and S3 support like this:
 
 ```bash
-chown -R <UID>:<GID> /srv/sftpgo/data /srv/sftpgo/config
-```
+docker build -t="drakkan/sftpgo" --build-arg FEATURES=nosqlite,nos3 .
+```
+
+Please take a look at the [build from source](./../../../docs/build-from-source.md) documentation for the complete list of the features that can be disabled.
+
+Now create the required folders on the host system, for example:
+
+```bash
+sudo mkdir -p /srv/sftpgo/data /srv/sftpgo/config /srv/sftpgo/backups
+```
+
+and give write access to them to the UID/GID defined inside the `Dockerfile`. You can choose to create a new user, on the host system, with a matching UID/GID pair, or simply do something like this:
+
+```bash
+sudo chown -R <UID>:<GID> /srv/sftpgo/data /srv/sftpgo/config /srv/sftpgo/backups
+```
+
+Download the default configuration file and edit it as you need:
+
+```bash
+sudo curl https://raw.githubusercontent.com/drakkan/sftpgo/master/sftpgo.json -o /srv/sftpgo/config/sftpgo.json
+```
+
+Initialize the configured provider. For PostgreSQL and MySQL providers you need to create the configured database and the `initprovider` command will create the required tables:
+
+```bash
+docker run --name sftpgo --mount type=bind,source=/srv/sftpgo/config,target=/app/config drakkan/sftpgo initprovider -c /app/config
+```
+
+and finally you can run the image using something like this:
+
+```bash
+docker rm sftpgo && docker run --name sftpgo -p 8080:8080 -p 2022:2022 --mount type=bind,source=/srv/sftpgo/data,target=/app/data --mount type=bind,source=/srv/sftpgo/config,target=/app/config --mount type=bind,source=/srv/sftpgo/backups,target=/app/backups drakkan/sftpgo
+```

docs/account.md

@@ -0,0 +1,65 @@
+# Account's configuration properties
+
+For each account, the following properties can be configured:
+
+- `username`
+- `password` used for password authentication. For users created using SFTPGo REST API, if the password has no known hashing algo prefix, it will be stored using argon2id. SFTPGo supports checking passwords stored with bcrypt, pbkdf2, md5crypt and sha512crypt too. For pbkdf2 the supported format is `$<algo>$<iterations>$<salt>$<hashed pwd base64 encoded>`, where algo is `pbkdf2-sha1` or `pbkdf2-sha256` or `pbkdf2-sha512` or `$pbkdf2-b64salt-sha256$`. For example the `pbkdf2-sha256` of the word `password` using 150000 iterations and `E86a9YMX3zC7` as salt must be stored as `$pbkdf2-sha256$150000$E86a9YMX3zC7$R5J62hsSq+pYw00hLLPKBbcGXmq7fj5+/M0IFoYtZbo=`. In pbkdf2 variant with `b64salt` the salt is base64 encoded. For bcrypt the format must be the one supported by golang's [crypto/bcrypt](https://godoc.org/golang.org/x/crypto/bcrypt) package, for example the password `secret` with cost `14` must be stored as `$2a$14$ajq8Q7fbtFRQvXpdCq7Jcuy.Rx1h/L4J60Otx.gyNLbAYctGMJ9tK`. For md5crypt and sha512crypt we support the format used in `/etc/shadow` with the `$1$` and `$6$` prefix, this is useful if you are migrating from Unix system user accounts. We support Apache md5crypt (`$apr1$` prefix) too. Using the REST API you can send a password hashed as bcrypt, pbkdf2, md5crypt or sha512crypt and it will be stored as is.
+- `public_keys` array of public keys. At least one public key or the password is mandatory.
+- `status` 1 means "active", 0 "inactive". An inactive account cannot login.
+- `expiration_date` expiration date as unix timestamp in milliseconds. An expired account cannot login. 0 means no expiration.
+- `home_dir` the user cannot upload or download files outside this directory. Must be an absolute path. A local home directory is required for Cloud Storage Backends too: in this case it will store temporary files.
+- `virtual_folders` list of mappings between virtual SFTP/SCP paths and local filesystem paths outside the user home directory. More information can be found [here](./virtual-folders.md)
+- `uid`, `gid`. If SFTPGo runs as root system user then the created files and directories will be assigned to this system uid/gid. Ignored on windows or if SFTPGo runs as non root user: in this case files and directories for all SFTP users will be owned by the system user that runs SFTPGo.
+- `max_sessions` maximum concurrent sessions. 0 means unlimited.
+- `quota_size` maximum size allowed as bytes. 0 means unlimited.
+- `quota_files` maximum number of files allowed. 0 means unlimited.
+- `permissions` for SFTP paths. The following per directory permissions are supported:
+  - `*` all permissions are granted
+  - `list` list items is allowed
+  - `download` download files is allowed
+  - `upload` upload files is allowed
+  - `overwrite` overwrite an existing file, while uploading, is allowed. `upload` permission is required to allow file overwrite
+  - `delete` delete files or directories is allowed
+  - `rename` rename a file or a directory is allowed if this permission is granted on source and target path. You can enable rename in a more controlled way granting `delete` permission on source directory and `upload`/`create_dirs`/`create_symlinks` permissions on target directory
+  - `create_dirs` create directories is allowed
+  - `create_symlinks` create symbolic links is allowed
+  - `chmod` changing file or directory permissions is allowed. On Windows, only the 0200 bit (owner writable) of mode is used; it controls whether the file's read-only attribute is set or cleared. The other bits are currently unused. Use mode 0400 for a read-only file and 0600 for a readable+writable file.
+  - `chown` changing file or directory owner and group is allowed. Changing owner and group is not supported on Windows.
+  - `chtimes` changing file or directory access and modification time is allowed
+- `upload_bandwidth` maximum upload bandwidth as KB/s, 0 means unlimited.
+- `download_bandwidth` maximum download bandwidth as KB/s, 0 means unlimited.
+- `allowed_ip`, List of IP/Mask allowed to login. Any IP address not contained in this list cannot login. IP/Mask must be in CIDR notation as defined in RFC 4632 and RFC 4291, for example "192.0.2.0/24" or "2001:db8::/32"
+- `denied_ip`, List of IP/Mask not allowed to login. If an IP address is both allowed and denied then login will be denied
+- `denied_login_methods`, List of login methods not allowed. To enable multi-step authentication you have to allow only multi-step login methods. The following login methods are supported:
+  - `publickey`
+  - `password`
+  - `keyboard-interactive`
+  - `publickey+password`
+  - `publickey+keyboard-interactive`
+- `file_extensions`, list of struct. These restrictions do not apply to files listing for performance reasons, so a denied file cannot be downloaded/overwritten/renamed but it will still be listed in the list of files. Please note that these restrictions can be easily bypassed. Each struct contains the following fields:
+  - `allowed_extensions`, list of, case insensitive, allowed files extension. Shell like expansion is not supported so you have to specify `.jpg` and not `*.jpg`. Any file that does not end with this suffix will be denied
+  - `denied_extensions`, list of, case insensitive, denied files extension. Denied file extensions are evaluated before the allowed ones
+  - `path`, SFTP/SCP path, if no other specific filter is defined, the filter apply for sub directories too. For example if filters are defined for the paths `/` and `/sub` then the filters for `/` are applied for any file outside the `/sub` directory
+- `fs_provider`, filesystem to serve via SFTP. Local filesystem and S3 Compatible Object Storage are supported
+- `s3_bucket`, required for S3 filesystem
+- `s3_region`, required for S3 filesystem. Must match the region for your bucket. You can find here the list of available [AWS regions](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). For example if your bucket is at `Frankfurt` you have to set the region to `eu-central-1`
+- `s3_access_key`
+- `s3_access_secret`, if provided it is stored encrypted (AES-256-GCM). You can leave access key and access secret blank to use credentials from environment
+- `s3_endpoint`, specifies a S3 endpoint (server) different from AWS. It is not required if you are connecting to AWS
+- `s3_storage_class`, leave blank to use the default or specify a valid AWS [storage class](https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html)
+- `s3_key_prefix`, allows to restrict access to the folder identified by this prefix and its contents
+- `s3_upload_part_size`, the buffer size for multipart uploads (MB). Zero means the default (5 MB). Minimum is 5
+- `s3_upload_concurrency` how many parts are uploaded in parallel
+- `gcs_bucket`, required for GCS filesystem
+- `gcs_credentials`, Google Cloud Storage JSON credentials base64 encoded
+- `gcs_automatic_credentials`, integer. Set to 1 to use Application Default Credentials strategy or set to 0 to use explicit credentials via `gcs_credentials`
+- `gcs_storage_class`
+- `gcs_key_prefix`, allows to restrict access to the folder identified by this prefix and its contents
+
+These properties are stored inside the data provider.
+
+If you want to use your existing accounts, you have these options:
+
+- If your accounts are aleady stored inside a supported database, you can create a database view. Since a view is read only, you have to disable user management and quota tracking so SFTPGo will never try to write to the view
+- you can import your users inside SFTPGo. Take a look at [sftpgo_api_cli.py](../examples/rest-api-cli#convert-users-from-other-stores "SFTPGo API CLI example"), it can convert and import users from Linux system users and Pure-FTPd/ProFTPD virtual users
+- you can use an external authentication program

docs/build-from-source.md

@@ -0,0 +1,47 @@
+# Build SFTPGo from source
+
+You can install the package to your [\$GOPATH](https://github.com/golang/go/wiki/GOPATH "GOPATH") with the [go tool](https://golang.org/cmd/go/ "go command") from shell:
+
+```bash
+go get -u github.com/drakkan/sftpgo
+```
+
+Or you can download the sources and use `go build`.
+
+Make sure [Git](https://git-scm.com/downloads) is installed on your machine and in your system's `PATH`.
+
+The following build tags are available:
+
+- `nogcs`, disable Google Cloud Storage backend, default enabled
+- `nos3`, disable S3 Compabible Object Storage backends, default enabled
+- `nobolt`, disable Bolt data provider, default enabled
+- `nomysql`, disable MySQL data provider, default enabled
+- `nopgsql`, disable PostgreSQL data provider, default enabled
+- `nosqlite`, disable SQLite data provider, default enabled
+- `noportable`, disable portable mode, default enabled
+- `nometrics`, disable Prometheus metrics, default enabled
+
+If no build tag is specified the build will include the default features.
+
+The optional [SQLite driver](https://github.com/mattn/go-sqlite3 "go-sqlite3") is a `CGO` package and so it requires a `C` compiler at build time.
+On Linux and macOS, a compiler is easy to install or already installed. On Windows, you need to download [MinGW-w64](https://sourceforge.net/projects/mingw-w64/files/) and build SFTPGo from its command prompt.
+
+The compiler is a build time only dependency. It is not required at runtime.
+
+Version info, such as git commit and build date, can be embedded setting the following string variables at build time:
+
+- `github.com/drakkan/sftpgo/version.commit`
+- `github.com/drakkan/sftpgo/version.date`
+
+For example, you can build using the following command:
+
+```bash
+go build -i -tags nogcs,nos3,nosqlite -ldflags "-s -w -X github.com/drakkan/sftpgo/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/version.date=`date -u +%FT%TZ`" -o sftpgo
+```
+
+You should get a version that includes git commit, build date and available features like this one:
+
+```bash
+$ ./sftpgo -v
+SFTPGo 0.9.6-dev-b30614e-dirty-2020-06-19T11:04:56Z +metrics -gcs -s3 +bolt +mysql +pgsql -sqlite +portable
+```

docs/custom-actions.md

@@ -0,0 +1,87 @@
+# Custom Actions
+
+The `actions` struct inside the "sftpd" configuration section allows to configure the actions for file operations and SSH commands.
+The `hook` can be defined as the absolute path of your program or an HTTP URL.
+
+The `upload` condition includes both uploads to new files and overwrite of existing files. If an upload is aborted for quota limits SFTPGo tries to remove the partial file, so if the notification reports a zero size file and a quota exceeded error the file has been deleted. The `ssh_cmd` condition will be triggered after a command is successfully executed via SSH. `scp` will trigger the `download` and `upload` conditions and not `ssh_cmd`.
+The notification will indicate if an error is detected and so, for example, a partial file is uploaded.
+The `pre-delete` action, if defined, will be called just before files deletion. If the external command completes with a zero exit status or the HTTP notification response code is `200` then SFTPGo will assume that the file was already deleted/moved and so it will not try to remove the file and it will not execute the hook defined for the `delete` action.
+
+If the `hook` defines a path to an external program, then this program is invoked with the following arguments:
+
+- `action`, string, possible values are: `download`, `upload`, `pre-delete`,`delete`, `rename`, `ssh_cmd`
+- `username`
+- `path` is the full filesystem path, can be empty for some ssh commands
+- `target_path`, non-empty for `rename` action and for `sftpgo-copy` SSH command
+- `ssh_cmd`, non-empty for `ssh_cmd` action
+
+The external program can also read the following environment variables:
+
+- `SFTPGO_ACTION`
+- `SFTPGO_ACTION_USERNAME`
+- `SFTPGO_ACTION_PATH`
+- `SFTPGO_ACTION_TARGET`, non-empty for `rename` `SFTPGO_ACTION`
+- `SFTPGO_ACTION_SSH_CMD`, non-empty for `ssh_cmd` `SFTPGO_ACTION`
+- `SFTPGO_ACTION_FILE_SIZE`, non-empty for `upload`, `download` and `delete` `SFTPGO_ACTION`
+- `SFTPGO_ACTION_FS_PROVIDER`, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend
+- `SFTPGO_ACTION_BUCKET`, non-empty for S3 and GCS backends
+- `SFTPGO_ACTION_ENDPOINT`, non-empty for S3 backend if configured
+- `SFTPGO_ACTION_STATUS`, integer. 0 means a generic error occurred. 1 means no error, 2 means quota exceeded error
+
+Previous global environment variables aren't cleared when the script is called.
+The program must finish within 30 seconds.
+
+If the `hook` defines an HTTP URL then this URL will be invoked as HTTP POST. The request body will contain a JSON serialized struct with the following fields:
+
+- `action`
+- `username`
+- `path`
+- `target_path`, not null for `rename` action
+- `ssh_cmd`, not null for `ssh_cmd` action
+- `file_size`, not null for `upload`, `download`, `delete` actions
+- `fs_provider`, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend
+- `bucket`, not null for S3 and GCS backends
+- `endpoint`, not null for S3 backend if configured
+- `status`, integer. 0 means a generic error occurred. 1 means no error, 2 means quota exceeded error
+
+The HTTP request will use the global configuration for HTTP clients.
+
+The `actions` struct inside the "data_provider" configuration section allows you to configure actions on user add, update, delete.
+
+Actions will not be fired for internal updates, such as the last login or the user quota fields, or after external authentication.
+
+If the `hook` defines a path to an external program, then this program is invoked with the following arguments:
+
+- `action`, string, possible values are: `add`, `update`, `delete`
+- `username`
+- `ID`
+- `status`
+- `expiration_date`
+- `home_dir`
+- `uid`
+- `gid`
+
+The external program can also read the following environment variables:
+
+- `SFTPGO_USER_ACTION`
+- `SFTPGO_USER_USERNAME`
+- `SFTPGO_USER_PASSWORD`, hashed password as stored inside the data provider, can be empty if the user does not login using a password
+- `SFTPGO_USER_ID`
+- `SFTPGO_USER_STATUS`
+- `SFTPGO_USER_EXPIRATION_DATE`
+- `SFTPGO_USER_HOME_DIR`
+- `SFTPGO_USER_UID`
+- `SFTPGO_USER_GID`
+- `SFTPGO_USER_QUOTA_FILES`
+- `SFTPGO_USER_QUOTA_SIZE`
+- `SFTPGO_USER_UPLOAD_BANDWIDTH`
+- `SFTPGO_USER_DOWNLOAD_BANDWIDTH`
+- `SFTPGO_USER_MAX_SESSIONS`
+- `SFTPGO_USER_FS_PROVIDER`
+
+Previous global environment variables aren't cleared when the script is called.
+The program must finish within 15 seconds.
+
+If the `hook` defines an HTTP URL then this URL will be invoked as HTTP POST. The action is added to the query string, for example `<hook>?action=update`, and the user is sent serialized as JSON inside the POST body with sensitive fields removed.
+
+The HTTP request will use the global configuration for HTTP clients.

docs/dynamic-user-mod.md

@@ -0,0 +1,53 @@
+# Dynamic user creation or modification
+
+Dynamic user creation or modification is supported via an external program or an HTTP URL that can be invoked just before the user login.
+To enable dynamic user modification, you must set the absolute path of your program or an HTTP URL using the `pre_login_hook` key in your configuration file.
+
+The external program can read the following environment variables to get info about the user trying to login:
+
+- `SFTPGO_LOGIND_USER`, it contains the user trying to login serialized as JSON. A JSON serialized user id equal to zero means the user does not exists inside SFTPGo
+- `SFTPGO_LOGIND_METHOD`, possible values are: `password`, `publickey` and `keyboard-interactive`
+
+The program must write, on its the standard output:
+
+- an empty string (or no response at all) if the user should not be created/updated
+- or the SFTPGo user, JSON serialized, if you want create or update the given user
+
+If the hook is an HTTP URL then it will be invoked as HTTP POST. The login method is added to the query string, for example `<http_url>?login_method=password`.
+The request body will contain the user trying to login serialized as JSON. If no modification is needed the HTTP response code must be 204, otherwise the response code must be 200 and the response body a valid SFTPGo user serialized as JSON.
+
+Actions defined for user's updates will not be executed in this case.
+
+The JSON response can include only the fields to update instead of the full user. For example, if you want to disable the user, you can return a response like this:
+
+```json
+{"status": 0}
+```
+
+Please note that if you want to create a new user, the pre-login hook response must include all the mandatory user fields.
+
+The program hook must finish within 30 seconds, the HTTP hook will use the global configuration for HTTP clients.
+
+If an error happens while executing the hook then login will be denied.
+
+"Dynamic user creation or modification" and "External Authentication" are mutally exclusive, they are quite similar, the difference is that "External Authentication" returns an already authenticated user while using "Dynamic users modification" you simply create or update a user. The authentication will be checked inside SFTPGo.
+In other words while using "External Authentication" the external program receives the credentials of the user trying to login (for example the clear text password) and it need to validate them. While using "Dynamic users modification" the pre-login program receives the user stored inside the dataprovider (it includes the hashed password if any) and it can modify it, after the modification SFTPGo will check the credentials of the user trying to login.
+
+Let's see a very basic example. Our sample program will grant access to the existing user `test_user` only in the time range 10:00-18:00. Other users will not be modified since the program will terminate with no output.
+
+```shell
+#!/bin/bash
+
+CURRENT_TIME=`date +%H:%M`
+if [[ "$SFTPGO_LOGIND_USER" =~ "\"test_user\"" ]]
+then
+  if [[ $CURRENT_TIME > "18:00" || $CURRENT_TIME < "10:00" ]]
+  then
+    echo '{"status":0}'
+  else
+    echo '{"status":1}'
+  fi
+fi
+```
+
+Please note that this is a demo program and it might not work in all cases. For example, the username should be obtained by parsing the JSON serialized user and not by searching the username inside the JSON as shown here.

docs/external-auth.md

@@ -0,0 +1,54 @@
+# External Authentication
+
+To enable external authentication, you must set the absolute path of your authentication program or an HTTP URL using the `external_auth_hook` key in your configuration file.
+
+The external program can read the following environment variables to get info about the user trying to authenticate:
+
+- `SFTPGO_AUTHD_USERNAME`
+- `SFTPGO_AUTHD_PASSWORD`, not empty for password authentication
+- `SFTPGO_AUTHD_PUBLIC_KEY`, not empty for public key authentication
+- `SFTPGO_AUTHD_KEYBOARD_INTERACTIVE`, not empty for keyboard interactive authentication
+
+Previous global environment variables aren't cleared when the script is called. The content of these variables is _not_ quoted. They may contain special characters. They are under the control of a possibly malicious remote user.
+The program must write, on its standard output, a valid SFTPGo user serialized as JSON if the authentication succeed or a user with an empty username if the authentication fails.
+
+If the hook is an HTTP URL then it will be invoked as HTTP POST. The request body will contain a JSON serialized struct with the following fields:
+
+- `username`
+- `password`, not empty for password authentication
+- `public_key`, not empty for public key authentication
+- `keyboard_interactive`, not empty for keyboard interactive authentication
+
+If authentication succeed the HTTP response code must be 200 and the response body a valid SFTPGo user serialized as JSON. If the authentication fails the HTTP response code must be != 200 or the response body must be empty.
+
+If the authentication succeeds, the user will be automatically added/updated inside the defined data provider. Actions defined for users added/updated will not be executed in this case.
+The external hook should check authentication only. If there are login restrictions such as user disabled, expired, or login allowed only from specific IP addresses, it is enough to populate the matching user fields, and these conditions will be checked in the same way as for built-in users.
+The program hook must finish within 30 seconds, the HTTP hook timeout will use the global configuration for HTTP clients.
+
+This method is slower than built-in authentication, but it's very flexible as anyone can easily write his own authentication hooks.
+You can also restrict the authentication scope for the hook using the `external_auth_scope` configuration key:
+
+- 0 means all supported authetication scopes. The external hook will be used for password, public key and keyboard interactive authentication
+- 1 means passwords only
+- 2 means public keys only
+- 4 means keyboard interactive only
+
+You can combine the scopes. For example, 3 means password and public key, 5 means password and keyboard interactive, and so on.
+
+Let's see a very basic example. Our sample authentication program will only accept user `test_user` with any password or public key.
+
+```shell
+#!/bin/sh
+
+if test "$SFTPGO_AUTHD_USERNAME" = "test_user"; then
+  echo '{"status":1,"username":"test_user","expiration_date":0,"home_dir":"/tmp/test_user","uid":0,"gid":0,"max_sessions":0,"quota_size":0,"quota_files":100000,"permissions":{"/":["*"],"/somedir":["list","download"]},"upload_bandwidth":0,"download_bandwidth":0,"filters":{"allowed_ip":[],"denied_ip":[]},"public_keys":[]}'
+else
+  echo '{"username":""}'
+fi
+```
+
+An example authentication program allowing to authenticate against an LDAP server can be found inside the source tree [ldapauth](../examples/ldapauth) directory.
+
+An example server, to use as HTTP authentication hook, allowing to authenticate against an LDAP server can be found inside the source tree [ldapauthserver](../examples/ldapauthserver) directory.
+
+If you have an external authentication hook that could be useful to others too, please let us know and/or please send a pull request.

docs/full-configuration.md

@@ -0,0 +1,153 @@
+# Configuring SFTPGo
+
+## Command line options
+
+The SFTPGo executable can be used this way:
+
+```console
+Usage:
+  sftpgo [command]
+
+Available Commands:
+  help         Help about any command
+  initprovider Initializes the configured data provider
+  portable     Serve a single directory
+  serve        Start the SFTP Server
+
+Flags:
+  -h, --help      help for sftpgo
+  -v, --version
+
+ Use "sftpgo [command] --help" for more information about a command
+```
+
+The `serve` command supports the following flags:
+
+- `--config-dir` string. Location of the config dir. This directory should contain the configuration file and is used as the base directory for any files that use a relative path (eg. the private keys for the SFTP server, the SQLite or bblot database if you use SQLite or bbolt as data provider). The default value is "." or the value of `SFTPGO_CONFIG_DIR` environment variable.
+- `--config-file` string. Name of the configuration file. It must be the name of a file stored in `config-dir`, not the absolute path to the configuration file. The specified file name must have no extension because we automatically append JSON, YAML, TOML, HCL and Java extensions when we search for the file. The default value is "sftpgo" (and therefore `sftpgo.json`, `sftpgo.yaml` and so on are searched) or the value of `SFTPGO_CONFIG_FILE` environment variable.
+- `--log-compress` boolean. Determine if the rotated log files should be compressed using gzip. Default `false` or the value of `SFTPGO_LOG_COMPRESS` environment variable (1 or `true`, 0 or `false`). It is unused if `log-file-path` is empty.
+- `--log-file-path` string. Location for the log file, default "sftpgo.log" or the value of `SFTPGO_LOG_FILE_PATH` environment variable. Leave empty to write logs to the standard error.
+- `--log-max-age` int. Maximum number of days to retain old log files. Default 28 or the value of `SFTPGO_LOG_MAX_AGE` environment variable. It is unused if `log-file-path` is empty.
+- `--log-max-backups` int. Maximum number of old log files to retain. Default 5 or the value of `SFTPGO_LOG_MAX_BACKUPS` environment variable. It is unused if `log-file-path` is empty.
+- `--log-max-size` int. Maximum size in megabytes of the log file before it gets rotated. Default 10 or the value of `SFTPGO_LOG_MAX_SIZE` environment variable. It is unused if `log-file-path` is empty.
+- `--log-verbose` boolean. Enable verbose logs. Default `true` or the value of `SFTPGO_LOG_VERBOSE` environment variable (1 or `true`, 0 or `false`).
+
+Log file can be rotated on demand sending a `SIGUSR1` signal on Unix based systems and using the command `sftpgo service rotatelogs` on Windows.
+
+If you don't configure any private host key, the daemon will use `id_rsa` and `id_ecdsa` in the configuration directory. If these files don't exist, the daemon will attempt to autogenerate them (if the user that executes SFTPGo has write access to the `config-dir`). The server supports any private key format supported by [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/keys.go#L33).
+
+## Configuration file
+
+The configuration file contains the following sections:
+
+- **"sftpd"**, the configuration for the SFTP server
+  - `bind_port`, integer. The port used for serving SFTP requests. Default: 2022
+  - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: ""
+  - `idle_timeout`, integer. Time in minutes after which an idle client will be disconnected. 0 means disabled. Default: 15
+  - `max_auth_tries` integer. Maximum number of authentication attempts permitted per connection. If set to a negative number, the number of attempts is unlimited. If set to zero, the number of attempts are limited to 6.
+  - `umask`, string. Umask for the new files and directories. This setting has no effect on Windows. Default: "0022"
+  - `banner`, string. Identification string used by the server. Leave empty to use the default banner. Default `SFTPGo_<version>`, for example `SSH-2.0-SFTPGo_0.9.5`
+  - `upload_mode` integer. 0 means standard: the files are uploaded directly to the requested path. 1 means atomic: files are uploaded to a temporary path and renamed to the requested path when the client ends the upload. Atomic mode avoids problems such as a web server that serves partial files when the files are being uploaded. In atomic mode, if there is an upload error, the temporary file is deleted and so the requested upload path will not contain a partial file. 2 means atomic with resume support: same as atomic but if there is an upload error, the temporary file is renamed to the requested path and not deleted. This way, a client can reconnect and resume the upload.
+  - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions. See the "Custom Actions" paragraph for more details
+    - `execute_on`, list of strings. Valid values are `download`, `upload`, `pre-delete`, `delete`, `rename`, `ssh_cmd`. Leave empty to disable actions.
+    - `command`, string. Deprecated please use `hook`.
+    - `http_notification_url`, a valid URL. Deprecated please use `hook`.
+    - `hook`, string. Absolute path to the command to execute or HTTP URL to notify.
+  - `keys`, struct array. Deprecated, please use `host_keys`.
+    - `private_key`, path to the private key file. It can be a path relative to the config dir or an absolute one.
+  - `host_keys`, list of strings. It contains the daemon's private host keys. Each host key can be defined as a path relative to the configuration directory or an absolute one. If empty, the daemon will search or try to generate `id_rsa` and `id_ecdsa` keys inside the configuration directory. If you configure absolute paths to files named `id_rsa` and/or `id_ecdsa` then SFTPGo will try to generate these keys using the default settings.
+  - `kex_algorithms`, list of strings. Available KEX (Key Exchange) algorithms in preference order. Leave empty to use default values. The supported values can be found here: [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/common.go#L46 "Supported kex algos")
+  - `ciphers`, list of strings. Allowed ciphers. Leave empty to use default values. The supported values can be found here: [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/common.go#L28 "Supported ciphers")
+  - `macs`, list of strings. Available MAC (message authentication code) algorithms in preference order. Leave empty to use default values. The supported values can be found here: [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/common.go#L84 "Supported MACs")
+  - `trusted_user_ca_keys`, list of public keys paths of certificate authorities that are trusted to sign user certificates for authentication. The paths can be absolute or relative to the configuration directory.
+  - `login_banner_file`, path to the login banner file. The contents of the specified file, if any, are sent to the remote user before authentication is allowed. It can be a path relative to the config dir or an absolute one. Leave empty to disable login banner.
+  - `setstat_mode`, integer. 0 means "normal mode": requests for changing permissions, owner/group and access/modification times are executed. 1 means "ignore mode": requests for changing permissions, owner/group and access/modification times are silently ignored.
+  - `enabled_ssh_commands`, list of enabled SSH commands. `*` enables all supported commands. More information can be found [here](./ssh-commands.md).
+  - `keyboard_interactive_auth_program`, string. Deprecated, please use `keyboard_interactive_auth_hook`.
+  - `keyboard_interactive_auth_hook`, string. Absolute path to an external program or an HTTP URL to invoke for keyboard interactive authentication. See the "Keyboard Interactive Authentication" paragraph for more details.
+  - `proxy_protocol`, integer. Support for [HAProxy PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt). If you are running SFTPGo behind a proxy server such as HAProxy, AWS ELB or NGNIX, you can enable the proxy protocol. It provides a convenient way to safely transport connection information such as a client's address across multiple layers of NAT or TCP proxies to get the real client IP address instead of the proxy IP. Both protocol versions 1 and 2 are supported. If the proxy protocol is enabled in SFTPGo then you have to enable the protocol in your proxy configuration too. For example, for HAProxy, add `send-proxy` or `send-proxy-v2` to each server configuration line. The following modes are supported:
+    - 0, disabled
+    - 1, enabled. Proxy header will be used and requests without proxy header will be accepted
+    - 2, required. Proxy header will be used and requests without proxy header will be rejected
+  - `proxy_allowed`, List of IP addresses and IP ranges allowed to send the proxy header:
+    - If `proxy_protocol` is set to 1 and we receive a proxy header from an IP that is not in the list then the connection will be accepted and the header will be ignored
+    - If `proxy_protocol` is set to 2 and we receive a proxy header from an IP that is not in the list then the connection will be rejected
+- **"data_provider"**, the configuration for the data provider
+  - `driver`, string. Supported drivers are `sqlite`, `mysql`, `postgresql`, `bolt`, `memory`
+  - `name`, string. Database name. For driver `sqlite` this can be the database name relative to the config dir or the absolute path to the SQLite database. For driver `memory` this is the (optional) path relative to the config dir or the absolute path to the users dump, obtained using the `dumpdata` REST API, to load. This dump will be loaded at startup and can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. The `memory` provider will not modify the provided file so quota usage and last login will not be persisted
+  - `host`, string. Database host. Leave empty for drivers `sqlite`, `bolt` and `memory`
+  - `port`, integer. Database port. Leave empty for drivers `sqlite`, `bolt` and `memory`
+  - `username`, string. Database user. Leave empty for drivers `sqlite`, `bolt` and `memory`
+  - `password`, string. Database password. Leave empty for drivers `sqlite`, `bolt` and `memory`
+  - `sslmode`, integer. Used for drivers `mysql` and `postgresql`. 0 disable SSL/TLS connections, 1 require ssl, 2 set ssl mode to `verify-ca` for driver `postgresql` and `skip-verify` for driver `mysql`, 3 set ssl mode to `verify-full` for driver `postgresql` and `preferred` for driver `mysql`
+  - `connectionstring`, string. Provide a custom database connection string. If not empty, this connection string will be used instead of building one using the previous parameters. Leave empty for drivers `bolt` and `memory`
+  - `sql_tables_prefix`, string. Prefix for SQL tables
+  - `manage_users`, integer. Set to 0 to disable users management, 1 to enable
+  - `track_quota`, integer. Set the preferred mode to track users quota between the following choices:
+    - 0, disable quota tracking. REST API to scan users home directories/virtual folders and update quota will do nothing
+    - 1, quota is updated each time a user uploads or deletes a file, even if the user has no quota restrictions
+    - 2, quota is updated each time a user uploads or deletes a file, but only for users with quota restrictions and for virtual folders. With this configuration, the `quota scan` and `folder_quota_scan` REST API can still be used to periodically update space usage for users without quota restrictions and for folders
+  - `pool_size`, integer. Sets the maximum number of open connections for `mysql` and `postgresql` driver. Default 0 (unlimited)
+  - `users_base_dir`, string. Users default base directory. If no home dir is defined while adding a new user, and this value is a valid absolute path, then the user home dir will be automatically defined as the path obtained joining the base dir and the username
+  - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions. See the "Custom Actions" paragraph for more details
+    - `execute_on`, list of strings. Valid values are `add`, `update`, `delete`. `update` action will not be fired for internal updates such as the last login or the user quota fields.
+    - `command`, string. Deprecated please use `hook`.
+    - `http_notification_url`, a valid URL. Deprecated please use `hook`.
+    - `hook`, string. Absolute path to the command to execute or HTTP URL to notify.
+  - `external_auth_program`, string. Deprecated, please use `external_auth_hook`.
+  - `external_auth_hook`, string. Absolute path to an external program or an HTTP URL to invoke for users authentication. See the "External Authentication" paragraph for more details. Leave empty to disable.
+  - `external_auth_scope`, integer. 0 means all supported authetication scopes (passwords, public keys and keyboard interactive). 1 means passwords only. 2 means public keys only. 4 means key keyboard interactive only. The flags can be combined, for example 6 means public keys and keyboard interactive
+  - `credentials_path`, string. It defines the directory for storing user provided credential files such as Google Cloud Storage credentials. This can be an absolute path or a path relative to the config dir
+  - `pre_login_program`, string. Deprecated, please use `pre_login_hook`.
+  - `pre_login_hook`, string. Absolute path to an external program or an HTTP URL to invoke to modify user details just before the login. See the "Dynamic user modification" paragraph for more details. Leave empty to disable.
+- **"httpd"**, the configuration for the HTTP server used to serve REST API and to expose the built-in web interface
+  - `bind_port`, integer. The port used for serving HTTP requests. Set to 0 to disable HTTP server. Default: 8080
+  - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "127.0.0.1"
+  - `templates_path`, string. Path to the HTML web templates. This can be an absolute path or a path relative to the config dir
+  - `static_files_path`, string. Path to the static files for the web interface. This can be an absolute path or a path relative to the config dir. If both `templates_path` and `static_files_path` are empty the built-in web interface will be disabled
+  - `backups_path`, string. Path to the backup directory. This can be an absolute path or a path relative to the config dir. We don't allow backups in arbitrary paths for security reasons
+  - `auth_user_file`, string. Path to a file used to store usernames and passwords for basic authentication. This can be an absolute path or a path relative to the config dir. We support HTTP basic authentication, and the file format must conform to the one generated using the Apache `htpasswd` tool. The supported password formats are bcrypt (`$2y$` prefix) and md5 crypt (`$apr1$` prefix). If empty, HTTP authentication is disabled.
+  - `certificate_file`, string. Certificate for HTTPS. This can be an absolute path or a path relative to the config dir.
+  - `certificate_key_file`, string. Private key matching the above certificate. This can be an absolute path or a path relative to the config dir. If both the certificate and the private key are provided, the server will expect HTTPS connections. Certificate and key files can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows.
+- **"http"**, the configuration for HTTP clients. HTTP clients are used for executing hooks such as the ones used for custom actions, external authentication and pre-login user modifications
+  - `timeout`, integer. Timeout specifies a time limit, in seconds, for requests.
+  - `ca_certificates`, list of strings. List of paths to extra CA certificates to trust. The paths can be absolute or relative to the config dir. Adding trusted CA certificates is a convenient way to use self-signed certificates without defeating the purpose of using TLS.
+  - `skip_tls_verify`, boolean. if enabled the HTTP client accepts any TLS certificate presented by the server and any host name in that certificate. In this mode, TLS is susceptible to man-in-the-middle attacks. This should be used only for testing.
+
+A full example showing the default config (in JSON format) can be found [here](../sftpgo.json).
+
+If you want to use a private host key that use an algorithm/setting different from the auto generated RSA/ECDSA keys, or more than two private keys, you can generate your own keys and replace the empty `keys` array with something like this:
+
+```json
+"host_keys": [
+  "id_rsa",
+  "id_ecdsa",
+  "id_ed25519"
+]
+```
+
+where `id_rsa`, `id_ecdsa` and `id_ed25519`, in this example, are files containing your generated keys. You can use absolute paths or paths relative to the configuration directory.
+
+If you want the default host keys generation in a directory different from the config dir, please specify absolute paths to files named `id_rsa` or `id_ecdsa` like this:
+
+```json
+"host_keys": [
+  "/etc/sftpgo/keys/id_rsa",
+  "/etc/sftpgo/keys/id_ecdsa"
+]
+```
+
+then SFTPGo will try to create `id_rsa` and `id_ecdsa`, if they are missing, inside the existing directory `/etc/sftpgo/keys`.
+
+The configuration can be read from JSON, TOML, YAML, HCL, envfile and Java properties config files. If your `config-file` flag is set to `sftpgo` (default value), you need to create a configuration file called `sftpgo.json` or `sftpgo.yaml` and so on inside `config-dir`.
+
+## Environment variables
+
+You can also override all the available configuration options using environment variables. SFTPGo will check for environment variables with a name matching the key uppercased and prefixed with the `SFTPGO_`. You need to use `__` to traverse a struct.
+
+Let's see some examples:
+
+- To set sftpd `bind_port`, you need to define the env var `SFTPGO_SFTPD__BIND_PORT`
+- To set the `execute_on` actions, you need to define the env var `SFTPGO_SFTPD__ACTIONS__EXECUTE_ON`. For example `SFTPGO_SFTPD__ACTIONS__EXECUTE_ON=upload,download`
+
+Please note that, to override configuration options with environment variables, a configuration file containing the options to override is required. You can, for example, deploy the default configuration file and then override the options you need to customize using environment variables.

docs/google-cloud-storage.md

@@ -0,0 +1,13 @@
+# Google Cloud Storage backend
+
+To connect SFTPGo to Google Cloud Storage you can use use the Application Default Credentials (ADC) strategy to try to find your application's credentials automatically or you can explicitly provide a JSON credentials file that you can obtain from the Google Cloud Console. Take a look [here](https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application) for details.
+
+Specifying a different `key_prefix`, you can assign different "folders" of the same bucket to different users. This is similar to a chroot directory for local filesystem. Each SFTP/SCP user can only access the assigned folder and its contents. The folder identified by `key_prefix` does not need to be pre-created.
+
+You can optionally specify a [storage class](https://cloud.google.com/storage/docs/storage-classes) too. Leave it blank to use the default storage class.
+
+The configured bucket must exist.
+
+Google Cloud Storage is exposed over HTTPS so if you are running SFTPGo as docker image please be sure to uncomment the line that install `ca-certificates`, inside your `Dockerfile`, to be able to properly verify certificate authorities.
+
+This backend is very similar to the [S3](./s3.md) backend, and it has the same limitations.

docs/keyboard-interactive.md

@@ -0,0 +1,164 @@
+# Keyboard Interactive Authentication
+
+Keyboard interactive authentication is, in general, a series of questions asked by the server with responses provided by the client.
+This authentication method is typically used for multi-factor authentication.
+There are no restrictions on the number of questions asked on a particular authentication stage; there are also no restrictions on the number of stages involving different sets of questions.
+
+To enable keyboard interactive authentication, you must set the absolute path of your authentication program or an HTTP URL using the  `keyboard_interactive_auth_hook` key in your configuration file.
+
+The external program can read the following environment variables to get info about the user trying to authenticate:
+
+- `SFTPGO_AUTHD_USERNAME`
+- `SFTPGO_AUTHD_PASSWORD`, this is the hashed password as stored inside the data provider
+
+Previous global environment variables aren't cleared when the script is called. The content of these variables is _not_ quoted. They may contain special characters.
+
+The program must write the questions on its standard output, in a single line, using the following struct JSON serialized:
+
+- `instruction`, string. A short description to show to the user that is trying to authenticate. Can be empty or omitted
+- `questions`, list of questions to be asked to the user
+- `echos` list of boolean flags corresponding to the questions (so the lengths of both lists must be the same) and indicating whether user's reply for a particular question should be echoed on the screen while they are typing: true if it should be echoed, or false if it should be hidden.
+- `check_password` optional integer. Ask exactly one question and set this field to 1 if the expected answer is the user password and you want that SFTPGo checks it for you. If the password is correct, the returned response to the program is `OK`. If the password is wrong, the program will be terminated and an authentication error will be returned to the user that is trying to authenticate.
+- `auth_result`, integer. Set this field to 1 to indicate successful authentication. 0 is ignored. Any other value means authentication error. If this field is found and it is different from 0 then SFTPGo will not read any other questions from the external program, and it will finalize the authentication.
+
+SFTPGo writes the user answers to the program standard input, one per line, in the same order as the questions.
+Please be sure that your program receives the answers for all the issued questions before asking for the next ones.
+
+Keyboard interactive authentication can be chained to the external authentication.
+The authentication must finish within 60 seconds.
+
+Let's see a very basic example. Our sample keyboard interactive authentication program will ask for 2 sets of questions and accept the user if the answer to the last question is `answer3`.
+
+```shell
+#!/bin/sh
+
+echo '{"questions":["Question1: ","Question2: "],"instruction":"This is a sample for keyboard interactive authentication","echos":[true,false]}'
+
+read ANSWER1
+read ANSWER2
+
+echo '{"questions":["Question3: "],"instruction":"","echos":[true]}'
+
+read ANSWER3
+
+if test "$ANSWER3" = "answer3"; then
+  echo '{"auth_result":1}'
+else
+  echo '{"auth_result":-1}'
+fi
+```
+
+and here is an example where SFTPGo checks the user password for you:
+
+```shell
+#!/bin/sh
+
+echo '{"questions":["Password: "],"instruction":"This is a sample for keyboard interactive authentication","echos":[false],"check_password":1}'
+
+read ANSWER1
+
+if test "$ANSWER1" != "OK"; then
+  exit 1
+fi
+
+echo '{"questions":["One time token: "],"instruction":"","echos":[false]}'
+
+read ANSWER2
+
+if test "$ANSWER2" = "token"; then
+  echo '{"auth_result":1}'
+else
+  echo '{"auth_result":-1}'
+fi
+```
+
+If the hook is an HTTP URL then it will be invoked as HTTP POST multiple times for each login request.
+The request body will contain a JSON struct with the following fields:
+
+- `request_id`, string. Unique request identifier
+- `username`, string
+- `password`, string. This is the hashed password as stored inside the data provider
+- `answers`, list of string. It will be null for the first request
+- `questions`, list of string. It will contains the previous asked questions. It will be null for the first request
+
+The HTTP response code must be 200 and the body must contain the same JSON struct described for the program.
+
+Let's see a basic sample, the configured hook is `http://127.0.0.1:8000/keyIntHookPwd`, as soon as the user try to login, SFTPGo makes this HTTP POST request:
+
+```shell
+POST /keyIntHookPwd HTTP/1.1
+Host: 127.0.0.1:8000
+User-Agent: Go-http-client/1.1
+Content-Length: 189
+Content-Type: application/json
+Accept-Encoding: gzip
+
+{"request_id":"bq1r5r7cdrpd2qtn25ng","username":"a","password":"$pbkdf2-sha512$150000$ClOPkLNujMTL$XktKy0xuJsOfMYBz+f2bIyPTdbvDTSnJ1q+7+zp/HPq5Qojwp6kcpSIiVHiwvbi8P6HFXI/D3UJv9BLcnQFqPA=="}
+```
+
+as you can see in this first requests `answers` and `questions` are null.
+
+Here is the response that instructs SFTPGo to ask for the user password and to check it:
+
+```shell
+HTTP/1.1 200 OK
+Date: Tue, 31 Mar 2020 21:15:24 GMT
+Server: WSGIServer/0.2 CPython/3.8.2
+Content-Type: application/json
+X-Frame-Options: SAMEORIGIN
+Content-Length: 143
+
+{"questions": ["Password: "], "check_password": 1, "instruction": "This is a sample for keyboard interactive authentication", "echos": [false]}
+```
+
+The user enters the correct password and so SFTPGo makes a new HTTP POST, please note that the `request_id` is the same of the previous request, this time the asked `questions` and the user's `answers` are not null:
+
+```shell
+POST /keyIntHookPwd HTTP/1.1
+Host: 127.0.0.1:8000
+User-Agent: Go-http-client/1.1
+Content-Length: 233
+Content-Type: application/json
+Accept-Encoding: gzip
+
+{"request_id":"bq1r5r7cdrpd2qtn25ng","username":"a","password":"$pbkdf2-sha512$150000$ClOPkLNujMTL$XktKy0xuJsOfMYBz+f2bIyPTdbvDTSnJ1q+7+zp/HPq5Qojwp6kcpSIiVHiwvbi8P6HFXI/D3UJv9BLcnQFqPA==","answers":["OK"],"questions":["Password: "]}
+```
+
+Here is the HTTP response that istructs SFTPGo to ask for a new question:
+
+```shell
+HTTP/1.1 200 OK
+Date: Tue, 31 Mar 2020 21:15:27 GMT
+Server: WSGIServer/0.2 CPython/3.8.2
+Content-Type: application/json
+X-Frame-Options: SAMEORIGIN
+Content-Length: 66
+
+{"questions": ["Question2: "], "instruction": "", "echos": [true]}
+```
+
+As soon as the user answer to this question, SFTPGo will make a new HTTP POST request with the user's answers:
+
+```shell
+POST /keyIntHookPwd HTTP/1.1
+Host: 127.0.0.1:8000
+User-Agent: Go-http-client/1.1
+Content-Length: 239
+Content-Type: application/json
+Accept-Encoding: gzip
+
+{"request_id":"bq1r5r7cdrpd2qtn25ng","username":"a","password":"$pbkdf2-sha512$150000$ClOPkLNujMTL$XktKy0xuJsOfMYBz+f2bIyPTdbvDTSnJ1q+7+zp/HPq5Qojwp6kcpSIiVHiwvbi8P6HFXI/D3UJv9BLcnQFqPA==","answers":["answer2"],"questions":["Question2: "]}
+```
+
+Here is the final HTTP response that allows the user login:
+
+```shell
+HTTP/1.1 200 OK
+Date: Tue, 31 Mar 2020 21:15:29 GMT
+Server: WSGIServer/0.2 CPython/3.8.2
+Content-Type: application/json
+X-Frame-Options: SAMEORIGIN
+Content-Length: 18
+
+{"auth_result": 1}
+```

docs/logs.md

@@ -0,0 +1,54 @@
+# Logs
+
+The log file is a stream of JSON structs. Each struct has a `sender` field that identifies the log type.
+
+The logs can be divided into the following categories:
+
+- **"app logs"**, internal logs used to debug SFTPGo:
+  - `sender` string. This is generally the package name that emits the log
+  - `time` string. Date/time with millisecond precision
+  - `level` string
+  - `message` string
+- **"transfer logs"**, SFTP/SCP transfer logs:
+  - `sender` string. `Upload` or `Download`
+  - `time` string. Date/time with millisecond precision
+  - `level` string
+  - `elapsed_ms`, int64. Elapsed time, as milliseconds, for the upload/download
+  - `size_bytes`, int64. Size, as bytes, of the download/upload
+  - `username`, string
+  - `file_path` string
+  - `connection_id` string. Unique connection identifier
+  - `protocol` string. `SFTP` or `SCP`
+- **"command logs"**, SFTP/SCP command logs:
+  - `sender` string. `Rename`, `Rmdir`, `Mkdir`, `Symlink`, `Remove`, `Chmod`, `Chown`, `Chtimes`, `SSHCommand`
+  - `level` string
+  - `username`, string
+  - `file_path` string
+  - `target_path` string
+  - `filemode` string. Valid for sender `Chmod` otherwise empty
+  - `uid` integer. Valid for sender `Chown` otherwise -1
+  - `gid` integer. Valid for sender `Chown` otherwise -1
+  - `access_time` datetime as YYYY-MM-DDTHH:MM:SS. Valid for sender `Chtimes` otherwise empty
+  - `modification_time` datetime as YYYY-MM-DDTHH:MM:SS. Valid for sender `Chtimes` otherwise empty
+  - `ssh_command`, string. Valid for sender `SSHCommand` otherwise empty
+  - `connection_id` string. Unique connection identifier
+  - `protocol` string. `SFTP`, `SCP` or `SSH`
+- **"http logs"**, REST API logs:
+  - `sender` string. `httpd`
+  - `level` string
+  - `remote_addr` string. IP and port of the remote client
+  - `proto` string, for example `HTTP/1.1`
+  - `method` string. HTTP method (`GET`, `POST`, `PUT`, `DELETE` etc.)
+  - `user_agent` string
+  - `uri` string. Full uri
+  - `resp_status` integer. HTTP response status code
+  - `resp_size` integer. Size in bytes of the HTTP response
+  - `elapsed_ms` int64. Elapsed time, as milliseconds, to complete the request
+  - `request_id` string. Unique request identifier
+- **"connection failed logs"**, logs for failed attempts to initialize a connection. A connection can fail for an authentication error or other errors such as a client abort or a timeout if the login does not happen in two minutes
+  - `sender` string. `connection_failed`
+  - `level` string
+  - `username`, string. Can be empty if the connection is closed before an authentication attempt
+  - `client_ip` string.
+  - `login_type` string. Can be `publickey`, `password`, `keyboard-interactive` or `no_auth_tryed`
+  - `error` string. Optional error description

docs/metrics.md

@@ -0,0 +1,18 @@
+# Metrics
+
+SFTPGo exposes [Prometheus](https://prometheus.io/) metrics at the `/metrics` HTTP endpoint.
+Several counters and gauges are available, for example:
+
+- Total uploads and downloads
+- Total upload and download size
+- Total upload and download errors
+- Total executed SSH commands
+- Total SSH command errors
+- Number of active connections
+- Data provider availability
+- Total successful and failed logins using password, public key, keyboard interactive authentication or supported multi-step authentications
+- Total HTTP requests served and totals for response code
+- Go's runtime details about GC, number of gouroutines and OS threads
+- Process information like CPU, memory, file descriptor usage and start time
+
+Please check the `/metrics` page for more details.

docs/performance.md

@@ -0,0 +1,164 @@
+# Performance
+
+SFTPGo can easily saturate a Gigabit connection on low end hardware with no special configuration, this is generally enough for most use cases.
+
+For Multi-Gig connections, some performance improvements and comparisons with OpenSSH have been discussed [here](https://github.com/drakkan/sftpgo/issues/69), most of them have been included in the master branch. To summarize:
+
+- In current state with all performance improvements applied, SFTP performance is very close to OpenSSH however CPU usage is higher. SCP performance match OpenSSH.
+- The main bottlenecks are the encryption and the messages authentication, so if you can use a fast cipher with implicit messages authentication, such as `aes128-gcm@openssh.com`, you will get a big performance boost.
+- SCP protocol is much simpler than SFTP and so, the multi-platform, SFTPGo's SCP implementation performs better than SFTP.
+- Load balancing with HAProxy can greatly improve the performance if CPU not become the bottleneck.
+
+## Benchmark
+
+### Hardware specification
+
+**Server** ||
+--- | --- |
+OS| Debian 10.2 x64 |
+CPU| Ryzen5 3600 |
+RAM| 64GB 2400MHz ECC |
+Disk| Ramdisk |
+Ethernet| Mellanox ConnectX-3 40GbE|
+
+**Client** ||
+--- | --- |
+OS| Ubuntu 19.10 x64 |
+CPU| Threadripper 1920X |
+RAM| 64GB 2400MHz ECC |
+Disk| Ramdisk |
+Ethernet| Mellanox ConnectX-3 40GbE|
+
+### Test configurations
+
+- `Baseline`: SFTPGo version 0.9.6.
+- `Devel`: SFTPGo commit b0ed1905918b9dcc22f9a20e89e354313f491734, compiled with Golang 1.14.2 .
+- `Optimized`: Various [optimizations](#Optimizations-applied) applied on top of `Devel`.
+- `Balanced`: Two optimized instances, running on localhost, load balanced by HAProxy 2.1.3.
+- `OpenSSH`: OpenSSH_7.9p1 Debian-10+deb10u2, OpenSSL 1.1.1d  10 Sep 2019
+
+Server's CPU is in Eco mode, you can expect better results in certain cases with a stronger CPU, especially multi-stream HAProxy balanced load.
+
+#### Cipher aes128-ctr
+
+The Message Authentication Code (MAC) used is `hmac-sha2-256`.
+
+##### SFTP
+
+Download:
+
+Stream|Baseline MB/s|Devel MB/s|Optimized MB/s|Balanced MB/s|OpenSSH MB/s|
+---|---|---|---|---|---|
+1|150|243|319|412|452|
+2|267|452|600|740|735|
+3|351|637|802|991|1045|
+4|414|811|1002|1192|1265|
+8|536|1451|1742|1552|1798|
+
+Upload:
+
+Stream|Baseline MB/s|Devel MB/s|Optimized MB/s|Balanced MB/s|OpenSSH MB/s|
+---|---|---|---|---|---|
+1|172|273|343|407|426|
+2|284|469|595|673|738|
+3|368|644|820|881|1090|
+4|446|851|1041|1026|1244|
+8|605|1210|1368|1273|1820|
+
+##### SCP
+
+Download:
+
+Stream|Baseline MB/s|Devel MB/s|Optimized MB/s|Balanced MB/s|OpenSSH MB/s|
+---|---|---|---|---|---|
+1|220|369|525|611|558|
+2|437|659|941|1048|856|
+3|635|1000|1365|1363|1201|
+4|787|1272|1664|1610|1415|
+8|1297|2129|2690|2100|1959|
+
+Upload:
+
+Stream|Baseline MB/s|Devel MB/s|Optimized MB/s|Balanced MB/s|OpenSSH MB/s|
+---|---|---|---|---|---|
+1|208|312|400|458|508|
+2|360|516|647|745|926|
+3|476|678|861|935|1254|
+4|576|836|1080|1099|1569|
+8|857|1161|1416|1433|2271|
+
+#### Cipher aes128-gcm@openssh.com
+
+With this cipher the messages authentication is implicit, no SHA256 computation is needed.
+
+##### SFTP
+
+Download:
+
+Stream|Baseline MB/s|Devel MB/s|Optimized MB/s|Balanced MB/s|OpenSSH MB/s|
+---|---|---|---|---|---|
+1|332|423|<--|583|443|
+2|533|755|<--|970|809|
+3|666|1045|<--|1249|1098|
+4|762|1276|<--|1461|1351|
+8|886|2064|<--|1825|1933|
+
+Upload:
+
+Stream|Baseline MB/s|Devel MB/s|Optimized MB/s|Balanced MB/s|OpenSSH MB/s|
+---|---|---|---|---|---|
+1|348|410|<--|527|469|
+2|596|729|<--|842|930|
+3|778|974|<--|1088|1341|
+4|886|1192|<--|1232|1494|
+8|1042|1578|<--|1433|1893|
+
+##### SCP
+
+Download:
+
+Stream|Baseline MB/s|Devel MB/s|Optimized MB/s|Balanced MB/s|OpenSSH MB/s|
+---|---|---|---|---|---|
+1|776|793|<--|832|578|
+2|1343|1415|<--|1435|938|
+3|1815|1878|<--|1877|1279|
+4|2192|2205|<--|2056|1567|
+8|3237|3287|<--|2493|2036|
+
+Upload:
+
+Stream|Baseline MB/s|Devel MB/s|Optimized MB/s|Balanced MB/s|OpenSSH MB/s|
+---|---|---|---|---|---|
+1|528|545|<--|608|584|
+2|872|849|<--|975|1019|
+3|1121|1138|<--|1217|1412|
+4|1367|1387|<--|1368|1755|
+8|1733|1744|<--|1664|2510|
+
+### Optimizations applied
+
+- AES-CTR optimization of Go compiler for x86_64, there is a [patch](https://go-review.googlesource.com/c/go/+/51670) that hasn't been merged yet, you can apply it yourself.
+
+### HAProxy configuration
+
+Here is the relevant HAProxy configuration used for the `Balanced` test configuration:
+
+```console
+frontend sftp
+    bind   :2222
+    mode   tcp
+    timeout  client  600s
+    default_backend sftpgo
+
+backend sftpgo
+    mode    tcp
+    balance roundrobin
+    timeout connect 10s
+    timeout server  600s
+    timeout queue   30s
+    option  tcp-check
+    tcp-check expect string SSH-2.0-
+
+    server sftpgo1 127.0.0.1:2022 check send-proxy-v2 weight 10 inter 10s rise 2 fall 3
+    server sftpgo2 127.0.0.1:2024 check send-proxy-v2 weight 10 inter 10s rise 2 fall 3
+```

docs/portable-mode.md

@@ -0,0 +1,58 @@
+# Portable mode
+
+SFTPGo allows to share a single directory on demand using the `portable` subcommand:
+
+```console
+sftpgo portable --help
+To serve the current working directory with auto generated credentials simply use:
+
+sftpgo portable
+
+Please take a look at the usage below to customize the serving parameters
+
+Usage:
+  sftpgo portable [flags]
+
+Flags:
+  -C, --advertise-credentials            If the SFTP service is advertised via multicast DNS, this flag allows to put username/password inside the advertised TXT record
+  -S, --advertise-service                Advertise SFTP service using multicast DNS
+      --allowed-extensions stringArray   Allowed file extensions case insensitive. The format is /dir::ext1,ext2. For example: "/somedir::.jpg,.png"
+      --denied-extensions stringArray    Denied file extensions case insensitive. The format is /dir::ext1,ext2. For example: "/somedir::.jpg,.png"
+  -d, --directory string                 Path to the directory to serve. This can be an absolute path or a path relative to the current directory (default ".")
+  -f, --fs-provider int                  0 means local filesystem, 1 Amazon S3 compatible, 2 Google Cloud Storage
+      --gcs-automatic-credentials int    0 means explicit credentials using a JSON credentials file, 1 automatic (default 1)
+      --gcs-bucket string
+      --gcs-credentials-file string      Google Cloud Storage JSON credentials file
+      --gcs-key-prefix string            Allows to restrict access to the virtual folder identified by this prefix and its contents
+      --gcs-storage-class string
+  -h, --help                             help for portable
+  -l, --log-file-path string             Leave empty to disable logging
+  -v, --log-verbose                      Enable verbose logs
+  -p, --password string                  Leave empty to use an auto generated value
+  -g, --permissions strings              User's permissions. "*" means any permission (default [list,download])
+  -k, --public-key strings
+      --s3-access-key string
+      --s3-access-secret string
+      --s3-bucket string
+      --s3-endpoint string
+      --s3-key-prefix string             Allows to restrict access to the virtual folder identified by this prefix and its contents
+      --s3-region string
+      --s3-storage-class string
+      --s3-upload-concurrency int        How many parts are uploaded in parallel (default 2)
+      --s3-upload-part-size int          The buffer size for multipart uploads (MB) (default 5)
+  -s, --sftpd-port int                   0 means a random non privileged port
+  -c, --ssh-commands strings             SSH commands to enable. "*" means any supported SSH command including scp (default [md5sum,sha1sum,cd,pwd])
+  -u, --username string                  Leave empty to use an auto generated value
+```
+
+In portable mode, SFTPGo can advertise the SFTP service and, optionally, the credentials via multicast DNS, so there is a standard way to discover the service and to automatically connect to it.
+
+Here is an example of the advertised service including credentials as seen using `avahi-browse`:
+
+```console
+= enp0s31f6 IPv4 SFTPGo portable 53705                         SFTP File Transfer   local
+   hostname = [p1.local]
+   address = [192.168.1.230]
+   port = [53705]
+   txt = ["password=EWOo6pJe" "user=user" "version=0.9.3-dev-b409523-dirty-2019-10-26T13:43:32Z"]
+```

docs/profiling.md

@@ -0,0 +1,23 @@
+# Profiling SFTPGo
+
+The built-in profiler lets you collect CPU profiles, traces, allocations and heap profiles that allow to identify and correct specific bottlenecks.
+You can enable the built-in profiler using the `--profiler` command flag.
+
+Profiling data are exposed via HTTP/HTTPS in the format expected by the [pprof](https://github.com/google/pprof/blob/master/doc/README.md) visualization tool. You can find the index page at the URL `/debug/pprof/`.
+
+The following profiles are available, you can obtain them via HTTP GET requests:
+
+- `allocs`, a sampling of all past memory allocations
+- `block`, stack traces that led to blocking on synchronization primitives
+- `goroutine`, stack traces of all current goroutines
+- `heap`, a sampling of memory allocations of live objects. You can specify the `gc` GET parameter to run GC before taking the heap sample
+- `mutex`, stack traces of holders of contended mutexes
+- `profile`, CPU profile. You can specify the duration in the `seconds` GET parameter. After you get the profile file, use the `go tool pprof` command to investigate the profile
+- `threadcreate`, stack traces that led to the creation of new OS threads
+- `trace`, a trace of execution of the current program. You can specify the duration in the `seconds` GET parameter. After you get the trace file, use the `go tool trace` command to investigate the trace
+
+For example you can:
+
+- download a 30 seconds CPU profile from the URL `/debug/pprof/profile?seconds=30`
+- download a sampling of memory allocations of live objects from the URL `/debug/pprof/heap?gc=1`
+- download a sampling of all past memory allocations from the URL `/debug/pprof/allocs`

docs/rest-api.md

@@ -0,0 +1,35 @@
+# REST API
+
+SFTPGo exposes REST API to manage, backup, and restore users and folders, and to get real time reports of the active connections with the ability to forcibly close a connection.
+
+If quota tracking is enabled in the configuration file, then the used size and number of files are updated each time a file is added/removed. If files are added/removed not using SFTP/SCP, or if you change `track_quota` from `2` to `1`, you can rescan the users home dir and update the used quota using the REST API.
+
+REST API can be protected using HTTP basic authentication and exposed via HTTPS. If you need more advanced security features, you can setup a reverse proxy using an HTTP Server such as Apache or NGNIX.
+
+For example, you can keep SFTPGo listening on localhost and expose it externally configuring a reverse proxy using Apache HTTP Server this way:
+
+```shell
+ProxyPass /api/v1 http://127.0.0.1:8080/api/v1
+ProxyPassReverse /api/v1 http://127.0.0.1:8080/api/v1
+```
+
+and you can add authentication with something like this:
+
+```shell
+<Location /api/v1>
+  AuthType Digest
+  AuthName "Private"
+  AuthDigestDomain "/api/v1"
+  AuthDigestProvider file
+  AuthUserFile "/etc/httpd/conf/auth_digest"
+  Require valid-user
+</Location>
+```
+
+and, of course, you can configure the web server to use HTTPS.
+
+The OpenAPI 3 schema for the exposed API can be found inside the source tree: [openapi.yaml](../httpd/schema/openapi.yaml "OpenAPI 3 specs").
+
+A sample CLI client for the REST API can be found inside the source tree [rest-api-cli](../examples/rest-api-cli) directory.
+
+You can also generate your own REST client in your preferred programming language, or even bash scripts, using an OpenAPI generator such as [swagger-codegen](https://github.com/swagger-api/swagger-codegen) or [OpenAPI Generator](https://openapi-generator.tech/)

docs/s3.md

@@ -0,0 +1,35 @@
+# S3 Compatible Object Storage backends
+
+To connect SFTPGo to AWS, you need to specify credentials, a `bucket` and a `region`. Here is the list of available [AWS regions](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). For example, if your bucket is at `Frankfurt`, you have to set the region to `eu-central-1`. You can specify an AWS [storage class](https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html) too. Leave it blank to use the default AWS storage class. An endpoint is required if you are connecting to a Compatible AWS Storage such as [MinIO](https://min.io/).
+
+AWS SDK has different options for credentials. [More Detail](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html). We support:
+
+1. Providing [Access Keys](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys).
+2. Use IAM roles for Amazon EC2
+3. Use IAM roles for tasks if your application uses an ECS task definition
+
+So, you need to provide access keys to activate option 1, or leave them blank to use the other ways to specify credentials.
+
+Most S3 backends require HTTPS connections so if you are running SFTPGo as docker image please be sure to uncomment the line that install `ca-certificates`, inside your `Dockerfile`, to be able to properly verify certificate authorities.
+
+Specifying a different `key_prefix`, you can assign different "folders" of the same bucket to different users. This is similar to a chroot directory for local filesystem. Each SFTP/SCP user can only access the assigned folder and its contents. The folder identified by `key_prefix` does not need to be pre-created.
+
+SFTPGo uses multipart uploads and parallel downloads for storing and retrieving files from S3.
+
+For multipart uploads you can customize the parts size and the upload concurrency. Please note that if the upload bandwidth between the SFTP client and SFTPGo is greater than the upload bandwidth between SFTPGo and S3 then the SFTP client have to wait for the upload of the last parts to S3 after it ends the file upload to SFTPGo, and it may time out. Keep this in mind if you customize these parameters.
+
+The configured bucket must exist.
+
+Some SFTP commands don't work over S3:
+
+- `symlink` and `chtimes` will fail
+- `chown` and `chmod` are silently ignored
+- upload resume is not supported
+- upload mode `atomic` is ignored since S3 uploads are already atomic
+
+Other notes:
+
+- `rename` is a two step operation: server-side copy and then deletion. So, it is not atomic as for local filesystem.
+- We don't support renaming non empty directories since we should rename all the contents too and this could take a long time: think about directories with thousands of files; for each file we should do an AWS API call.
+- For server side encryption, you have to configure the mapped bucket to automatically encrypt objects.
+- A local home directory is still required to store temporary files.

docs/service.md

@@ -0,0 +1,110 @@
+# Running SFTPGo as a service
+
+## Linux
+
+For Linux, a `systemd` sample [service](../init/sftpgo.service "systemd service") can be found inside the source tree.
+
+Here are some basic instructions to run SFTPGo as service, please run the following commands from the directory where you downloaded SFTPGo:
+
+```bash
+# create the required directories
+sudo mkdir -p /etc/sftpgo \
+  /var/lib/sftpgo
+
+# install sftpgo executable
+sudo install -Dm755 sftpgo /usr/bin/sftpgo
+# install the default configuration file, edit it if required
+sudo install -Dm644 sftpgo.json /etc/sftpgo/
+# override some configuration keys using environment variables
+sudo sh -c 'echo "SFTPGO_HTTPD__TEMPLATES_PATH=/var/lib/sftpgo/templates" > /etc/sftpgo/sftpgo.env'
+sudo sh -c 'echo "SFTPGO_HTTPD__STATIC_FILES_PATH=/var/lib/sftpgo/static" >> /etc/sftpgo/sftpgo.env'
+sudo sh -c 'echo "SFTPGO_HTTPD__BACKUPS_PATH=/var/lib/sftpgo/backups" >> /etc/sftpgo/sftpgo.env'
+sudo sh -c 'echo "SFTPGO_DATA_PROVIDER__CREDENTIALS_PATH=/var/lib/sftpgo/credentials" >> /etc/sftpgo/sftpgo.env'
+# install static files and templates for the web UI
+sudo cp -r static templates /var/lib/sftpgo/
+# initialize the configured data provider
+# if you want to use MySQL or PostgreSQL you need to create the configured database before running the initprovider command
+sudo /usr/bin/sftpgo initprovider -c /etc/sftpgo/
+# install the systemd service
+sudo install -Dm644 init/sftpgo.service /etc/systemd/system
+# start the service
+sudo systemctl start sftpgo
+# verify that the service is started
+sudo systemctl status sftpgo
+# automatically start sftpgo on boot
+sudo systemctl enable sftpgo
+# optional, install the REST API CLI. It requires python-requests to run
+sudo install -Dm755 examples/rest-api-cli/sftpgo_api_cli.py /usr/bin/sftpgo_api_cli
+```
+
+## macOS
+
+For macOS, a `launchd` sample [service](../init/com.github.drakkan.sftpgo.plist "launchd plist") can be found inside the source tree. The `launchd` plist assumes that SFTPGo has `/usr/local/opt/sftpgo` as base directory.
+
+Here are some basic instructions to run SFTPGo as service, please run the following commands from the directory where you downloaded SFTPGo:
+
+```bash
+# create the required directories
+sudo mkdir -p /usr/local/opt/sftpgo/init \
+  /usr/local/opt/sftpgo/var/lib \
+  /usr/local/opt/sftpgo/var/log \
+  /usr/local/opt/sftpgo/etc \
+  /usr/local/opt/sftpgo/bin
+
+# install sftpgo executable
+sudo cp sftpgo /usr/local/opt/sftpgo/bin/
+# install the launchd service
+sudo cp init/com.github.drakkan.sftpgo.plist /usr/local/opt/sftpgo/init/
+sudo chown root:wheel /usr/local/opt/sftpgo/init/com.github.drakkan.sftpgo.plist
+# install the default configuration file, edit it if required
+sudo cp sftpgo.json /usr/local/opt/sftpgo/etc/
+# install static files and templates for the web UI
+sudo cp -r static templates /usr/local/opt/sftpgo/var/lib/
+# initialize the configured data provider
+# if you want to use MySQL or PostgreSQL you need to create the configured database before running the initprovider command
+sudo /usr/local/opt/sftpgo/bin/sftpgo initprovider -c /usr/local/opt/sftpgo/etc/
+# add sftpgo to the launch daemons
+sudo ln -s /usr/local/opt/sftpgo/init/com.github.drakkan.sftpgo.plist /Library/LaunchDaemons/com.github.drakkan.sftpgo.plist
+# start the service and enable it to start on boot
+sudo launchctl load -w /Library/LaunchDaemons/com.github.drakkan.sftpgo.plist
+# verify that the service is started
+sudo launchctl list com.github.drakkan.sftpgo
+# optional, install the REST API CLI. It requires python-requests to run, this python module is not installed by default
+sudo cp examples/rest-api-cli/sftpgo_api_cli.py /usr/local/opt/sftpgo/bin/
+```
+
+## Windows
+
+On Windows, you can register SFTPGo as Windows Service. Take a look at the CLI usage to learn how to do this:
+
+```powershell
+PS> sftpgo.exe service --help
+Install, Uninstall, Start, Stop, Reload and retrieve status for SFTPGo Windows Service
+
+Usage:
+  sftpgo service [command]
+
+Available Commands:
+  install     Install SFTPGo as Windows Service
+  reload      Reload the SFTPGo Windows Service sending a "paramchange" request
+  rotatelogs  Signal to the running service to close the existing log file and immediately create a new one
+  start       Start SFTPGo Windows Service
+  status      Retrieve the status for the SFTPGo Windows Service
+  stop        Stop SFTPGo Windows Service
+  uninstall   Uninstall SFTPGo Windows Service
+
+Flags:
+  -h, --help   help for service
+
+Use "sftpgo service [command] --help" for more information about a command.
+```
+
+The `install` subcommand accepts the same flags that are valid for `serve`.
+
+After installing as a Windows Service, please remember to allow network access to the SFTPGo executable using something like this:
+
+```powershell
+PS> netsh advfirewall firewall add rule name="SFTPGo Service" dir=in action=allow program="C:\Program Files\SFTPGo\sftpgo.exe"
+```
+
+(Or through the Windows Firewall GUI.)

docs/ssh-commands.md

@@ -0,0 +1,33 @@
+# SSH commands
+
+Some SSH commands are implemented directly inside SFTPGo, while for others we use system commands that need to be installed and in your system's `PATH`.
+
+For system commands we have no direct control on file creation/deletion and so there are some limitations:
+
+- we cannot allow them if the target directory contains virtual folders or file extensions filters
+- system commands work only on local filyestem
+- we cannot avoid to leak real filesystem paths
+- quota check is suboptimal
+
+ If quota is enabled and SFTPGO receives a system command, the used size and number of files are checked at the command start and not while new files are created/deleted. While the command is running the number of files is not checked, the remaining size is calculated as the difference between the max allowed quota and the used one, and it is checked against the bytes transferred via SSH. The command is aborted if it uploads more bytes than the remaining allowed size calculated at the command start. Anyway, we only see the bytes that the remote command sends to the local one via SSH. These bytes contain both protocol commands and files, and so the size of the files is different from the size trasferred via SSH: for example, a command can send compressed files, or a protocol command (few bytes) could delete a big file. To mitigate these issues, quotas are recalculated at the command end with a full scan of the directory specified for the system command. This could be heavy for big directories. If you need system commands and quotas you could consider disabling quota restrictions and periodically update quota usage yourself using the REST API.
+
+ For these reasons we should limit system commands usage as much as possibile, we currently support the following system commands:
+
+- `git-receive-pack`, `git-upload-pack`, `git-upload-archive`. These commands enable support for Git repositories over SSH. They need to be installed and in your system's `PATH`.
+- `rsync`. The `rsync` command needs to be installed and in your system's `PATH`. We cannot avoid that rsync creates symlinks, so if the user has the permission to create symlinks, we add the option `--safe-links` to the received rsync command if it is not already set. This should prevent creating symlinks that point outside the home dir. If the user cannot create symlinks, we add the option `--munge-links` if it is not already set. This should make symlinks unusable (but manually recoverable).
+
+SFTPGo support the following built-in SSH commands:
+
+- `scp`, SFTPGo implements the SCP protocol so we can support it for cloud filesystems too and we can avoid the other system commands limitations. SCP between two remote hosts is supported using the `-3` scp option.
+- `md5sum`, `sha1sum`, `sha256sum`, `sha384sum`, `sha512sum`. Useful to check message digests for uploaded files.
+- `cd`, `pwd`. Some SFTP clients do not support the SFTP SSH_FXP_REALPATH packet type, so they use `cd` and `pwd` SSH commands to get the initial directory. Currently `cd` does nothing and `pwd` always returns the `/` path.
+- `sftpgo-copy`. This is a built-in copy implementation. It allows server side copy for files and directories. The first argument is the source file/directory and the second one is the destination file/directory, for example `sftpgo-copy <src> <dst>`. The command will fail if the destination exists. Copy for directories spanning virtual folders is not supported. Only local filesystem is supported: recursive copy for Cloud Storage filesystems requires a new request for every file in any case, so a real server side copy is not possibile.
+- `sftpgo-remove`. This is a built-in remove implementation. It allows to remove single files and to recursively remove directories. The first argument is the file/directory to remove, for example `sftpgo-remove <dst>`. Only local filesystem is supported: recursive remove for Cloud Storage filesystems requires a new request for every file in any case, so a server side remove is not possibile.
+
+The following SSH commands are enabled by default:
+
+- `md5sum`
+- `sha1sum`
+- `cd`
+- `pwd`
+- `scp`

docs/virtual-folders.md

@@ -0,0 +1,31 @@
+# Virtual Folders
+
+A virtual folder is a mapping between a SFTP/SCP virtual path and a filesystem path outside the user home directory.
+The specified paths must be absolute and the virtual path cannot be "/", it must be a sub directory.
+The parent directory to the specified virtual path must exist. SFTPGo will try to automatically create any missing parent directory for the configured virtual folders at user login.
+
+For each virtual folder, the following properties can be configured:
+
+- `mapped_path`, the full absolute path to the filesystem path to expose as virtual folder
+- `virtual_path`, the SFTP/SCP absolute path to use to expose the mapped path
+- `quota_size`, maximum size allowed as bytes. 0 means unlimited, -1 included in user quota
+- `quota_files`, maximum number of files allowed. 0 means unlimited, -1 included in user quota
+
+For example if you configure `/tmp/mapped` or `C:\mapped` as mapped path and `/vfolder` as virtual path then SFTP/SCP users can access the mapped path via the `/vfolder` SFTP path.
+
+The same virtual folder, identified by the `mapped_path`, can be shared among users and different folder quota limits for each user are supported.
+Folder quota limits can also be included inside the user quota but in this case the folder is considered "private" and sharing it with other users will break user quota calculation.
+
+You don't need to create virtual folders, inside the data provider, to associate them to the users: any missing virtual folder will be automatically created when you add/update an user. You only have to create the folder on the filesystem.
+
+Using the REST API you can:
+
+- monitor folders quota usage
+- scan quota for folders
+- inspect the relationships among users and folders
+- delete a virtual folder. SFTPGo removes folders from the data provider, no files deletion will occur
+
+If you remove a folder, from the data provider, any users relationships will be cleared up. If the deleted folder is included inside the user quota you need to do a user quota scan to update its quota. An orphan virtual folder will not be automatically deleted since if you add it again later then a quota scan is needed and it could be quite expensive, anyway you can easily list the orphan folders using the REST API and delete them if they are not needed anymore.
+
+Overlapping virtual paths are not allowed for the same user, overlapping mapped paths are allowed only if quota tracking is globally disabled inside the configuration file (`track_quota` must be set to `0`).
+Virtual folders are supported for local filesystem only.

docs/web-admin.md

@@ -0,0 +1,8 @@
+# Web Admin
+
+You can easily build your own interface using the exposed REST API. Anyway, SFTPGo also provides a very basic built-in web interface that allows you to manage users and connections.
+With the default `httpd` configuration, the web admin is available at the following URL:
+
+[http://127.0.0.1:8080/web](http://127.0.0.1:8080/web)
+
+The web interface can be protected using HTTP basic authentication and exposed via HTTPS. If you need more advanced security features, you can setup a reverse proxy as explained for the [REST API](./rest-api.md).

examples/ldapauth/README.md

@@ -0,0 +1,48 @@
+# LDAPAuth
+
+This is an example for an external authentication program. It performs authentication against an LDAP server.
+It is tested against [389ds](https://directory.fedoraproject.org/) and can be used as starting point to authenticate using any LDAP server including Active Directory.
+
+You need to change the LDAP connection parameters and the user search query to match your environment.
+You can build this example using the following command:
+
+```console
+go build -i -ldflags "-s -w" -o ldapauth
+```
+
+This program assumes that the 389ds schema was extended to add support for public keys using the following ldif file placed in `/etc/dirsrv/schema/98openssh-ldap.ldif`:
+
+```console
+dn: cn=schema
+changetype: modify
+add: attributetypes
+attributetypes: ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' DESC 'MANDATORY: OpenSSH Public key' EQUALITY octetStringMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 )
+-
+add: objectclasses
+objectClasses: ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY DESC 'MANDATORY: OpenSSH LPK objectclass' MUST ( uid ) MAY ( sshPublicKey ) )
+-
+
+dn: cn=sshpublickey,cn=default indexes,cn=config,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: sshpublickey
+nsIndexType: eq
+nsIndexType: pres
+nsSystemIndex: false
+objectClass: top
+objectClass: nsIndex
+
+dn: cn=sshpublickey_self_manage,ou=groups,dc=example,dc=com
+changetype: add
+objectClass: top
+objectClass: groupofuniquenames
+cn: sshpublickey_self_manage
+description: Members of this group gain the ability to edit their own sshPublicKey field
+
+dn: dc=example,dc=com
+changetype: modify
+add: aci
+aci: (targetattr = "sshPublicKey") (version 3.0; acl "Allow members of sshpublickey_self_manage to edit their keys"; allow(write) (groupdn = "ldap:///cn=sshpublickey_self_manage,ou=groups,dc=example,dc=com" and userdn="ldap:///self" ); )
+-
+```
+
+Please feel free to send pull requests to improve this example authentication program, thanks!

examples/ldapauth/go.mod

@@ -0,0 +1,10 @@
+module github.com/drakkan/ldapauth
+
+go 1.14
+
+require (
+	github.com/go-asn1-ber/asn1-ber v1.4.1 // indirect
+	github.com/go-ldap/ldap/v3 v3.1.8
+	golang.org/x/crypto v0.0.0-20200406173513-056763e48d71
+	golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa // indirect
+)

examples/ldapauth/go.sum

@@ -0,0 +1,13 @@
+github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-asn1-ber/asn1-ber v1.4.1 h1:qP/QDxOtmMoJVgXHCXNzDpA0+wkgYB2x5QoLMVOciyw=
+github.com/go-asn1-ber/asn1-ber v1.4.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-ldap/ldap/v3 v3.1.8 h1:5vU/2jOh9HqprwXp8aF915s9p6Z8wmbSEVF7/gdTFhM=
+github.com/go-ldap/ldap/v3 v3.1.8/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200406173513-056763e48d71 h1:DOmugCavvUtnUD114C1Wh+UgTgQZ4pMLzXxi1pSt+/Y=
+golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

examples/ldapauth/main.go

@@ -0,0 +1,135 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+
+	"github.com/go-ldap/ldap/v3"
+	"golang.org/x/crypto/ssh"
+)
+
+const (
+	bindUsername = "cn=Directory Manager"
+	bindPassword = "YOUR_ADMIN_PASSWORD_HERE"
+	bindURL      = "ldap://192.168.1.103:389"
+)
+
+type userFilters struct {
+	DeniedLoginMethods []string `json:"denied_login_methods,omitempty"`
+}
+
+type minimalSFTPGoUser struct {
+	Status      int                 `json:"status,omitempty"`
+	Username    string              `json:"username"`
+	HomeDir     string              `json:"home_dir,omitempty"`
+	UID         int                 `json:"uid,omitempty"`
+	GID         int                 `json:"gid,omitempty"`
+	Permissions map[string][]string `json:"permissions"`
+	Filters     userFilters         `json:"filters"`
+}
+
+func exitError() {
+	u := minimalSFTPGoUser{
+		Username: "",
+	}
+	json, _ := json.Marshal(u)
+	fmt.Printf("%v\n", string(json))
+	os.Exit(1)
+}
+
+func printSuccessResponse(username, homeDir string, uid, gid int) {
+	u := minimalSFTPGoUser{
+		Username: username,
+		HomeDir:  homeDir,
+		UID:      uid,
+		GID:      gid,
+		Status:   1,
+	}
+	u.Permissions = make(map[string][]string)
+	u.Permissions["/"] = []string{"*"}
+	// uncomment the next line to require publickey+password authentication
+	//u.Filters.DeniedLoginMethods = []string{"publickey", "password", "keyboard-interactive", "publickey+keyboard-interactive"}
+	json, _ := json.Marshal(u)
+	fmt.Printf("%v\n", string(json))
+	os.Exit(0)
+}
+
+func main() {
+	// get credentials from env vars
+	username := os.Getenv("SFTPGO_AUTHD_USERNAME")
+	password := os.Getenv("SFTPGO_AUTHD_PASSWORD")
+	publickey := os.Getenv("SFTPGO_AUTHD_PUBLIC_KEY")
+	l, err := ldap.DialURL(bindURL)
+	if err != nil {
+		exitError()
+	}
+	defer l.Close()
+	// bind to the ldap server with an account that can read users
+	err = l.Bind(bindUsername, bindPassword)
+	if err != nil {
+		exitError()
+	}
+
+	// search the user trying to login and fetch some attributes, this search string is tested against 389ds using the default configuration
+	searchRequest := ldap.NewSearchRequest(
+		"dc=example,dc=com",
+		ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+		fmt.Sprintf("(&(objectClass=nsPerson)(uid=%s))", username),
+		[]string{"dn", "homeDirectory", "uidNumber", "gidNumber", "nsSshPublicKey"},
+		nil,
+	)
+
+	sr, err := l.Search(searchRequest)
+	if err != nil {
+		exitError()
+	}
+
+	// we expect exactly one user
+	if len(sr.Entries) != 1 {
+		exitError()
+	}
+
+	if len(publickey) > 0 {
+		// check public key
+		userKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publickey))
+		if err != nil {
+			exitError()
+		}
+		authOk := false
+		for _, k := range sr.Entries[0].GetAttributeValues("nsSshPublicKey") {
+			key, _, _, _, err := ssh.ParseAuthorizedKey([]byte(k))
+			// we skip an invalid public key stored inside the LDAP server
+			if err != nil {
+				continue
+			}
+			if bytes.Equal(key.Marshal(), userKey.Marshal()) {
+				authOk = true
+				break
+			}
+		}
+		if !authOk {
+			exitError()
+		}
+	} else {
+		// bind to the LDAP server with the user dn and the given password to check the password
+		userdn := sr.Entries[0].DN
+		err = l.Bind(userdn, password)
+		if err != nil {
+			exitError()
+		}
+	}
+
+	uid, err := strconv.Atoi(sr.Entries[0].GetAttributeValue("uidNumber"))
+	if err != nil {
+		exitError()
+	}
+	gid, err := strconv.Atoi(sr.Entries[0].GetAttributeValue("gidNumber"))
+	if err != nil {
+		exitError()
+	}
+	// return the authenticated user
+	printSuccessResponse(username, sr.Entries[0].GetAttributeValue("homeDirectory"), uid, gid)
+}

examples/ldapauthserver/README.md

@@ -0,0 +1,11 @@
+# LDAPAuthServer
+
+This is an example for an HTTP server to use as external authentication HTTP hook. It performs authentication against an LDAP server.
+It is tested against [389ds](https://directory.fedoraproject.org/) and can be used as starting point to authenticate using any LDAP server including Active Directory.
+
+You can configure the server using the [ldapauth.toml](./ldapauth.toml) configuration file.
+You can build this example using the following command:
+
+```console
+go build -i -ldflags "-s -w" -o ldapauthserver
+```

examples/ldapauthserver/cmd/root.go

@@ -0,0 +1,137 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/drakkan/sftpgo/ldapauthserver/config"
+	"github.com/drakkan/sftpgo/ldapauthserver/utils"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+const (
+	logSender           = "cmd"
+	configDirFlag       = "config-dir"
+	configDirKey        = "config_dir"
+	configFileFlag      = "config-file"
+	configFileKey       = "config_file"
+	logFilePathFlag     = "log-file-path"
+	logFilePathKey      = "log_file_path"
+	logMaxSizeFlag      = "log-max-size"
+	logMaxSizeKey       = "log_max_size"
+	logMaxBackupFlag    = "log-max-backups"
+	logMaxBackupKey     = "log_max_backups"
+	logMaxAgeFlag       = "log-max-age"
+	logMaxAgeKey        = "log_max_age"
+	logCompressFlag     = "log-compress"
+	logCompressKey      = "log_compress"
+	logVerboseFlag      = "log-verbose"
+	logVerboseKey       = "log_verbose"
+	profilerFlag        = "profiler"
+	profilerKey         = "profiler"
+	defaultConfigDir    = "."
+	defaultConfigName   = config.DefaultConfigName
+	defaultLogFile      = "ldapauth.log"
+	defaultLogMaxSize   = 10
+	defaultLogMaxBackup = 5
+	defaultLogMaxAge    = 28
+	defaultLogCompress  = false
+	defaultLogVerbose   = true
+)
+
+var (
+	configDir     string
+	configFile    string
+	logFilePath   string
+	logMaxSize    int
+	logMaxBackups int
+	logMaxAge     int
+	logCompress   bool
+	logVerbose    bool
+
+	rootCmd = &cobra.Command{
+		Use:   "ldapauthserver",
+		Short: "LDAP Authentication Server for SFTPGo",
+	}
+)
+
+func init() {
+	version := utils.GetAppVersion()
+	rootCmd.Flags().BoolP("version", "v", false, "")
+	rootCmd.Version = version.GetVersionAsString()
+	rootCmd.SetVersionTemplate(`{{printf "LDAP Authentication Server version: "}}{{printf "%s" .Version}}
+`)
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func addConfigFlags(cmd *cobra.Command) {
+	viper.SetDefault(configDirKey, defaultConfigDir)
+	viper.BindEnv(configDirKey, "LDAPAUTH_CONFIG_DIR")
+	cmd.Flags().StringVarP(&configDir, configDirFlag, "c", viper.GetString(configDirKey),
+		"Location for the config dir. This directory should contain the \"ldapauth\" configuration file or the configured "+
+			"config-file. This flag can be set using LDAPAUTH_CONFIG_DIR env var too.")
+	viper.BindPFlag(configDirKey, cmd.Flags().Lookup(configDirFlag))
+
+	viper.SetDefault(configFileKey, defaultConfigName)
+	viper.BindEnv(configFileKey, "LDAPAUTH_CONFIG_FILE")
+	cmd.Flags().StringVarP(&configFile, configFileFlag, "f", viper.GetString(configFileKey),
+		"Name for the configuration file. It must be the name of a file stored in config-dir not the absolute path to the "+
+			"configuration file. The specified file name must have no extension we automatically load JSON, YAML, TOML, HCL and "+
+			"Java properties. Therefore if you set \"ldapauth\" then \"ldapauth.toml\", \"ldapauth.yaml\" and so on are searched. "+
+			"This flag can be set using LDAPAUTH_CONFIG_FILE env var too.")
+	viper.BindPFlag(configFileKey, cmd.Flags().Lookup(configFileFlag))
+}
+
+func addServeFlags(cmd *cobra.Command) {
+	addConfigFlags(cmd)
+
+	viper.SetDefault(logFilePathKey, defaultLogFile)
+	viper.BindEnv(logFilePathKey, "LDAPAUTH_LOG_FILE_PATH")
+	cmd.Flags().StringVarP(&logFilePath, logFilePathFlag, "l", viper.GetString(logFilePathKey),
+		"Location for the log file. Leave empty to write logs to the standard output. This flag can be set using LDAPAUTH_LOG_FILE_PATH "+
+			"env var too.")
+	viper.BindPFlag(logFilePathKey, cmd.Flags().Lookup(logFilePathFlag))
+
+	viper.SetDefault(logMaxSizeKey, defaultLogMaxSize)
+	viper.BindEnv(logMaxSizeKey, "LDAPAUTH_LOG_MAX_SIZE")
+	cmd.Flags().IntVarP(&logMaxSize, logMaxSizeFlag, "s", viper.GetInt(logMaxSizeKey),
+		"Maximum size in megabytes of the log file before it gets rotated. This flag can be set using LDAPAUTH_LOG_MAX_SIZE "+
+			"env var too. It is unused if log-file-path is empty.")
+	viper.BindPFlag(logMaxSizeKey, cmd.Flags().Lookup(logMaxSizeFlag))
+
+	viper.SetDefault(logMaxBackupKey, defaultLogMaxBackup)
+	viper.BindEnv(logMaxBackupKey, "LDAPAUTH_LOG_MAX_BACKUPS")
+	cmd.Flags().IntVarP(&logMaxBackups, "log-max-backups", "b", viper.GetInt(logMaxBackupKey),
+		"Maximum number of old log files to retain. This flag can be set using LDAPAUTH_LOG_MAX_BACKUPS env var too. "+
+			"It is unused if log-file-path is empty.")
+	viper.BindPFlag(logMaxBackupKey, cmd.Flags().Lookup(logMaxBackupFlag))
+
+	viper.SetDefault(logMaxAgeKey, defaultLogMaxAge)
+	viper.BindEnv(logMaxAgeKey, "LDAPAUTH_LOG_MAX_AGE")
+	cmd.Flags().IntVarP(&logMaxAge, "log-max-age", "a", viper.GetInt(logMaxAgeKey),
+		"Maximum number of days to retain old log files. This flag can be set using LDAPAUTH_LOG_MAX_AGE env var too. "+
+			"It is unused if log-file-path is empty.")
+	viper.BindPFlag(logMaxAgeKey, cmd.Flags().Lookup(logMaxAgeFlag))
+
+	viper.SetDefault(logCompressKey, defaultLogCompress)
+	viper.BindEnv(logCompressKey, "LDAPAUTH_LOG_COMPRESS")
+	cmd.Flags().BoolVarP(&logCompress, logCompressFlag, "z", viper.GetBool(logCompressKey), "Determine if the rotated "+
+		"log files should be compressed using gzip. This flag can be set using LDAPAUTH_LOG_COMPRESS env var too. "+
+		"It is unused if log-file-path is empty.")
+	viper.BindPFlag(logCompressKey, cmd.Flags().Lookup(logCompressFlag))
+
+	viper.SetDefault(logVerboseKey, defaultLogVerbose)
+	viper.BindEnv(logVerboseKey, "LDAPAUTH_LOG_VERBOSE")
+	cmd.Flags().BoolVarP(&logVerbose, logVerboseFlag, "v", viper.GetBool(logVerboseKey), "Enable verbose logs. "+
+		"This flag can be set using LDAPAUTH_LOG_VERBOSE env var too.")
+	viper.BindPFlag(logVerboseKey, cmd.Flags().Lookup(logVerboseFlag))
+}

examples/ldapauthserver/cmd/serve.go

@@ -0,0 +1,49 @@
+package cmd
+
+import (
+	"path/filepath"
+
+	"github.com/drakkan/sftpgo/ldapauthserver/config"
+	"github.com/drakkan/sftpgo/ldapauthserver/httpd"
+	"github.com/drakkan/sftpgo/ldapauthserver/logger"
+	"github.com/drakkan/sftpgo/ldapauthserver/utils"
+	"github.com/rs/zerolog"
+	"github.com/spf13/cobra"
+)
+
+var (
+	serveCmd = &cobra.Command{
+		Use:   "serve",
+		Short: "Start the LDAP Authentication Server",
+		Long: `To start the server with the default values for the command line flags simply use:
+
+ldapauthserver serve
+
+Please take a look at the usage below to customize the startup options`,
+		Run: func(cmd *cobra.Command, args []string) {
+			startServer()
+		},
+	}
+)
+
+func init() {
+	rootCmd.AddCommand(serveCmd)
+	addServeFlags(serveCmd)
+}
+
+func startServer() error {
+	logLevel := zerolog.DebugLevel
+	if !logVerbose {
+		logLevel = zerolog.InfoLevel
+	}
+	if !filepath.IsAbs(logFilePath) && utils.IsFileInputValid(logFilePath) {
+		logFilePath = filepath.Join(configDir, logFilePath)
+	}
+	logger.InitLogger(logFilePath, logMaxSize, logMaxBackups, logMaxAge, logCompress, logLevel)
+	version := utils.GetAppVersion()
+	logger.Info(logSender, "", "starting LDAP Auth Server %v, config dir: %v, config file: %v, log max size: %v log max backups: %v "+
+		"log max age: %v log verbose: %v, log compress: %v", version.GetVersionAsString(), configDir, configFile, logMaxSize,
+		logMaxBackups, logMaxAge, logVerbose, logCompress)
+	config.LoadConfig(configDir, configFile)
+	return httpd.StartHTTPServer(configDir, config.GetHTTPDConfig())
+}

examples/ldapauthserver/config/config.go

@@ -0,0 +1,158 @@
+package config
+
+import (
+	"strings"
+
+	"github.com/drakkan/sftpgo/ldapauthserver/logger"
+	"github.com/spf13/viper"
+)
+
+const (
+	logSender = "config"
+	// DefaultConfigName defines the name for the default config file.
+	// This is the file name without extension, we use viper and so we
+	// support all the config files format supported by viper
+	DefaultConfigName = "ldapauth"
+	// ConfigEnvPrefix defines a prefix that ENVIRONMENT variables will use
+	configEnvPrefix = "ldapauth"
+)
+
+// HTTPDConfig defines configuration for the HTTPD server
+type HTTPDConfig struct {
+	BindAddress        string `mapstructure:"bind_address"`
+	BindPort           int    `mapstructure:"bind_port"`
+	AuthUserFile       string `mapstructure:"auth_user_file"`
+	CertificateFile    string `mapstructure:"certificate_file"`
+	CertificateKeyFile string `mapstructure:"certificate_key_file"`
+}
+
+// LDAPConfig defines the configuration parameters for LDAP connections and searches
+type LDAPConfig struct {
+	BaseDN             string   `mapstructure:"basedn"`
+	BindURL            string   `mapstructure:"bind_url"`
+	BindUsername       string   `mapstructure:"bind_username"`
+	BindPassword       string   `mapstructure:"bind_password"`
+	SearchFilter       string   `mapstructure:"search_filter"`
+	SearchBaseAttrs    []string `mapstructure:"search_base_attrs"`
+	DefaultUID         int      `mapstructure:"default_uid"`
+	DefaultGID         int      `mapstructure:"default_gid"`
+	ForceDefaultUID    bool     `mapstructure:"force_default_uid"`
+	ForceDefaultGID    bool     `mapstructure:"force_default_gid"`
+	InsecureSkipVerify bool     `mapstructure:"insecure_skip_verify"`
+	CACertificates     []string `mapstructure:"ca_certificates"`
+}
+
+type appConfig struct {
+	HTTPD HTTPDConfig `mapstructure:"httpd"`
+	LDAP  LDAPConfig  `mapstructure:"ldap"`
+}
+
+var conf appConfig
+
+func init() {
+	conf = appConfig{
+		HTTPD: HTTPDConfig{
+			BindAddress:        "",
+			BindPort:           9000,
+			AuthUserFile:       "",
+			CertificateFile:    "",
+			CertificateKeyFile: "",
+		},
+		LDAP: LDAPConfig{
+			BaseDN:       "dc=example,dc=com",
+			BindURL:      "ldap://192.168.1.103:389",
+			BindUsername: "cn=Directory Manager",
+			BindPassword: "YOUR_ADMIN_PASSWORD_HERE",
+			SearchFilter: "(&(objectClass=nsPerson)(uid=%s))",
+			SearchBaseAttrs: []string{
+				"dn",
+				"homeDirectory",
+				"uidNumber",
+				"gidNumber",
+				"nsSshPublicKey",
+			},
+			DefaultUID:         0,
+			DefaultGID:         0,
+			ForceDefaultUID:    true,
+			ForceDefaultGID:    true,
+			InsecureSkipVerify: false,
+			CACertificates:     nil,
+		},
+	}
+	viper.SetEnvPrefix(configEnvPrefix)
+	replacer := strings.NewReplacer(".", "__")
+	viper.SetEnvKeyReplacer(replacer)
+	viper.SetConfigName(DefaultConfigName)
+	viper.AutomaticEnv()
+	viper.AllowEmptyEnv(true)
+}
+
+// GetHomeDirectory returns the configured name for the LDAP field to use as home directory
+func (l *LDAPConfig) GetHomeDirectory() string {
+	if len(l.SearchBaseAttrs) > 1 {
+		return l.SearchBaseAttrs[1]
+	}
+	return "homeDirectory"
+}
+
+// GetUIDNumber returns the configured name for the LDAP field to use as UID
+func (l *LDAPConfig) GetUIDNumber() string {
+	if len(l.SearchBaseAttrs) > 2 {
+		return l.SearchBaseAttrs[2]
+	}
+	return "uidNumber"
+}
+
+// GetGIDNumber returns the configured name for the LDAP field to use as GID
+func (l *LDAPConfig) GetGIDNumber() string {
+	if len(l.SearchBaseAttrs) > 3 {
+		return l.SearchBaseAttrs[3]
+	}
+	return "gidNumber"
+}
+
+// GetPublicKey returns the configured name for the LDAP field to use as public keys
+func (l *LDAPConfig) GetPublicKey() string {
+	if len(l.SearchBaseAttrs) > 4 {
+		return l.SearchBaseAttrs[4]
+	}
+	return "nsSshPublicKey"
+}
+
+// GetHTTPDConfig returns the configuration for the HTTP server
+func GetHTTPDConfig() HTTPDConfig {
+	return conf.HTTPD
+}
+
+// GetLDAPConfig returns LDAP related settings
+func GetLDAPConfig() LDAPConfig {
+	return conf.LDAP
+}
+
+func getRedactedConf() appConfig {
+	c := conf
+	return c
+}
+
+// LoadConfig loads the configuration
+func LoadConfig(configDir, configName string) error {
+	var err error
+	viper.AddConfigPath(configDir)
+	viper.AddConfigPath(".")
+	viper.SetConfigName(configName)
+	if err = viper.ReadInConfig(); err != nil {
+		logger.Warn(logSender, "", "error loading configuration file: %v. Default configuration will be used: %+v",
+			err, getRedactedConf())
+		logger.WarnToConsole("error loading configuration file: %v. Default configuration will be used.", err)
+		return err
+	}
+	err = viper.Unmarshal(&conf)
+	if err != nil {
+		logger.Warn(logSender, "", "error parsing configuration file: %v. Default configuration will be used: %+v",
+			err, getRedactedConf())
+		logger.WarnToConsole("error parsing configuration file: %v. Default configuration will be used.", err)
+		return err
+	}
+	logger.Debug(logSender, "", "config file used: '%#v', config loaded: %+v", viper.ConfigFileUsed(), getRedactedConf())
+	return err
+}

examples/ldapauthserver/go.mod

@@ -0,0 +1,26 @@
+module github.com/drakkan/sftpgo/ldapauthserver
+
+go 1.14
+
+require (
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.4.1 // indirect
+	github.com/go-chi/chi v4.1.1+incompatible
+	github.com/go-chi/render v1.0.1
+	github.com/go-ldap/ldap/v3 v3.1.8
+	github.com/mitchellh/mapstructure v1.2.2 // indirect
+	github.com/nathanaelle/password/v2 v2.0.1
+	github.com/pelletier/go-toml v1.7.0 // indirect
+	github.com/rs/zerolog v1.18.0
+	github.com/spf13/afero v1.2.2 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/cobra v1.0.0
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.6.3
+	golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a
+	golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f // indirect
+	golang.org/x/text v0.3.2 // indirect
+	gopkg.in/ini.v1 v1.55.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
+)

examples/ldapauthserver/go.sum

@@ -0,0 +1,228 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-asn1-ber/asn1-ber v1.3.1 h1:gvPdv/Hr++TRFCl0UbPFHC54P9N9jgsRPnmnr419Uck=
+github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-asn1-ber/asn1-ber v1.4.1 h1:qP/QDxOtmMoJVgXHCXNzDpA0+wkgYB2x5QoLMVOciyw=
+github.com/go-asn1-ber/asn1-ber v1.4.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-chi/chi v4.1.1+incompatible h1:MmTgB0R8Bt/jccxp+t6S/1VGIKdJw5J74CK/c9tTfA4=
+github.com/go-chi/chi v4.1.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
+github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-ldap/ldap/v3 v3.1.8 h1:5vU/2jOh9HqprwXp8aF915s9p6Z8wmbSEVF7/gdTFhM=
+github.com/go-ldap/ldap/v3 v3.1.8/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4=
+github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nathanaelle/password/v2 v2.0.1 h1:ItoCTdsuIWzilYmllQPa3DR3YoCXcpfxScWLqr8Ii2s=
+github.com/nathanaelle/password/v2 v2.0.1/go.mod h1:eaoT+ICQEPNtikBRIAatN8ThWwMhVG+r1jTw60BvPJk=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI=
+github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8=
+github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
+github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs=
+github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a h1:y6sBfNd1b9Wy08a6K1Z1DZc4aXABUN5TKjkYhz7UKmo=
+golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8=
+golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ=
+gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

examples/ldapauthserver/httpd/auth.go

@@ -0,0 +1,146 @@
+package httpd
+
+import (
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"sync"
+
+	unixcrypt "github.com/nathanaelle/password/v2"
+
+	"github.com/drakkan/sftpgo/ldapauthserver/logger"
+	"github.com/drakkan/sftpgo/ldapauthserver/utils"
+	"golang.org/x/crypto/bcrypt"
+)
+
+const (
+	authenticationHeader = "WWW-Authenticate"
+	authenticationRealm  = "LDAP Auth Server"
+	unauthResponse       = "Unauthorized"
+)
+
+var (
+	md5CryptPwdPrefixes = []string{"$1$", "$apr1$"}
+	bcryptPwdPrefixes   = []string{"$2a$", "$2$", "$2x$", "$2y$", "$2b$"}
+)
+
+type httpAuthProvider interface {
+	getHashedPassword(username string) (string, bool)
+	isEnabled() bool
+}
+
+type basicAuthProvider struct {
+	Path  string
+	Info  os.FileInfo
+	Users map[string]string
+	lock  *sync.RWMutex
+}
+
+func newBasicAuthProvider(authUserFile string) (httpAuthProvider, error) {
+	basicAuthProvider := basicAuthProvider{
+		Path:  authUserFile,
+		Info:  nil,
+		Users: make(map[string]string),
+		lock:  new(sync.RWMutex),
+	}
+	return &basicAuthProvider, basicAuthProvider.loadUsers()
+}
+
+func (p *basicAuthProvider) isEnabled() bool {
+	return len(p.Path) > 0
+}
+
+func (p *basicAuthProvider) isReloadNeeded(info os.FileInfo) bool {
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+	return p.Info == nil || p.Info.ModTime() != info.ModTime() || p.Info.Size() != info.Size()
+}
+
+func (p *basicAuthProvider) loadUsers() error {
+	if !p.isEnabled() {
+		return nil
+	}
+	info, err := os.Stat(p.Path)
+	if err != nil {
+		logger.Debug(logSender, "", "unable to stat basic auth users file: %v", err)
+		return err
+	}
+	if p.isReloadNeeded(info) {
+		r, err := os.Open(p.Path)
+		if err != nil {
+			logger.Debug(logSender, "", "unable to open basic auth users file: %v", err)
+			return err
+		}
+		defer r.Close()
+		reader := csv.NewReader(r)
+		reader.Comma = ':'
+		reader.Comment = '#'
+		reader.TrimLeadingSpace = true
+		records, err := reader.ReadAll()
+		if err != nil {
+			logger.Debug(logSender, "", "unable to parse basic auth users file: %v", err)
+			return err
+		}
+		p.lock.Lock()
+		defer p.lock.Unlock()
+		p.Users = make(map[string]string)
+		for _, record := range records {
+			if len(record) == 2 {
+				p.Users[record[0]] = record[1]
+			}
+		}
+		logger.Debug(logSender, "", "number of users loaded for httpd basic auth: %v", len(p.Users))
+		p.Info = info
+	}
+	return nil
+}
+
+func (p *basicAuthProvider) getHashedPassword(username string) (string, bool) {
+	err := p.loadUsers()
+	if err != nil {
+		return "", false
+	}
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+	pwd, ok := p.Users[username]
+	return pwd, ok
+}
+
+func checkAuth(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !validateCredentials(r) {
+			w.Header().Set(authenticationHeader, fmt.Sprintf("Basic realm=\"%v\"", authenticationRealm))
+			sendAPIResponse(w, r, errors.New(unauthResponse), "", http.StatusUnauthorized)
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+func validateCredentials(r *http.Request) bool {
+	if !httpAuth.isEnabled() {
+		return true
+	}
+	username, password, ok := r.BasicAuth()
+	if !ok {
+		return false
+	}
+	if hashedPwd, ok := httpAuth.getHashedPassword(username); ok {
+		if utils.IsStringPrefixInSlice(hashedPwd, bcryptPwdPrefixes) {
+			err := bcrypt.CompareHashAndPassword([]byte(hashedPwd), []byte(password))
+			return err == nil
+		}
+		if utils.IsStringPrefixInSlice(hashedPwd, md5CryptPwdPrefixes) {
+			crypter, ok := unixcrypt.MD5.CrypterFound(hashedPwd)
+			if !ok {
+				err := errors.New("cannot found matching MD5 crypter")
+				logger.Debug(logSender, "", "error comparing password with MD5 crypt hash: %v", err)
+				return false
+			}
+			return crypter.Verify([]byte(password))
+		}
+	}
+	return false
+}

examples/ldapauthserver/httpd/httpd.go

@@ -0,0 +1,148 @@
+package httpd
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
+	"time"
+
+	"github.com/drakkan/sftpgo/ldapauthserver/config"
+	"github.com/drakkan/sftpgo/ldapauthserver/logger"
+	"github.com/drakkan/sftpgo/ldapauthserver/utils"
+	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/middleware"
+	"github.com/go-chi/render"
+)
+
+const (
+	logSender      = "httpd"
+	versionPath    = "/api/v1/version"
+	checkAuthPath  = "/api/v1/check_auth"
+	maxRequestSize = 1 << 18 // 256KB
+)
+
+var (
+	ldapConfig config.LDAPConfig
+	httpAuth   httpAuthProvider
+	certMgr    *certManager
+	rootCAs    *x509.CertPool
+)
+
+// StartHTTPServer initializes and starts the HTTP Server
+func StartHTTPServer(configDir string, httpConfig config.HTTPDConfig) error {
+	var err error
+	authUserFile := getConfigPath(httpConfig.AuthUserFile, configDir)
+	httpAuth, err = newBasicAuthProvider(authUserFile)
+	if err != nil {
+		return err
+	}
+
+	router := chi.NewRouter()
+	router.Use(middleware.RequestID)
+	router.Use(middleware.RealIP)
+	router.Use(logger.NewStructuredLogger(logger.GetLogger()))
+	router.Use(middleware.Recoverer)
+
+	router.NotFound(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		sendAPIResponse(w, r, nil, "Not Found", http.StatusNotFound)
+	}))
+
+	router.MethodNotAllowed(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		sendAPIResponse(w, r, nil, "Method not allowed", http.StatusMethodNotAllowed)
+	}))
+
+	router.Get(versionPath, func(w http.ResponseWriter, r *http.Request) {
+		render.JSON(w, r, utils.GetAppVersion())
+	})
+
+	router.Group(func(router chi.Router) {
+		router.Use(checkAuth)
+
+		router.Post(checkAuthPath, checkSFTPGoUserAuth)
+	})
+
+	ldapConfig = config.GetLDAPConfig()
+	loadCACerts(configDir)
+
+	certificateFile := getConfigPath(httpConfig.CertificateFile, configDir)
+	certificateKeyFile := getConfigPath(httpConfig.CertificateKeyFile, configDir)
+
+	httpServer := &http.Server{
+		Addr:           fmt.Sprintf("%s:%d", httpConfig.BindAddress, httpConfig.BindPort),
+		Handler:        router,
+		ReadTimeout:    70 * time.Second,
+		WriteTimeout:   70 * time.Second,
+		IdleTimeout:    120 * time.Second,
+		MaxHeaderBytes: 1 << 16, // 64KB
+	}
+	if len(certificateFile) > 0 && len(certificateKeyFile) > 0 {
+		certMgr, err = newCertManager(certificateFile, certificateKeyFile)
+		if err != nil {
+			return err
+		}
+		config := &tls.Config{
+			GetCertificate: certMgr.GetCertificateFunc(),
+		}
+		httpServer.TLSConfig = config
+		return httpServer.ListenAndServeTLS("", "")
+	}
+	return httpServer.ListenAndServe()
+}
+
+func sendAPIResponse(w http.ResponseWriter, r *http.Request, err error, message string, code int) {
+	var errorString string
+	if err != nil {
+		errorString = err.Error()
+	}
+	resp := apiResponse{
+		Error:      errorString,
+		Message:    message,
+		HTTPStatus: code,
+	}
+	ctx := context.WithValue(r.Context(), render.StatusCtxKey, code)
+	render.JSON(w, r.WithContext(ctx), resp)
+}
+
+func loadCACerts(configDir string) error {
+	var err error
+	rootCAs, err = x509.SystemCertPool()
+	if err != nil {
+		rootCAs = x509.NewCertPool()
+	}
+	for _, ca := range ldapConfig.CACertificates {
+		caPath := getConfigPath(ca, configDir)
+		certs, err := ioutil.ReadFile(caPath)
+		if err != nil {
+			logger.Warn(logSender, "", "error loading ca cert %#v: %v", caPath, err)
+			return err
+		}
+		if !rootCAs.AppendCertsFromPEM(certs) {
+			logger.Warn(logSender, "", "unable to add ca cert %#v", caPath)
+		} else {
+			logger.Debug(logSender, "", "ca cert %#v added to the trusted certificates", caPath)
+		}
+	}
+
+	return nil
+}
+
+// ReloadTLSCertificate reloads the TLS certificate and key from the configured paths
+func ReloadTLSCertificate() {
+	if certMgr != nil {
+		certMgr.loadCertificate()
+	}
+}
+
+func getConfigPath(name, configDir string) string {
+	if !utils.IsFileInputValid(name) {
+		return ""
+	}
+	if len(name) > 0 && !filepath.IsAbs(name) {
+		return filepath.Join(configDir, name)
+	}
+	return name
+}

examples/ldapauthserver/httpd/ldapauth.go

@@ -0,0 +1,143 @@
+package httpd
+
+import (
+	"bytes"
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/drakkan/sftpgo/ldapauthserver/logger"
+	"github.com/go-chi/chi/middleware"
+	"github.com/go-chi/render"
+	"github.com/go-ldap/ldap/v3"
+	"golang.org/x/crypto/ssh"
+)
+
+func getSFTPGoUser(entry *ldap.Entry, username string) (SFTPGoUser, error) {
+	var err error
+	var user SFTPGoUser
+	uid := ldapConfig.DefaultUID
+	gid := ldapConfig.DefaultGID
+	status := 1
+
+	if !ldapConfig.ForceDefaultUID {
+		uid, err = strconv.Atoi(entry.GetAttributeValue(ldapConfig.GetUIDNumber()))
+		if err != nil {
+			return user, err
+		}
+	}
+
+	if !ldapConfig.ForceDefaultGID {
+		uid, err = strconv.Atoi(entry.GetAttributeValue(ldapConfig.GetGIDNumber()))
+		if err != nil {
+			return user, err
+		}
+	}
+
+	sftpgoUser := SFTPGoUser{
+		Username: username,
+		HomeDir:  entry.GetAttributeValue(ldapConfig.GetHomeDirectory()),
+		UID:      uid,
+		GID:      gid,
+		Status:   status,
+	}
+	sftpgoUser.Permissions = make(map[string][]string)
+	sftpgoUser.Permissions["/"] = []string{"*"}
+	return sftpgoUser, nil
+}
+
+func checkSFTPGoUserAuth(w http.ResponseWriter, r *http.Request) {
+	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
+	var authReq externalAuthRequest
+	err := render.DecodeJSON(r.Body, &authReq)
+	if err != nil {
+		logger.Warn(logSender, middleware.GetReqID(r.Context()), "error decoding auth request: %v", err)
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+	l, err := ldap.DialURL(ldapConfig.BindURL, ldap.DialWithTLSConfig(&tls.Config{
+		InsecureSkipVerify: ldapConfig.InsecureSkipVerify,
+		RootCAs:            rootCAs,
+	}))
+	if err != nil {
+		logger.Warn(logSender, middleware.GetReqID(r.Context()), "error connecting to the LDAP server: %v", err)
+		sendAPIResponse(w, r, err, "Error connecting to the LDAP server", http.StatusInternalServerError)
+		return
+	}
+	defer l.Close()
+
+	err = l.Bind(ldapConfig.BindUsername, ldapConfig.BindPassword)
+	if err != nil {
+		logger.Warn(logSender, middleware.GetReqID(r.Context()), "error binding to the LDAP server: %v", err)
+		sendAPIResponse(w, r, err, "Error binding to the LDAP server", http.StatusInternalServerError)
+		return
+	}
+
+	searchRequest := ldap.NewSearchRequest(
+		ldapConfig.BaseDN,
+		ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+		strings.Replace(ldapConfig.SearchFilter, "%s", authReq.Username, 1),
+		ldapConfig.SearchBaseAttrs,
+		nil,
+	)
+
+	sr, err := l.Search(searchRequest)
+	if err != nil {
+		logger.Warn(logSender, middleware.GetReqID(r.Context()), "error searching LDAP user %#v: %v", authReq.Username, err)
+		sendAPIResponse(w, r, err, "Error searching LDAP user", http.StatusInternalServerError)
+		return
+	}
+
+	if len(sr.Entries) != 1 {
+		logger.Warn(logSender, middleware.GetReqID(r.Context()), "expected one user, found: %v", len(sr.Entries))
+		sendAPIResponse(w, r, nil, fmt.Sprintf("Expected one user, found: %v", len(sr.Entries)), http.StatusNotFound)
+		return
+	}
+
+	if len(authReq.PublicKey) > 0 {
+		userKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(authReq.PublicKey))
+		if err != nil {
+			logger.Warn(logSender, middleware.GetReqID(r.Context()), "invalid public key for user %#v: %v", authReq.Username, err)
+			sendAPIResponse(w, r, err, "Invalid public key", http.StatusBadRequest)
+			return
+		}
+		authOk := false
+		for _, k := range sr.Entries[0].GetAttributeValues(ldapConfig.GetPublicKey()) {
+			key, _, _, _, err := ssh.ParseAuthorizedKey([]byte(k))
+			// we skip an invalid public key stored inside the LDAP server
+			if err != nil {
+				continue
+			}
+			if bytes.Equal(key.Marshal(), userKey.Marshal()) {
+				authOk = true
+				break
+			}
+		}
+		if !authOk {
+			logger.Warn(logSender, middleware.GetReqID(r.Context()), "public key authentication failed for user: %#v", authReq.Username)
+			sendAPIResponse(w, r, nil, "public key authentication failed", http.StatusForbidden)
+			return
+		}
+	} else {
+		// bind to the LDAP server with the user dn and the given password to check the password
+		userdn := sr.Entries[0].DN
+		err = l.Bind(userdn, authReq.Password)
+		if err != nil {
+			logger.Warn(logSender, middleware.GetReqID(r.Context()), "password authentication failed for user: %#v", authReq.Username)
+			sendAPIResponse(w, r, nil, "password authentication failed", http.StatusForbidden)
+			return
+		}
+	}
+
+	user, err := getSFTPGoUser(sr.Entries[0], authReq.Username)
+	if err != nil {
+		logger.Warn(logSender, middleware.GetReqID(r.Context()), "get user from LDAP entry failed for username %#v: %v",
+			authReq.Username, err)
+		sendAPIResponse(w, r, err, "mapping LDAP user failed", http.StatusInternalServerError)
+		return
+	}
+
+	render.JSON(w, r, user)
+}

examples/ldapauthserver/httpd/models.go

@@ -0,0 +1,109 @@
+package httpd
+
+type apiResponse struct {
+	Error      string `json:"error"`
+	Message    string `json:"message"`
+	HTTPStatus int    `json:"status"`
+}
+
+type externalAuthRequest struct {
+	Username  string `json:"username"`
+	Password  string `json:"password"`
+	PublicKey string `json:"public_key"`
+}
+
+// SFTPGoExtensionsFilter defines filters based on file extensions
+type SFTPGoExtensionsFilter struct {
+	Path              string   `json:"path"`
+	AllowedExtensions []string `json:"allowed_extensions,omitempty"`
+	DeniedExtensions  []string `json:"denied_extensions,omitempty"`
+}
+
+// SFTPGoUserFilters defines additional restrictions for an SFTPGo user
+type SFTPGoUserFilters struct {
+	AllowedIP          []string                 `json:"allowed_ip,omitempty"`
+	DeniedIP           []string                 `json:"denied_ip,omitempty"`
+	DeniedLoginMethods []string                 `json:"denied_login_methods,omitempty"`
+	FileExtensions     []SFTPGoExtensionsFilter `json:"file_extensions,omitempty"`
+}
+
+// S3FsConfig defines the configuration for S3 based filesystem
+type S3FsConfig struct {
+	Bucket            string `json:"bucket,omitempty"`
+	KeyPrefix         string `json:"key_prefix,omitempty"`
+	Region            string `json:"region,omitempty"`
+	AccessKey         string `json:"access_key,omitempty"`
+	AccessSecret      string `json:"access_secret,omitempty"`
+	Endpoint          string `json:"endpoint,omitempty"`
+	StorageClass      string `json:"storage_class,omitempty"`
+	UploadPartSize    int64  `json:"upload_part_size,omitempty"`
+	UploadConcurrency int    `json:"upload_concurrency,omitempty"`
+}
+
+// GCSFsConfig defines the configuration for Google Cloud Storage based filesystem
+type GCSFsConfig struct {
+	Bucket               string `json:"bucket,omitempty"`
+	KeyPrefix            string `json:"key_prefix,omitempty"`
+	Credentials          string `json:"credentials,omitempty"`
+	AutomaticCredentials int    `json:"automatic_credentials,omitempty"`
+	StorageClass         string `json:"storage_class,omitempty"`
+}
+
+// SFTPGoFilesystem defines cloud storage filesystem details
+type SFTPGoFilesystem struct {
+	// 0 local filesystem, 1 Amazon S3 compatible, 2 Google Cloud Storage
+	Provider  int         `json:"provider"`
+	S3Config  S3FsConfig  `json:"s3config,omitempty"`
+	GCSConfig GCSFsConfig `json:"gcsconfig,omitempty"`
+}
+
+type virtualFolder struct {
+	VirtualPath string `json:"virtual_path"`
+	MappedPath  string `json:"mapped_path"`
+}
+
+// SFTPGoUser defines an SFTPGo user
+type SFTPGoUser struct {
+	// Database unique identifier
+	ID int64 `json:"id"`
+	// 1 enabled, 0 disabled (login is not allowed)
+	Status int `json:"status"`
+	// Username
+	Username string `json:"username"`
+	// Account expiration date as unix timestamp in milliseconds. An expired account cannot login.
+	// 0 means no expiration
+	ExpirationDate int64    `json:"expiration_date"`
+	Password       string   `json:"password,omitempty"`
+	PublicKeys     []string `json:"public_keys,omitempty"`
+	HomeDir        string   `json:"home_dir"`
+	// Mapping between virtual paths and filesystem paths outside the home directory. Supported for local filesystem only
+	VirtualFolders []virtualFolder `json:"virtual_folders,omitempty"`
+	// If sftpgo runs as root system user then the created files and directories will be assigned to this system UID
+	UID int `json:"uid"`
+	// If sftpgo runs as root system user then the created files and directories will be assigned to this system GID
+	GID int `json:"gid"`
+	// Maximum concurrent sessions. 0 means unlimited
+	MaxSessions int `json:"max_sessions"`
+	// Maximum size allowed as bytes. 0 means unlimited
+	QuotaSize int64 `json:"quota_size"`
+	// Maximum number of files allowed. 0 means unlimited
+	QuotaFiles int `json:"quota_files"`
+	// List of the granted permissions
+	Permissions map[string][]string `json:"permissions"`
+	// Used quota as bytes
+	UsedQuotaSize int64 `json:"used_quota_size"`
+	// Used quota as number of files
+	UsedQuotaFiles int `json:"used_quota_files"`
+	// Last quota update as unix timestamp in milliseconds
+	LastQuotaUpdate int64 `json:"last_quota_update"`
+	// Maximum upload bandwidth as KB/s, 0 means unlimited
+	UploadBandwidth int64 `json:"upload_bandwidth"`
+	// Maximum download bandwidth as KB/s, 0 means unlimited
+	DownloadBandwidth int64 `json:"download_bandwidth"`
+	// Last login as unix timestamp in milliseconds
+	LastLogin int64 `json:"last_login"`
+	// Additional restrictions
+	Filters SFTPGoUserFilters `json:"filters"`
+	// Filesystem configuration details
+	FsConfig SFTPGoFilesystem `json:"filesystem"`
+}

examples/ldapauthserver/httpd/tlsutils.go

@@ -0,0 +1,50 @@
+package httpd
+
+import (
+	"crypto/tls"
+	"sync"
+
+	"github.com/drakkan/sftpgo/ldapauthserver/logger"
+)
+
+type certManager struct {
+	cert     *tls.Certificate
+	certPath string
+	keyPath  string
+	lock     *sync.RWMutex
+}
+
+func (m *certManager) loadCertificate() error {
+	newCert, err := tls.LoadX509KeyPair(m.certPath, m.keyPath)
+	if err != nil {
+		logger.Warn(logSender, "", "unable to load https certificate: %v", err)
+		return err
+	}
+	logger.Debug(logSender, "", "https certificate successfully loaded")
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.cert = &newCert
+	return nil
+}
+
+func (m *certManager) GetCertificateFunc() func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+	return func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+		m.lock.RLock()
+		defer m.lock.RUnlock()
+		return m.cert, nil
+	}
+}
+
+func newCertManager(certificateFile, certificateKeyFile string) (*certManager, error) {
+	manager := &certManager{
+		cert:     nil,
+		certPath: certificateFile,
+		keyPath:  certificateKeyFile,
+		lock:     new(sync.RWMutex),
+	}
+	err := manager.loadCertificate()
+	if err != nil {
+		return nil, err
+	}
+	return manager, nil
+}

examples/ldapauthserver/ldapauth.toml

@@ -0,0 +1,33 @@
+[httpd]
+bind_address = ""
+bind_port = 9000
+# Path to a file used to store usernames and passwords for basic authentication. It can be generated using the Apache htpasswd tool
+auth_user_file = ""
+# If both the certificate and the private key are provided, the server will expect HTTPS connections
+certificate_file = ""
+certificate_key_file = ""
+
+[ldap]
+basedn = "dc=example,dc=com"
+bind_url = "ldap://127.0.0.1:389"
+bind_username = "cn=Directory Manager"
+bind_password = "YOUR_ADMIN_PASSWORD_HERE"
+search_filter = "(&(objectClass=nsPerson)(uid=%s))"
+# you can change the name of the search base attributes to adapt them to your schema but the order must remain the same
+search_base_attrs = [
+    "dn",
+    "homeDirectory",
+    "uidNumber",
+    "gidNumber",
+    "nsSshPublicKey"
+]
+default_uid = 0
+default_gid = 0
+force_default_uid = true
+force_default_gid = true
+# if true, ldaps accepts any certificate presented by the LDAP server and any host name in that certificate.
+# This should be used only for testing
+insecure_skip_verify = false
+# list of root CA to use for ldaps connections
+# If you use a self signed certificate is better to add the root CA to this list than set insecure_skip_verify to true
+ca_certificates = []

examples/ldapauthserver/logger/logger.go

@@ -0,0 +1,127 @@
+package logger
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+
+	"github.com/rs/zerolog"
+	lumberjack "gopkg.in/natefinch/lumberjack.v2"
+)
+
+const (
+	dateFormat = "2006-01-02T15:04:05.000" // YYYY-MM-DDTHH:MM:SS.ZZZ
+)
+
+var (
+	logger        zerolog.Logger
+	consoleLogger zerolog.Logger
+)
+
+// GetLogger get the configured logger instance
+func GetLogger() *zerolog.Logger {
+	return &logger
+}
+
+// InitLogger initialize loggers
+func InitLogger(logFilePath string, logMaxSize, logMaxBackups, logMaxAge int, logCompress bool, level zerolog.Level) {
+	zerolog.TimeFieldFormat = dateFormat
+	if isLogFilePathValid(logFilePath) {
+		logger = zerolog.New(&lumberjack.Logger{
+			Filename:   logFilePath,
+			MaxSize:    logMaxSize,
+			MaxBackups: logMaxBackups,
+			MaxAge:     logMaxAge,
+			Compress:   logCompress,
+		})
+		EnableConsoleLogger(level)
+	} else {
+		logger = zerolog.New(logSyncWrapper{
+			output: os.Stdout,
+			lock:   new(sync.Mutex)})
+		consoleLogger = zerolog.Nop()
+	}
+	logger.Level(level)
+}
+
+// DisableLogger disable the main logger.
+// ConsoleLogger will not be affected
+func DisableLogger() {
+	logger = zerolog.Nop()
+}
+
+// EnableConsoleLogger enables the console logger
+func EnableConsoleLogger(level zerolog.Level) {
+	consoleOutput := zerolog.ConsoleWriter{
+		Out:        os.Stdout,
+		TimeFormat: dateFormat,
+		NoColor:    runtime.GOOS == "windows",
+	}
+	consoleLogger = zerolog.New(consoleOutput).With().Timestamp().Logger().Level(level)
+}
+
+// Debug logs at debug level for the specified sender
+func Debug(prefix, requestID string, format string, v ...interface{}) {
+	logger.Debug().
+		Timestamp().
+		Str("sender", prefix).
+		Str("request_id", requestID).
+		Msg(fmt.Sprintf(format, v...))
+}
+
+// Info logs at info level for the specified sender
+func Info(prefix, requestID string, format string, v ...interface{}) {
+	logger.Info().
+		Timestamp().
+		Str("sender", prefix).
+		Str("request_id", requestID).
+		Msg(fmt.Sprintf(format, v...))
+}
+
+// Warn logs at warn level for the specified sender
+func Warn(prefix, requestID string, format string, v ...interface{}) {
+	logger.Warn().
+		Timestamp().
+		Str("sender", prefix).
+		Str("request_id", requestID).
+		Msg(fmt.Sprintf(format, v...))
+}
+
+// Error logs at error level for the specified sender
+func Error(prefix, requestID string, format string, v ...interface{}) {
+	logger.Error().
+		Timestamp().
+		Str("sender", prefix).
+		Str("request_id", requestID).
+		Msg(fmt.Sprintf(format, v...))
+}
+
+// DebugToConsole logs at debug level to stdout
+func DebugToConsole(format string, v ...interface{}) {
+	consoleLogger.Debug().Msg(fmt.Sprintf(format, v...))
+}
+
+// InfoToConsole logs at info level to stdout
+func InfoToConsole(format string, v ...interface{}) {
+	consoleLogger.Info().Msg(fmt.Sprintf(format, v...))
+}
+
+// WarnToConsole logs at info level to stdout
+func WarnToConsole(format string, v ...interface{}) {
+	consoleLogger.Warn().Msg(fmt.Sprintf(format, v...))
+}
+
+// ErrorToConsole logs at error level to stdout
+func ErrorToConsole(format string, v ...interface{}) {
+	consoleLogger.Error().Msg(fmt.Sprintf(format, v...))
+}
+
+func isLogFilePathValid(logFilePath string) bool {
+	cleanInput := filepath.Clean(logFilePath)
+	if cleanInput == "." || cleanInput == ".." {
+		return false
+	}
+	return true
+}

examples/ldapauthserver/logger/request_logger.go

@@ -0,0 +1,73 @@
+package logger
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/go-chi/chi/middleware"
+	"github.com/rs/zerolog"
+)
+
+// StructuredLogger defines a simple wrapper around zerolog logger.
+// It implements chi.middleware.LogFormatter interface
+type StructuredLogger struct {
+	Logger *zerolog.Logger
+}
+
+// StructuredLoggerEntry ...
+type StructuredLoggerEntry struct {
+	Logger *zerolog.Logger
+	fields map[string]interface{}
+}
+
+// NewStructuredLogger returns a chi.middleware.RequestLogger using our StructuredLogger.
+// This structured logger is called by the chi.middleware.Logger handler to log each HTTP request
+func NewStructuredLogger(logger *zerolog.Logger) func(next http.Handler) http.Handler {
+	return middleware.RequestLogger(&StructuredLogger{logger})
+}
+
+// NewLogEntry creates a new log entry for an HTTP request
+func (l *StructuredLogger) NewLogEntry(r *http.Request) middleware.LogEntry {
+	scheme := "http"
+	if r.TLS != nil {
+		scheme = "https"
+	}
+
+	fields := map[string]interface{}{
+		"remote_addr": r.RemoteAddr,
+		"proto":       r.Proto,
+		"method":      r.Method,
+		"user_agent":  r.UserAgent(),
+		"uri":         fmt.Sprintf("%s://%s%s", scheme, r.Host, r.RequestURI)}
+
+	reqID := middleware.GetReqID(r.Context())
+	if reqID != "" {
+		fields["request_id"] = reqID
+	}
+
+	return &StructuredLoggerEntry{Logger: l.Logger, fields: fields}
+}
+
+// Write logs a new entry at the end of the HTTP request
+func (l *StructuredLoggerEntry) Write(status, bytes int, header http.Header, elapsed time.Duration, extra interface{}) {
+	l.Logger.Info().
+		Timestamp().
+		Str("sender", "httpd").
+		Fields(l.fields).
+		Int("resp_status", status).
+		Int("resp_size", bytes).
+		Int64("elapsed_ms", elapsed.Nanoseconds()/1000000).
+		Msg("")
+}
+
+// Panic logs panics
+func (l *StructuredLoggerEntry) Panic(v interface{}, stack []byte) {
+	l.Logger.Error().
+		Timestamp().
+		Str("sender", "httpd").
+		Fields(l.fields).
+		Str("stack", string(stack)).
+		Str("panic", fmt.Sprintf("%+v", v)).
+		Msg("")
+}

examples/ldapauthserver/logger/sync_wrapper.go

@@ -0,0 +1,17 @@
+package logger
+
+import (
+	"os"
+	"sync"
+)
+
+type logSyncWrapper struct {
+	lock   *sync.Mutex
+	output *os.File
+}
+
+func (l logSyncWrapper) Write(b []byte) (n int, err error) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+	return l.output.Write(b)
+}

examples/ldapauthserver/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/drakkan/sftpgo/ldapauthserver/cmd"
+
+func main() {
+	cmd.Execute()
+}

examples/ldapauthserver/utils/utils.go

@@ -0,0 +1,28 @@
+package utils
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+// IsFileInputValid returns true this is a valid file name.
+// This method must be used before joining a file name, generally provided as
+// user input, with a directory
+func IsFileInputValid(fileInput string) bool {
+	cleanInput := filepath.Clean(fileInput)
+	if cleanInput == "." || cleanInput == ".." {
+		return false
+	}
+	return true
+}
+
+// IsStringPrefixInSlice searches a string prefix in a slice and returns true
+// if a matching prefix is found
+func IsStringPrefixInSlice(obj string, list []string) bool {
+	for _, v := range list {
+		if strings.HasPrefix(obj, v) {
+			return true
+		}
+	}
+	return false
+}

examples/ldapauthserver/utils/version.go

@@ -1,6 +1,6 @@
 package utils
 
-const version = "0.9.2"
+const version = "0.1.0-dev"
 
 var (
 	commit      = ""
@@ -15,6 +15,14 @@ type VersionInfo struct {
 	CommitHash string `json:"commit_hash"`
 }
 
+func init() {
+	versionInfo = VersionInfo{
+		Version:    version,
+		CommitHash: commit,
+		BuildDate:  date,
+	}
+}
+
 // GetVersionAsString returns the string representation of the VersionInfo struct
 func (v *VersionInfo) GetVersionAsString() string {
 	versionString := v.Version
@@ -27,10 +35,7 @@ func (v *VersionInfo) GetVersionAsString() string {
 	return versionString
 }
 
-func init() {
-	versionInfo = VersionInfo{
-		Version:    version,
-		CommitHash: commit,
-		BuildDate:  date,
-	}
+// GetAppVersion returns VersionInfo struct
+func GetAppVersion() VersionInfo {
+	return versionInfo
 }

examples/rest-api-cli/README.md

@@ -0,0 +1,618 @@
+# REST API CLI client
+
+`sftpgo_api_cli.py` is a very simple command line client for `SFTPGo` REST API written in python.
+
+It has the following requirements:
+
+- python3 or python2
+- python [Requests](https://2.python-requests.org/en/master/ "Requests") module
+- Optionally, if the python module [Pygments](http://pygments.org/ "Pygments") 1.5 or above is installed, the JSON responses will be highlighted with colors.
+
+You can see the usage with the following command:
+
+```console
+python sftpgo_api_cli.py --help
+```
+
+and
+
+```console
+python sftpgo_api_cli.py [sub-command] --help
+```
+
+Basically there is a sub command for each REST API and the following global arguments:
+
+- `-d`, `--debug`, default disabled, print useful debug info.
+- `-b`, `--base-url`, default `http://127.0.0.1:8080`. Base URL for SFTPGo REST API
+- `-a`, `--auth-type`, HTTP auth type. Supported HTTP auth type are `basic` and `digest`. Default none
+- `-u`, `--auth-user`, user for HTTP authentication
+- `-p`, `--auth-password`, password for HTTP authentication
+- `-i`, `--insecure`, enable to ignore verifying the SSL certificate. Default disabled
+- `-t`, `--no-color`, disable color highligth for JSON responses. You need python pygments module 1.5 or above for this to work. Default disabled if pygments is found and you aren't on Windows, otherwise enabled.
+- `-c`, `--color`, enable color highligth for JSON responses. You need python pygments module 1.5 or above for this to work. Default enabled if `pygments` is found and you aren't on Windows, otherwise disabled. Please read the note at the end of this doc for colors in Windows command prompt.
+
+For each subcommand `--help` shows the available arguments, try for example:
+
+```python sftpgo_api_cli.py add_user --help```
+
+Additionally it can convert users to the SFTPGo format from some supported users stores
+
+Let's see a sample usage for each REST API.
+
+## Add user
+
+Command:
+
+```console
+python sftpgo_api_cli.py add-user test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 33 --gid 1000 --max-sessions 2 --quota-size 0 --quota-files 3 --permissions "list" "download" "upload" "delete" "rename" "create_dirs" "overwrite" --subdirs-permissions "/dir1::list,download" "/dir2::*" --upload-bandwidth 100 --download-bandwidth 60 --status 0 --expiration-date 2019-01-01 --allowed-ip "192.168.1.1/32" --fs S3 --s3-bucket test --s3-region eu-west-1 --s3-access-key accesskey --s3-access-secret secret --s3-endpoint "http://127.0.0.1:9000" --s3-storage-class Standard --s3-key-prefix "vfolder/" --s3-upload-part-size 10 --s3-upload-concurrency 4 --denied-login-methods "password" "keyboard-interactive" --allowed-extensions "/dir1::.jpg,.png" "/dir2::.rar,.png" --denied-extensions "/dir3::.zip,.rar"
+```
+
+Output:
+
+```json
+{
+  "download_bandwidth": 60,
+  "expiration_date": 1546297200000,
+  "filesystem": {
+    "gcsconfig": {},
+    "provider": 1,
+    "s3config": {
+      "access_key": "accesskey",
+      "access_secret": "$aes$6c088ba12b0b261247c8cf331c46d9260b8e58002957d89ad1c0495e3af665cd0227",
+      "bucket": "test",
+      "endpoint": "http://127.0.0.1:9000",
+      "key_prefix": "vfolder/",
+      "region": "eu-west-1",
+      "storage_class": "Standard",
+      "upload_concurrency": 4,
+      "upload_part_size": 10
+    }
+  },
+  "filters": {
+    "allowed_ip": [
+      "192.168.1.1/32"
+    ],
+    "denied_login_methods": [
+      "password",
+      "keyboard-interactive"
+    ],
+    "file_extensions": [
+      {
+        "allowed_extensions": [
+          ".jpg",
+          ".png"
+        ],
+        "path": "/dir1"
+      },
+      {
+        "allowed_extensions": [
+          ".rar",
+          ".png"
+        ],
+        "path": "/dir2"
+      },
+      {
+        "denied_extensions": [
+          ".zip",
+          ".rar"
+        ],
+        "path": "/dir3"
+      }
+    ]
+  },
+  "gid": 1000,
+  "home_dir": "/tmp/test_home_dir",
+  "id": 9576,
+  "last_login": 0,
+  "last_quota_update": 0,
+  "max_sessions": 2,
+  "permissions": {
+    "/": [
+      "list",
+      "download",
+      "upload",
+      "delete",
+      "rename",
+      "create_dirs",
+      "overwrite"
+    ],
+    "/dir1": [
+      "list",
+      "download"
+    ],
+    "/dir2": [
+      "*"
+    ]
+  },
+  "quota_files": 3,
+  "quota_size": 0,
+  "status": 0,
+  "uid": 33,
+  "upload_bandwidth": 100,
+  "used_quota_files": 0,
+  "used_quota_size": 0,
+  "username": "test_username"
+}
+```
+
+## Update user
+
+Command:
+
+```console
+python sftpgo_api_cli.py update-user 9576 test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 0 --gid 33 --max-sessions 3 --quota-size 0 --quota-files 4 --permissions "*" --subdirs-permissions "/dir1::list,download,create_symlinks" --upload-bandwidth 90 --download-bandwidth 80 --status 1 --expiration-date "" --allowed-ip "" --denied-ip "192.168.1.0/24" --denied-login-methods "" --fs local --virtual-folders "/vdir1::/tmp/mapped1::-1::-1" "/vdir2::/tmp/mapped2::100::104857600" --allowed-extensions "" --denied-extensions ""
+```
+
+Output:
+
+```json
+{
+  "error": "",
+  "message": "User updated",
+  "status": 200
+}
+```
+
+## Get user by id
+
+Command:
+
+```console
+python sftpgo_api_cli.py get-user-by-id 9576
+```
+
+Output:
+
+```json
+{
+  "download_bandwidth": 80,
+  "expiration_date": 0,
+  "filesystem": {
+    "gcsconfig": {},
+    "provider": 0,
+    "s3config": {}
+  },
+  "filters": {
+    "denied_ip": [
+      "192.168.1.0/24"
+    ]
+  },
+  "gid": 33,
+  "home_dir": "/tmp/test_home_dir",
+  "id": 9576,
+  "last_login": 0,
+  "last_quota_update": 0,
+  "max_sessions": 3,
+  "permissions": {
+    "/": [
+      "*"
+    ],
+    "/dir1": [
+      "list",
+      "download",
+      "create_symlinks"
+    ]
+  },
+  "quota_files": 4,
+  "quota_size": 0,
+  "status": 1,
+  "uid": 0,
+  "upload_bandwidth": 90,
+  "used_quota_files": 0,
+  "used_quota_size": 0,
+  "username": "test_username",
+  "virtual_folders": [
+    {
+      "id": 1,
+      "last_quota_update": 0,
+      "mapped_path": "/tmp/mapped1",
+      "quota_files": -1,
+      "quota_size": -1,
+      "used_quota_files": 0,
+      "used_quota_size": 0,
+      "virtual_path": "/vdir1"
+    },
+    {
+      "id": 2,
+      "last_quota_update": 0,
+      "mapped_path": "/tmp/mapped2",
+      "quota_files": 100,
+      "quota_size": 104857600,
+      "used_quota_files": 0,
+      "used_quota_size": 0,
+      "virtual_path": "/vdir2"
+    }
+  ]
+}
+```
+
+## Get users
+
+Command:
+
+```console
+python sftpgo_api_cli.py get-users --limit 1 --offset 0 --username test_username --order DESC
+```
+
+Output:
+
+```json
+[
+  {
+    "download_bandwidth": 80,
+    "expiration_date": 0,
+    "filesystem": {
+      "gcsconfig": {},
+      "provider": 0,
+      "s3config": {}
+    },
+    "filters": {
+      "denied_ip": [
+        "192.168.1.0/24"
+      ]
+    },
+    "gid": 33,
+    "home_dir": "/tmp/test_home_dir",
+    "id": 9576,
+    "last_login": 0,
+    "last_quota_update": 0,
+    "max_sessions": 3,
+    "permissions": {
+      "/": [
+        "*"
+      ],
+      "/dir1": [
+        "list",
+        "download",
+        "create_symlinks"
+      ]
+    },
+    "quota_files": 4,
+    "quota_size": 0,
+    "status": 1,
+    "uid": 0,
+    "upload_bandwidth": 90,
+    "used_quota_files": 0,
+    "used_quota_size": 0,
+    "username": "test_username",
+    "virtual_folders": [
+      {
+        "exclude_from_quota": false,
+        "mapped_path": "/tmp/mapped1",
+        "virtual_path": "/vdir1"
+      },
+      {
+        "exclude_from_quota": true,
+        "mapped_path": "/tmp/mapped2",
+        "virtual_path": "/vdir2"
+      }
+    ]
+  }
+]
+```
+
+## Get active connections
+
+Command:
+
+```console
+python sftpgo_api_cli.py get-connections
+```
+
+Output:
+
+```json
+[
+  {
+    "active_transfers": [
+      {
+        "last_activity": 1577197485561,
+        "operation_type": "upload",
+        "path": "/test_upload.tar.gz",
+        "size": 1540096,
+        "start_time": 1577197471372
+      }
+    ],
+    "client_version": "SSH-2.0-OpenSSH_8.1",
+    "connection_id": "f82cfec6a391ad673edd4ae9a144f32ccb59456139f8e1185b070134fffbab7c",
+    "connection_time": 1577197433003,
+    "last_activity": 1577197485561,
+    "protocol": "SFTP",
+    "remote_address": "127.0.0.1:43714",
+    "ssh_command": "",
+    "username": "test_username"
+  }
+]
+```
+
+## Get folders
+
+Command:
+
+```console
+python sftpgo_api_cli.py get-folders --limit 1 --offset 0 --folder-path /tmp/mapped1 --order DESC
+```
+
+Output:
+
+```json
+[
+  {
+    "id": 1,
+    "last_quota_update": 1591563422870,
+    "mapped_path": "/tmp/mapped1",
+    "used_quota_files": 1,
+    "used_quota_size": 13313790,
+    "users": [
+      "test_username"
+    ]
+  }
+]
+```
+
+## Add folder
+
+```console
+python sftpgo_api_cli.py add-folder /tmp/mapped_folder
+```
+
+Output:
+
+```json
+{
+  "id": 4,
+  "last_quota_update": 0,
+  "mapped_path": "/tmp/mapped_folder",
+  "used_quota_files": 0,
+  "used_quota_size": 0
+}
+```
+
+## Close connection
+
+Command:
+
+```console
+python sftpgo_api_cli.py close-connection f82cfec6a391ad673edd4ae9a144f32ccb59456139f8e1185b070134fffbab7c
+```
+
+Output:
+
+```json
+{
+  "error": "",
+  "message": "Connection closed",
+  "status": 200
+}
+```
+
+## Get quota scans
+
+Command:
+
+```console
+python sftpgo_api_cli.py get-quota-scans
+```
+
+## Start quota scan
+
+Command:
+
+```console
+python sftpgo_api_cli.py start-quota-scan test_username
+```
+
+Output:
+
+```json
+{
+  "status": 201,
+  "message": "Scan started",
+  "error": ""
+}
+```
+
+## Get folder quota scans
+
+Command:
+
+```console
+python sftpgo_api_cli.py get-folders-quota-scans
+```
+
+## Start folder quota scan
+
+Command:
+
+```console
+python sftpgo_api_cli.py start-folder-quota-scan /tmp/mapped_folder
+```
+
+Output:
+
+```json
+{
+  "status": 201,
+  "message": "Scan started",
+  "error": ""
+}
+```
+
+## Update quota usage
+
+Command:
+
+```console
+python sftpgo_api_cli.py -d update-quota-usage a -S 123 -F 1 -M reset
+```
+
+Output:
+
+```json
+{
+  "error": "",
+  "message": "Quota updated",
+  "status": 200
+}
+```
+
+## Update folder quota usage
+
+Command:
+
+```console
+python sftpgo_api_cli.py -d update-quota-usage /tmp/mapped_folder -S 123 -F 1 -M add
+```
+
+Output:
+
+```json
+{
+  "error": "",
+  "message": "Quota updated",
+  "status": 200
+}
+```
+
+## Delete user
+
+Command:
+
+```console
+python sftpgo_api_cli.py delete-user 9576
+```
+
+Output:
+
+```json
+{
+  "error": "",
+  "message": "User deleted",
+  "status": 200
+}
+```
+
+## Delete folder
+
+```console
+python sftpgo_api_cli.py delete-folder /tmp/mapped_folder
+```
+
+Output:
+
+```json
+{
+  "error": "",
+  "message": "Folder deleted",
+  "status": 200
+}
+```
+
+## Get version
+
+Command:
+
+```console
+python sftpgo_api_cli.py get-version
+```
+
+Output:
+
+```json
+{
+  "build_date": "2019-12-24T14:17:47Z",
+  "commit_hash": "f8fd5c0-dirty",
+  "version": "0.9.4-dev"
+}
+```
+
+## Get provider status
+
+Command:
+
+```console
+python sftpgo_api_cli.py get-provider-status
+```
+
+Output:
+
+```json
+{
+  "error": "",
+  "message": "Alive",
+  "status": 200
+}
+```
+
+## Backup data
+
+Command:
+
+```console
+python sftpgo_api_cli.py dumpdata backup.json --indent 1
+```
+
+Output:
+
+```json
+{
+  "error": "",
+  "message": "Data saved",
+  "status": 200
+}
+```
+
+## Restore data
+
+Command:
+
+```console
+python sftpgo_api_cli.py loaddata /app/data/backups/backup.json --scan-quota 2 --mode 0
+```
+
+Output:
+
+```json
+{
+  "error": "",
+  "message": "Data restored",
+  "status": 200
+}
+```
+
+## Convert users from other stores
+
+You can convert users to the SFTPGo format from the following users stores:
+
+- Linux users stored in `shadow`/`passwd` files
+- Pure-FTPd virtual users generated using `pure-pw` CLI
+- ProFTPD users generated using `ftpasswd` CLI
+
+For details give a look at the `convert-users` subcommand usage:
+
+```console
+python sftpgo_api_cli.py convert-users --help
+```
+
+Let's see some examples:
+
+```console
+python sftpgo_api_cli.py convert-users "" unix-passwd unix_users.json --min-uid 500 --force-uid 1000 --force-gid 1000
+```
+
+```console
+python sftpgo_api_cli.py convert-users pureftpd.passwd pure-ftpd pure_users.json --usernames "user1" "user2"
+```
+
+```console
+python sftpgo_api_cli.py convert-users proftpd.passwd proftpd pro_users.json
+```
+
+The json file generated using the `convert-users` subcommand can be used as input for the `loaddata` subcommand.
+
+Please note that when importing Linux/Unix users the input file is not required: `/etc/passwd` and `/etc/shadow` are automatically parsed. `/etc/shadow` read permission is is typically granted to the `root` user, so you need to execute the `convert-users` subcommand as `root`.
+
+## Colors highlight for Windows command prompt
+
+If your Windows command prompt does not recognize ANSI/VT100 escape sequences you can download [ANSICON](https://github.com/adoxa/ansicon "ANSICON") extract proper files depending on your Windows OS, and install them using `ansicon -i`.
+Thats all. From now on, your Windows command prompt will be aware of ANSI colors.

examples/rest-api-cli/sftpgo_api_cli.py

@@ -0,0 +1,804 @@
+#!/usr/bin/env python
+import argparse
+import base64
+from datetime import datetime
+import json
+import platform
+import sys
+import time
+
+import requests
+
+try:
+	import urllib.parse as urlparse
+except ImportError:
+	import urlparse
+
+try:
+	import pygments
+	from pygments.lexers import JsonLexer
+	from pygments.formatters import TerminalFormatter
+except ImportError:
+	pygments = None
+
+try:
+	import pwd
+	import spwd
+except ImportError:
+	pwd = None
+
+
+class SFTPGoApiRequests:
+
+	def __init__(self, debug, baseUrl, authType, authUser, authPassword, secure, no_color):
+		self.userPath = urlparse.urljoin(baseUrl, '/api/v1/user')
+		self.folderPath = urlparse.urljoin(baseUrl, '/api/v1/folder')
+		self.quotaScanPath = urlparse.urljoin(baseUrl, '/api/v1/quota_scan')
+		self.folderQuotaScanPath = urlparse.urljoin(baseUrl, '/api/v1/folder_quota_scan')
+		self.activeConnectionsPath = urlparse.urljoin(baseUrl, '/api/v1/connection')
+		self.versionPath = urlparse.urljoin(baseUrl, '/api/v1/version')
+		self.providerStatusPath = urlparse.urljoin(baseUrl, '/api/v1/providerstatus')
+		self.dumpDataPath = urlparse.urljoin(baseUrl, '/api/v1/dumpdata')
+		self.loadDataPath = urlparse.urljoin(baseUrl, '/api/v1/loaddata')
+		self.updateUsedQuotaPath = urlparse.urljoin(baseUrl, "/api/v1/quota_update")
+		self.updateFolderUsedQuotaPath = urlparse.urljoin(baseUrl, "/api/v1/folder_quota_update")
+		self.debug = debug
+		if authType == 'basic':
+			self.auth = requests.auth.HTTPBasicAuth(authUser, authPassword)
+		elif authType == 'digest':
+			self.auth = requests.auth.HTTPDigestAuth(authUser, authPassword)
+		else:
+			self.auth = None
+		self.verify = secure
+		self.no_color = no_color
+
+	def formatAsJSON(self, text):
+		if not text:
+			return ''
+		json_string = json.dumps(json.loads(text), sort_keys=True, indent=2)
+		if not self.no_color and pygments:
+			return pygments.highlight(json_string, JsonLexer(), TerminalFormatter())
+		return json_string
+
+	def printResponse(self, r):
+		if 'content-type' in r.headers and 'application/json' in r.headers['content-type']:
+			if self.debug:
+				if pygments is None:
+					print('')
+					print('Response color highlight is not available: you need pygments 1.5 or above.')
+				print('')
+				print('Executed request: {} {} - request body: {}'.format(
+					r.request.method, r.url, self.formatAsJSON(r.request.body)))
+				print('')
+				print('Got response, status code: {} body:'.format(r.status_code))
+			print(self.formatAsJSON(r.text))
+		else:
+			print(r.text)
+
+	def buildUserObject(self, user_id=0, username='', password='', public_keys=[], home_dir='', uid=0, gid=0,
+					max_sessions=0, quota_size=0, quota_files=0, permissions={}, upload_bandwidth=0, download_bandwidth=0,
+					status=1, expiration_date=0, allowed_ip=[], denied_ip=[], fs_provider='local', s3_bucket='',
+					s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class='',
+					s3_key_prefix='', gcs_bucket='', gcs_key_prefix='', gcs_storage_class='', gcs_credentials_file='',
+					gcs_automatic_credentials='automatic', denied_login_methods=[], virtual_folders=[],
+					denied_extensions=[], allowed_extensions=[], s3_upload_part_size=0, s3_upload_concurrency=0):
+		user = {'id':user_id, 'username':username, 'uid':uid, 'gid':gid,
+			'max_sessions':max_sessions, 'quota_size':quota_size, 'quota_files':quota_files,
+			'upload_bandwidth':upload_bandwidth, 'download_bandwidth':download_bandwidth,
+			'status':status, 'expiration_date':expiration_date}
+		if password is not None:
+			user.update({'password':password})
+		if public_keys:
+			if len(public_keys) == 1 and not public_keys[0]:
+				user.update({'public_keys':[]})
+			else:
+				user.update({'public_keys':public_keys})
+		if home_dir:
+			user.update({'home_dir':home_dir})
+		if permissions:
+			user.update({'permissions':permissions})
+		if virtual_folders:
+			user.update({'virtual_folders':self.buildVirtualFolders(virtual_folders)})
+		if allowed_ip or denied_ip or denied_login_methods or allowed_extensions or denied_extensions:
+			user.update({'filters':self.buildFilters(allowed_ip, denied_ip, denied_login_methods, denied_extensions,
+													allowed_extensions)})
+		user.update({'filesystem':self.buildFsConfig(fs_provider, s3_bucket, s3_region, s3_access_key, s3_access_secret,
+													s3_endpoint, s3_storage_class, s3_key_prefix, gcs_bucket,
+													gcs_key_prefix, gcs_storage_class, gcs_credentials_file,
+													gcs_automatic_credentials, s3_upload_part_size, s3_upload_concurrency)})
+		return user
+
+	def buildVirtualFolders(self, vfolders):
+		result = []
+		for f in vfolders:
+			if '::' in f:
+				vpath = ''
+				mapped_path = ''
+				quota_files = 0
+				quota_size = 0
+				values = f.split('::')
+				if len(values) > 1:
+					vpath = values[0]
+					mapped_path = values[1]
+				if len(values) > 2:
+					try:
+						quota_files = int(values[2])
+					except:
+						pass
+				if len(values) > 3:
+					try:
+						quota_size = int(values[3])
+					except:
+						pass
+				if vpath and mapped_path:
+					result.append({"virtual_path":vpath, "mapped_path":mapped_path,
+								"quota_files":quota_files, "quota_size":quota_size})
+		return result
+
+	def buildPermissions(self, root_perms, subdirs_perms):
+		permissions = {}
+		if root_perms:
+			permissions.update({'/':root_perms})
+		for p in subdirs_perms:
+			if '::' in p:
+				directory = None
+				values = []
+				for value in p.split('::'):
+					if directory is None:
+						directory = value
+					else:
+						values = [v.strip() for v in value.split(',') if v.strip()]
+				if directory:
+					permissions.update({directory:values})
+		return permissions
+
+	def buildFilters(self, allowed_ip, denied_ip, denied_login_methods, denied_extensions, allowed_extensions):
+		filters = {}
+		if allowed_ip:
+			if len(allowed_ip) == 1 and not allowed_ip[0]:
+				filters.update({'allowed_ip':[]})
+			else:
+				filters.update({'allowed_ip':allowed_ip})
+		if denied_ip:
+			if len(denied_ip) == 1 and not denied_ip[0]:
+				filters.update({'denied_ip':[]})
+			else:
+				filters.update({'denied_ip':denied_ip})
+		if denied_login_methods:
+			if len(denied_login_methods) == 1 and not denied_login_methods[0]:
+				filters.update({'denied_login_methods':[]})
+			else:
+				filters.update({'denied_login_methods':denied_login_methods})
+		extensions_filter = []
+		extensions_denied = []
+		extensions_allowed = []
+		if denied_extensions:
+			for e in denied_extensions:
+				if '::' in e:
+					directory = None
+					values = []
+					for value in e.split('::'):
+						if directory is None:
+							directory = value
+						else:
+							values = [v.strip() for v in value.split(',') if v.strip()]
+					if directory:
+						extensions_denied.append({'path':directory, 'denied_extensions':values,
+												'allowed_extensions':[]})
+		if allowed_extensions:
+			for e in allowed_extensions:
+				if '::' in e:
+					directory = None
+					values = []
+					for value in e.split('::'):
+						if directory is None:
+							directory = value
+						else:
+							values = [v.strip() for v in value.split(',') if v.strip()]
+					if directory:
+						extensions_allowed.append({'path':directory, 'allowed_extensions':values,
+												'denied_extensions':[]})
+		if extensions_allowed and extensions_denied:
+			for allowed in extensions_allowed:
+				for denied in extensions_denied:
+					if allowed.get('path') == denied.get('path'):
+						allowed.update({'denied_extensions':denied.get('denied_extensions')})
+				extensions_filter.append(allowed)
+			for denied in extensions_denied:
+				found = False
+				for allowed in extensions_allowed:
+					if allowed.get('path') == denied.get('path'):
+						found = True
+				if not found:
+					extensions_filter.append(denied)
+		elif extensions_allowed:
+			extensions_filter = extensions_allowed
+		elif extensions_denied:
+			extensions_filter = extensions_denied
+		if allowed_extensions or denied_extensions:
+			filters.update({'file_extensions':extensions_filter})
+		return filters
+
+	def buildFsConfig(self, fs_provider, s3_bucket, s3_region, s3_access_key, s3_access_secret, s3_endpoint,
+					s3_storage_class, s3_key_prefix, gcs_bucket, gcs_key_prefix, gcs_storage_class,
+					gcs_credentials_file, gcs_automatic_credentials, s3_upload_part_size, s3_upload_concurrency):
+		fs_config = {'provider':0}
+		if fs_provider == 'S3':
+			s3config = {'bucket':s3_bucket, 'region':s3_region, 'access_key':s3_access_key, 'access_secret':
+					s3_access_secret, 'endpoint':s3_endpoint, 'storage_class':s3_storage_class, 'key_prefix':
+					s3_key_prefix, 'upload_part_size':s3_upload_part_size, 'upload_concurrency':s3_upload_concurrency}
+			fs_config.update({'provider':1, 's3config':s3config})
+		elif fs_provider == 'GCS':
+			gcsconfig = {'bucket':gcs_bucket, 'key_prefix':gcs_key_prefix, 'storage_class':gcs_storage_class}
+			if gcs_automatic_credentials == "automatic":
+				gcsconfig.update({'automatic_credentials':1})
+			else:
+				gcsconfig.update({'automatic_credentials':0})
+			if gcs_credentials_file:
+				with open(gcs_credentials_file) as creds:
+					gcsconfig.update({'credentials':base64.b64encode(creds.read().encode('UTF-8')).decode('UTF-8'),
+									'automatic_credentials':0})
+			fs_config.update({'provider':2, 'gcsconfig':gcsconfig})
+		return fs_config
+
+	def getUsers(self, limit=100, offset=0, order='ASC', username=''):
+		r = requests.get(self.userPath, params={'limit':limit, 'offset':offset, 'order':order,
+											'username':username}, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def getUserByID(self, user_id):
+		r = requests.get(urlparse.urljoin(self.userPath, 'user/' + str(user_id)), auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def addUser(self, username='', password='', public_keys='', home_dir='', uid=0, gid=0, max_sessions=0, quota_size=0,
+			quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1, expiration_date=0,
+			subdirs_permissions=[], allowed_ip=[], denied_ip=[], fs_provider='local', s3_bucket='', s3_region='',
+			s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class='', s3_key_prefix='', gcs_bucket='',
+			gcs_key_prefix='', gcs_storage_class='', gcs_credentials_file='', gcs_automatic_credentials='automatic',
+			denied_login_methods=[], virtual_folders=[], denied_extensions=[], allowed_extensions=[],
+			s3_upload_part_size=0, s3_upload_concurrency=0):
+		u = self.buildUserObject(0, username, password, public_keys, home_dir, uid, gid, max_sessions,
+			quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth,
+			status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region, s3_access_key,
+			s3_access_secret, s3_endpoint, s3_storage_class, s3_key_prefix, gcs_bucket, gcs_key_prefix, gcs_storage_class,
+			gcs_credentials_file, gcs_automatic_credentials, denied_login_methods, virtual_folders, denied_extensions,
+			allowed_extensions, s3_upload_part_size, s3_upload_concurrency)
+		r = requests.post(self.userPath, json=u, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def updateUser(self, user_id, username='', password='', public_keys='', home_dir='', uid=0, gid=0, max_sessions=0,
+				quota_size=0, quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1,
+				expiration_date=0, subdirs_permissions=[], allowed_ip=[], denied_ip=[], fs_provider='local',
+				s3_bucket='', s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class='',
+				s3_key_prefix='', gcs_bucket='', gcs_key_prefix='', gcs_storage_class='', gcs_credentials_file='',
+				gcs_automatic_credentials='automatic', denied_login_methods=[], virtual_folders=[], denied_extensions=[],
+				allowed_extensions=[], s3_upload_part_size=0, s3_upload_concurrency=0):
+		u = self.buildUserObject(user_id, username, password, public_keys, home_dir, uid, gid, max_sessions,
+			quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth,
+			status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region, s3_access_key,
+			s3_access_secret, s3_endpoint, s3_storage_class, s3_key_prefix, gcs_bucket, gcs_key_prefix, gcs_storage_class,
+			gcs_credentials_file, gcs_automatic_credentials, denied_login_methods, virtual_folders, denied_extensions,
+			allowed_extensions, s3_upload_part_size, s3_upload_concurrency)
+		r = requests.put(urlparse.urljoin(self.userPath, 'user/' + str(user_id)), json=u, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def deleteUser(self, user_id):
+		r = requests.delete(urlparse.urljoin(self.userPath, 'user/' + str(user_id)), auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def updateQuotaUsage(self, username, used_quota_size, used_quota_files, mode):
+		req = {"username":username, "used_quota_files":used_quota_files, "used_quota_size":used_quota_size}
+		r = requests.put(self.updateUsedQuotaPath, params={'mode':mode}, json=req, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def updateFolderQuotaUsage(self, mapped_path, used_quota_size, used_quota_files, mode):
+		req = {"mapped_path":mapped_path, "used_quota_files":used_quota_files, "used_quota_size":used_quota_size}
+		r = requests.put(self.updateFolderUsedQuotaPath, params={'mode':mode}, json=req, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def getConnections(self):
+		r = requests.get(self.activeConnectionsPath, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def closeConnection(self, connectionID):
+		r = requests.delete(urlparse.urljoin(self.activeConnectionsPath, 'connection/' + str(connectionID)), auth=self.auth)
+		self.printResponse(r)
+
+	def getQuotaScans(self):
+		r = requests.get(self.quotaScanPath, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def startQuotaScan(self, username):
+		u = self.buildUserObject(0, username)
+		r = requests.post(self.quotaScanPath, json=u, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def getFoldersQuotaScans(self):
+		r = requests.get(self.folderQuotaScanPath, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def startFolderQuotaScan(self, mapped_path):
+		f = {"mapped_path":mapped_path}
+		r = requests.post(self.folderQuotaScanPath, json=f, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def addFolder(self, mapped_path):
+		f = {"mapped_path":mapped_path}
+		r = requests.post(self.folderPath, json=f, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def deleteFolder(self, mapped_path):
+		r = requests.delete(self.folderPath, params={'folder_path':mapped_path}, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def getFolders(self, limit=100, offset=0, order='ASC', mapped_path=''):
+		r = requests.get(self.folderPath, params={'limit':limit, 'offset':offset, 'order':order,
+											'folder_path':mapped_path}, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def getVersion(self):
+		r = requests.get(self.versionPath, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def getProviderStatus(self):
+		r = requests.get(self.providerStatusPath, auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def dumpData(self, output_file, indent):
+		r = requests.get(self.dumpDataPath, params={'output_file':output_file, 'indent':indent},
+						auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+	def loadData(self, input_file, scan_quota, mode):
+		r = requests.get(self.loadDataPath, params={'input_file':input_file, 'scan_quota':scan_quota,
+												'mode':mode},
+						auth=self.auth, verify=self.verify)
+		self.printResponse(r)
+
+
+class ConvertUsers:
+
+	def __init__(self, input_file, users_format, output_file, min_uid, max_uid, usernames, force_uid, force_gid):
+		self.input_file = input_file
+		self.users_format = users_format
+		self.output_file = output_file
+		self.min_uid = min_uid
+		self.max_uid = max_uid
+		self.usernames = usernames
+		self.force_uid = force_uid
+		self.force_gid = force_gid
+		self.SFTPGoUsers = []
+
+	def setSFTPGoRestApi(self, api):
+		self.SFTPGoRestAPI = api
+
+	def addUser(self, user):
+		user['id'] = len(self.SFTPGoUsers) + 1
+		print('')
+		print('New user imported: {}'.format(user))
+		print('')
+		self.SFTPGoUsers.append(user)
+
+	def saveUsers(self):
+		if self.SFTPGoUsers:
+			data = {'users':self.SFTPGoUsers}
+			jsonData = json.dumps(data)
+			with open(self.output_file, 'w') as f:
+				f.write(jsonData)
+			print()
+			print('Number of users saved to "{}": {}. You can import them using loaddata'.format(self.output_file,
+																								len(self.SFTPGoUsers)))
+			print()
+			sys.exit(0)
+		else:
+			print('No user imported')
+			sys.exit(1)
+
+	def convert(self):
+		if self.users_format == 'unix-passwd':
+			self.convertFromUnixPasswd()
+		elif self.users_format == 'pure-ftpd':
+			self.convertFromPureFTPD()
+		else:
+			self.convertFromProFTPD()
+		self.saveUsers()
+
+	def isUserValid(self, username, uid):
+		if self.usernames and not username in self.usernames:
+			return False
+		if self.min_uid >= 0 and uid < self.min_uid:
+			return False
+		if self.max_uid >= 0 and uid > self.max_uid:
+			return False
+		return True
+
+	def convertFromUnixPasswd(self):
+		days_from_epoch_time = time.time() / 86400
+		for user in pwd.getpwall():
+			username = user.pw_name
+			password = user.pw_passwd
+			uid = user.pw_uid
+			gid = user.pw_gid
+			home_dir = user.pw_dir
+			status = 1
+			expiration_date = 0
+			if not self.isUserValid(username, uid):
+				continue
+			if self.force_uid >= 0:
+				uid = self.force_uid
+			if self.force_gid >= 0:
+				gid = self.force_gid
+			# FIXME: if the passwords aren't in /etc/shadow they are probably DES encrypted and we don't support them
+			if password == 'x' or password == '*':
+				user_info = spwd.getspnam(username)
+				password = user_info.sp_pwdp
+				if not password or password == '!!':
+					print('cannot import user "{}" without a password'.format(username))
+					continue
+				if user_info.sp_inact > 0:
+					last_pwd_change_diff = days_from_epoch_time - user_info.sp_lstchg
+					if last_pwd_change_diff > user_info.sp_inact:
+						status = 0
+				if user_info.sp_expire > 0:
+					expiration_date = user_info.sp_expire * 86400
+			permissions = self.SFTPGoRestAPI.buildPermissions(['*'], [])
+			self.addUser(self.SFTPGoRestAPI.buildUserObject(0, username, password, [], home_dir, uid, gid, 0, 0, 0,
+														permissions, 0, 0, status, expiration_date))
+
+	def convertFromProFTPD(self):
+		with open(self.input_file, 'r') as f:
+			for line in f:
+				fields = line.split(':')
+				if len(fields) > 6:
+					username = fields[0]
+					password = fields[1]
+					uid = int(fields[2])
+					gid = int(fields[3])
+					home_dir = fields[5]
+					if not self.isUserValid(username, uid, gid):
+						continue
+					if self.force_uid >= 0:
+						uid = self.force_uid
+					if self.force_gid >= 0:
+						gid = self.force_gid
+					permissions = self.SFTPGoRestAPI.buildPermissions(['*'], [])
+					self.addUser(self.SFTPGoRestAPI.buildUserObject(0, username, password, [], home_dir, uid, gid, 0, 0,
+																0, permissions, 0, 0, 1, 0))
+
+	def convertPureFTPDIP(self, fields):
+		result = []
+		if not fields:
+			return result
+		for v in fields.split(','):
+			ip_mask = v.strip()
+			if not ip_mask:
+				continue
+			if ip_mask.count('.') < 3 and ip_mask.count(':') < 3:
+				print('cannot import pure-ftpd IP: {}'.format(ip_mask))
+				continue
+			if '/' not in ip_mask:
+				ip_mask += '/32'
+			result.append(ip_mask)
+		return result
+
+	def convertFromPureFTPD(self):
+		with open(self.input_file, 'r') as f:
+			for line in f:
+				fields = line.split(':')
+				if len(fields) > 16:
+					username = fields[0]
+					password = fields[1]
+					uid = int(fields[2])
+					gid = int(fields[3])
+					home_dir = fields[5]
+					upload_bandwidth = 0
+					if fields[6]:
+						upload_bandwidth = int(int(fields[6]) / 1024)
+					download_bandwidth = 0
+					if fields[7]:
+						download_bandwidth = int(int(fields[7]) / 1024)
+					max_sessions = 0
+					if fields[10]:
+						max_sessions = int(fields[10])
+					quota_files = 0
+					if fields[11]:
+						quota_files = int(fields[11])
+					quota_size = 0
+					if fields[12]:
+						quota_size = int(fields[12])
+					allowed_ip = self.convertPureFTPDIP(fields[15])
+					denied_ip = self.convertPureFTPDIP(fields[16])
+					if not self.isUserValid(username, uid, gid):
+						continue
+					if self.force_uid >= 0:
+						uid = self.force_uid
+					if self.force_gid >= 0:
+						gid = self.force_gid
+					permissions = self.SFTPGoRestAPI.buildPermissions(['*'], [])
+					self.addUser(self.SFTPGoRestAPI.buildUserObject(0, username, password, [], home_dir, uid, gid,
+																max_sessions, quota_size, quota_files, permissions,
+																upload_bandwidth, download_bandwidth, 1, 0, allowed_ip,
+																denied_ip))
+
+
+def validDate(s):
+	if not s:
+		return datetime.fromtimestamp(0)
+	try:
+		return datetime.strptime(s, '%Y-%m-%d')
+	except ValueError:
+		msg = 'Not a valid date: "{0}".'.format(s)
+		raise argparse.ArgumentTypeError(msg)
+
+
+def getDatetimeAsMillisSinceEpoch(dt):
+	epoch = datetime.fromtimestamp(0)
+	return int((dt - epoch).total_seconds() * 1000)
+
+
+def addCommonUserArguments(parser):
+	parser.add_argument('username', type=str)
+	parser.add_argument('-P', '--password', type=str, default=None, help='Default: %(default)s')
+	parser.add_argument('-K', '--public-keys', type=str, nargs='+', default=[], help='Public keys or SSH user certificates. ' +
+					'Default: %(default)s')
+	parser.add_argument('-H', '--home-dir', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--uid', type=int, default=0, help='Default: %(default)s')
+	parser.add_argument('--gid', type=int, default=0, help='Default: %(default)s')
+	parser.add_argument('-C', '--max-sessions', type=int, default=0,
+					help='Maximum concurrent sessions. 0 means unlimited. Default: %(default)s')
+	parser.add_argument('-S', '--quota-size', type=int, default=0,
+					help='Maximum size allowed as bytes. 0 means unlimited. Default: %(default)s')
+	parser.add_argument('-F', '--quota-files', type=int, default=0, help='default: %(default)s')
+	parser.add_argument('-G', '--permissions', type=str, nargs='+', default=[],
+					choices=['*', 'list', 'download', 'upload', 'overwrite', 'delete', 'rename', 'create_dirs',
+							'create_symlinks', 'chmod', 'chown', 'chtimes'], help='Permissions for the root directory '
+							+'(/). Default: %(default)s')
+	parser.add_argument('-L', '--denied-login-methods', type=str, nargs='+', default=[],
+					choices=['', 'publickey', 'password', 'keyboard-interactive', 'publickey+password',
+							'publickey+keyboard-interactive'], help='Default: %(default)s')
+	parser.add_argument('--subdirs-permissions', type=str, nargs='*', default=[], help='Permissions for subdirs. '
+					+'For example: "/somedir::list,download" "/otherdir/subdir::*" Default: %(default)s')
+	parser.add_argument('--virtual-folders', type=str, nargs='*', default=[], help='Virtual folder mapping. For example: '
+					+'"/vpath::/home/adir" "/vpath::C:\adir::[quota_file]::[quota_size]". Quota parameters -1 means '
+					+'included inside user quota, 0 means unlimited. Ignored for non local filesystems. Default: %(default)s')
+	parser.add_argument('-U', '--upload-bandwidth', type=int, default=0,
+					help='Maximum upload bandwidth as KB/s, 0 means unlimited. Default: %(default)s')
+	parser.add_argument('-D', '--download-bandwidth', type=int, default=0,
+					help='Maximum download bandwidth as KB/s, 0 means unlimited. Default: %(default)s')
+	parser.add_argument('--status', type=int, choices=[0, 1], default=1,
+							help='User\'s status. 1 enabled, 0 disabled. Default: %(default)s')
+	parser.add_argument('-E', '--expiration-date', type=validDate, default='',
+					help='Expiration date as YYYY-MM-DD, empty string means no expiration. Default: %(default)s')
+	parser.add_argument('-Y', '--allowed-ip', type=str, nargs='+', default=[],
+					help='Allowed IP/Mask in CIDR notation. For example "192.168.2.0/24" or "2001:db8::/32". Default: %(default)s')
+	parser.add_argument('-N', '--denied-ip', type=str, nargs='+', default=[],
+					help='Denied IP/Mask in CIDR notation. For example "192.168.2.0/24" or "2001:db8::/32". Default: %(default)s')
+	parser.add_argument('--denied-extensions', type=str, nargs='*', default=[], help='Denied file extensions case insensitive. '
+					+'The format is /dir::ext1,ext2. For example: "/somedir::.jpg,.png" "/otherdir/subdir::.zip,.rar". ' +
+					'You have to set both denied and allowed extensions to update existing values or none to preserve them.' +
+					' If you only set allowed or denied extensions the missing one is assumed to be an empty list. Default: %(default)s')
+	parser.add_argument('--allowed-extensions', type=str, nargs='*', default=[], help='Allowed file extensions case insensitive. '
+					+'The format is /dir::ext1,ext2. For example: "/somedir::.jpg,.png" "/otherdir/subdir::.zip,.rar". ' +
+					'Default: %(default)s')
+	parser.add_argument('--fs', type=str, default='local', choices=['local', 'S3', 'GCS'],
+					help='Filesystem provider. Default: %(default)s')
+	parser.add_argument('--s3-bucket', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--s3-key-prefix', type=str, default='', help='Virtual root directory. If non empty only this ' +
+					'directory and its contents will be available. Cannot start with "/". For example "folder/subfolder/".' +
+					' Default: %(default)s')
+	parser.add_argument('--s3-region', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--s3-access-key', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--s3-access-secret', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--s3-endpoint', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--s3-storage-class', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--s3-upload-part-size', type=int, default=0, help='The buffer size for multipart uploads (MB). ' +
+					'Zero means the default (5 MB). Minimum is 5. Default: %(default)s')
+	parser.add_argument('--s3-upload-concurrency', type=int, default=0, help='How many parts are uploaded in parallel. ' +
+					'Zero means the default (2). Default: %(default)s')
+	parser.add_argument('--gcs-bucket', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--gcs-key-prefix', type=str, default='', help='Virtual root directory. If non empty only this ' +
+					'directory and its contents will be available. Cannot start with "/". For example "folder/subfolder/".' +
+					' Default: %(default)s')
+	parser.add_argument('--gcs-storage-class', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--gcs-credentials-file', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--gcs-automatic-credentials', type=str, default='automatic', choices=['explicit', 'automatic'],
+					help='If you provide a credentials file this argument will be setted to "explicit". Default: %(default)s')
+
+
+if __name__ == '__main__':
+	parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+	parser.add_argument('-b', '--base-url', type=str, default='http://127.0.0.1:8080',
+					help='Base URL for SFTPGo REST API. Default: %(default)s')
+	parser.add_argument('-a', '--auth-type', type=str, default=None, choices=['basic', 'digest'],
+					help='HTTP authentication type. Default: %(default)s')
+	parser.add_argument('-u', '--auth-user', type=str, default='',
+					help='User for HTTP authentication. Default: %(default)s')
+	parser.add_argument('-p', '--auth-password', type=str, default='',
+					help='Password for HTTP authentication. Default: %(default)s')
+	parser.add_argument('-d', '--debug', dest='debug', action='store_true')
+	parser.set_defaults(debug=False)
+	parser.add_argument('-i', '--insecure', dest='secure', action='store_false',
+					help='Set to false to ignore verifying the SSL certificate')
+	parser.set_defaults(secure=True)
+	has_colors_default = pygments is not None and platform.system() != 'Windows'
+	group = parser.add_mutually_exclusive_group(required=False)
+	group.add_argument('-t', '--no-color', dest='no_color', action='store_true', default=(not has_colors_default),
+					help='Disable color highlight for JSON responses. You need python pygments module 1.5 or above to have highlighted output')
+	group.add_argument('-c', '--color', dest='no_color', action='store_false', default=has_colors_default,
+					help='Enable color highlight for JSON responses. You need python pygments module 1.5 or above to have highlighted output')
+	parser.add_argument_group(group)
+
+	subparsers = parser.add_subparsers(dest='command', help='sub-command --help')
+	subparsers.required = True
+
+	parserAddUser = subparsers.add_parser('add-user', help='Add a new SFTP user')
+	addCommonUserArguments(parserAddUser)
+
+	parserUpdateUser = subparsers.add_parser('update-user', help='Update an existing user')
+	parserUpdateUser.add_argument('id', type=int, help='User\'s ID to update')
+	addCommonUserArguments(parserUpdateUser)
+
+	parserDeleteUser = subparsers.add_parser('delete-user', help='Delete an existing user')
+	parserDeleteUser.add_argument('id', type=int, help='User\'s ID to delete')
+
+	parserGetUsers = subparsers.add_parser('get-users', help='Returns an array with one or more SFTP users')
+	parserGetUsers.add_argument('-L', '--limit', type=int, default=100, choices=range(1, 501),
+							help='Maximum allowed value is 500. Default: %(default)s', metavar='[1...500]')
+	parserGetUsers.add_argument('-O', '--offset', type=int, default=0, help='Default: %(default)s')
+	parserGetUsers.add_argument('-U', '--username', type=str, default='', help='Default: %(default)s')
+	parserGetUsers.add_argument('-S', '--order', type=str, choices=['ASC', 'DESC'], default='ASC',
+							help='default: %(default)s')
+
+	parserGetUserByID = subparsers.add_parser('get-user-by-id', help='Find user by ID')
+	parserGetUserByID.add_argument('id', type=int)
+
+	parserGetConnections = subparsers.add_parser('get-connections',
+													help='Get the active users and info about their uploads/downloads')
+
+	parserCloseConnection = subparsers.add_parser('close-connection', help='Terminate an active SFTP/SCP connection')
+	parserCloseConnection.add_argument('connectionID', type=str)
+
+	parserGetQuotaScans = subparsers.add_parser('get-quota-scans', help='Get the active quota scans for users home directories')
+
+	parserStartQuotaScan = subparsers.add_parser('start-quota-scan', help='Start a new user quota scan')
+	addCommonUserArguments(parserStartQuotaScan)
+
+	parserGetFolderQuotaScans = subparsers.add_parser('get-folders-quota-scans', help='Get the active quota scans for folders')
+
+	parserStartFolderQuotaScan = subparsers.add_parser('start-folder-quota-scan', help='Start a new folder quota scan')
+	parserStartFolderQuotaScan.add_argument('folder_path', type=str)
+
+	parserGetFolders = subparsers.add_parser('get-folders', help='Returns an array with one or more folders')
+	parserGetFolders.add_argument('-L', '--limit', type=int, default=100, choices=range(1, 501),
+							help='Maximum allowed value is 500. Default: %(default)s', metavar='[1...500]')
+	parserGetFolders.add_argument('-O', '--offset', type=int, default=0, help='Default: %(default)s')
+	parserGetFolders.add_argument('-P', '--folder-path', type=str, default='', help='Default: %(default)s')
+	parserGetFolders.add_argument('-S', '--order', type=str, choices=['ASC', 'DESC'], default='ASC',
+							help='default: %(default)s')
+
+	parserAddFolder = subparsers.add_parser('add-folder', help='Add a new folder')
+	parserAddFolder.add_argument('folder_path', type=str)
+
+	parserDeleteFolder = subparsers.add_parser('delete-folder', help='Delete an existing folder')
+	parserDeleteFolder.add_argument('folder_path', type=str)
+
+	parserGetVersion = subparsers.add_parser('get-version', help='Get version details')
+
+	parserGetProviderStatus = subparsers.add_parser('get-provider-status', help='Get data provider status')
+
+	parserDumpData = subparsers.add_parser('dumpdata', help='Backup SFTPGo data serializing them as JSON')
+	parserDumpData.add_argument('output_file', type=str)
+	parserDumpData.add_argument('-I', '--indent', type=int, choices=[0, 1], default=0,
+							help='0 means no indentation. 1 means format the output JSON. Default: %(default)s')
+
+	parserLoadData = subparsers.add_parser('loaddata', help='Restore SFTPGo data from a JSON backup')
+	parserLoadData.add_argument('input_file', type=str)
+	parserLoadData.add_argument('-Q', '--scan-quota', type=int, choices=[0, 1, 2], default=0,
+							help='0 means no quota scan after a user is added/updated. 1 means always scan quota. 2 ' +
+							'means scan quota if the user has quota restrictions. Default: %(default)s')
+	parserLoadData.add_argument('-M', '--mode', type=int, choices=[0, 1], default=0,
+							help='0 means new users are added, existing users are updated. 1 means new users are added,' +
+							' existing users are not modified. Default: %(default)s')
+
+	parserUpdateQuotaUsage = subparsers.add_parser('update-quota-usage', help='Update the user used quota limits')
+	parserUpdateQuotaUsage.add_argument('username', type=str)
+	parserUpdateQuotaUsage.add_argument('-M', '--mode', type=str, choices=["add", "reset"], default="reset",
+							help='the update mode specifies if the given quota usage values should be added or ' +
+							'replace the current ones. Default: %(default)s')
+	parserUpdateQuotaUsage.add_argument('-S', '--used_quota_size', type=int, default=0, help='Default: %(default)s')
+	parserUpdateQuotaUsage.add_argument('-F', '--used_quota_files', type=int, default=0, help='Default: %(default)s')
+
+	parserUpdateFolderQuotaUsage = subparsers.add_parser('update-folder-quota-usage', help='Update the folder used quota limits')
+	parserUpdateFolderQuotaUsage.add_argument('folder_path', type=str)
+	parserUpdateFolderQuotaUsage.add_argument('-M', '--mode', type=str, choices=["add", "reset"], default="reset",
+							help='the update mode specifies if the given quota usage values should be added or ' +
+							'replace the current ones. Default: %(default)s')
+	parserUpdateFolderQuotaUsage.add_argument('-S', '--used_quota_size', type=int, default=0, help='Default: %(default)s')
+	parserUpdateFolderQuotaUsage.add_argument('-F', '--used_quota_files', type=int, default=0, help='Default: %(default)s')
+
+	parserConvertUsers = subparsers.add_parser('convert-users', help='Convert users to a JSON format suitable to use ' +
+											'with loadddata')
+	supportedUsersFormats = []
+	help_text = ''
+	if pwd is not None:
+		supportedUsersFormats.append('unix-passwd')
+		help_text = 'To import from unix-passwd format you need the permission to read /etc/shadow that is typically granted to the root user only'
+	supportedUsersFormats.append('pure-ftpd')
+	supportedUsersFormats.append('proftpd')
+	parserConvertUsers.add_argument('input_file', type=str)
+	parserConvertUsers.add_argument('users_format', type=str, choices=supportedUsersFormats, help=help_text)
+	parserConvertUsers.add_argument('output_file', type=str)
+	parserConvertUsers.add_argument('--min-uid', type=int, default=-1, help='if >= 0 only import users with UID greater ' +
+								'or equal to this value. Default: %(default)s')
+	parserConvertUsers.add_argument('--max-uid', type=int, default=-1, help='if >= 0 only import users with UID lesser ' +
+								'or equal to this value. Default: %(default)s')
+	parserConvertUsers.add_argument('--usernames', type=str, nargs='+', default=[], help='Only import users with these ' +
+								'usernames. Default: %(default)s')
+	parserConvertUsers.add_argument('--force-uid', type=int, default=-1, help='if >= 0 the imported users will have this UID in SFTPGo. Default: %(default)s')
+	parserConvertUsers.add_argument('--force-gid', type=int, default=-1, help='if >= 0 the imported users will have this GID in SFTPGp. Default: %(default)s')
+
+	args = parser.parse_args()
+
+	api = SFTPGoApiRequests(args.debug, args.base_url, args.auth_type, args.auth_user, args.auth_password, args.secure,
+						 args.no_color)
+
+	if args.command == 'add-user':
+		api.addUser(args.username, args.password, args.public_keys, args.home_dir, args.uid, args.gid, args.max_sessions,
+				args.quota_size, args.quota_files, args.permissions, args.upload_bandwidth, args.download_bandwidth,
+				args.status, getDatetimeAsMillisSinceEpoch(args.expiration_date), args.subdirs_permissions, args.allowed_ip,
+				args.denied_ip, args.fs, args.s3_bucket, args.s3_region, args.s3_access_key, args.s3_access_secret,
+				args.s3_endpoint, args.s3_storage_class, args.s3_key_prefix, args.gcs_bucket, args.gcs_key_prefix,
+				args.gcs_storage_class, args.gcs_credentials_file, args.gcs_automatic_credentials,
+				args.denied_login_methods, args.virtual_folders, args.denied_extensions, args.allowed_extensions,
+				args.s3_upload_part_size, args.s3_upload_concurrency)
+	elif args.command == 'update-user':
+		api.updateUser(args.id, args.username, args.password, args.public_keys, args.home_dir, args.uid, args.gid,
+					args.max_sessions, args.quota_size, args.quota_files, args.permissions, args.upload_bandwidth,
+					args.download_bandwidth, args.status, getDatetimeAsMillisSinceEpoch(args.expiration_date),
+					args.subdirs_permissions, args.allowed_ip, args.denied_ip, args.fs, args.s3_bucket, args.s3_region,
+					args.s3_access_key, args.s3_access_secret, args.s3_endpoint, args.s3_storage_class,
+					args.s3_key_prefix, args.gcs_bucket, args.gcs_key_prefix, args.gcs_storage_class,
+					args.gcs_credentials_file, args.gcs_automatic_credentials, args.denied_login_methods,
+					args.virtual_folders, args.denied_extensions, args.allowed_extensions, args.s3_upload_part_size,
+					args.s3_upload_concurrency)
+	elif args.command == 'delete-user':
+		api.deleteUser(args.id)
+	elif args.command == 'get-users':
+		api.getUsers(args.limit, args.offset, args.order, args.username)
+	elif args.command == 'get-user-by-id':
+		api.getUserByID(args.id)
+	elif args.command == 'get-connections':
+		api.getConnections()
+	elif args.command == 'close-connection':
+		api.closeConnection(args.connectionID)
+	elif args.command == 'get-quota-scans':
+		api.getQuotaScans()
+	elif args.command == 'start-quota-scan':
+		api.startQuotaScan(args.username)
+	elif args.command == 'get-folders':
+		api.getFolders(args.limit, args.offset, args.order, args.folder_path)
+	elif args.command == 'add-folder':
+		api.addFolder(args.folder_path)
+	elif args.command == 'delete-folder':
+		api.deleteFolder(args.folder_path)
+	elif args.command == 'get-folders-quota-scans':
+		api.getFoldersQuotaScans()
+	elif args.command == 'start-folder-quota-scan':
+		api.startFolderQuotaScan(args.folder_path)
+	elif args.command == 'get-version':
+		api.getVersion()
+	elif args.command == 'get-provider-status':
+		api.getProviderStatus()
+	elif args.command == 'dumpdata':
+		api.dumpData(args.output_file, args.indent)
+	elif args.command == 'loaddata':
+		api.loadData(args.input_file, args.scan_quota, args.mode)
+	elif args.command == 'update-quota-usage':
+		api.updateQuotaUsage(args.username, args.used_quota_size, args.used_quota_files, args.mode)
+	elif args.command == 'update-folder-quota-usage':
+		api.updateFolderQuotaUsage(args.folder_path, args.used_quota_size, args.used_quota_files, args.mode)
+	elif args.command == 'convert-users':
+		convertUsers = ConvertUsers(args.input_file, args.users_format, args.output_file, args.min_uid, args.max_uid,
+								args.usernames, args.force_uid, args.force_gid)
+		convertUsers.setSFTPGoRestApi(api)
+		convertUsers.convert()

fail2ban/filters/sftpgo.conf

@@ -0,0 +1,15 @@
+[INCLUDES]
+before = common.conf
+
+[DEFAULT]
+_daemon = sftpgo
+
+[Definition]
+
+# By default, first authenticate method is public_key and must be excluded from the filter to avoid false positives failed attempts
+failregex = ^.*"sender":"connection_failed","client_ip":"<HOST>","username":".*","login_type":"password".*"}$
+
+ignoreregex =
+
+[Init]
+journalmatch = _SYSTEMD_UNIT=sftpgo.service + _COMM=sftpgo

fail2ban/jails/sftpgo.conf

@@ -0,0 +1,10 @@
+[sftpgo]
+enabled = true
+port = 2022
+filter = sftpgo
+action = iptables-allports[name=sftpgo]
+logpath = 
+backend = systemd
+maxretry = 5
+bantime = 600
+findtime = 86400

go.mod

@@ -1,32 +1,53 @@
 module github.com/drakkan/sftpgo
 
-go 1.12
+go 1.13
 
 require (
-	github.com/alexedwards/argon2id v0.0.0-20190612080829-01a59b2b8802
-	github.com/go-chi/chi v4.0.2+incompatible
+	cloud.google.com/go v0.60.0 // indirect
+	cloud.google.com/go/storage v1.10.0
+	github.com/alexedwards/argon2id v0.0.0-20200522061839-9369edc04b05
+	github.com/aws/aws-sdk-go v1.33.1
+	github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/go-chi/render v1.0.1
-	github.com/go-sql-driver/mysql v1.4.1
-	github.com/lib/pq v1.2.0
-	github.com/magiconair/properties v1.8.1 // indirect
-	github.com/mattn/go-sqlite3 v1.11.0
-	github.com/nathanaelle/password v1.0.0
-	github.com/pelletier/go-toml v1.4.0 // indirect
-	github.com/pkg/sftp v1.10.2-0.20190913011139-8fc59612f2b0
-	github.com/prometheus/client_golang v1.1.0
-	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
-	github.com/prometheus/common v0.7.0 // indirect
-	github.com/prometheus/procfs v0.0.4 // indirect
+	github.com/go-sql-driver/mysql v1.5.0
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/grandcat/zeroconf v1.0.0
+	github.com/lib/pq v1.7.0
+	github.com/mattn/go-sqlite3 v1.14.0
+	github.com/miekg/dns v1.1.29 // indirect
+	github.com/mitchellh/mapstructure v1.3.2 // indirect
+	github.com/nathanaelle/password/v2 v2.0.1
+	github.com/otiai10/copy v1.2.0
+	github.com/pelletier/go-toml v1.8.0 // indirect
+	github.com/pires/go-proxyproto v0.1.3
+	github.com/pkg/sftp v1.11.1-0.20200310224833-18dc4db7a456
+	github.com/prometheus/client_golang v1.7.1
 	github.com/rs/xid v1.2.1
-	github.com/rs/zerolog v1.15.0
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cobra v0.0.5
+	github.com/rs/zerolog v1.19.0
+	github.com/spf13/afero v1.3.1 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/cobra v1.0.0
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.4.0
-	go.etcd.io/bbolt v1.3.3
-	golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7
-	golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect
-	golang.org/x/sys v0.0.0-20190913121621-c3b328c6e5a7
-	google.golang.org/appengine v1.6.2 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.7.0
+	github.com/stretchr/testify v1.6.1
+	go.etcd.io/bbolt v1.3.5
+	go.opencensus.io v0.22.4 // indirect
+	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+	golang.org/x/net v0.0.0-20200625001655-4c5254603344 // indirect
+	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae
+	golang.org/x/text v0.3.3 // indirect
+	golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347 // indirect
+	google.golang.org/api v0.28.0
+	google.golang.org/genproto v0.0.0-20200702021140-07506425bd67 // indirect
+	google.golang.org/grpc v1.30.0 // indirect
+	gopkg.in/ini.v1 v1.57.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
+
+replace (
+	github.com/pkg/sftp => github.com/drakkan/sftp v0.0.0-20200705201813-118ca5720446
+	golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20200705203859-05ad140ecdbd
+)

go.sum

@@ -1,71 +1,205 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.60.0 h1:R+tDlceO7Ss+zyvtsdhTxacDyZ1k99xwskQ4FT7ruoM=
+cloud.google.com/go v0.60.0/go.mod h1:yw2G51M9IfRboUH61Us8GqCeF1PzPblB823Mn2q2eAU=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alexedwards/argon2id v0.0.0-20190612080829-01a59b2b8802 h1:RwMM1q/QSKYIGbHfOkf843hE8sSUJtf1dMwFPtEDmm0=
-github.com/alexedwards/argon2id v0.0.0-20190612080829-01a59b2b8802/go.mod h1:4dsm7ufQm1Gwl8S2ss57u+2J7KlxIL2QUmFGlGtWogY=
+github.com/alexedwards/argon2id v0.0.0-20200522061839-9369edc04b05 h1:votg1faEmwABhCeJ4tiBrvwk4BWftQGkEtFy5iuI7rU=
+github.com/alexedwards/argon2id v0.0.0-20200522061839-9369edc04b05/go.mod h1:GFtu6vaWaRJV5EvSFaVqgq/3Iq95xyYElBV/aupGzUo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aws/aws-sdk-go v1.33.1 h1:yz9XmNzPshz/lhfAZvLfMnIS9HPo8+boGRcWqDVX+T0=
+github.com/aws/aws-sdk-go v1.33.1/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
+github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/drakkan/crypto v0.0.0-20200705203859-05ad140ecdbd h1:7DQ8ayx6QylOxQrQ6oHdO+gk1cqtaINUekLrwD9gGNc=
+github.com/drakkan/crypto v0.0.0-20200705203859-05ad140ecdbd/go.mod h1:v3bhWOXGYda7H5d2s5t9XA6th3fxW3s0MQxU1R96G/w=
+github.com/drakkan/sftp v0.0.0-20200705201813-118ca5720446 h1:GxI4rQ487aXpKd4RkqZJ4aeJ0/1uksscNlaZ/a3FlN0=
+github.com/drakkan/sftp v0.0.0-20200705201813-118ca5720446/go.mod h1:PIrgHN0+qgDmYTNiwryjoEqmXo9tv8aMwQ//Yg1xwIs=
+github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d h1:8RvCRWer7TB2n+DKhW4uW15hRiqPmabSnSyYhju/Nuw=
+github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d/go.mod h1:+JPhBw5JdJrSF80r6xsSg1TYHjyAGxYs4X24VyUdMZU=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
-github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
+github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
 github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
 github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
-github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE=
+github.com/grandcat/zeroconf v1.0.0/go.mod h1:lTKmG1zh86XyCoUeIHSA4FJMBwCJiQmGfcP2PdzytEs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
+github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -73,170 +207,413 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
-github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY=
+github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
-github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
+github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
+github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg=
+github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/nathanaelle/password v1.0.0 h1:1Etka3uuBvATlCb72f7P5vsgedus+C91Fgff1oMloq0=
-github.com/nathanaelle/password v1.0.0/go.mod h1:wt9xV3xwQmc3Qi0ofowmzR7N+kF1L4cguCuWjAfdj1Q=
+github.com/nathanaelle/password/v2 v2.0.1 h1:ItoCTdsuIWzilYmllQPa3DR3YoCXcpfxScWLqr8Ii2s=
+github.com/nathanaelle/password/v2 v2.0.1/go.mod h1:eaoT+ICQEPNtikBRIAatN8ThWwMhVG+r1jTw60BvPJk=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k=
+github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
+github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=
-github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw=
+github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs=
+github.com/pires/go-proxyproto v0.1.3 h1:2XEuhsQluSNA5QIQkiUv8PfgZ51sNYIQkq/yFquiSQM=
+github.com/pires/go-proxyproto v0.1.3/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/sftp v1.10.2-0.20190913011139-8fc59612f2b0 h1:uSPJdIuCJRCOQWSER5+E2xC2KwmgKR4XFGiOSlJdsqA=
-github.com/pkg/sftp v1.10.2-0.20190913011139-8fc59612f2b0/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
-github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
-github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
-github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
-github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
-github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/prometheus/procfs v0.0.4 h1:w8DjqFMJDjuVwdZBQoOozr4MVWOnwF7RcL/7uxBjY78=
-github.com/prometheus/procfs v0.0.4/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
-github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY=
-github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
-github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/rs/zerolog v1.19.0 h1:hYz4ZVdUgjXTBUmrkrw55j1nHx68LfOKIQk5IYtyScg=
+github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/afero v1.3.1 h1:GPTpEAuNr98px18yNQ66JllNil98wfRZ/5Ukny8FeQA=
+github.com/spf13/afero v1.3.1/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
-github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
+github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
-github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190913121621-c3b328c6e5a7 h1:wYqz/tQaWUgGKyx+B/rssSE6wkIKdY5Ee6ryOmzarIg=
-golang.org/x/sys v0.0.0-20190913121621-c3b328c6e5a7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0 h1:jMF5hhVfMkTZwHW1SDpKq5CkgWLXOb31Foaca9Zr3oM=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.6.2 h1:j8RI1yW0SkI+paT6uGwMlrMI/6zwYA6/CFil8rxOzGI=
-google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200626011028-ee7919e894b5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200702021140-07506425bd67 h1:4BC1C1i30F3MZeiIO6y6IIo4DxrtOwITK87bQl3lhFA=
+google.golang.org/genproto v0.0.0-20200702021140-07506425bd67/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
+gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

httpclient/httpclient.go

@@ -0,0 +1,92 @@
+package httpclient
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
+	"time"
+
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/utils"
+)
+
+// Config defines the configuration for HTTP clients.
+// HTTP clients are used for executing hooks such as the ones used for
+// custom actions, external authentication and pre-login user modifications
+type Config struct {
+	// Timeout specifies a time limit, in seconds, for requests
+	Timeout int64 `json:"timeout" mapstructure:"timeout"`
+	// CACertificates defines extra CA certificates to trust.
+	// The paths can be absolute or relative to the config dir.
+	// Adding trusted CA certificates is a convenient way to use self-signed
+	// certificates without defeating the purpose of using TLS
+	CACertificates []string `json:"ca_certificates" mapstructure:"ca_certificates"`
+	// if enabled the HTTP client accepts any TLS certificate presented by
+	// the server and any host name in that certificate.
+	// In this mode, TLS is susceptible to man-in-the-middle attacks.
+	// This should be used only for testing.
+	SkipTLSVerify   bool `json:"skip_tls_verify" mapstructure:"skip_tls_verify"`
+	customTransport *http.Transport
+}
+
+const logSender = "httpclient"
+
+var httpConfig Config
+
+// Initialize configures HTTP clients
+func (c Config) Initialize(configDir string) {
+	httpConfig = c
+	rootCAs := c.loadCACerts(configDir)
+	customTransport := http.DefaultTransport.(*http.Transport).Clone()
+	if customTransport.TLSClientConfig != nil {
+		customTransport.TLSClientConfig.RootCAs = rootCAs
+	} else {
+		customTransport.TLSClientConfig = &tls.Config{
+			RootCAs: rootCAs,
+		}
+	}
+	customTransport.TLSClientConfig.InsecureSkipVerify = c.SkipTLSVerify
+	httpConfig.customTransport = customTransport
+}
+
+// loadCACerts returns system cert pools and try to add the configured
+// CA certificates to it
+func (c Config) loadCACerts(configDir string) *x509.CertPool {
+	rootCAs, err := x509.SystemCertPool()
+	if err != nil {
+		rootCAs = x509.NewCertPool()
+	}
+
+	for _, ca := range c.CACertificates {
+		if !utils.IsFileInputValid(ca) {
+			logger.Warn(logSender, "", "unable to load invalid CA certificate: %#v", ca)
+			logger.WarnToConsole("unable to load invalid CA certificate: %#v", ca)
+			continue
+		}
+		if !filepath.IsAbs(ca) {
+			ca = filepath.Join(configDir, ca)
+		}
+		certs, err := ioutil.ReadFile(ca)
+		if err != nil {
+			logger.Warn(logSender, "", "unable to load CA certificate: %v", err)
+			logger.WarnToConsole("unable to load CA certificate: %#v", err)
+		}
+		if rootCAs.AppendCertsFromPEM(certs) {
+			logger.Debug(logSender, "", "CA certificate %#v added to the trusted certificates", ca)
+		} else {
+			logger.Warn(logSender, "", "unable to add CA certificate %#v to the trusted cetificates", ca)
+			logger.WarnToConsole("unable to add CA certificate %#v to the trusted cetificates", ca)
+		}
+	}
+	return rootCAs
+}
+
+// GetHTTPClient returns an HTTP client with the configured parameters
+func GetHTTPClient() *http.Client {
+	return &http.Client{
+		Timeout:   time.Duration(httpConfig.Timeout) * time.Second,
+		Transport: httpConfig.customTransport,
+	}
+}

httpd/api_folder.go

@@ -0,0 +1,101 @@
+package httpd
+
+import (
+	"errors"
+	"net/http"
+	"strconv"
+
+	"github.com/go-chi/render"
+
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+func getFolders(w http.ResponseWriter, r *http.Request) {
+	var err error
+	limit := 100
+	offset := 0
+	order := dataprovider.OrderASC
+	folderPath := ""
+	if _, ok := r.URL.Query()["limit"]; ok {
+		limit, err = strconv.Atoi(r.URL.Query().Get("limit"))
+		if err != nil {
+			err = errors.New("Invalid limit")
+			sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+			return
+		}
+		if limit > 500 {
+			limit = 500
+		}
+	}
+	if _, ok := r.URL.Query()["offset"]; ok {
+		offset, err = strconv.Atoi(r.URL.Query().Get("offset"))
+		if err != nil {
+			err = errors.New("Invalid offset")
+			sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+			return
+		}
+	}
+	if _, ok := r.URL.Query()["order"]; ok {
+		order = r.URL.Query().Get("order")
+		if order != dataprovider.OrderASC && order != dataprovider.OrderDESC {
+			err = errors.New("Invalid order")
+			sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+			return
+		}
+	}
+	if _, ok := r.URL.Query()["folder_path"]; ok {
+		folderPath = r.URL.Query().Get("folder_path")
+	}
+	folders, err := dataprovider.GetFolders(dataProvider, limit, offset, order, folderPath)
+	if err == nil {
+		render.JSON(w, r, folders)
+	} else {
+		sendAPIResponse(w, r, err, "", http.StatusInternalServerError)
+	}
+}
+
+func addFolder(w http.ResponseWriter, r *http.Request) {
+	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
+	var folder vfs.BaseVirtualFolder
+	err := render.DecodeJSON(r.Body, &folder)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+	err = dataprovider.AddFolder(dataProvider, folder)
+	if err == nil {
+		folder, err = dataprovider.GetFolderByPath(dataProvider, folder.MappedPath)
+		if err == nil {
+			render.JSON(w, r, folder)
+		} else {
+			sendAPIResponse(w, r, err, "", getRespStatus(err))
+		}
+	} else {
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+	}
+}
+
+func deleteFolderByPath(w http.ResponseWriter, r *http.Request) {
+	var folderPath string
+	if _, ok := r.URL.Query()["folder_path"]; ok {
+		folderPath = r.URL.Query().Get("folder_path")
+	}
+	if len(folderPath) == 0 {
+		err := errors.New("a non-empty folder path is required")
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+
+	folder, err := dataprovider.GetFolderByPath(dataProvider, folderPath)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+		return
+	}
+	err = dataprovider.DeleteFolder(dataProvider, folder)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusInternalServerError)
+	} else {
+		sendAPIResponse(w, r, err, "Folder deleted", http.StatusOK)
+	}
+}

httpd/api_maintenance.go

@@ -0,0 +1,194 @@
+package httpd
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/sftpd"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+func dumpData(w http.ResponseWriter, r *http.Request) {
+	var outputFile, indent string
+	if _, ok := r.URL.Query()["output_file"]; ok {
+		outputFile = strings.TrimSpace(r.URL.Query().Get("output_file"))
+	}
+	if _, ok := r.URL.Query()["indent"]; ok {
+		indent = strings.TrimSpace(r.URL.Query().Get("indent"))
+	}
+	if len(outputFile) == 0 {
+		sendAPIResponse(w, r, errors.New("Invalid or missing output_file"), "", http.StatusBadRequest)
+		return
+	}
+	if filepath.IsAbs(outputFile) {
+		sendAPIResponse(w, r, fmt.Errorf("Invalid output_file %#v: it must be a relative path", outputFile), "", http.StatusBadRequest)
+		return
+	}
+	if strings.Contains(outputFile, "..") {
+		sendAPIResponse(w, r, fmt.Errorf("Invalid output_file %#v", outputFile), "", http.StatusBadRequest)
+		return
+	}
+	outputFile = filepath.Join(backupsPath, outputFile)
+	err := os.MkdirAll(filepath.Dir(outputFile), 0700)
+	if err != nil {
+		logger.Warn(logSender, "", "dumping data error: %v, output file: %#v", err, outputFile)
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+		return
+	}
+	logger.Debug(logSender, "", "dumping data to: %#v", outputFile)
+
+	backup, err := dataprovider.DumpData(dataProvider)
+	if err != nil {
+		logger.Warn(logSender, "", "dumping data error: %v, output file: %#v", err, outputFile)
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+		return
+	}
+	var dump []byte
+	if indent == "1" {
+		dump, err = json.MarshalIndent(backup, "", "  ")
+	} else {
+		dump, err = json.Marshal(backup)
+	}
+	if err == nil {
+		err = ioutil.WriteFile(outputFile, dump, 0600)
+	}
+	if err != nil {
+		logger.Warn(logSender, "", "dumping data error: %v, output file: %#v", err, outputFile)
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+		return
+	}
+	logger.Debug(logSender, "", "dumping data completed, output file: %#v, error: %v", outputFile, err)
+	sendAPIResponse(w, r, err, "Data saved", http.StatusOK)
+}
+
+func loadData(w http.ResponseWriter, r *http.Request) {
+	inputFile, scanQuota, mode, err := getLoaddataOptions(r)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+	if !filepath.IsAbs(inputFile) {
+		sendAPIResponse(w, r, fmt.Errorf("Invalid input_file %#v: it must be an absolute path", inputFile), "", http.StatusBadRequest)
+		return
+	}
+	fi, err := os.Stat(inputFile)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+		return
+	}
+	if fi.Size() > maxRestoreSize {
+		sendAPIResponse(w, r, err, fmt.Sprintf("Unable to restore input file: %#v size too big: %v/%v bytes",
+			inputFile, fi.Size(), maxRestoreSize), http.StatusBadRequest)
+		return
+	}
+
+	content, err := ioutil.ReadFile(inputFile)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+		return
+	}
+	var dump dataprovider.BackupData
+	err = json.Unmarshal(content, &dump)
+	if err != nil {
+		sendAPIResponse(w, r, err, fmt.Sprintf("Unable to parse input file: %#v", inputFile), http.StatusBadRequest)
+		return
+	}
+
+	if err = restoreFolders(dump.Folders, inputFile, scanQuota); err != nil {
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+		return
+	}
+
+	if err = restoreUsers(dump.Users, inputFile, mode, scanQuota); err != nil {
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+		return
+	}
+
+	logger.Debug(logSender, "", "backup restored, users: %v", len(dump.Users))
+	sendAPIResponse(w, r, err, "Data restored", http.StatusOK)
+}
+
+func getLoaddataOptions(r *http.Request) (string, int, int, error) {
+	var inputFile string
+	var err error
+	scanQuota := 0
+	restoreMode := 0
+	if _, ok := r.URL.Query()["input_file"]; ok {
+		inputFile = strings.TrimSpace(r.URL.Query().Get("input_file"))
+	}
+	if _, ok := r.URL.Query()["scan_quota"]; ok {
+		scanQuota, err = strconv.Atoi(r.URL.Query().Get("scan_quota"))
+		if err != nil {
+			err = fmt.Errorf("invalid scan_quota: %v", err)
+		}
+	}
+	if _, ok := r.URL.Query()["mode"]; ok {
+		restoreMode, err = strconv.Atoi(r.URL.Query().Get("mode"))
+		if err != nil {
+			err = fmt.Errorf("invalid mode: %v", err)
+		}
+	}
+	return inputFile, scanQuota, restoreMode, err
+}
+
+func restoreFolders(folders []vfs.BaseVirtualFolder, inputFile string, scanQuota int) error {
+	for _, folder := range folders {
+		_, err := dataprovider.GetFolderByPath(dataProvider, folder.MappedPath)
+		if err == nil {
+			logger.Debug(logSender, "", "folder %#v already exists, restore not needed", folder.MappedPath)
+			continue
+		}
+		folder.Users = nil
+		err = dataprovider.AddFolder(dataProvider, folder)
+		logger.Debug(logSender, "", "adding new folder: %+v, dump file: %#v, error: %v", folder, inputFile, err)
+		if err != nil {
+			return err
+		}
+		if scanQuota >= 1 {
+			if sftpd.AddVFolderQuotaScan(folder.MappedPath) {
+				logger.Debug(logSender, "", "starting quota scan for restored folder: %#v", folder.MappedPath)
+				go doFolderQuotaScan(folder) //nolint:errcheck
+			}
+		}
+	}
+	return nil
+}
+
+func restoreUsers(users []dataprovider.User, inputFile string, mode, scanQuota int) error {
+	for _, user := range users {
+		u, err := dataprovider.UserExists(dataProvider, user.Username)
+		if err == nil {
+			if mode == 1 {
+				logger.Debug(logSender, "", "loaddata mode 1, existing user %#v not updated", u.Username)
+				continue
+			}
+			user.ID = u.ID
+			err = dataprovider.UpdateUser(dataProvider, user)
+			user.Password = "[redacted]"
+			logger.Debug(logSender, "", "restoring existing user: %+v, dump file: %#v, error: %v", user, inputFile, err)
+		} else {
+			err = dataprovider.AddUser(dataProvider, user)
+			user.Password = "[redacted]"
+			logger.Debug(logSender, "", "adding new user: %+v, dump file: %#v, error: %v", user, inputFile, err)
+		}
+		if err != nil {
+			return err
+		}
+		if scanQuota == 1 || (scanQuota == 2 && user.HasQuotaRestrictions()) {
+			if sftpd.AddQuotaScan(user.Username) {
+				logger.Debug(logSender, "", "starting quota scan for restored user: %#v", user.Username)
+				go doQuotaScan(user) //nolint:errcheck
+			}
+		}
+	}
+	return nil
+}

httpd/api_quota.go

@@ -0,0 +1,194 @@
+package httpd
+
+import (
+	"errors"
+	"net/http"
+
+	"github.com/go-chi/render"
+
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/sftpd"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+const (
+	quotaUpdateModeAdd   = "add"
+	quotaUpdateModeReset = "reset"
+)
+
+func getQuotaScans(w http.ResponseWriter, r *http.Request) {
+	render.JSON(w, r, sftpd.GetQuotaScans())
+}
+
+func getVFolderQuotaScans(w http.ResponseWriter, r *http.Request) {
+	render.JSON(w, r, sftpd.GetVFoldersQuotaScans())
+}
+
+func updateUserQuotaUsage(w http.ResponseWriter, r *http.Request) {
+	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
+	var u dataprovider.User
+	err := render.DecodeJSON(r.Body, &u)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+	if u.UsedQuotaFiles < 0 || u.UsedQuotaSize < 0 {
+		sendAPIResponse(w, r, errors.New("Invalid used quota parameters, negative values are not allowed"),
+			"", http.StatusBadRequest)
+		return
+	}
+	mode, err := getQuotaUpdateMode(r)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+	user, err := dataprovider.UserExists(dataProvider, u.Username)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+		return
+	}
+	if mode == quotaUpdateModeAdd && !user.HasQuotaRestrictions() && dataprovider.GetQuotaTracking() == 2 {
+		sendAPIResponse(w, r, errors.New("this user has no quota restrictions, only reset mode is supported"),
+			"", http.StatusBadRequest)
+		return
+	}
+	if !sftpd.AddQuotaScan(user.Username) {
+		sendAPIResponse(w, r, err, "A quota scan is in progress for this user", http.StatusConflict)
+		return
+	}
+	defer sftpd.RemoveQuotaScan(user.Username) //nolint:errcheck
+	err = dataprovider.UpdateUserQuota(dataProvider, user, u.UsedQuotaFiles, u.UsedQuotaSize, mode == quotaUpdateModeReset)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+	} else {
+		sendAPIResponse(w, r, err, "Quota updated", http.StatusOK)
+	}
+}
+
+func updateVFolderQuotaUsage(w http.ResponseWriter, r *http.Request) {
+	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
+	var f vfs.BaseVirtualFolder
+	err := render.DecodeJSON(r.Body, &f)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+	if f.UsedQuotaFiles < 0 || f.UsedQuotaSize < 0 {
+		sendAPIResponse(w, r, errors.New("Invalid used quota parameters, negative values are not allowed"),
+			"", http.StatusBadRequest)
+		return
+	}
+	mode, err := getQuotaUpdateMode(r)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+	folder, err := dataprovider.GetFolderByPath(dataProvider, f.MappedPath)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+		return
+	}
+	if !sftpd.AddVFolderQuotaScan(folder.MappedPath) {
+		sendAPIResponse(w, r, err, "A quota scan is in progress for this folder", http.StatusConflict)
+		return
+	}
+	defer sftpd.RemoveVFolderQuotaScan(folder.MappedPath) //nolint:errcheck
+	err = dataprovider.UpdateVirtualFolderQuota(dataProvider, folder, f.UsedQuotaFiles, f.UsedQuotaSize, mode == quotaUpdateModeReset)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+	} else {
+		sendAPIResponse(w, r, err, "Quota updated", http.StatusOK)
+	}
+}
+
+func startQuotaScan(w http.ResponseWriter, r *http.Request) {
+	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
+	if dataprovider.GetQuotaTracking() == 0 {
+		sendAPIResponse(w, r, nil, "Quota tracking is disabled!", http.StatusForbidden)
+		return
+	}
+	var u dataprovider.User
+	err := render.DecodeJSON(r.Body, &u)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+	user, err := dataprovider.UserExists(dataProvider, u.Username)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+		return
+	}
+	if sftpd.AddQuotaScan(user.Username) {
+		go doQuotaScan(user) //nolint:errcheck
+		sendAPIResponse(w, r, err, "Scan started", http.StatusCreated)
+	} else {
+		sendAPIResponse(w, r, err, "Another scan is already in progress", http.StatusConflict)
+	}
+}
+
+func startVFolderQuotaScan(w http.ResponseWriter, r *http.Request) {
+	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
+	if dataprovider.GetQuotaTracking() == 0 {
+		sendAPIResponse(w, r, nil, "Quota tracking is disabled!", http.StatusForbidden)
+		return
+	}
+	var f vfs.BaseVirtualFolder
+	err := render.DecodeJSON(r.Body, &f)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+	folder, err := dataprovider.GetFolderByPath(dataProvider, f.MappedPath)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+		return
+	}
+	if sftpd.AddVFolderQuotaScan(folder.MappedPath) {
+		go doFolderQuotaScan(folder) //nolint:errcheck
+		sendAPIResponse(w, r, err, "Scan started", http.StatusCreated)
+	} else {
+		sendAPIResponse(w, r, err, "Another scan is already in progress", http.StatusConflict)
+	}
+}
+
+func doQuotaScan(user dataprovider.User) error {
+	defer sftpd.RemoveQuotaScan(user.Username) //nolint:errcheck
+	fs, err := user.GetFilesystem("")
+	if err != nil {
+		logger.Warn(logSender, "", "unable scan quota for user %#v error creating filesystem: %v", user.Username, err)
+		return err
+	}
+	numFiles, size, err := fs.ScanRootDirContents()
+	if err != nil {
+		logger.Warn(logSender, "", "error scanning user home dir %#v: %v", user.Username, err)
+		return err
+	}
+	err = dataprovider.UpdateUserQuota(dataProvider, user, numFiles, size, true)
+	logger.Debug(logSender, "", "user home dir scanned, user: %#v, error: %v", user.Username, err)
+	return err
+}
+
+func doFolderQuotaScan(folder vfs.BaseVirtualFolder) error {
+	defer sftpd.RemoveVFolderQuotaScan(folder.MappedPath) //nolint:errcheck
+	fs := vfs.NewOsFs("", "", nil).(vfs.OsFs)
+	numFiles, size, err := fs.GetDirSize(folder.MappedPath)
+	if err != nil {
+		logger.Warn(logSender, "", "error scanning folder %#v: %v", folder.MappedPath, err)
+		return err
+	}
+	err = dataprovider.UpdateVirtualFolderQuota(dataProvider, folder, numFiles, size, true)
+	logger.Debug(logSender, "", "virtual folder %#v scanned, error: %v", folder.MappedPath, err)
+	return err
+}
+
+func getQuotaUpdateMode(r *http.Request) (string, error) {
+	mode := quotaUpdateModeReset
+	if _, ok := r.URL.Query()["mode"]; ok {
+		mode = r.URL.Query().Get("mode")
+		if mode != quotaUpdateModeReset && mode != quotaUpdateModeAdd {
+			return "", errors.New("Invalid mode")
+		}
+	}
+	return mode, nil
+}