Mirrors/sftpgo
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2025-12-07 06:40:54 +03:00
Code Issues Packages Projects Releases Wiki Activity

add basic S3-Compatible Object Storage support

Browse Source 
we have now an interface for filesystem backeds, this make easy to add
new filesystem backends
This commit is contained in:
Nicola Murino
2020-01-19 07:41:05 +01:00
parent 0b42dbc3c3
commit a4834f4a83
40 changed files with 2315 additions and 420 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
scripts/README.md
View File

@@ -44,7 +44,7 @@ Let's see a sample usage for each REST API.
Command:
```
python sftpgo_api_cli.py add-user test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 33 --gid 1000 --max-sessions 2 --quota-size 0 --quota-files 3 --permissions "list" "download" "upload" "delete" "rename" "create_dirs" "overwrite" --subdirs-permissions "/dir1:list,download" "/dir2:*" --upload-bandwidth 100 --download-bandwidth 60 --status 0 --expiration-date 2019-01-01 --allowed-ip "192.168.1.1/32"
python sftpgo_api_cli.py add-user test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 33 --gid 1000 --max-sessions 2 --quota-size 0 --quota-files 3 --permissions "list" "download" "upload" "delete" "rename" "create_dirs" "overwrite" --subdirs-permissions "/dir1:list,download" "/dir2:*" --upload-bandwidth 100 --download-bandwidth 60 --status 0 --expiration-date 2019-01-01 --allowed-ip "192.168.1.1/32" --fs S3 --s3-bucket test --s3-region eu-west-1 --s3-access-key accesskey --s3-access-secret secret --s3-endpoint "http://127.0.0.1:9000" --s3-storage-class Standard
```
Output:
@@ -53,6 +53,17 @@ Output:
{
"download_bandwidth": 60,
"expiration_date": 1546297200000,
"filesystem": {
"provider": 1,
"s3config": {
"access_key": "accesskey",
"access_secret": "$aes$6c088ba12b0b261247c8cf331c46d9260b8e58002957d89ad1c0495e3af665cd0227",
"bucket": "test",
"endpoint": "http://127.0.0.1:9000",
"region": "eu-west-1",
"storage_class": "Standard"
}
},
"filters": {
"allowed_ip": [
"192.168.1.1/32"
@@ -99,7 +110,7 @@ Output:
Command:
```
python sftpgo_api_cli.py update-user 9576 test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 0 --gid 33 --max-sessions 3 --quota-size 0 --quota-files 4 --permissions "*" --subdirs-permissions "/dir1:list,download,create_symlinks" --upload-bandwidth 90 --download-bandwidth 80 --status 1 --expiration-date "" --allowed-ip "" --denied-ip "192.168.1.0/24"
python sftpgo_api_cli.py update-user 9576 test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 0 --gid 33 --max-sessions 3 --quota-size 0 --quota-files 4 --permissions "*" --subdirs-permissions "/dir1:list,download,create_symlinks" --upload-bandwidth 90 --download-bandwidth 80 --status 1 --expiration-date "" --allowed-ip "" --denied-ip "192.168.1.0/24" --fs local
```
Output:
@@ -126,6 +137,10 @@ Output:
{
"download_bandwidth": 80,
"expiration_date": 0,
"filesystem": {
"provider": 0,
"s3config": {}
},
"filters": {
"allowed_ip": [],
"denied_ip": [
@@ -174,6 +189,10 @@ Output:
{
"download_bandwidth": 80,
"expiration_date": 0,
"filesystem": {
"provider": 0,
"s3config": {}
},
"filters": {
"allowed_ip": [],
"denied_ip": [

67
scripts/sftpgo_api_cli.py
View File

@@ -70,9 +70,10 @@ class SFTPGoApiRequests:
else:
print(r.text)
def buildUserObject(self, user_id=0, username="", password="", public_keys=[], home_dir="", uid=0,
gid=0, max_sessions=0, quota_size=0, quota_files=0, permissions={}, upload_bandwidth=0,
download_bandwidth=0, status=1, expiration_date=0, allowed_ip=[], denied_ip=[]):
def buildUserObject(self, user_id=0, username="", password="", public_keys=[], home_dir="", uid=0, gid=0,
max_sessions=0, quota_size=0, quota_files=0, permissions={}, upload_bandwidth=0, download_bandwidth=0,
status=1, expiration_date=0, allowed_ip=[], denied_ip=[], fs_provider='local', s3_bucket='',
s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''):
user = {"id":user_id, "username":username, "uid":uid, "gid":gid,
"max_sessions":max_sessions, "quota_size":quota_size, "quota_files":quota_files,
"upload_bandwidth":upload_bandwidth, "download_bandwidth":download_bandwidth,
@@ -90,6 +91,8 @@ class SFTPGoApiRequests:
user.update({"permissions":permissions})
if allowed_ip or denied_ip:
user.update({"filters":self.buildFilters(allowed_ip, denied_ip)})
user.update({"filesystem":self.buildFsConfig(fs_provider, s3_bucket, s3_region, s3_access_key,
s3_access_secret, s3_endpoint, s3_storage_class)})
return user
def buildPermissions(self, root_perms, subdirs_perms):
@@ -113,16 +116,25 @@ class SFTPGoApiRequests:
filters = {}
if allowed_ip:
if len(allowed_ip) == 1 and not allowed_ip[0]:
filters.update({"allowed_ip":[]})
filters.update({'allowed_ip':[]})
else:
filters.update({"allowed_ip":allowed_ip})
filters.update({'allowed_ip':allowed_ip})
if denied_ip:
if len(denied_ip) == 1 and not denied_ip[0]:
filters.update({"denied_ip":[]})
filters.update({'denied_ip':[]})
else:
filters.update({"denied_ip":denied_ip})
filters.update({'denied_ip':denied_ip})
return filters
def buildFsConfig(self, fs_provider, s3_bucket, s3_region, s3_access_key, s3_access_secret, s3_endpoint,
s3_storage_class):
fs_config = {'provider':0}
if fs_provider == 'S3':
s3config = {'bucket':s3_bucket, 'region':s3_region, 'access_key':s3_access_key, 'access_secret':
s3_access_secret, 'endpoint':s3_endpoint, 'storage_class':s3_storage_class}
fs_config.update({'provider':1, 's3config':s3config})
return fs_config
def getUsers(self, limit=100, offset=0, order="ASC", username=""):
r = requests.get(self.userPath, params={"limit":limit, "offset":offset, "order":order,
"username":username}, auth=self.auth, verify=self.verify)
@@ -132,22 +144,25 @@ class SFTPGoApiRequests:
r = requests.get(urlparse.urljoin(self.userPath, "user/" + str(user_id)), auth=self.auth, verify=self.verify)
self.printResponse(r)
def addUser(self, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0,
quota_size=0, quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1,
expiration_date=0, subdirs_permissions=[], allowed_ip=[], denied_ip=[]):
def addUser(self, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0, quota_size=0,
quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1, expiration_date=0,
subdirs_permissions=[], allowed_ip=[], denied_ip=[], fs_provider='local', s3_bucket='', s3_region='',
s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''):
u = self.buildUserObject(0, username, password, public_keys, home_dir, uid, gid, max_sessions,
quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth,
status, expiration_date, allowed_ip, denied_ip)
status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region,
s3_access_key, s3_access_secret, s3_endpoint, s3_storage_class)
r = requests.post(self.userPath, json=u, auth=self.auth, verify=self.verify)
self.printResponse(r)
def updateUser(self, user_id, username="", password="", public_keys="", home_dir="", uid=0, gid=0,
max_sessions=0, quota_size=0, quota_files=0, perms=[], upload_bandwidth=0,
download_bandwidth=0, status=1, expiration_date=0, subdirs_permissions=[],
allowed_ip=[], denied_ip=[]):
def updateUser(self, user_id, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0,
quota_size=0, quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1,
expiration_date=0, subdirs_permissions=[], allowed_ip=[], denied_ip=[], fs_provider='local',
s3_bucket='', s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''):
u = self.buildUserObject(user_id, username, password, public_keys, home_dir, uid, gid, max_sessions,
quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth,
status, expiration_date, allowed_ip, denied_ip)
status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region, s3_access_key,
s3_access_secret, s3_endpoint, s3_storage_class)
r = requests.put(urlparse.urljoin(self.userPath, "user/" + str(user_id)), json=u, auth=self.auth, verify=self.verify)
self.printResponse(r)
@@ -238,7 +253,7 @@ class ConvertUsers:
self.convertFromProFTPD()
self.saveUsers()
def isUserValid(self, username, uid, gid):
def isUserValid(self, username, uid):
if self.usernames and not username in self.usernames:
return False
if self.min_uid >= 0 and uid < self.min_uid:
@@ -257,7 +272,7 @@ class ConvertUsers:
home_dir = user.pw_dir
status = 1
expiration_date = 0
if not self.isUserValid(username, uid, gid):
if not self.isUserValid(username, uid):
continue
if self.force_uid >= 0:
uid = self.force_uid
@@ -375,7 +390,7 @@ def addCommonUserArguments(parser):
parser.add_argument('username', type=str)
parser.add_argument('-P', '--password', type=str, default=None, help='Default: %(default)s')
parser.add_argument('-K', '--public-keys', type=str, nargs='+', default=[], help='Default: %(default)s')
parser.add_argument('-H', '--home-dir', type=str, default="", help='Default: %(default)s')
parser.add_argument('-H', '--home-dir', type=str, default='', help='Default: %(default)s')
parser.add_argument('--uid', type=int, default=0, help='Default: %(default)s')
parser.add_argument('--gid', type=int, default=0, help='Default: %(default)s')
parser.add_argument('-C', '--max-sessions', type=int, default=0,
@@ -401,6 +416,14 @@ def addCommonUserArguments(parser):
help='Allowed IP/Mask in CIDR notation. For example "192.168.2.0/24" or "2001:db8::/32". Default: %(default)s')
parser.add_argument('-N', '--denied-ip', type=str, nargs='+', default=[],
help='Denied IP/Mask in CIDR notation. For example "192.168.2.0/24" or "2001:db8::/32". Default: %(default)s')
parser.add_argument('--fs', type=str, default='local', choices=['local', 'S3'],
help='Filesystem provider. Default: %(default)s')
parser.add_argument('--s3-bucket', type=str, default='', help='Default: %(default)s')
parser.add_argument('--s3-region', type=str, default='', help='Default: %(default)s')
parser.add_argument('--s3-access-key', type=str, default='', help='Default: %(default)s')
parser.add_argument('--s3-access-secret', type=str, default='', help='Default: %(default)s')
parser.add_argument('--s3-endpoint', type=str, default='', help='Default: %(default)s')
parser.add_argument('--s3-storage-class', type=str, default='', help='Default: %(default)s')
if __name__ == '__main__':
@@ -503,12 +526,14 @@ if __name__ == '__main__':
api.addUser(args.username, args.password, args.public_keys, args.home_dir, args.uid, args.gid, args.max_sessions,
args.quota_size, args.quota_files, args.permissions, args.upload_bandwidth, args.download_bandwidth,
args.status, getDatetimeAsMillisSinceEpoch(args.expiration_date), args.subdirs_permissions, args.allowed_ip,
args.denied_ip)
args.denied_ip, args.fs, args.s3_bucket, args.s3_region, args.s3_access_key, args.s3_access_secret,
args.s3_endpoint, args.s3_storage_class)
elif args.command == 'update-user':
api.updateUser(args.id, args.username, args.password, args.public_keys, args.home_dir, args.uid, args.gid,
args.max_sessions, args.quota_size, args.quota_files, args.permissions, args.upload_bandwidth,
args.download_bandwidth, args.status, getDatetimeAsMillisSinceEpoch(args.expiration_date),
args.subdirs_permissions, args.allowed_ip, args.denied_ip)
args.subdirs_permissions, args.allowed_ip, args.denied_ip, args.fs, args.s3_bucket, args.s3_region,
args.s3_access_key, args.s3_access_secret, args.s3_endpoint, args.s3_storage_class)
elif args.command == 'delete-user':
api.deleteUser(args.id)
elif args.command == 'get-users':