Mirrors/sshj
1
0
Fork 0
mirror of https://github.com/hierynomus/sshj.git synced 2025-12-08 16:18:05 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added SFTP file transfer resume support on both PUT and GET. (#775)

Browse Source 
* Added SFTP file transfer resume support on both PUT and GET. Internally SFTPFileTransfer has a few sanity checks to fall back to full replacement even if the resume flag is set. 

SCP file transfers have not been changed to support this at this time.

* Added JUnit tests for issue-700

* Throw SCPException when attempting to resume SCP transfers.

* Licensing

* Small bug resuming a completed file was restarting since the bytes were equal.

* Enhanced test cases to validate the expected bytes transferred for each scenario are the actual bytes transferred.

* Removed author info which was pre-filled from company IDE template

* Added "fall through" comment for switch

* Changed the API for requesting a resume from a boolean flag with some internal decisions to be a user-specified long byte offset. This is cleaner but puts the onus on the caller to know exactly what they're asking for in their circumstance, which is ultimately better for a library like sshj.

* Reverted some now-unnecessary changes to SFTPFileTransfer.Uploader.prepareFile()

* Fix gradle exclude path for test files

Co-authored-by: Jeroen van Erp <jeroen@hierynomus.com>
This commit is contained in:
Brent Tyler
2022-05-27 06:05:41 -05:00
committed by GitHub
parent d7e402c557
commit 3de0302c84
12 changed files with 398 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/net/schmizz/sshj/sftp/RemoteFile.java
View File

@@ -261,7 +261,16 @@ public class RemoteFile
private boolean eof;
public ReadAheadRemoteFileInputStream(int maxUnconfirmedReads) {
this(maxUnconfirmedReads, 0L, -1L);
this(maxUnconfirmedReads, 0L);
}
/**
*
* @param maxUnconfirmedReads Maximum number of unconfirmed requests to send
* @param fileOffset Initial offset in file to read from
*/
public ReadAheadRemoteFileInputStream(int maxUnconfirmedReads, long fileOffset) {
this(maxUnconfirmedReads, fileOffset, -1L);
}
/**

20
src/main/java/net/schmizz/sshj/sftp/SFTPClient.java
View File

@@ -232,21 +232,41 @@ public class SFTPClient
throws IOException {
xfer.download(source, dest);
}
public void get(String source, String dest, long byteOffset)
throws IOException {
xfer.download(source, dest, byteOffset);
}
public void put(String source, String dest)
throws IOException {
xfer.upload(source, dest);
}
public void put(String source, String dest, long byteOffset)
throws IOException {
xfer.upload(source, dest, byteOffset);
}
public void get(String source, LocalDestFile dest)
throws IOException {
xfer.download(source, dest);
}
public void get(String source, LocalDestFile dest, long byteOffset)
throws IOException {
xfer.download(source, dest, byteOffset);
}
public void put(LocalSourceFile source, String dest)
throws IOException {
xfer.upload(source, dest);
}
public void put(LocalSourceFile source, String dest, long byteOffset)
throws IOException {
xfer.upload(source, dest, byteOffset);
}
@Override
public void close()

83
src/main/java/net/schmizz/sshj/sftp/SFTPFileTransfer.java
View File

@@ -50,25 +50,47 @@ public class SFTPFileTransfer
@Override
public void upload(String source, String dest)
throws IOException {
upload(new FileSystemFile(source), dest);
upload(source, dest, 0);
}
@Override
public void upload(String source, String dest, long byteOffset)
throws IOException {
upload(new FileSystemFile(source), dest, byteOffset);
}
@Override
public void download(String source, String dest)
throws IOException {
download(source, new FileSystemFile(dest));
download(source, dest, 0);
}
@Override
public void download(String source, String dest, long byteOffset)
throws IOException {
download(source, new FileSystemFile(dest), byteOffset);
}
@Override
public void upload(LocalSourceFile localFile, String remotePath) throws IOException {
new Uploader(localFile, remotePath).upload(getTransferListener());
upload(localFile, remotePath, 0);
}
@Override
public void upload(LocalSourceFile localFile, String remotePath, long byteOffset) throws IOException {
new Uploader(localFile, remotePath).upload(getTransferListener(), byteOffset);
}
@Override
public void download(String source, LocalDestFile dest) throws IOException {
download(source, dest, 0);
}
@Override
public void download(String source, LocalDestFile dest, long byteOffset) throws IOException {
final PathComponents pathComponents = engine.getPathHelper().getComponents(source);
final FileAttributes attributes = engine.stat(source);
new Downloader().download(getTransferListener(), new RemoteResourceInfo(pathComponents, attributes), dest);
new Downloader().download(getTransferListener(), new RemoteResourceInfo(pathComponents, attributes), dest, byteOffset);
}
public void setUploadFilter(LocalFileFilter uploadFilter) {
@@ -92,7 +114,8 @@ public class SFTPFileTransfer
@SuppressWarnings("PMD.MissingBreakInSwitch")
private void download(final TransferListener listener,
final RemoteResourceInfo remote,
final LocalDestFile local) throws IOException {
final LocalDestFile local,
final long byteOffset) throws IOException {
final LocalDestFile adjustedFile;
switch (remote.getAttributes().getType()) {
case DIRECTORY:
@@ -101,8 +124,9 @@ public class SFTPFileTransfer
case UNKNOWN:
log.warn("Server did not supply information about the type of file at `{}` " +
"-- assuming it is a regular file!", remote.getPath());
// fall through
case REGULAR:
adjustedFile = downloadFile(listener.file(remote.getName(), remote.getAttributes().getSize()), remote, local);
adjustedFile = downloadFile(listener.file(remote.getName(), remote.getAttributes().getSize()), remote, local, byteOffset);
break;
default:
throw new IOException(remote + " is not a regular file or directory");
@@ -119,7 +143,7 @@ public class SFTPFileTransfer
final RemoteDirectory rd = engine.openDir(remote.getPath());
try {
for (RemoteResourceInfo rri : rd.scan(getDownloadFilter()))
download(listener, rri, adjusted.getChild(rri.getName()));
download(listener, rri, adjusted.getChild(rri.getName()), 0); // not supporting individual byte offsets for these files
} finally {
rd.close();
}
@@ -128,13 +152,15 @@ public class SFTPFileTransfer
private LocalDestFile downloadFile(final StreamCopier.Listener listener,
final RemoteResourceInfo remote,
final LocalDestFile local)
final LocalDestFile local,
final long byteOffset)
throws IOException {
final LocalDestFile adjusted = local.getTargetFile(remote.getName());
final RemoteFile rf = engine.open(remote.getPath());
try {
final RemoteFile.ReadAheadRemoteFileInputStream rfis = rf.new ReadAheadRemoteFileInputStream(16);
final OutputStream os = adjusted.getOutputStream();
log.debug("Attempting to download {} with offset={}", remote.getPath(), byteOffset);
final RemoteFile.ReadAheadRemoteFileInputStream rfis = rf.new ReadAheadRemoteFileInputStream(16, byteOffset);
final OutputStream os = adjusted.getOutputStream(byteOffset != 0);
try {
new StreamCopier(rfis, os, engine.getLoggerFactory())
.bufSize(engine.getSubsystem().getLocalMaxPacketSize())
@@ -173,17 +199,17 @@ public class SFTPFileTransfer
this.remote = remote;
}
private void upload(final TransferListener listener) throws IOException {
private void upload(final TransferListener listener, long byteOffset) throws IOException {
if (source.isDirectory()) {
makeDirIfNotExists(remote); // Ensure that the directory exists
uploadDir(listener.directory(source.getName()), source, remote);
setAttributes(source, remote);
} else if (source.isFile() && isDirectory(remote)) {
String adjustedRemote = engine.getPathHelper().adjustForParent(this.remote, source.getName());
uploadFile(listener.file(source.getName(), source.getLength()), source, adjustedRemote);
uploadFile(listener.file(source.getName(), source.getLength()), source, adjustedRemote, byteOffset);
setAttributes(source, adjustedRemote);
} else if (source.isFile()) {
uploadFile(listener.file(source.getName(), source.getLength()), source, remote);
uploadFile(listener.file(source.getName(), source.getLength()), source, remote, byteOffset);
setAttributes(source, remote);
} else {
throw new IOException(source + " is not a file or directory");
@@ -192,13 +218,14 @@ public class SFTPFileTransfer
private void upload(final TransferListener listener,
final LocalSourceFile local,
final String remote)
final String remote,
final long byteOffset)
throws IOException {
final String adjustedPath;
if (local.isDirectory()) {
adjustedPath = uploadDir(listener.directory(local.getName()), local, remote);
} else if (local.isFile()) {
adjustedPath = uploadFile(listener.file(local.getName(), local.getLength()), local, remote);
adjustedPath = uploadFile(listener.file(local.getName(), local.getLength()), local, remote, byteOffset);
} else {
throw new IOException(local + " is not a file or directory");
}
@@ -217,22 +244,34 @@ public class SFTPFileTransfer
throws IOException {
makeDirIfNotExists(remote);
for (LocalSourceFile f : local.getChildren(getUploadFilter()))
upload(listener, f, engine.getPathHelper().adjustForParent(remote, f.getName()));
upload(listener, f, engine.getPathHelper().adjustForParent(remote, f.getName()), 0); // not supporting individual byte offsets for these files
return remote;
}
private String uploadFile(final StreamCopier.Listener listener,
final LocalSourceFile local,
final String remote)
final String remote,
final long byteOffset)
throws IOException {
final String adjusted = prepareFile(local, remote);
final String adjusted = prepareFile(local, remote, byteOffset);
RemoteFile rf = null;
InputStream fis = null;
RemoteFile.RemoteFileOutputStream rfos = null;
EnumSet<OpenMode> modes;
try {
rf = engine.open(adjusted, EnumSet.of(OpenMode.WRITE, OpenMode.CREAT, OpenMode.TRUNC));
if (byteOffset == 0) {
// Starting at the beginning, overwrite/create
modes = EnumSet.of(OpenMode.WRITE, OpenMode.CREAT, OpenMode.TRUNC);
} else {
// Starting at some offset, append
modes = EnumSet.of(OpenMode.WRITE, OpenMode.APPEND);
}
log.debug("Attempting to upload {} with offset={}", local.getName(), byteOffset);
rf = engine.open(adjusted, modes);
fis = local.getInputStream();
rfos = rf.new RemoteFileOutputStream(0, 16);
fis.skip(byteOffset);
rfos = rf.new RemoteFileOutputStream(byteOffset, 16);
new StreamCopier(fis, rfos, engine.getLoggerFactory())
.bufSize(engine.getSubsystem().getRemoteMaxPacketSize() - rf.getOutgoingPacketOverhead())
.keepFlushing(false)
@@ -294,7 +333,7 @@ public class SFTPFileTransfer
}
}
private String prepareFile(final LocalSourceFile local, final String remote)
private String prepareFile(final LocalSourceFile local, final String remote, final long byteOffset)
throws IOException {
final FileAttributes attrs;
try {
@@ -309,7 +348,7 @@ public class SFTPFileTransfer
if (attrs.getMode().getType() == FileMode.Type.DIRECTORY) {
throw new IOException("Trying to upload file " + local.getName() + " to path " + remote + " but that is a directory");
} else {
log.debug("probeFile: {} is a {} file that will be replaced", remote, attrs.getMode().getType());
log.debug("probeFile: {} is a {} file that will be {}", remote, attrs.getMode().getType(), byteOffset > 0 ? "resumed" : "replaced");
return remote;
}
}

8
src/main/java/net/schmizz/sshj/xfer/FileSystemFile.java
View File

@@ -71,7 +71,13 @@ public class FileSystemFile
@Override
public OutputStream getOutputStream()
throws IOException {
return new FileOutputStream(file);
return getOutputStream(false);
}
@Override
public OutputStream getOutputStream(boolean append)
throws IOException {
return new FileOutputStream(file, append);
}
@Override

50
src/main/java/net/schmizz/sshj/xfer/FileTransfer.java
View File

@@ -31,6 +31,19 @@ public interface FileTransfer {
void upload(String localPath, String remotePath)
throws IOException;
/**
* This is meant to delegate to {@link #upload(LocalSourceFile, String)} with the {@code localPath} wrapped as e.g.
* a {@link FileSystemFile}. Appends to existing if {@code byteOffset} &gt; 0.
*
* @param localPath
* @param remotePath
* @param byteOffset
*
* @throws IOException
*/
void upload(String localPath, String remotePath, long byteOffset)
throws IOException;
/**
* This is meant to delegate to {@link #download(String, LocalDestFile)} with the {@code localPath} wrapped as e.g.
* a {@link FileSystemFile}.
@@ -43,6 +56,19 @@ public interface FileTransfer {
void download(String remotePath, String localPath)
throws IOException;
/**
* This is meant to delegate to {@link #download(String, LocalDestFile)} with the {@code localPath} wrapped as e.g.
* a {@link FileSystemFile}. Appends to existing if {@code byteOffset} &gt; 0.
*
* @param localPath
* @param remotePath
* @param byteOffset
*
* @throws IOException
*/
void download(String remotePath, String localPath, long byteOffset)
throws IOException;
/**
* Upload {@code localFile} to {@code remotePath}.
*
@@ -54,6 +80,18 @@ public interface FileTransfer {
void upload(LocalSourceFile localFile, String remotePath)
throws IOException;
/**
* Upload {@code localFile} to {@code remotePath}. Appends to existing if {@code byteOffset} &gt; 0.
*
* @param localFile
* @param remotePath
* @param byteOffset
*
* @throws IOException
*/
void upload(LocalSourceFile localFile, String remotePath, long byteOffset)
throws IOException;
/**
* Download {@code remotePath} to {@code localFile}.
*
@@ -65,6 +103,18 @@ public interface FileTransfer {
void download(String remotePath, LocalDestFile localFile)
throws IOException;
/**
* Download {@code remotePath} to {@code localFile}. Appends to existing if {@code byteOffset} &gt; 0.
*
* @param localFile
* @param remotePath
* @param byteOffset
*
* @throws IOException
*/
void download(String remotePath, LocalDestFile localFile, long byteOffset)
throws IOException;
TransferListener getTransferListener();
void setTransferListener(TransferListener listener);

5
src/main/java/net/schmizz/sshj/xfer/LocalDestFile.java
View File

@@ -20,8 +20,13 @@ import java.io.OutputStream;
public interface LocalDestFile {
long getLength();
OutputStream getOutputStream()
throws IOException;
OutputStream getOutputStream(boolean append)
throws IOException;
/** @return A child file/directory of this directory with given {@code name}. */
LocalDestFile getChild(String name);

36
src/main/java/net/schmizz/sshj/xfer/scp/SCPFileTransfer.java
View File

@@ -52,24 +52,49 @@ public class SCPFileTransfer
@Override
public void upload(String localPath, String remotePath)
throws IOException {
newSCPUploadClient().copy(new FileSystemFile(localPath), remotePath);
upload(localPath, remotePath, 0);
}
@Override
public void upload(String localFile, String remotePath, long byteOffset)
throws IOException {
upload(new FileSystemFile(localFile), remotePath, byteOffset);
}
@Override
public void download(String remotePath, String localPath)
throws IOException {
download(remotePath, new FileSystemFile(localPath));
download(remotePath, localPath, 0);
}
@Override
public void download(String remotePath, String localPath, long byteOffset) throws IOException {
download(remotePath, new FileSystemFile(localPath), byteOffset);
}
@Override
public void download(String remotePath, LocalDestFile localFile)
throws IOException {
download(remotePath, localFile, 0);
}
@Override
public void download(String remotePath, LocalDestFile localFile, long byteOffset)
throws IOException {
checkByteOffsetSupport(byteOffset);
newSCPDownloadClient().copy(remotePath, localFile);
}
@Override
public void upload(LocalSourceFile localFile, String remotePath)
throws IOException {
upload(localFile, remotePath, 0);
}
@Override
public void upload(LocalSourceFile localFile, String remotePath, long byteOffset)
throws IOException {
checkByteOffsetSupport(byteOffset);
newSCPUploadClient().copy(localFile, remotePath);
}
@@ -79,4 +104,11 @@ public class SCPFileTransfer
}
return this;
}
private void checkByteOffsetSupport(long byteOffset) throws IOException {
// TODO - implement byte offsets on SCP, if possible.
if (byteOffset > 0) {
throw new SCPException("Byte offset on SCP file transfers is not supported.");
}
}
}