Skip to content
This repository has been archived by the owner on Feb 10, 2023. It is now read-only.

Commit

Permalink
Merge pull request #2 from Statflo/multipart-upload
Browse files Browse the repository at this point in the history
Multipart upload
  • Loading branch information
FabioBatSilva authored Aug 15, 2018
2 parents 7199500 + 2601467 commit 2b4b42b
Show file tree
Hide file tree
Showing 9 changed files with 136 additions and 10 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,5 @@ nbdist/
.nb-gradle/

.env*
.vscode
.vscode
.volumes/
18 changes: 17 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,20 @@ include $(MAKEFILE_DIR)/targets.mk
### Application specific targets can be added to this file ###
-include targets.mk

### Or application sepcific targets can be added here ###
### Or application sepcific targets can be added here ###

start:
@echo ">>> Starting sftpd"
@docker-compose -f docker-compose-local.yml up sftpd

stop:
@echo ">>> Stopping docker-compose"
@docker-compose -f docker-compose-local.yml stop

logs:
@echo ">>> Starting application"
@docker-compose -f docker-compose-local.yml logs -f --tail="all"

mysql-connect:
@echo ">>> Connecting to mysql"
@docker-compose -f docker-compose-local.yml exec mysql mysql -uroot -proot sftpd
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ Running the app
AWS_REGION="ca-central-1" AWS_ACCESS_KEY_ID="ACCESSKEYHERE" AWS_SECRET_ACCESS_KEY="SECRETKEYHERE" java -jar target/s3-sftp-1.0.1-SNAPSHOT.jar
```

### Running the app localy
```
make start AWS_ACCESS_KEY_ID=ACCESSKEYHERE AWS_SECRET_ACCESS_KEY=SECRETKEYHERE
```

### How to configure your app

Example configuration:
Expand Down
30 changes: 30 additions & 0 deletions docker-compose-local.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
version: '2.1'

services:
mysql:
image: mysql:5.7
restart: always
environment:
MYSQL_DATABASE: sftpd
MYSQL_ROOT_PASSWORD: root
healthcheck:
test: mysql -uroot -proot -e 'SHOW TABLES' sftpd

sftpd:
image: maven:3-jdk-8-alpine
working_dir: /usr/share/statflo
command: mvn clean spring-boot:run
ports:
- "2222:2222"
environment:
- AWS_REGION=ca-central-1
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID- }
- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY- }

- SPRING_PROFILES_ACTIVE=default,local
volumes:
- "./:/usr/share/statflo"
- "./.volumes/m2:/root/.m2"
depends_on:
mysql:
condition: service_healthy
15 changes: 14 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,13 @@
<org.apache.sshd.version>1.6.0</org.apache.sshd.version>
</properties>

<repositories>
<repository>
<id>jitpack.io</id>
<url>https://jitpack.io</url>
</repository>
</repositories>

<dependencies>

<dependency>
Expand Down Expand Up @@ -61,12 +68,18 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>
<!--
<dependency>
<groupId>com.upplication</groupId>
<artifactId>s3fs</artifactId>
<version>2.2.2</version>
</dependency>
-->
<dependency>
<groupId>com.github.Statflo</groupId>
<artifactId>Amazon-S3-FileSystem-NIO2</artifactId>
<version>master-SNAPSHOT</version>
</dependency>

<dependency>
<groupId>com.amazonaws</groupId>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.dataexchange.server.aws;

import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSCredentialsProvider;
Expand Down Expand Up @@ -39,4 +40,18 @@ public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest) throws S

return super.copyObject(copyObjectRequest);
}

@Override
public InitiateMultipartUploadResult initiateMultipartUpload(final InitiateMultipartUploadRequest request) throws SdkClientException, AmazonServiceException {
ObjectMetadata metadata = request.getObjectMetadata();

if (metadata == null) {
metadata = new ObjectMetadata();
}

metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
request.setObjectMetadata(metadata);

return super.initiateMultipartUpload(request);
}
}
13 changes: 8 additions & 5 deletions src/main/java/com/dataexchange/server/conf/SftpServer.java
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@
@EnableConfigurationProperties(SftpServer.SftpServerConfiguration.class)
public class SftpServer {

public static final Map<String, ?> ENV = ImmutableMap.<String, Object>builder()
.put(S3FileSystemProvider.AMAZON_S3_FACTORY_CLASS, CustomAmazonS3ClientFactory.class.getName())
.put(S3FileSystemProvider.MULTIPART_UPLOAD_ENABLED, "true")
.build();

@Value("${app.sftp.aws.bucket-name}")
private String bucketName;

Expand Down Expand Up @@ -106,10 +111,8 @@ public SshServer sshServer(PasswordAuthenticator passwordAuthenticator, Tracking


private Path getS3BucketPath() throws URISyntaxException, IOException {
Map<String, ?> env = ImmutableMap.<String, Object>builder()
.put(S3FileSystemProvider.AMAZON_S3_FACTORY_CLASS, CustomAmazonS3ClientFactory.class.getName()).build();

FileSystem fileSystem = FileSystems.newFileSystem(new URI("s3:///"), env, Thread.currentThread().getContextClassLoader());
final URI uri = new URI("s3:///");
final FileSystem fileSystem = FileSystems.newFileSystem(uri, ENV, Thread.currentThread().getContextClassLoader());

return fileSystem.getPath("/" + bucketName);
}
Expand Down Expand Up @@ -198,6 +201,6 @@ public String getPassword() {
public void setPassword(String password) {
this.password = password;
}

}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.dataexchange.server.sshd.file;

import com.dataexchange.server.conf.SftpServer;
import org.apache.sshd.common.file.root.RootedFileSystemProvider;
import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
import org.apache.sshd.common.session.Session;
Expand All @@ -9,7 +10,6 @@
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.util.Collections;

public class UserRootedFileSystemFactory extends VirtualFileSystemFactory {

Expand All @@ -25,7 +25,7 @@ public FileSystem createFileSystem(Session session) throws IOException {
throw new InvalidPathException(username, "Cannot resolve home directory");
}

return new RootedFileSystemProvider().newFileSystem(dir, Collections.emptyMap());
return new RootedFileSystemProvider().newFileSystem(dir, SftpServer.ENV);
}

@Override
Expand Down
43 changes: 43 additions & 0 deletions src/main/resources/application-local.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
app:
sftp:
users:
-
username: test
password: test
aws:
bucket-name: "test-sftp-uploader"
hostkey-algorithm: "RSA"
hostkey-private: |
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEArXpNaJAOXsgNe8OIFzWrKMC0D5yE3GV8TPjdAWy8n7MjOole
DbEH5DzO1fKlLMCQGITjZuouSY3yE8zMC9xs4hE5e+KTSStreK1E9X3csSszuOLr
+olj/C3ZcMKNq2s0TPBu691cRnTwVaL1V75V32rfCHz8NShCxyYXNiahycrOL2Bm
9Cu8AMZcm1e9POgqABZbP2cP5MDy+Vw4RMw8XI8G6ClCQBlJayNAhqMDDw+80A/Z
MBt3OyDOJ7o2xq/sJAIhppck+Fw43W04BIQwIrmEucBs1c6Np/+ePvk5jKhII5iE
2+7nQizwwUJQmzdmTcciLJ8CMhZ7FDUobXFHwQIDAQABAoIBAQCtLbq495Zg6DUN
qFTjVqBXPoiCTS4VFvrPnGCEW5ixlJNGjHtoD6JhJpMU4fYihtlSoNailoU+yIUx
XvETpijMRYWtNrLyCHZxMgf1SoBWHVq051PL9/voc5zuBEqJJaxAYoVY0NKdQ3/o
/ZUO0uPG9VlRj/P23QpM5KmnVUmNt1L6nml9v9J/yeO/GlMtv5FEwJm11uHeeWGn
Qfh/4B6hyDXsCOCAhD5C7G4hWChzs6q/IRI+swiocSf5S5u6N2h1HdZz3W0Q92lQ
BAFypWjCPDlMHzRgbtMUKQ2n8bv78svEe7kAgZvEX8dqnE0EJGLlJP2fo1nv6BQt
VmWMTMsBAoGBANQHndt+gXx75amGfTYyIK51dXahagfMiz4JGPBrzrZ4zOW10opU
zvvB0p2ApaKr8+jZtRLxHZToof/8Cck2RUodwS00he/Ikbz7GdwqDoiyWbjVTaut
+HJNn3zUxHSMDTHxU+DOVrIb79vqUweAYlq2ManPRzi7yDVkZv3ssr0FAoGBANF0
AoNV8w8gGDfV8O6P/wTSXmkRfvk8WkTOkOoOHCTHXt4QuTZ5Q6v7ad0BlSj/A+sN
qScpn0GQsDNdgqSCoXdXDgKWjUqKvcwN+SLq5f0uPeGFkL3GuGraK+jAaxjXml4q
eb3MoSzMmlHXO7LorbNXcYXen180j34jFgcOijyNAoGAcXoeFzIulPQuZzuOWeoc
KaK9mh8rgz8KqsntqgbY3Dns12gnZVeca4tBcIBApb5HzTBvQ9L91QtoYHuAtUqx
bMhd8haxNDLxJOZy4a00mB5z/INUjXSPioVWvVWWSxYFndwd2ywzpCFljxprE5G3
kfr+u3YvFQMTVNgGcxIyWw0CgYBps/Q4t72MWpQeSZfnrT/z/P6sY6oCuhSMOPsl
Q62sWflkdGgkN5qRRrYsVMokAF7H1wMQwo6elhh2M7DkuRAnnAV0BOqVPeN/AWZD
2uvqk9Wi5LchLxKOyj/ynHjv9o7vCtyZjpWgeDlBKC/2oDpZEb424xzm/S2VaLuL
IClfWQKBgF/aQ0Jxhg1LCTK/gZmqyxXyvxu7x2Utedxt7hI6ia3eSvfXowgcOXMT
mfixttpPUobQhk43y5/h3N7gFGKxZQV6/ZFeBG7Dvck2NWgutNwqvbhFtfhVf37E
PP+E8udcWsXfU2ZjPAIrApFNIwIYtEOsisy2QTZkN8BaTKcHq6+S
-----END RSA PRIVATE KEY-----
spring:
datasource:
url: "jdbc:mysql://mysql/sftpd?useSSL=false&serverTimezone=UTC"
username: "root"
password: "root"

0 comments on commit 2b4b42b

Please sign in to comment.