Java Awssdk S3 Tải object xuống với S3TransferManager

import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.SizeConstant;

public class MyUtils {

	public static S3TransferManager createS3TransferManager(Region region) {
		AwsCredentials credentials = AwsBasicCredentials.create("accessKeyId", "secretAccessKey");
		AwsCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(credentials);

		S3AsyncClient s3AsyncClient = S3AsyncClient.crtBuilder() //
				.credentialsProvider(credentialsProvider) //
				.region(region) //
				.targetThroughputInGbps(20.0) //
				.minimumPartSizeInBytes(10 * SizeConstant.MB) //
				.build();

		return S3TransferManager.builder().s3Client(s3AsyncClient).build();
	}
}

<!-- https://mvnrepository.com/artifact/software.amazon.awssdk/s3 -->
<dependency>
	<groupId>software.amazon.awssdk</groupId>
	<artifactId>s3</artifactId>
	<version>2.21.10</version>
</dependency>

<!-- https://mvnrepository.com/artifact/software.amazon.awssdk/s3-transfer-manager -->
<dependency>
	<groupId>software.amazon.awssdk</groupId>
	<artifactId>s3-transfer-manager</artifactId>
	<version>2.21.10</version>
</dependency>

<!-- https://mvnrepository.com/artifact/software.amazon.awssdk.crt/aws-crt -->
<dependency>
	<groupId>software.amazon.awssdk.crt</groupId>
	<artifactId>aws-crt</artifactId>
	<version>0.28.0</version>
</dependency>

<!-- Log Library -->
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
<dependency>
	<groupId>org.slf4j</groupId>
	<artifactId>slf4j-api</artifactId>
	<version>2.0.9</version>
</dependency>

<!-- Log Library -->
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-simple -->
<dependency>
	<groupId>org.slf4j</groupId>
	<artifactId>slf4j-simple</artifactId>
	<version>2.0.9</version>
</dependency>

package org.o7planning.java_14249_awssdk_s3;

import java.nio.file.Paths;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletionException;

import org.o7planning.java_14249_awssdk_s3.utils.MyUtils;

import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload;
import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest;
import software.amazon.awssdk.transfer.s3.model.FileDownload;
import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener;

public class DownloadOneFileExample1 {
	private static final Region MY_BUCKET_REGION = Region.EU_CENTRAL_1;
	private static final String BUCKET_NAME = "my-bucket-name";

	public static void downloadFile(S3TransferManager transferManager, String bucketName, //
			String key, String filePath) {
		DownloadFileRequest downloadFileRequest = DownloadFileRequest.builder()
				.getObjectRequest(req -> req.bucket(bucketName).key(key)) // bucketName & key
				.destination(Paths.get(filePath))
				// Attaching a LoggingTransferListener that will log the progress
				.addTransferListener(LoggingTransferListener.create()).build();

		FileDownload fileDownload = transferManager.downloadFile(downloadFileRequest);
		try {
			// Wait for the transfer to complete
			CompletedFileDownload completedFileDownload = fileDownload //
					.completionFuture() // CompletableFuture<CompletedFileDownload>
					.join(); // Wait until completed, and return value.
			//
			System.out.println("SuccessfullyCompleted");
			System.out.println(" --> " + completedFileDownload.toString());
		} catch (CancellationException e) {
			System.out.println("CancellationException");
			System.out.println(" --> (It could be by the user calling the pause method.)");
		} catch (CompletionException e) {
			System.out.println("Completed with error: " + e);
		}
	}

	public static void main(String[] args) {
		// There are several ways to create an S3TransferManager
		// @See my article 14231.
		S3TransferManager transferManager = MyUtils.createS3TransferManager(MY_BUCKET_REGION);
		//
		// Test download a video file 11.9MB
		//
		downloadFile(transferManager, //
				BUCKET_NAME, // bucketName
				"static/videos/sample-15s.mp4", // key
				"/Volumes/New/Test/download/sample-15s.mp4" // filePath to save.
		);
	}
}

package org.o7planning.java_14249_awssdk_s3;

import java.nio.file.Paths;

import org.o7planning.java_14249_awssdk_s3.utils.MyUtils;

import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryDownload;
import software.amazon.awssdk.transfer.s3.model.DirectoryDownload;
import software.amazon.awssdk.transfer.s3.model.DownloadDirectoryRequest;
import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener;

public class DownloadMultiFilesExample1 {

	private static final Region MY_BUCKET_REGION = Region.EU_CENTRAL_1;
	private static final String BUCKET_NAME = "my-bucket-name";

	public static Integer downloadMultiFiles(S3TransferManager transferManager, //
			String bucketName, String keyPrefix, String destinationDirectory) {
		// Create DownloadDirectoryRequest
		DownloadDirectoryRequest downloadDirectoryRequest = DownloadDirectoryRequest.builder() //
				.destination(Paths.get(destinationDirectory))//
				.bucket(bucketName) //
				// Only download objects with prefix
				.listObjectsV2RequestTransformer(l -> l.prefix(keyPrefix)) //
				// Optional, add LoggingTransferListener for each file download.
				.downloadFileRequestTransformer(
						request -> request.addTransferListener(LoggingTransferListener.create()))
				.build();

		DirectoryDownload directoryDownload = transferManager.downloadDirectory(downloadDirectoryRequest);

		// Wait for the transfer to complete
		CompletedDirectoryDownload completedDirectoryDownload = directoryDownload //
				.completionFuture() // CompletableFuture<CompletedFileDownload>
				.join(); // Wait until completed, and return value.
		// Print out any failed downloads
		completedDirectoryDownload.failedTransfers().forEach(System.err::println);

		return completedDirectoryDownload.failedTransfers().size();
	}

	public static void main(String[] args) {
		// There are several ways to create an S3TransferManager
		// @See my article 14231.
		S3TransferManager transferManager = MyUtils.createS3TransferManager(MY_BUCKET_REGION);
		//
		// Test download all files with keyPrefix:
		//
		int failCount = downloadMultiFiles(transferManager, //
				BUCKET_NAME, // bucketName
				"static/images/", // keyPrefix
				"/Volumes/New/Test/download/" // destinationDirectory
		);
		System.out.println("failCount: " + failCount);
	}
}

package org.o7planning.java_14249_awssdk_s3;

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletionException;

import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload;
import software.amazon.awssdk.transfer.s3.model.FileDownload;
import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload;
import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener;
import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest;

public class ResumableFileDownloadThread extends Thread {
	private S3TransferManager transferManager;
	private String bucketName;
	private String key;
	private String filePath;

	private FileDownload fileDownload;
	private boolean isPause;
	private boolean successfullyCompleted;

	public ResumableFileDownloadThread(S3TransferManager transferManager, //
			String bucketName, String key, String filePath) {
		super();
		this.transferManager = transferManager;
		this.bucketName = bucketName;
		this.key = key;
		this.filePath = filePath;
	}

	private String resumableStateFile(String key) {
		return key.replace('/', '_') + "_resumableFileDownload.json";
	}

	public void pauseDownload() {
		if (fileDownload == null || isPause || successfullyCompleted) {
			return;
		}
		System.out.println("--> User calls pauseDownload!");

		// Pause the download
		ResumableFileDownload resumableFileDownload = fileDownload.pause();
		isPause = true;
		// Optionally, persist the resumableFileDownload
		Path path = Paths.get(this.resumableStateFile(key));
		resumableFileDownload.serializeToFile(path);
	}

	public void resumeDownload() {
		if (fileDownload == null || !isPause || successfullyCompleted) {
			return;
		}
		isPause = false;
		System.out.println("--> User calls resumeDownload!");

		Path path = Paths.get(this.resumableStateFile(key));
		// Retrieve the resumableFileDownload from the file
		//
		// WARNING: Test with: Awssdk 2.21.10
		// The "TransferListener" is not restored. (Need to wait for newer version of
		// Awssdk)
		//
		ResumableFileDownload persistedResumableFileDownload = ResumableFileDownload.fromFile(path);

		// Resume the download
		this.fileDownload = transferManager.resumeDownloadFile(persistedResumableFileDownload);

		this.handleCompletionState(fileDownload);
	}

	@Override
	public void run() {
		DownloadFileRequest downloadFileRequest = DownloadFileRequest.builder()
				.getObjectRequest(req -> req.bucket(bucketName).key(key)) // bucketName & key
				.destination(Paths.get(filePath))
				// Attaching a LoggingTransferListener that will log the progress
				.addTransferListener(LoggingTransferListener.create())//
				.build();
		// Initiate the transfer
		this.fileDownload = transferManager.downloadFile(downloadFileRequest);

		this.handleCompletionState(fileDownload);
	}

	private void handleCompletionState(FileDownload fileDownload) {
		try {
			CompletedFileDownload completedFileDownload = fileDownload //
					.completionFuture() // CompletableFuture<CompletedDirectoryDownload>
					.join(); // Wait until completed, and return value.
			//
			successfullyCompleted = true;
			System.out.println("handleCompletionState: SuccessfullyCompleted");
			System.out.println(" --> " + completedFileDownload.toString());
		} catch (CancellationException e) {
			System.out.println("handleCompletionState: CancellationException");
			System.out.println(" --> (It could be by the user calling the pause method.)");
		} catch (CompletionException e) {
			System.out.println("handleCompletionState: Completed with error: " + e);
		}
	}
}

package org.o7planning.java_14249_awssdk_s3;

import software.amazon.awssdk.transfer.s3.progress.TransferListener;
import software.amazon.awssdk.transfer.s3.progress.TransferListener.Context.BytesTransferred;
import software.amazon.awssdk.transfer.s3.progress.TransferListener.Context.TransferComplete;
import software.amazon.awssdk.transfer.s3.progress.TransferListener.Context.TransferFailed;
import software.amazon.awssdk.transfer.s3.progress.TransferListener.Context.TransferInitiated;
import software.amazon.awssdk.transfer.s3.progress.TransferProgressSnapshot;

public class MyTransferListener implements TransferListener {

	@Override
	public void bytesTransferred(BytesTransferred context) {
		TransferProgressSnapshot progress = context.progressSnapshot();
		System.out.println("MyTransferListener.bytesTransferred: " //
				+ (progress.ratioTransferred().getAsDouble() * 100) + " %");
	}

	@Override
	public void transferInitiated(TransferInitiated context) {
		TransferProgressSnapshot progress = context.progressSnapshot();
		System.out.println("MyTransferListener.transferInitiated: " //
				+ progress.ratioTransferred().getAsDouble());
	}

	@Override
	public void transferComplete(TransferComplete context) {
		TransferProgressSnapshot progress = context.progressSnapshot();
		System.out.println("MyTransferListener.bytesTransferred: " //
				+ progress.ratioTransferred().getAsDouble());
	}

	@Override
	public void transferFailed(TransferFailed context) {
		System.out.println("MyTransferListener.transferFailed (Or pause called): " //
				+ context.exception().toString());
	}
}

package org.o7planning.java_14249_awssdk_s3;

import java.time.Duration;

import org.o7planning.java_14249_awssdk_s3.utils.MyUtils;

import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.transfer.s3.S3TransferManager;

public class ResumableFileDownloadThreadTest {
	private static final Region MY_BUCKET_REGION = Region.EU_CENTRAL_1;
	private static final String BUCKET_NAME = "my-bucket-name";

	public static void main(String[] args) {
		S3TransferManager transferManager = MyUtils.createS3TransferManager(MY_BUCKET_REGION);

		// Test with a large File...
		ResumableFileDownloadThread downloadThread = new ResumableFileDownloadThread(transferManager, //
				BUCKET_NAME, // bucketName
				"static/videos/sample-16m.mp4", // key   (171MB)
				"/Volumes/New/Test/download/sample-16m.mp4" // filePath
		);
		//
		downloadThread.start();

		try {
			Thread.sleep(Duration.ofSeconds(60));
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		// The User calls pauseDownload method.
		downloadThread.pauseDownload();
		try {
			Thread.sleep(Duration.ofSeconds(3));
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		// The User calls resumeDownload method.
		downloadThread.resumeDownload();
	} 
}