JAVA-批量下载Minio数据到本地（G）

package com.slice;

import io.minio.*;
import io.minio.errors.*;
import io.minio.messages.Item;
import java.io.File;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;

import io.minio.*;
import io.minio.messages.Item;
import java.io.*;
import java.nio.file.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicLong;

public class MinIOFolderDownloader {
    private final MinioClient minioClient;
    private final ExecutorService executorService;
    private final AtomicLong totalFiles = new AtomicLong(0);
    private final AtomicLong completedFiles = new AtomicLong(0);

    public MinIOFolderDownloader(String endpoint, String accessKey, String secretKey, int threadCount) {
        this.minioClient = MinioClient.builder()
                .endpoint(endpoint)
                .credentials(accessKey, secretKey)
                .build();
        this.executorService = Executors.newFixedThreadPool(threadCount);
    }

    public void downloadFolder(String bucketName, String folderPath, String localDir) throws IOException, ServerException, InsufficientDataException, ErrorResponseException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException {
        Path localBasePath = Paths.get(localDir);
        Files.createDirectories(localBasePath);

        Iterable<Result<Item>> results = minioClient.listObjects(
                ListObjectsArgs.builder()
                        .bucket(bucketName)
                        .prefix(folderPath)
                        .recursive(true)
                        .build());

        for (Result<Item> result : results) {
            Item item = result.get();
            if (!item.isDir()) {
                totalFiles.incrementAndGet();
                executorService.submit(() -> downloadFile(bucketName, item.objectName(), localBasePath));
            }
        }
        printProgress();
    }

    private void downloadFile(String bucketName, String objectName, Path localBasePath) {
        try {
            Path localPath = localBasePath.resolve(objectName);
            Files.createDirectories(localPath.getParent());

            File localFile = localPath.toFile();
            StatObjectResponse remoteStat = minioClient.statObject(
                    StatObjectArgs.builder()
                            .bucket(bucketName)
                            .object(objectName)
                            .build());

            // 检查文件是否已存在且完整
            if (localFile.exists()) {
                if (isFileComplete(localFile, remoteStat)) {
                    System.out.println("跳过已存在文件: " + objectName);
                    completedFiles.incrementAndGet();
                    return;
                }

                // 断点续传实现
                if (localFile.length() < remoteStat.size()) {
                    System.out.printf("继续下载 %s (%d/%d bytes)%n",
                            objectName, localFile.length(), remoteStat.size());
                    resumeDownload(bucketName, objectName, localFile, remoteStat.size());
                    completedFiles.incrementAndGet();
                    return;
                }
            }

            // 全新下载
            System.out.println("开始下载: " + objectName);
            minioClient.downloadObject(
                    DownloadObjectArgs.builder()
                            .bucket(bucketName)
                            .object(objectName)
                            .filename(localPath.toString())
                            .build());
            completedFiles.incrementAndGet();
        } catch (Exception e) {
            System.err.println("下载失败 " + objectName + ": " + e.getMessage());
        }
    }

    private boolean isFileComplete(File localFile, StatObjectResponse remoteStat) {
        return localFile.length() == remoteStat.size() &&
                localFile.lastModified() >= remoteStat.lastModified().toInstant().toEpochMilli();
    }

    private void resumeDownload(String bucketName, String objectName, File localFile, long remoteSize) throws Exception {
        try (RandomAccessFile output = new RandomAccessFile(localFile, "rw");
             InputStream stream = minioClient.getObject(
                     GetObjectArgs.builder()
                             .bucket(bucketName)
                             .object(objectName)
                             .offset(localFile.length())
                             .build())) {

            output.seek(localFile.length());
            byte[] buffer = new byte[8192];
            int bytesRead;
            while ((bytesRead = stream.read(buffer)) != -1) {
                output.write(buffer, 0, bytesRead);
            }
        }
    }

    private void printProgress() {
        new Thread(() -> {
            while (completedFiles.get() < totalFiles.get()) {
                System.out.printf("\r进度: %d/%d (%.2f%%)",
                        completedFiles.get(),
                        totalFiles.get(),
                        (completedFiles.get() * 100.0 / totalFiles.get()));
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    return;
                }
            }
            System.out.println("\n下载完成!");
        }).start();
    }

    public void shutdown() {
        executorService.shutdown();
    }

    public static void main(String[] args) throws IOException {
        MinIOFolderDownloader downloader = new MinIOFolderDownloader(
                "url",
                "testminio",
                "testminio",
                100);

        try {
            downloader.downloadFolder("slbqxcpdl", "Encast24hr-10min/", "D://qxminio/slbqxcpdl");
        } catch (ServerException e) {
            e.printStackTrace();
        } catch (InsufficientDataException e) {
            e.printStackTrace();
        } catch (ErrorResponseException e) {
            e.printStackTrace();
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        } catch (InvalidKeyException e) {
            e.printStackTrace();
        } catch (InvalidResponseException e) {
            e.printStackTrace();
        } catch (XmlParserException e) {
            e.printStackTrace();
        } catch (InternalException e) {
            e.printStackTrace();
        } finally {
            downloader.shutdown();
        }
    }
}