第七章：S3 SDK 使用

7.1 SDK 概述

MinIO 100% 兼容 AWS S3 API，支持所有主流 AWS SDK。

支持的 SDK
├── AWS SDK for Python (boto3)
├── AWS SDK for Java
├── AWS SDK for Go
├── AWS SDK for JavaScript (Node.js)
├── AWS SDK for .NET
├── AWS CLI
└── S3cmd

7.2 Python SDK (boto3)

7.2.1 安装配置

# 安装 boto3
pip install boto3

# 配置凭证
mkdir -p ~/.aws
cat > ~/.aws/credentials << 'EOF'
[minio]
aws_access_key_id = admin
aws_secret_access_key = password123
EOF

cat > ~/.aws/config << 'EOF'
[profile minio]
region = us-east-1
s3 =
  endpoint_url = http://localhost:9000
  signature_version = s3v4
  addressing_style = path
EOF

7.2.2 基本操作

import boto3
from botocore.config import Config

# 创建 S3 客户端
s3 = boto3.client(
    's3',
    endpoint_url='http://localhost:9000',
    aws_access_key_id='admin',
    aws_secret_access_key='password123'
)

# 创建存储桶
s3.create_bucket(Bucket='mybucket')

# 列出存储桶
response = s3.list_buckets()
for bucket in response['Buckets']:
    print(f"Bucket: {bucket['Name']}")

# 上传文件
s3.upload_file('/path/to/file.txt', 'mybucket', 'object-key')

# 下载文件
s3.download_file('mybucket', 'object-key', '/path/to/download.txt')

# 列出对象
response = s3.list_objects_v2(Bucket='mybucket')
for obj in response.get('Contents', []):
    print(f"Key: {obj['Key']}, Size: {obj['Size']}")

# 删除对象
s3.delete_object(Bucket='mybucket', Key='object-key')

7.2.3 高级操作

import boto3
from botocore.config import Config
from botocore.exceptions import ClientError

# 创建带配置的客户端
s3 = boto3.client(
    's3',
    endpoint_url='http://localhost:9000',
    aws_access_key_id='admin',
    aws_secret_access_key='password123',
    config=Config(
        max_pool_connections=100,
        retries={'max_attempts': 3}
    )
)

# 分段上传大文件
def upload_large_file(file_path, bucket, object_key):
    config = boto3.s3.transfer.S3TransferConfig(
        multipart_threshold=100 * 1024 * 1024,  # 100MB
        multipart_chunksize=50 * 1024 * 1024    # 50MB per chunk
    )
    transfer = boto3.s3.transfer.S3Transfer(s3, config)
    transfer.upload_file(file_path, bucket, object_key)

# 生成预签名 URL
url = s3.generate_presigned_url(
    'get_object',
    Params={'Bucket': 'mybucket', 'Key': 'object-key'},
    ExpiresIn=3600
)
print(f"Download URL: {url}")

# 上传带元数据的对象
s3.put_object(
    Bucket='mybucket',
    Key='document.txt',
    Body=b'file content',
    Metadata={
        'project': 'analytics',
        'owner': 'admin'
    }
)

# 启用版本控制
s3.put_bucket_versioning(
    Bucket='mybucket',
    VersioningConfiguration={'Status': 'Enabled'}
)

# 复制对象
s3.copy_object(
    Bucket='dest-bucket',
    Key='new-object-key',
    CopySource={'Bucket': 'source-bucket', 'Key': 'source-key'}
)

7.3 Java SDK

7.3.1 Maven 依赖

<dependency>
    <groupId>software.amazon.awssdk</groupId>
    <artifactId>s3</artifactId>
    <version>2.21.0</version>
</dependency>

7.3.2 基本操作

import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.*;
import java.net.URI;

public class MinioExample {
    public static void main(String[] args) {
        // 创建 S3 客户端
        S3Client s3 = S3Client.builder()
            .endpointOverride(URI.create("http://localhost:9000"))
            .region(Region.US_EAST_1)
            .credentialsProvider(() -> AwsBasicCredentials.create(
                "admin", "password123"))
            .forcePathStyle(true)
            .build();

        // 创建存储桶
        s3.createBucket(CreateBucketRequest.builder()
            .bucket("mybucket")
            .build());

        // 上传文件
        s3.putObject(PutObjectRequest.builder()
            .bucket("mybucket")
            .key("test.txt")
            .build(),
            software.amazon.awssdk.core.sync.RequestBody
                .fromString("Hello MinIO!"));

        // 下载文件
        GetObjectResponse response = s3.getObject(GetObjectRequest.builder()
            .bucket("mybucket")
            .key("test.txt")
            .build());

        System.out.println(response.readUtf8());

        // 列出对象
        ListObjectsV2Response listResponse = s3.listObjectsV2(
            ListObjectsV2Request.builder()
                .bucket("mybucket")
                .build());

        listResponse.contents().forEach(obj ->
            System.out.println(obj.key()));
    }
}

7.4 Go SDK

7.4.1 安装

go get github.com/aws/aws-sdk-go-v2
go get github.com/aws/aws-sdk-go-v2/config
go get github.com/aws/aws-sdk-go-v2/service/s3

7.4.2 基本操作

package main

import (
    "context"
    "fmt"
    "io"
    "os"

    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/config"
    "github.com/aws/aws-sdk-go-v2/credentials"
    "github.com/aws/aws-sdk-go-v2/service/s3"
)

func main() {
    ctx := context.Background()

    // 创建配置
    cfg, err := config.WithCredentialsProvider(
        credentials.NewStaticCredentialsProvider(
            "admin",
            "password123",
            "",
        ),
    ).WithRegion("us-east-1")

    if err != nil {
        panic(err)
    }

    // 创建 S3 客户端
    client := s3.New(s3.Options{
        BaseEndpoint: aws.String("http://localhost:9000"),
        Credentials:   credentials.NewStaticCredentialsProvider("admin", "password123", ""),
        UsePathStyle:  true,
    })

    // 创建存储桶
    _, err = client.CreateBucket(ctx, &s3.CreateBucketInput{
        Bucket: aws.String("mybucket"),
    })
    if err != nil {
        fmt.Printf("Error: %v\n", err)
    }

    // 上传文件
    file, _ := os.Open("/path/to/file.txt")
    defer file.Close()

    _, err = client.PutObject(ctx, &s3.PutObjectInput{
        Bucket: aws.String("mybucket"),
        Key:    aws.String("object-key"),
        Body:   file,
    })

    // 下载文件
    result, err := client.GetObject(ctx, &s3.GetObjectInput{
        Bucket: aws.String("mybucket"),
        Key:    aws.String("object-key"),
    })
    if err != nil {
        panic(err)
    }
    defer result.Body.Close()

    body, _ := io.ReadAll(result.Body)
    fmt.Println(string(body))
}

7.5 JavaScript/Node.js SDK

7.5.1 安装

npm install @aws-sdk/client-s3
npm install @aws-sdk/s3-request-presigner

7.5.2 基本操作

const { S3Client, PutObjectCommand, GetObjectCommand, ListBucketsCommand } = require('@aws-sdk/client-s3');
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');

// 创建 S3 客户端
const s3Client = new S3Client({
  endpoint: 'http://localhost:9000',
  region: 'us-east-1',
  credentials: {
    accessKeyId: 'admin',
    secretAccessKey: 'password123'
  },
  forcePathStyle: true
});

async function main() {
  // 创建存储桶
  await s3Client.send(new CreateBucketCommand({
    Bucket: 'mybucket'
  }));

  // 上传文件
  const fs = require('fs');
  const fileContent = fs.readFileSync('/path/to/file.txt');

  await s3Client.send(new PutObjectCommand({
    Bucket: 'mybucket',
    Key: 'object-key',
    Body: fileContent,
    ContentType: 'text/plain'
  }));

  // 生成预签名 URL
  const command = new GetObjectCommand({
    Bucket: 'mybucket',
    Key: 'object-key'
  });
  const signedUrl = await getSignedUrl(s3Client, command, { expiresIn: 3600 });
  console.log('Signed URL:', signedUrl);

  // 列出存储桶
  const listResponse = await s3Client.send(new ListBucketsCommand({}));
  console.log('Buckets:', listResponse.Buckets);
}

main();

7.6 AWS CLI 使用

7.6.1 配置 MinIO 别名

# 配置 MinIO 别名
aws configure set aws_access_key_id admin --profile minio
aws configure set aws_secret_access_key password123 --profile minio
aws configure set region us-east-1 --profile minio
aws configure set endpoint_url http://localhost:9000 --profile minio

7.6.2 AWS CLI 操作

# 使用 MinIO 配置执行命令
aws --profile minio s3 mb s3://mybucket

# 上传文件
aws --profile minio s3 cp /path/to/file.txt s3://mybucket/

# 下载文件
aws --profile minio s3 cp s3://mybucket/file.txt /path/to/

# 列出对象
aws --profile minio s3 ls s3://mybucket/

# 同步目录
aws --profile minio s3 sync /local/path/ s3://mybucket/

# 生成预签名 URL
aws --profile minio s3 presign s3://mybucket/file.txt

# 设置存储桶策略
aws --profile minio s3api put-bucket-policy \
  --bucket mybucket \
  --policy file://policy.json

7.7 错误处理

7.7.1 常见错误

import boto3
from botocore.exceptions import ClientError, NoCredentialsError

s3 = boto3.client(
    's3',
    endpoint_url='http://localhost:9000',
    aws_access_key_id='admin',
    aws_secret_access_key='password123'
)

try:
    s3.head_bucket(Bucket='mybucket')
except ClientError as e:
    error_code = e.response['Error']['Code']
    if error_code == '404':
        print("Bucket not found")
    elif error_code == '403':
        print("Access denied")
    else:
        print(f"Error: {e}")

except NoCredentialsError:
    print("Credentials not found")

7.7.2 重试机制

from botocore.config import Config
from botocore.retries import standard

s3 = boto3.client(
    's3',
    endpoint_url='http://localhost:9000',
    aws_access_key_id='admin',
    aws_secret_access_key='password123',
    config=Config(
        retries={
            'max_attempts': 3,
            'mode': 'standard'
        }
    )
)

7.8 总结

本章介绍了使用 AWS S3 SDK 在 MinIO 上进行开发的各种方法，包括 Python、Java、Go 和 JavaScript SDK。MinIO 的 S3 兼容性使得开发者可以使用熟悉的 AWS 工具和库来操作 MinIO 存储。下一章将学习存储桶的高级管理。