opendal/services/s3/config.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::fmt::Debug;
19use std::fmt::Formatter;
20
21use serde::Deserialize;
22use serde::Serialize;
23
24/// Config for Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support.
25#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
26#[serde(default)]
27#[non_exhaustive]
28pub struct S3Config {
29 /// root of this backend.
30 ///
31 /// All operations will happen under this root.
32 ///
33 /// default to `/` if not set.
34 pub root: Option<String>,
35 /// bucket name of this backend.
36 ///
37 /// required.
38 pub bucket: String,
39 /// is bucket versioning enabled for this bucket
40 pub enable_versioning: bool,
41 /// endpoint of this backend.
42 ///
43 /// Endpoint must be full uri, e.g.
44 ///
45 /// - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com`
46 /// - Cloudflare R2: `https://<ACCOUNT_ID>.r2.cloudflarestorage.com`
47 /// - Aliyun OSS: `https://{region}.aliyuncs.com`
48 /// - Tencent COS: `https://cos.{region}.myqcloud.com`
49 /// - Minio: `http://127.0.0.1:9000`
50 ///
51 /// If user inputs endpoint without scheme like "s3.amazonaws.com", we
52 /// will prepend "https://" before it.
53 ///
54 /// - If endpoint is set, we will take user's input first.
55 /// - If not, we will try to load it from environment.
56 /// - If still not set, default to `https://s3.amazonaws.com`.
57 pub endpoint: Option<String>,
58 /// Region represent the signing region of this endpoint. This is required
59 /// if you are using the default AWS S3 endpoint.
60 ///
61 /// If using a custom endpoint,
62 /// - If region is set, we will take user's input first.
63 /// - If not, we will try to load it from environment.
64 pub region: Option<String>,
65
66 /// access_key_id of this backend.
67 ///
68 /// - If access_key_id is set, we will take user's input first.
69 /// - If not, we will try to load it from environment.
70 pub access_key_id: Option<String>,
71 /// secret_access_key of this backend.
72 ///
73 /// - If secret_access_key is set, we will take user's input first.
74 /// - If not, we will try to load it from environment.
75 pub secret_access_key: Option<String>,
76 /// session_token (aka, security token) of this backend.
77 ///
78 /// This token will expire after sometime, it's recommended to set session_token
79 /// by hand.
80 pub session_token: Option<String>,
81 /// role_arn for this backend.
82 ///
83 /// If `role_arn` is set, we will use already known config as source
84 /// credential to assume role with `role_arn`.
85 pub role_arn: Option<String>,
86 /// external_id for this backend.
87 pub external_id: Option<String>,
88 /// role_session_name for this backend.
89 pub role_session_name: Option<String>,
90 /// Disable config load so that opendal will not load config from
91 /// environment.
92 ///
93 /// For examples:
94 ///
95 /// - envs like `AWS_ACCESS_KEY_ID`
96 /// - files like `~/.aws/config`
97 pub disable_config_load: bool,
98 /// Disable load credential from ec2 metadata.
99 ///
100 /// This option is used to disable the default behavior of opendal
101 /// to load credential from ec2 metadata, a.k.a, IMDSv2
102 pub disable_ec2_metadata: bool,
103 /// Allow anonymous will allow opendal to send request without signing
104 /// when credential is not loaded.
105 pub allow_anonymous: bool,
106 /// server_side_encryption for this backend.
107 ///
108 /// Available values: `AES256`, `aws:kms`.
109 pub server_side_encryption: Option<String>,
110 /// server_side_encryption_aws_kms_key_id for this backend
111 ///
112 /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id`
113 /// is not set, S3 will use aws managed kms key to encrypt data.
114 /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id`
115 /// is a valid kms key id, S3 will use the provided kms key to encrypt data.
116 /// - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an error will be
117 /// returned.
118 /// - If `server_side_encryption` is not `aws:kms`, setting `server_side_encryption_aws_kms_key_id`
119 /// is a noop.
120 pub server_side_encryption_aws_kms_key_id: Option<String>,
121 /// server_side_encryption_customer_algorithm for this backend.
122 ///
123 /// Available values: `AES256`.
124 pub server_side_encryption_customer_algorithm: Option<String>,
125 /// server_side_encryption_customer_key for this backend.
126 ///
127 /// Value: BASE64-encoded key that matches algorithm specified in
128 /// `server_side_encryption_customer_algorithm`.
129 pub server_side_encryption_customer_key: Option<String>,
130 /// Set server_side_encryption_customer_key_md5 for this backend.
131 ///
132 /// Value: MD5 digest of key specified in `server_side_encryption_customer_key`.
133 pub server_side_encryption_customer_key_md5: Option<String>,
134 /// default storage_class for this backend.
135 ///
136 /// Available values:
137 /// - `DEEP_ARCHIVE`
138 /// - `GLACIER`
139 /// - `GLACIER_IR`
140 /// - `INTELLIGENT_TIERING`
141 /// - `ONEZONE_IA`
142 /// - `EXPRESS_ONEZONE`
143 /// - `OUTPOSTS`
144 /// - `REDUCED_REDUNDANCY`
145 /// - `STANDARD`
146 /// - `STANDARD_IA`
147 ///
148 /// S3 compatible services don't support all of them
149 pub default_storage_class: Option<String>,
150 /// Enable virtual host style so that opendal will send API requests
151 /// in virtual host style instead of path style.
152 ///
153 /// - By default, opendal will send API to `https://s3.us-east-1.amazonaws.com/bucket_name`
154 /// - Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com`
155 pub enable_virtual_host_style: bool,
156 /// Set maximum batch operations of this backend.
157 ///
158 /// Some compatible services have a limit on the number of operations in a batch request.
159 /// For example, R2 could return `Internal Error` while batch delete 1000 files.
160 ///
161 /// Please tune this value based on services' document.
162 #[deprecated(
163 since = "0.52.0",
164 note = "Please use `delete_max_size` instead of `batch_max_operations`"
165 )]
166 pub batch_max_operations: Option<usize>,
167 /// Set the maximum delete size of this backend.
168 ///
169 /// Some compatible services have a limit on the number of operations in a batch request.
170 /// For example, R2 could return `Internal Error` while batch delete 1000 files.
171 ///
172 /// Please tune this value based on services' document.
173 pub delete_max_size: Option<usize>,
174 /// Disable stat with override so that opendal will not send stat request with override queries.
175 ///
176 /// For example, R2 doesn't support stat with `response_content_type` query.
177 pub disable_stat_with_override: bool,
178 /// Checksum Algorithm to use when sending checksums in HTTP headers.
179 /// This is necessary when writing to AWS S3 Buckets with Object Lock enabled for example.
180 ///
181 /// Available options:
182 /// - "crc32c"
183 pub checksum_algorithm: Option<String>,
184 /// Disable write with if match so that opendal will not send write request with if match headers.
185 ///
186 /// For example, Ceph RADOS S3 doesn't support write with if match.
187 pub disable_write_with_if_match: bool,
188
189 /// Enable write with append so that opendal will send write request with append headers.
190 pub enable_write_with_append: bool,
191
192 /// OpenDAL uses List Objects V2 by default to list objects.
193 /// However, some legacy services do not yet support V2.
194 /// This option allows users to switch back to the older List Objects V1.
195 pub disable_list_objects_v2: bool,
196}
197
198impl Debug for S3Config {
199 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
200 let mut d = f.debug_struct("S3Config");
201
202 d.field("root", &self.root)
203 .field("bucket", &self.bucket)
204 .field("endpoint", &self.endpoint)
205 .field("region", &self.region);
206
207 d.finish_non_exhaustive()
208 }
209}