opendal/types/operator/builder.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19use std::sync::Arc;
20
21use crate::layers::*;
22use crate::raw::*;
23use crate::*;
24
25/// # Operator build API
26///
27/// Operator should be built via [`OperatorBuilder`]. We recommend to use [`Operator::new`] to get started:
28///
29/// ```
30/// # use anyhow::Result;
31/// use opendal::services::Fs;
32/// use opendal::Operator;
33/// async fn test() -> Result<()> {
34/// // Create fs backend builder.
35/// let builder = Fs::default().root("/tmp");
36///
37/// // Build an `Operator` to start operating the storage.
38/// let op: Operator = Operator::new(builder)?.finish();
39///
40/// Ok(())
41/// }
42/// ```
43impl Operator {
44 /// Create a new operator with input builder.
45 ///
46 /// OpenDAL will call `builder.build()` internally, so we don't need
47 /// to import `opendal::Builder` trait.
48 ///
49 /// # Examples
50 ///
51 /// Read more backend init examples in [examples](https://github.com/apache/opendal/tree/main/examples).
52 ///
53 /// ```
54 /// # use anyhow::Result;
55 /// use opendal::services::Fs;
56 /// use opendal::Operator;
57 /// async fn test() -> Result<()> {
58 /// // Create fs backend builder.
59 /// let builder = Fs::default().root("/tmp");
60 ///
61 /// // Build an `Operator` to start operating the storage.
62 /// let op: Operator = Operator::new(builder)?.finish();
63 ///
64 /// Ok(())
65 /// }
66 /// ```
67 #[allow(clippy::new_ret_no_self)]
68 pub fn new<B: Builder>(ab: B) -> Result<OperatorBuilder<impl Access>> {
69 let acc = ab.build()?;
70 Ok(OperatorBuilder::new(acc))
71 }
72
73 /// Create a new operator from given config.
74 ///
75 /// # Examples
76 ///
77 /// ```
78 /// # use anyhow::Result;
79 /// use std::collections::HashMap;
80 ///
81 /// use opendal::services::MemoryConfig;
82 /// use opendal::Operator;
83 /// async fn test() -> Result<()> {
84 /// let cfg = MemoryConfig::default();
85 ///
86 /// // Build an `Operator` to start operating the storage.
87 /// let op: Operator = Operator::from_config(cfg)?.finish();
88 ///
89 /// Ok(())
90 /// }
91 /// ```
92 pub fn from_config<C: Configurator>(cfg: C) -> Result<OperatorBuilder<impl Access>> {
93 let builder = cfg.into_builder();
94 let acc = builder.build()?;
95 Ok(OperatorBuilder::new(acc))
96 }
97
98 /// Create a new operator from given iterator in static dispatch.
99 ///
100 /// # Notes
101 ///
102 /// `from_iter` generates a `OperatorBuilder` which allows adding layer in zero-cost way.
103 ///
104 /// # Examples
105 ///
106 /// ```
107 /// # use anyhow::Result;
108 /// use std::collections::HashMap;
109 ///
110 /// use opendal::services::Fs;
111 /// use opendal::Operator;
112 /// async fn test() -> Result<()> {
113 /// let map = HashMap::from([
114 /// // Set the root for fs, all operations will happen under this root.
115 /// //
116 /// // NOTE: the root must be absolute path.
117 /// ("root".to_string(), "/tmp".to_string()),
118 /// ]);
119 ///
120 /// // Build an `Operator` to start operating the storage.
121 /// let op: Operator = Operator::from_iter::<Fs>(map)?.finish();
122 ///
123 /// Ok(())
124 /// }
125 /// ```
126 #[allow(clippy::should_implement_trait)]
127 pub fn from_iter<B: Builder>(
128 iter: impl IntoIterator<Item = (String, String)>,
129 ) -> Result<OperatorBuilder<impl Access>> {
130 let builder = B::Config::from_iter(iter)?.into_builder();
131 let acc = builder.build()?;
132 Ok(OperatorBuilder::new(acc))
133 }
134
135 /// Create a new operator via given scheme and iterator of config value in dynamic dispatch.
136 ///
137 /// # Notes
138 ///
139 /// `via_iter` generates a `Operator` which allows building operator without generic type.
140 ///
141 /// # Examples
142 ///
143 /// ```
144 /// # use anyhow::Result;
145 /// use std::collections::HashMap;
146 ///
147 /// use opendal::Operator;
148 /// use opendal::Scheme;
149 /// async fn test() -> Result<()> {
150 /// let map = [
151 /// // Set the root for fs, all operations will happen under this root.
152 /// //
153 /// // NOTE: the root must be absolute path.
154 /// ("root".to_string(), "/tmp".to_string()),
155 /// ];
156 ///
157 /// // Build an `Operator` to start operating the storage.
158 /// let op: Operator = Operator::via_iter(Scheme::Fs, map)?;
159 ///
160 /// Ok(())
161 /// }
162 /// ```
163 #[allow(unused_variables, unreachable_code)]
164 pub fn via_iter(
165 scheme: Scheme,
166 iter: impl IntoIterator<Item = (String, String)>,
167 ) -> Result<Operator> {
168 let op = match scheme {
169 #[cfg(feature = "services-aliyun-drive")]
170 Scheme::AliyunDrive => Self::from_iter::<services::AliyunDrive>(iter)?.finish(),
171 #[cfg(feature = "services-alluxio")]
172 Scheme::Alluxio => Self::from_iter::<services::Alluxio>(iter)?.finish(),
173 #[cfg(feature = "services-compfs")]
174 Scheme::Compfs => Self::from_iter::<services::Compfs>(iter)?.finish(),
175 #[cfg(feature = "services-upyun")]
176 Scheme::Upyun => Self::from_iter::<services::Upyun>(iter)?.finish(),
177 #[cfg(feature = "services-koofr")]
178 Scheme::Koofr => Self::from_iter::<services::Koofr>(iter)?.finish(),
179 #[cfg(feature = "services-yandex-disk")]
180 Scheme::YandexDisk => Self::from_iter::<services::YandexDisk>(iter)?.finish(),
181 #[cfg(feature = "services-pcloud")]
182 Scheme::Pcloud => Self::from_iter::<services::Pcloud>(iter)?.finish(),
183 #[cfg(feature = "services-azblob")]
184 Scheme::Azblob => Self::from_iter::<services::Azblob>(iter)?.finish(),
185 #[cfg(feature = "services-azdls")]
186 Scheme::Azdls => Self::from_iter::<services::Azdls>(iter)?.finish(),
187 #[cfg(feature = "services-azfile")]
188 Scheme::Azfile => Self::from_iter::<services::Azfile>(iter)?.finish(),
189 #[cfg(feature = "services-b2")]
190 Scheme::B2 => Self::from_iter::<services::B2>(iter)?.finish(),
191 #[cfg(feature = "services-cacache")]
192 Scheme::Cacache => Self::from_iter::<services::Cacache>(iter)?.finish(),
193 #[cfg(feature = "services-cos")]
194 Scheme::Cos => Self::from_iter::<services::Cos>(iter)?.finish(),
195 #[cfg(feature = "services-d1")]
196 Scheme::D1 => Self::from_iter::<services::D1>(iter)?.finish(),
197 #[cfg(feature = "services-dashmap")]
198 Scheme::Dashmap => Self::from_iter::<services::Dashmap>(iter)?.finish(),
199 #[cfg(feature = "services-dropbox")]
200 Scheme::Dropbox => Self::from_iter::<services::Dropbox>(iter)?.finish(),
201 #[cfg(feature = "services-etcd")]
202 Scheme::Etcd => Self::from_iter::<services::Etcd>(iter)?.finish(),
203 #[cfg(feature = "services-foundationdb")]
204 Scheme::Foundationdb => Self::from_iter::<services::Foundationdb>(iter)?.finish(),
205 #[cfg(feature = "services-fs")]
206 Scheme::Fs => Self::from_iter::<services::Fs>(iter)?.finish(),
207 #[cfg(feature = "services-ftp")]
208 Scheme::Ftp => Self::from_iter::<services::Ftp>(iter)?.finish(),
209 #[cfg(feature = "services-gcs")]
210 Scheme::Gcs => Self::from_iter::<services::Gcs>(iter)?.finish(),
211 #[cfg(feature = "services-ghac")]
212 Scheme::Ghac => Self::from_iter::<services::Ghac>(iter)?.finish(),
213 #[cfg(feature = "services-gridfs")]
214 Scheme::Gridfs => Self::from_iter::<services::Gridfs>(iter)?.finish(),
215 #[cfg(feature = "services-github")]
216 Scheme::Github => Self::from_iter::<services::Github>(iter)?.finish(),
217 #[cfg(feature = "services-hdfs")]
218 Scheme::Hdfs => Self::from_iter::<services::Hdfs>(iter)?.finish(),
219 #[cfg(feature = "services-http")]
220 Scheme::Http => Self::from_iter::<services::Http>(iter)?.finish(),
221 #[cfg(feature = "services-huggingface")]
222 Scheme::Huggingface => Self::from_iter::<services::Huggingface>(iter)?.finish(),
223 #[cfg(feature = "services-ipfs")]
224 Scheme::Ipfs => Self::from_iter::<services::Ipfs>(iter)?.finish(),
225 #[cfg(feature = "services-ipmfs")]
226 Scheme::Ipmfs => Self::from_iter::<services::Ipmfs>(iter)?.finish(),
227 #[cfg(feature = "services-memcached")]
228 Scheme::Memcached => Self::from_iter::<services::Memcached>(iter)?.finish(),
229 #[cfg(feature = "services-memory")]
230 Scheme::Memory => Self::from_iter::<services::Memory>(iter)?.finish(),
231 #[cfg(feature = "services-mini-moka")]
232 Scheme::MiniMoka => Self::from_iter::<services::MiniMoka>(iter)?.finish(),
233 #[cfg(feature = "services-moka")]
234 Scheme::Moka => Self::from_iter::<services::Moka>(iter)?.finish(),
235 #[cfg(feature = "services-monoiofs")]
236 Scheme::Monoiofs => Self::from_iter::<services::Monoiofs>(iter)?.finish(),
237 #[cfg(feature = "services-mysql")]
238 Scheme::Mysql => Self::from_iter::<services::Mysql>(iter)?.finish(),
239 #[cfg(feature = "services-obs")]
240 Scheme::Obs => Self::from_iter::<services::Obs>(iter)?.finish(),
241 #[cfg(feature = "services-onedrive")]
242 Scheme::Onedrive => Self::from_iter::<services::Onedrive>(iter)?.finish(),
243 #[cfg(feature = "services-postgresql")]
244 Scheme::Postgresql => Self::from_iter::<services::Postgresql>(iter)?.finish(),
245 #[cfg(feature = "services-gdrive")]
246 Scheme::Gdrive => Self::from_iter::<services::Gdrive>(iter)?.finish(),
247 #[cfg(feature = "services-oss")]
248 Scheme::Oss => Self::from_iter::<services::Oss>(iter)?.finish(),
249 #[cfg(feature = "services-persy")]
250 Scheme::Persy => Self::from_iter::<services::Persy>(iter)?.finish(),
251 #[cfg(feature = "services-redis")]
252 Scheme::Redis => Self::from_iter::<services::Redis>(iter)?.finish(),
253 #[cfg(feature = "services-rocksdb")]
254 Scheme::Rocksdb => Self::from_iter::<services::Rocksdb>(iter)?.finish(),
255 #[cfg(feature = "services-s3")]
256 Scheme::S3 => Self::from_iter::<services::S3>(iter)?.finish(),
257 #[cfg(feature = "services-seafile")]
258 Scheme::Seafile => Self::from_iter::<services::Seafile>(iter)?.finish(),
259 #[cfg(feature = "services-sftp")]
260 Scheme::Sftp => Self::from_iter::<services::Sftp>(iter)?.finish(),
261 #[cfg(feature = "services-sled")]
262 Scheme::Sled => Self::from_iter::<services::Sled>(iter)?.finish(),
263 #[cfg(feature = "services-sqlite")]
264 Scheme::Sqlite => Self::from_iter::<services::Sqlite>(iter)?.finish(),
265 #[cfg(feature = "services-swift")]
266 Scheme::Swift => Self::from_iter::<services::Swift>(iter)?.finish(),
267 #[cfg(feature = "services-tikv")]
268 Scheme::Tikv => Self::from_iter::<services::Tikv>(iter)?.finish(),
269 #[cfg(feature = "services-vercel-artifacts")]
270 Scheme::VercelArtifacts => Self::from_iter::<services::VercelArtifacts>(iter)?.finish(),
271 #[cfg(feature = "services-vercel-blob")]
272 Scheme::VercelBlob => Self::from_iter::<services::VercelBlob>(iter)?.finish(),
273 #[cfg(feature = "services-webdav")]
274 Scheme::Webdav => Self::from_iter::<services::Webdav>(iter)?.finish(),
275 #[cfg(feature = "services-webhdfs")]
276 Scheme::Webhdfs => Self::from_iter::<services::Webhdfs>(iter)?.finish(),
277 #[cfg(feature = "services-redb")]
278 Scheme::Redb => Self::from_iter::<services::Redb>(iter)?.finish(),
279 #[cfg(feature = "services-mongodb")]
280 Scheme::Mongodb => Self::from_iter::<services::Mongodb>(iter)?.finish(),
281 #[cfg(feature = "services-hdfs-native")]
282 Scheme::HdfsNative => Self::from_iter::<services::HdfsNative>(iter)?.finish(),
283 #[cfg(feature = "services-lakefs")]
284 Scheme::Lakefs => Self::from_iter::<services::Lakefs>(iter)?.finish(),
285 v => {
286 return Err(Error::new(
287 ErrorKind::Unsupported,
288 "scheme is not enabled or supported",
289 )
290 .with_context("scheme", v))
291 }
292 };
293
294 Ok(op)
295 }
296
297 /// Create a new operator from given map.
298 ///
299 /// # Notes
300 ///
301 /// from_map is using static dispatch layers which is zero cost. via_map is
302 /// using dynamic dispatch layers which has a bit runtime overhead with an
303 /// extra vtable lookup and unable to inline. But from_map requires generic
304 /// type parameter which is not always easy to be used.
305 ///
306 /// # Examples
307 ///
308 /// ```
309 /// # use anyhow::Result;
310 /// use std::collections::HashMap;
311 ///
312 /// use opendal::services::Fs;
313 /// use opendal::Operator;
314 /// async fn test() -> Result<()> {
315 /// let map = HashMap::from([
316 /// // Set the root for fs, all operations will happen under this root.
317 /// //
318 /// // NOTE: the root must be absolute path.
319 /// ("root".to_string(), "/tmp".to_string()),
320 /// ]);
321 ///
322 /// // Build an `Operator` to start operating the storage.
323 /// let op: Operator = Operator::from_map::<Fs>(map)?.finish();
324 ///
325 /// Ok(())
326 /// }
327 /// ```
328 #[deprecated = "use from_iter instead"]
329 pub fn from_map<B: Builder>(
330 map: HashMap<String, String>,
331 ) -> Result<OperatorBuilder<impl Access>> {
332 Self::from_iter::<B>(map)
333 }
334
335 /// Create a new operator from given scheme and map.
336 ///
337 /// # Notes
338 ///
339 /// from_map is using static dispatch layers which is zero cost. via_map is
340 /// using dynamic dispatch layers which has a bit runtime overhead with an
341 /// extra vtable lookup and unable to inline. But from_map requires generic
342 /// type parameter which is not always easy to be used.
343 ///
344 /// # Examples
345 ///
346 /// ```
347 /// # use anyhow::Result;
348 /// use std::collections::HashMap;
349 ///
350 /// use opendal::Operator;
351 /// use opendal::Scheme;
352 /// async fn test() -> Result<()> {
353 /// let map = HashMap::from([
354 /// // Set the root for fs, all operations will happen under this root.
355 /// //
356 /// // NOTE: the root must be absolute path.
357 /// ("root".to_string(), "/tmp".to_string()),
358 /// ]);
359 ///
360 /// // Build an `Operator` to start operating the storage.
361 /// let op: Operator = Operator::via_map(Scheme::Fs, map)?;
362 ///
363 /// Ok(())
364 /// }
365 /// ```
366 #[deprecated = "use via_iter instead"]
367 pub fn via_map(scheme: Scheme, map: HashMap<String, String>) -> Result<Operator> {
368 Self::via_iter(scheme, map)
369 }
370
371 /// Create a new layer with dynamic dispatch.
372 ///
373 /// Please note that `Layer` can modify internal contexts such as `HttpClient`
374 /// and `Runtime` for the operator. Therefore, it is recommended to add layers
375 /// before interacting with the storage. Adding or duplicating layers after
376 /// accessing the storage may result in unexpected behavior.
377 ///
378 /// # Notes
379 ///
380 /// `OperatorBuilder::layer()` is using static dispatch which is zero
381 /// cost. `Operator::layer()` is using dynamic dispatch which has a
382 /// bit runtime overhead with an extra vtable lookup and unable to
383 /// inline.
384 ///
385 /// It's always recommended to use `OperatorBuilder::layer()` instead.
386 ///
387 /// # Examples
388 ///
389 /// ```no_run
390 /// # use std::sync::Arc;
391 /// # use anyhow::Result;
392 /// use opendal::layers::LoggingLayer;
393 /// use opendal::services::Fs;
394 /// use opendal::Operator;
395 ///
396 /// # async fn test() -> Result<()> {
397 /// let op = Operator::new(Fs::default())?.finish();
398 /// let op = op.layer(LoggingLayer::default());
399 /// // All operations will go through the new_layer
400 /// let _ = op.read("test_file").await?;
401 /// # Ok(())
402 /// # }
403 /// ```
404 #[must_use]
405 pub fn layer<L: Layer<Accessor>>(self, layer: L) -> Self {
406 Self::from_inner(Arc::new(
407 TypeEraseLayer.layer(layer.layer(self.into_inner())),
408 ))
409 }
410}
411
412/// OperatorBuilder is a typed builder to build an Operator.
413///
414/// # Notes
415///
416/// OpenDAL uses static dispatch internally and only performs dynamic
417/// dispatch at the outmost type erase layer. OperatorBuilder is the only
418/// public API provided by OpenDAL come with generic parameters.
419///
420/// It's required to call `finish` after the operator built.
421///
422/// # Examples
423///
424/// For users who want to support many services, we can build a helper function like the following:
425///
426/// ```
427/// use std::collections::HashMap;
428///
429/// use opendal::layers::LoggingLayer;
430/// use opendal::layers::RetryLayer;
431/// use opendal::services;
432/// use opendal::Builder;
433/// use opendal::Operator;
434/// use opendal::Result;
435/// use opendal::Scheme;
436///
437/// fn init_service<B: Builder>(cfg: HashMap<String, String>) -> Result<Operator> {
438/// let op = Operator::from_map::<B>(cfg)?
439/// .layer(LoggingLayer::default())
440/// .layer(RetryLayer::new())
441/// .finish();
442///
443/// Ok(op)
444/// }
445///
446/// async fn init(scheme: Scheme, cfg: HashMap<String, String>) -> Result<()> {
447/// let _ = match scheme {
448/// Scheme::S3 => init_service::<services::S3>(cfg)?,
449/// Scheme::Fs => init_service::<services::Fs>(cfg)?,
450/// _ => todo!(),
451/// };
452///
453/// Ok(())
454/// }
455/// ```
456pub struct OperatorBuilder<A: Access> {
457 accessor: A,
458}
459
460impl<A: Access> OperatorBuilder<A> {
461 /// Create a new operator builder.
462 #[allow(clippy::new_ret_no_self)]
463 pub fn new(accessor: A) -> OperatorBuilder<impl Access> {
464 // Make sure error context layer has been attached.
465 OperatorBuilder { accessor }
466 .layer(ErrorContextLayer)
467 .layer(CompleteLayer)
468 .layer(CorrectnessCheckLayer)
469 }
470
471 /// Create a new layer with static dispatch.
472 ///
473 /// # Notes
474 ///
475 /// `OperatorBuilder::layer()` is using static dispatch which is zero
476 /// cost. `Operator::layer()` is using dynamic dispatch which has a
477 /// bit runtime overhead with an extra vtable lookup and unable to
478 /// inline.
479 ///
480 /// It's always recommended to use `OperatorBuilder::layer()` instead.
481 ///
482 /// # Examples
483 ///
484 /// ```no_run
485 /// # use std::sync::Arc;
486 /// # use anyhow::Result;
487 /// use opendal::layers::LoggingLayer;
488 /// use opendal::services::Fs;
489 /// use opendal::Operator;
490 ///
491 /// # async fn test() -> Result<()> {
492 /// let op = Operator::new(Fs::default())?
493 /// .layer(LoggingLayer::default())
494 /// .finish();
495 /// // All operations will go through the new_layer
496 /// let _ = op.read("test_file").await?;
497 /// # Ok(())
498 /// # }
499 /// ```
500 #[must_use]
501 pub fn layer<L: Layer<A>>(self, layer: L) -> OperatorBuilder<L::LayeredAccess> {
502 OperatorBuilder {
503 accessor: layer.layer(self.accessor),
504 }
505 }
506
507 /// Finish the building to construct an Operator.
508 pub fn finish(self) -> Operator {
509 let ob = self.layer(TypeEraseLayer);
510 Operator::from_inner(Arc::new(ob.accessor) as Accessor)
511 }
512}