opendal/types/operator/
builder.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19use std::sync::Arc;
20
21use crate::layers::*;
22use crate::raw::*;
23use crate::*;
24
25/// # Operator build API
26///
27/// Operator should be built via [`OperatorBuilder`]. We recommend to use [`Operator::new`] to get started:
28///
29/// ```
30/// # use anyhow::Result;
31/// use opendal::services::Fs;
32/// use opendal::Operator;
33/// async fn test() -> Result<()> {
34///     // Create fs backend builder.
35///     let builder = Fs::default().root("/tmp");
36///
37///     // Build an `Operator` to start operating the storage.
38///     let op: Operator = Operator::new(builder)?.finish();
39///
40///     Ok(())
41/// }
42/// ```
43impl Operator {
44    /// Create a new operator with input builder.
45    ///
46    /// OpenDAL will call `builder.build()` internally, so we don't need
47    /// to import `opendal::Builder` trait.
48    ///
49    /// # Examples
50    ///
51    /// Read more backend init examples in [examples](https://github.com/apache/opendal/tree/main/examples).
52    ///
53    /// ```
54    /// # use anyhow::Result;
55    /// use opendal::services::Fs;
56    /// use opendal::Operator;
57    /// async fn test() -> Result<()> {
58    ///     // Create fs backend builder.
59    ///     let builder = Fs::default().root("/tmp");
60    ///
61    ///     // Build an `Operator` to start operating the storage.
62    ///     let op: Operator = Operator::new(builder)?.finish();
63    ///
64    ///     Ok(())
65    /// }
66    /// ```
67    #[allow(clippy::new_ret_no_self)]
68    pub fn new<B: Builder>(ab: B) -> Result<OperatorBuilder<impl Access>> {
69        let acc = ab.build()?;
70        Ok(OperatorBuilder::new(acc))
71    }
72
73    /// Create a new operator from given config.
74    ///
75    /// # Examples
76    ///
77    /// ```
78    /// # use anyhow::Result;
79    /// use std::collections::HashMap;
80    ///
81    /// use opendal::services::MemoryConfig;
82    /// use opendal::Operator;
83    /// async fn test() -> Result<()> {
84    ///     let cfg = MemoryConfig::default();
85    ///
86    ///     // Build an `Operator` to start operating the storage.
87    ///     let op: Operator = Operator::from_config(cfg)?.finish();
88    ///
89    ///     Ok(())
90    /// }
91    /// ```
92    pub fn from_config<C: Configurator>(cfg: C) -> Result<OperatorBuilder<impl Access>> {
93        let builder = cfg.into_builder();
94        let acc = builder.build()?;
95        Ok(OperatorBuilder::new(acc))
96    }
97
98    /// Create a new operator from given iterator in static dispatch.
99    ///
100    /// # Notes
101    ///
102    /// `from_iter` generates a `OperatorBuilder` which allows adding layer in zero-cost way.
103    ///
104    /// # Examples
105    ///
106    /// ```
107    /// # use anyhow::Result;
108    /// use std::collections::HashMap;
109    ///
110    /// use opendal::services::Fs;
111    /// use opendal::Operator;
112    /// async fn test() -> Result<()> {
113    ///     let map = HashMap::from([
114    ///         // Set the root for fs, all operations will happen under this root.
115    ///         //
116    ///         // NOTE: the root must be absolute path.
117    ///         ("root".to_string(), "/tmp".to_string()),
118    ///     ]);
119    ///
120    ///     // Build an `Operator` to start operating the storage.
121    ///     let op: Operator = Operator::from_iter::<Fs>(map)?.finish();
122    ///
123    ///     Ok(())
124    /// }
125    /// ```
126    #[allow(clippy::should_implement_trait)]
127    pub fn from_iter<B: Builder>(
128        iter: impl IntoIterator<Item = (String, String)>,
129    ) -> Result<OperatorBuilder<impl Access>> {
130        let builder = B::Config::from_iter(iter)?.into_builder();
131        let acc = builder.build()?;
132        Ok(OperatorBuilder::new(acc))
133    }
134
135    /// Create a new operator via given scheme and iterator of config value in dynamic dispatch.
136    ///
137    /// # Notes
138    ///
139    /// `via_iter` generates a `Operator` which allows building operator without generic type.
140    ///
141    /// # Examples
142    ///
143    /// ```
144    /// # use anyhow::Result;
145    /// use std::collections::HashMap;
146    ///
147    /// use opendal::Operator;
148    /// use opendal::Scheme;
149    /// async fn test() -> Result<()> {
150    ///     let map = [
151    ///         // Set the root for fs, all operations will happen under this root.
152    ///         //
153    ///         // NOTE: the root must be absolute path.
154    ///         ("root".to_string(), "/tmp".to_string()),
155    ///     ];
156    ///
157    ///     // Build an `Operator` to start operating the storage.
158    ///     let op: Operator = Operator::via_iter(Scheme::Fs, map)?;
159    ///
160    ///     Ok(())
161    /// }
162    /// ```
163    #[allow(unused_variables, unreachable_code)]
164    pub fn via_iter(
165        scheme: Scheme,
166        iter: impl IntoIterator<Item = (String, String)>,
167    ) -> Result<Operator> {
168        let op = match scheme {
169            #[cfg(feature = "services-aliyun-drive")]
170            Scheme::AliyunDrive => Self::from_iter::<services::AliyunDrive>(iter)?.finish(),
171            #[cfg(feature = "services-alluxio")]
172            Scheme::Alluxio => Self::from_iter::<services::Alluxio>(iter)?.finish(),
173            #[cfg(feature = "services-compfs")]
174            Scheme::Compfs => Self::from_iter::<services::Compfs>(iter)?.finish(),
175            #[cfg(feature = "services-upyun")]
176            Scheme::Upyun => Self::from_iter::<services::Upyun>(iter)?.finish(),
177            #[cfg(feature = "services-koofr")]
178            Scheme::Koofr => Self::from_iter::<services::Koofr>(iter)?.finish(),
179            #[cfg(feature = "services-yandex-disk")]
180            Scheme::YandexDisk => Self::from_iter::<services::YandexDisk>(iter)?.finish(),
181            #[cfg(feature = "services-pcloud")]
182            Scheme::Pcloud => Self::from_iter::<services::Pcloud>(iter)?.finish(),
183            #[cfg(feature = "services-azblob")]
184            Scheme::Azblob => Self::from_iter::<services::Azblob>(iter)?.finish(),
185            #[cfg(feature = "services-azdls")]
186            Scheme::Azdls => Self::from_iter::<services::Azdls>(iter)?.finish(),
187            #[cfg(feature = "services-azfile")]
188            Scheme::Azfile => Self::from_iter::<services::Azfile>(iter)?.finish(),
189            #[cfg(feature = "services-b2")]
190            Scheme::B2 => Self::from_iter::<services::B2>(iter)?.finish(),
191            #[cfg(feature = "services-cacache")]
192            Scheme::Cacache => Self::from_iter::<services::Cacache>(iter)?.finish(),
193            #[cfg(feature = "services-cos")]
194            Scheme::Cos => Self::from_iter::<services::Cos>(iter)?.finish(),
195            #[cfg(feature = "services-d1")]
196            Scheme::D1 => Self::from_iter::<services::D1>(iter)?.finish(),
197            #[cfg(feature = "services-dashmap")]
198            Scheme::Dashmap => Self::from_iter::<services::Dashmap>(iter)?.finish(),
199            #[cfg(feature = "services-dropbox")]
200            Scheme::Dropbox => Self::from_iter::<services::Dropbox>(iter)?.finish(),
201            #[cfg(feature = "services-etcd")]
202            Scheme::Etcd => Self::from_iter::<services::Etcd>(iter)?.finish(),
203            #[cfg(feature = "services-foundationdb")]
204            Scheme::Foundationdb => Self::from_iter::<services::Foundationdb>(iter)?.finish(),
205            #[cfg(feature = "services-fs")]
206            Scheme::Fs => Self::from_iter::<services::Fs>(iter)?.finish(),
207            #[cfg(feature = "services-ftp")]
208            Scheme::Ftp => Self::from_iter::<services::Ftp>(iter)?.finish(),
209            #[cfg(feature = "services-gcs")]
210            Scheme::Gcs => Self::from_iter::<services::Gcs>(iter)?.finish(),
211            #[cfg(feature = "services-ghac")]
212            Scheme::Ghac => Self::from_iter::<services::Ghac>(iter)?.finish(),
213            #[cfg(feature = "services-gridfs")]
214            Scheme::Gridfs => Self::from_iter::<services::Gridfs>(iter)?.finish(),
215            #[cfg(feature = "services-github")]
216            Scheme::Github => Self::from_iter::<services::Github>(iter)?.finish(),
217            #[cfg(feature = "services-hdfs")]
218            Scheme::Hdfs => Self::from_iter::<services::Hdfs>(iter)?.finish(),
219            #[cfg(feature = "services-http")]
220            Scheme::Http => Self::from_iter::<services::Http>(iter)?.finish(),
221            #[cfg(feature = "services-huggingface")]
222            Scheme::Huggingface => Self::from_iter::<services::Huggingface>(iter)?.finish(),
223            #[cfg(feature = "services-ipfs")]
224            Scheme::Ipfs => Self::from_iter::<services::Ipfs>(iter)?.finish(),
225            #[cfg(feature = "services-ipmfs")]
226            Scheme::Ipmfs => Self::from_iter::<services::Ipmfs>(iter)?.finish(),
227            #[cfg(feature = "services-memcached")]
228            Scheme::Memcached => Self::from_iter::<services::Memcached>(iter)?.finish(),
229            #[cfg(feature = "services-memory")]
230            Scheme::Memory => Self::from_iter::<services::Memory>(iter)?.finish(),
231            #[cfg(feature = "services-mini-moka")]
232            Scheme::MiniMoka => Self::from_iter::<services::MiniMoka>(iter)?.finish(),
233            #[cfg(feature = "services-moka")]
234            Scheme::Moka => Self::from_iter::<services::Moka>(iter)?.finish(),
235            #[cfg(feature = "services-monoiofs")]
236            Scheme::Monoiofs => Self::from_iter::<services::Monoiofs>(iter)?.finish(),
237            #[cfg(feature = "services-mysql")]
238            Scheme::Mysql => Self::from_iter::<services::Mysql>(iter)?.finish(),
239            #[cfg(feature = "services-obs")]
240            Scheme::Obs => Self::from_iter::<services::Obs>(iter)?.finish(),
241            #[cfg(feature = "services-onedrive")]
242            Scheme::Onedrive => Self::from_iter::<services::Onedrive>(iter)?.finish(),
243            #[cfg(feature = "services-postgresql")]
244            Scheme::Postgresql => Self::from_iter::<services::Postgresql>(iter)?.finish(),
245            #[cfg(feature = "services-gdrive")]
246            Scheme::Gdrive => Self::from_iter::<services::Gdrive>(iter)?.finish(),
247            #[cfg(feature = "services-oss")]
248            Scheme::Oss => Self::from_iter::<services::Oss>(iter)?.finish(),
249            #[cfg(feature = "services-persy")]
250            Scheme::Persy => Self::from_iter::<services::Persy>(iter)?.finish(),
251            #[cfg(feature = "services-redis")]
252            Scheme::Redis => Self::from_iter::<services::Redis>(iter)?.finish(),
253            #[cfg(feature = "services-rocksdb")]
254            Scheme::Rocksdb => Self::from_iter::<services::Rocksdb>(iter)?.finish(),
255            #[cfg(feature = "services-s3")]
256            Scheme::S3 => Self::from_iter::<services::S3>(iter)?.finish(),
257            #[cfg(feature = "services-seafile")]
258            Scheme::Seafile => Self::from_iter::<services::Seafile>(iter)?.finish(),
259            #[cfg(feature = "services-sftp")]
260            Scheme::Sftp => Self::from_iter::<services::Sftp>(iter)?.finish(),
261            #[cfg(feature = "services-sled")]
262            Scheme::Sled => Self::from_iter::<services::Sled>(iter)?.finish(),
263            #[cfg(feature = "services-sqlite")]
264            Scheme::Sqlite => Self::from_iter::<services::Sqlite>(iter)?.finish(),
265            #[cfg(feature = "services-swift")]
266            Scheme::Swift => Self::from_iter::<services::Swift>(iter)?.finish(),
267            #[cfg(feature = "services-tikv")]
268            Scheme::Tikv => Self::from_iter::<services::Tikv>(iter)?.finish(),
269            #[cfg(feature = "services-vercel-artifacts")]
270            Scheme::VercelArtifacts => Self::from_iter::<services::VercelArtifacts>(iter)?.finish(),
271            #[cfg(feature = "services-vercel-blob")]
272            Scheme::VercelBlob => Self::from_iter::<services::VercelBlob>(iter)?.finish(),
273            #[cfg(feature = "services-webdav")]
274            Scheme::Webdav => Self::from_iter::<services::Webdav>(iter)?.finish(),
275            #[cfg(feature = "services-webhdfs")]
276            Scheme::Webhdfs => Self::from_iter::<services::Webhdfs>(iter)?.finish(),
277            #[cfg(feature = "services-redb")]
278            Scheme::Redb => Self::from_iter::<services::Redb>(iter)?.finish(),
279            #[cfg(feature = "services-mongodb")]
280            Scheme::Mongodb => Self::from_iter::<services::Mongodb>(iter)?.finish(),
281            #[cfg(feature = "services-hdfs-native")]
282            Scheme::HdfsNative => Self::from_iter::<services::HdfsNative>(iter)?.finish(),
283            #[cfg(feature = "services-lakefs")]
284            Scheme::Lakefs => Self::from_iter::<services::Lakefs>(iter)?.finish(),
285            v => {
286                return Err(Error::new(
287                    ErrorKind::Unsupported,
288                    "scheme is not enabled or supported",
289                )
290                .with_context("scheme", v))
291            }
292        };
293
294        Ok(op)
295    }
296
297    /// Create a new operator from given map.
298    ///
299    /// # Notes
300    ///
301    /// from_map is using static dispatch layers which is zero cost. via_map is
302    /// using dynamic dispatch layers which has a bit runtime overhead with an
303    /// extra vtable lookup and unable to inline. But from_map requires generic
304    /// type parameter which is not always easy to be used.
305    ///
306    /// # Examples
307    ///
308    /// ```
309    /// # use anyhow::Result;
310    /// use std::collections::HashMap;
311    ///
312    /// use opendal::services::Fs;
313    /// use opendal::Operator;
314    /// async fn test() -> Result<()> {
315    ///     let map = HashMap::from([
316    ///         // Set the root for fs, all operations will happen under this root.
317    ///         //
318    ///         // NOTE: the root must be absolute path.
319    ///         ("root".to_string(), "/tmp".to_string()),
320    ///     ]);
321    ///
322    ///     // Build an `Operator` to start operating the storage.
323    ///     let op: Operator = Operator::from_map::<Fs>(map)?.finish();
324    ///
325    ///     Ok(())
326    /// }
327    /// ```
328    #[deprecated = "use from_iter instead"]
329    pub fn from_map<B: Builder>(
330        map: HashMap<String, String>,
331    ) -> Result<OperatorBuilder<impl Access>> {
332        Self::from_iter::<B>(map)
333    }
334
335    /// Create a new operator from given scheme and map.
336    ///
337    /// # Notes
338    ///
339    /// from_map is using static dispatch layers which is zero cost. via_map is
340    /// using dynamic dispatch layers which has a bit runtime overhead with an
341    /// extra vtable lookup and unable to inline. But from_map requires generic
342    /// type parameter which is not always easy to be used.
343    ///
344    /// # Examples
345    ///
346    /// ```
347    /// # use anyhow::Result;
348    /// use std::collections::HashMap;
349    ///
350    /// use opendal::Operator;
351    /// use opendal::Scheme;
352    /// async fn test() -> Result<()> {
353    ///     let map = HashMap::from([
354    ///         // Set the root for fs, all operations will happen under this root.
355    ///         //
356    ///         // NOTE: the root must be absolute path.
357    ///         ("root".to_string(), "/tmp".to_string()),
358    ///     ]);
359    ///
360    ///     // Build an `Operator` to start operating the storage.
361    ///     let op: Operator = Operator::via_map(Scheme::Fs, map)?;
362    ///
363    ///     Ok(())
364    /// }
365    /// ```
366    #[deprecated = "use via_iter instead"]
367    pub fn via_map(scheme: Scheme, map: HashMap<String, String>) -> Result<Operator> {
368        Self::via_iter(scheme, map)
369    }
370
371    /// Create a new layer with dynamic dispatch.
372    ///
373    /// Please note that `Layer` can modify internal contexts such as `HttpClient`
374    /// and `Runtime` for the operator. Therefore, it is recommended to add layers
375    /// before interacting with the storage. Adding or duplicating layers after
376    /// accessing the storage may result in unexpected behavior.
377    ///
378    /// # Notes
379    ///
380    /// `OperatorBuilder::layer()` is using static dispatch which is zero
381    /// cost. `Operator::layer()` is using dynamic dispatch which has a
382    /// bit runtime overhead with an extra vtable lookup and unable to
383    /// inline.
384    ///
385    /// It's always recommended to use `OperatorBuilder::layer()` instead.
386    ///
387    /// # Examples
388    ///
389    /// ```no_run
390    /// # use std::sync::Arc;
391    /// # use anyhow::Result;
392    /// use opendal::layers::LoggingLayer;
393    /// use opendal::services::Fs;
394    /// use opendal::Operator;
395    ///
396    /// # async fn test() -> Result<()> {
397    /// let op = Operator::new(Fs::default())?.finish();
398    /// let op = op.layer(LoggingLayer::default());
399    /// // All operations will go through the new_layer
400    /// let _ = op.read("test_file").await?;
401    /// # Ok(())
402    /// # }
403    /// ```
404    #[must_use]
405    pub fn layer<L: Layer<Accessor>>(self, layer: L) -> Self {
406        Self::from_inner(Arc::new(
407            TypeEraseLayer.layer(layer.layer(self.into_inner())),
408        ))
409    }
410}
411
412/// OperatorBuilder is a typed builder to build an Operator.
413///
414/// # Notes
415///
416/// OpenDAL uses static dispatch internally and only performs dynamic
417/// dispatch at the outmost type erase layer. OperatorBuilder is the only
418/// public API provided by OpenDAL come with generic parameters.
419///
420/// It's required to call `finish` after the operator built.
421///
422/// # Examples
423///
424/// For users who want to support many services, we can build a helper function like the following:
425///
426/// ```
427/// use std::collections::HashMap;
428///
429/// use opendal::layers::LoggingLayer;
430/// use opendal::layers::RetryLayer;
431/// use opendal::services;
432/// use opendal::Builder;
433/// use opendal::Operator;
434/// use opendal::Result;
435/// use opendal::Scheme;
436///
437/// fn init_service<B: Builder>(cfg: HashMap<String, String>) -> Result<Operator> {
438///     let op = Operator::from_map::<B>(cfg)?
439///         .layer(LoggingLayer::default())
440///         .layer(RetryLayer::new())
441///         .finish();
442///
443///     Ok(op)
444/// }
445///
446/// async fn init(scheme: Scheme, cfg: HashMap<String, String>) -> Result<()> {
447///     let _ = match scheme {
448///         Scheme::S3 => init_service::<services::S3>(cfg)?,
449///         Scheme::Fs => init_service::<services::Fs>(cfg)?,
450///         _ => todo!(),
451///     };
452///
453///     Ok(())
454/// }
455/// ```
456pub struct OperatorBuilder<A: Access> {
457    accessor: A,
458}
459
460impl<A: Access> OperatorBuilder<A> {
461    /// Create a new operator builder.
462    #[allow(clippy::new_ret_no_self)]
463    pub fn new(accessor: A) -> OperatorBuilder<impl Access> {
464        // Make sure error context layer has been attached.
465        OperatorBuilder { accessor }
466            .layer(ErrorContextLayer)
467            .layer(CompleteLayer)
468            .layer(CorrectnessCheckLayer)
469    }
470
471    /// Create a new layer with static dispatch.
472    ///
473    /// # Notes
474    ///
475    /// `OperatorBuilder::layer()` is using static dispatch which is zero
476    /// cost. `Operator::layer()` is using dynamic dispatch which has a
477    /// bit runtime overhead with an extra vtable lookup and unable to
478    /// inline.
479    ///
480    /// It's always recommended to use `OperatorBuilder::layer()` instead.
481    ///
482    /// # Examples
483    ///
484    /// ```no_run
485    /// # use std::sync::Arc;
486    /// # use anyhow::Result;
487    /// use opendal::layers::LoggingLayer;
488    /// use opendal::services::Fs;
489    /// use opendal::Operator;
490    ///
491    /// # async fn test() -> Result<()> {
492    /// let op = Operator::new(Fs::default())?
493    ///     .layer(LoggingLayer::default())
494    ///     .finish();
495    /// // All operations will go through the new_layer
496    /// let _ = op.read("test_file").await?;
497    /// # Ok(())
498    /// # }
499    /// ```
500    #[must_use]
501    pub fn layer<L: Layer<A>>(self, layer: L) -> OperatorBuilder<L::LayeredAccess> {
502        OperatorBuilder {
503            accessor: layer.layer(self.accessor),
504        }
505    }
506
507    /// Finish the building to construct an Operator.
508    pub fn finish(self) -> Operator {
509        let ob = self.layer(TypeEraseLayer);
510        Operator::from_inner(Arc::new(ob.accessor) as Accessor)
511    }
512}