opendal/types/operator/builder.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19use std::sync::Arc;
20
21use crate::layers::*;
22use crate::raw::*;
23use crate::*;
24
25/// # Operator build API
26///
27/// Operator should be built via [`OperatorBuilder`]. We recommend to use [`Operator::new`] to get started:
28///
29/// ```
30/// # use anyhow::Result;
31/// use opendal::services::Fs;
32/// use opendal::Operator;
33/// async fn test() -> Result<()> {
34/// // Create fs backend builder.
35/// let builder = Fs::default().root("/tmp");
36///
37/// // Build an `Operator` to start operating the storage.
38/// let op: Operator = Operator::new(builder)?.finish();
39///
40/// Ok(())
41/// }
42/// ```
43impl Operator {
44 /// Create a new operator with input builder.
45 ///
46 /// OpenDAL will call `builder.build()` internally, so we don't need
47 /// to import `opendal::Builder` trait.
48 ///
49 /// # Examples
50 ///
51 /// Read more backend init examples in [examples](https://github.com/apache/opendal/tree/main/examples).
52 ///
53 /// ```
54 /// # use anyhow::Result;
55 /// use opendal::services::Fs;
56 /// use opendal::Operator;
57 /// async fn test() -> Result<()> {
58 /// // Create fs backend builder.
59 /// let builder = Fs::default().root("/tmp");
60 ///
61 /// // Build an `Operator` to start operating the storage.
62 /// let op: Operator = Operator::new(builder)?.finish();
63 ///
64 /// Ok(())
65 /// }
66 /// ```
67 #[allow(clippy::new_ret_no_self)]
68 pub fn new<B: Builder>(ab: B) -> Result<OperatorBuilder<impl Access>> {
69 let acc = ab.build()?;
70 Ok(OperatorBuilder::new(acc))
71 }
72
73 /// Create a new operator from given config.
74 ///
75 /// # Examples
76 ///
77 /// ```
78 /// # use anyhow::Result;
79 /// use std::collections::HashMap;
80 ///
81 /// use opendal::services::MemoryConfig;
82 /// use opendal::Operator;
83 /// async fn test() -> Result<()> {
84 /// let cfg = MemoryConfig::default();
85 ///
86 /// // Build an `Operator` to start operating the storage.
87 /// let op: Operator = Operator::from_config(cfg)?.finish();
88 ///
89 /// Ok(())
90 /// }
91 /// ```
92 pub fn from_config<C: Configurator>(cfg: C) -> Result<OperatorBuilder<impl Access>> {
93 let builder = cfg.into_builder();
94 let acc = builder.build()?;
95 Ok(OperatorBuilder::new(acc))
96 }
97
98 /// Create a new operator from given iterator in static dispatch.
99 ///
100 /// # Notes
101 ///
102 /// `from_iter` generates a `OperatorBuilder` which allows adding layer in zero-cost way.
103 ///
104 /// # Examples
105 ///
106 /// ```
107 /// # use anyhow::Result;
108 /// use std::collections::HashMap;
109 ///
110 /// use opendal::services::Fs;
111 /// use opendal::Operator;
112 /// async fn test() -> Result<()> {
113 /// let map = HashMap::from([
114 /// // Set the root for fs, all operations will happen under this root.
115 /// //
116 /// // NOTE: the root must be absolute path.
117 /// ("root".to_string(), "/tmp".to_string()),
118 /// ]);
119 ///
120 /// // Build an `Operator` to start operating the storage.
121 /// let op: Operator = Operator::from_iter::<Fs>(map)?.finish();
122 ///
123 /// Ok(())
124 /// }
125 /// ```
126 #[allow(clippy::should_implement_trait)]
127 pub fn from_iter<B: Builder>(
128 iter: impl IntoIterator<Item = (String, String)>,
129 ) -> Result<OperatorBuilder<impl Access>> {
130 let builder = B::Config::from_iter(iter)?.into_builder();
131 let acc = builder.build()?;
132 Ok(OperatorBuilder::new(acc))
133 }
134
135 /// Create a new operator via given scheme and iterator of config value in dynamic dispatch.
136 ///
137 /// # Notes
138 ///
139 /// `via_iter` generates a `Operator` which allows building operator without generic type.
140 ///
141 /// # Examples
142 ///
143 /// ```
144 /// # use anyhow::Result;
145 /// use std::collections::HashMap;
146 ///
147 /// use opendal::Operator;
148 /// use opendal::Scheme;
149 /// async fn test() -> Result<()> {
150 /// let map = [
151 /// // Set the root for fs, all operations will happen under this root.
152 /// //
153 /// // NOTE: the root must be absolute path.
154 /// ("root".to_string(), "/tmp".to_string()),
155 /// ];
156 ///
157 /// // Build an `Operator` to start operating the storage.
158 /// let op: Operator = Operator::via_iter(Scheme::Fs, map)?;
159 ///
160 /// Ok(())
161 /// }
162 /// ```
163 #[allow(unused_variables, unreachable_code)]
164 pub fn via_iter(
165 scheme: Scheme,
166 iter: impl IntoIterator<Item = (String, String)>,
167 ) -> Result<Operator> {
168 let op = match scheme {
169 #[cfg(feature = "services-aliyun-drive")]
170 Scheme::AliyunDrive => Self::from_iter::<services::AliyunDrive>(iter)?.finish(),
171 #[cfg(feature = "services-atomicserver")]
172 Scheme::Atomicserver => Self::from_iter::<services::Atomicserver>(iter)?.finish(),
173 #[cfg(feature = "services-alluxio")]
174 Scheme::Alluxio => Self::from_iter::<services::Alluxio>(iter)?.finish(),
175 #[cfg(feature = "services-compfs")]
176 Scheme::Compfs => Self::from_iter::<services::Compfs>(iter)?.finish(),
177 #[cfg(feature = "services-upyun")]
178 Scheme::Upyun => Self::from_iter::<services::Upyun>(iter)?.finish(),
179 #[cfg(feature = "services-koofr")]
180 Scheme::Koofr => Self::from_iter::<services::Koofr>(iter)?.finish(),
181 #[cfg(feature = "services-yandex-disk")]
182 Scheme::YandexDisk => Self::from_iter::<services::YandexDisk>(iter)?.finish(),
183 #[cfg(feature = "services-pcloud")]
184 Scheme::Pcloud => Self::from_iter::<services::Pcloud>(iter)?.finish(),
185 #[cfg(feature = "services-azblob")]
186 Scheme::Azblob => Self::from_iter::<services::Azblob>(iter)?.finish(),
187 #[cfg(feature = "services-azdls")]
188 Scheme::Azdls => Self::from_iter::<services::Azdls>(iter)?.finish(),
189 #[cfg(feature = "services-azfile")]
190 Scheme::Azfile => Self::from_iter::<services::Azfile>(iter)?.finish(),
191 #[cfg(feature = "services-b2")]
192 Scheme::B2 => Self::from_iter::<services::B2>(iter)?.finish(),
193 #[cfg(feature = "services-cacache")]
194 Scheme::Cacache => Self::from_iter::<services::Cacache>(iter)?.finish(),
195 #[cfg(feature = "services-cos")]
196 Scheme::Cos => Self::from_iter::<services::Cos>(iter)?.finish(),
197 #[cfg(feature = "services-d1")]
198 Scheme::D1 => Self::from_iter::<services::D1>(iter)?.finish(),
199 #[cfg(feature = "services-dashmap")]
200 Scheme::Dashmap => Self::from_iter::<services::Dashmap>(iter)?.finish(),
201 #[cfg(feature = "services-dropbox")]
202 Scheme::Dropbox => Self::from_iter::<services::Dropbox>(iter)?.finish(),
203 #[cfg(feature = "services-etcd")]
204 Scheme::Etcd => Self::from_iter::<services::Etcd>(iter)?.finish(),
205 #[cfg(feature = "services-foundationdb")]
206 Scheme::Foundationdb => Self::from_iter::<services::Foundationdb>(iter)?.finish(),
207 #[cfg(feature = "services-fs")]
208 Scheme::Fs => Self::from_iter::<services::Fs>(iter)?.finish(),
209 #[cfg(feature = "services-ftp")]
210 Scheme::Ftp => Self::from_iter::<services::Ftp>(iter)?.finish(),
211 #[cfg(feature = "services-gcs")]
212 Scheme::Gcs => Self::from_iter::<services::Gcs>(iter)?.finish(),
213 #[cfg(feature = "services-ghac")]
214 Scheme::Ghac => Self::from_iter::<services::Ghac>(iter)?.finish(),
215 #[cfg(feature = "services-gridfs")]
216 Scheme::Gridfs => Self::from_iter::<services::Gridfs>(iter)?.finish(),
217 #[cfg(feature = "services-github")]
218 Scheme::Github => Self::from_iter::<services::Github>(iter)?.finish(),
219 #[cfg(feature = "services-hdfs")]
220 Scheme::Hdfs => Self::from_iter::<services::Hdfs>(iter)?.finish(),
221 #[cfg(feature = "services-http")]
222 Scheme::Http => Self::from_iter::<services::Http>(iter)?.finish(),
223 #[cfg(feature = "services-huggingface")]
224 Scheme::Huggingface => Self::from_iter::<services::Huggingface>(iter)?.finish(),
225 #[cfg(feature = "services-ipfs")]
226 Scheme::Ipfs => Self::from_iter::<services::Ipfs>(iter)?.finish(),
227 #[cfg(feature = "services-ipmfs")]
228 Scheme::Ipmfs => Self::from_iter::<services::Ipmfs>(iter)?.finish(),
229 #[cfg(feature = "services-icloud")]
230 Scheme::Icloud => Self::from_iter::<services::Icloud>(iter)?.finish(),
231 #[cfg(feature = "services-memcached")]
232 Scheme::Memcached => Self::from_iter::<services::Memcached>(iter)?.finish(),
233 #[cfg(feature = "services-memory")]
234 Scheme::Memory => Self::from_iter::<services::Memory>(iter)?.finish(),
235 #[cfg(feature = "services-mini-moka")]
236 Scheme::MiniMoka => Self::from_iter::<services::MiniMoka>(iter)?.finish(),
237 #[cfg(feature = "services-moka")]
238 Scheme::Moka => Self::from_iter::<services::Moka>(iter)?.finish(),
239 #[cfg(feature = "services-monoiofs")]
240 Scheme::Monoiofs => Self::from_iter::<services::Monoiofs>(iter)?.finish(),
241 #[cfg(feature = "services-mysql")]
242 Scheme::Mysql => Self::from_iter::<services::Mysql>(iter)?.finish(),
243 #[cfg(feature = "services-obs")]
244 Scheme::Obs => Self::from_iter::<services::Obs>(iter)?.finish(),
245 #[cfg(feature = "services-onedrive")]
246 Scheme::Onedrive => Self::from_iter::<services::Onedrive>(iter)?.finish(),
247 #[cfg(feature = "services-postgresql")]
248 Scheme::Postgresql => Self::from_iter::<services::Postgresql>(iter)?.finish(),
249 #[cfg(feature = "services-gdrive")]
250 Scheme::Gdrive => Self::from_iter::<services::Gdrive>(iter)?.finish(),
251 #[cfg(feature = "services-oss")]
252 Scheme::Oss => Self::from_iter::<services::Oss>(iter)?.finish(),
253 #[cfg(feature = "services-persy")]
254 Scheme::Persy => Self::from_iter::<services::Persy>(iter)?.finish(),
255 #[cfg(feature = "services-redis")]
256 Scheme::Redis => Self::from_iter::<services::Redis>(iter)?.finish(),
257 #[cfg(feature = "services-rocksdb")]
258 Scheme::Rocksdb => Self::from_iter::<services::Rocksdb>(iter)?.finish(),
259 #[cfg(feature = "services-s3")]
260 Scheme::S3 => Self::from_iter::<services::S3>(iter)?.finish(),
261 #[cfg(feature = "services-seafile")]
262 Scheme::Seafile => Self::from_iter::<services::Seafile>(iter)?.finish(),
263 #[cfg(feature = "services-sftp")]
264 Scheme::Sftp => Self::from_iter::<services::Sftp>(iter)?.finish(),
265 #[cfg(feature = "services-sled")]
266 Scheme::Sled => Self::from_iter::<services::Sled>(iter)?.finish(),
267 #[cfg(feature = "services-sqlite")]
268 Scheme::Sqlite => Self::from_iter::<services::Sqlite>(iter)?.finish(),
269 #[cfg(feature = "services-swift")]
270 Scheme::Swift => Self::from_iter::<services::Swift>(iter)?.finish(),
271 #[cfg(feature = "services-tikv")]
272 Scheme::Tikv => Self::from_iter::<services::Tikv>(iter)?.finish(),
273 #[cfg(feature = "services-vercel-artifacts")]
274 Scheme::VercelArtifacts => Self::from_iter::<services::VercelArtifacts>(iter)?.finish(),
275 #[cfg(feature = "services-vercel-blob")]
276 Scheme::VercelBlob => Self::from_iter::<services::VercelBlob>(iter)?.finish(),
277 #[cfg(feature = "services-webdav")]
278 Scheme::Webdav => Self::from_iter::<services::Webdav>(iter)?.finish(),
279 #[cfg(feature = "services-webhdfs")]
280 Scheme::Webhdfs => Self::from_iter::<services::Webhdfs>(iter)?.finish(),
281 #[cfg(feature = "services-redb")]
282 Scheme::Redb => Self::from_iter::<services::Redb>(iter)?.finish(),
283 #[cfg(feature = "services-mongodb")]
284 Scheme::Mongodb => Self::from_iter::<services::Mongodb>(iter)?.finish(),
285 #[cfg(feature = "services-hdfs-native")]
286 Scheme::HdfsNative => Self::from_iter::<services::HdfsNative>(iter)?.finish(),
287 #[cfg(feature = "services-lakefs")]
288 Scheme::Lakefs => Self::from_iter::<services::Lakefs>(iter)?.finish(),
289 #[cfg(feature = "services-nebula-graph")]
290 Scheme::NebulaGraph => Self::from_iter::<services::NebulaGraph>(iter)?.finish(),
291 v => {
292 return Err(Error::new(
293 ErrorKind::Unsupported,
294 "scheme is not enabled or supported",
295 )
296 .with_context("scheme", v))
297 }
298 };
299
300 Ok(op)
301 }
302
303 /// Create a new operator from given map.
304 ///
305 /// # Notes
306 ///
307 /// from_map is using static dispatch layers which is zero cost. via_map is
308 /// using dynamic dispatch layers which has a bit runtime overhead with an
309 /// extra vtable lookup and unable to inline. But from_map requires generic
310 /// type parameter which is not always easy to be used.
311 ///
312 /// # Examples
313 ///
314 /// ```
315 /// # use anyhow::Result;
316 /// use std::collections::HashMap;
317 ///
318 /// use opendal::services::Fs;
319 /// use opendal::Operator;
320 /// async fn test() -> Result<()> {
321 /// let map = HashMap::from([
322 /// // Set the root for fs, all operations will happen under this root.
323 /// //
324 /// // NOTE: the root must be absolute path.
325 /// ("root".to_string(), "/tmp".to_string()),
326 /// ]);
327 ///
328 /// // Build an `Operator` to start operating the storage.
329 /// let op: Operator = Operator::from_map::<Fs>(map)?.finish();
330 ///
331 /// Ok(())
332 /// }
333 /// ```
334 #[deprecated = "use from_iter instead"]
335 pub fn from_map<B: Builder>(
336 map: HashMap<String, String>,
337 ) -> Result<OperatorBuilder<impl Access>> {
338 Self::from_iter::<B>(map)
339 }
340
341 /// Create a new operator from given scheme and map.
342 ///
343 /// # Notes
344 ///
345 /// from_map is using static dispatch layers which is zero cost. via_map is
346 /// using dynamic dispatch layers which has a bit runtime overhead with an
347 /// extra vtable lookup and unable to inline. But from_map requires generic
348 /// type parameter which is not always easy to be used.
349 ///
350 /// # Examples
351 ///
352 /// ```
353 /// # use anyhow::Result;
354 /// use std::collections::HashMap;
355 ///
356 /// use opendal::Operator;
357 /// use opendal::Scheme;
358 /// async fn test() -> Result<()> {
359 /// let map = HashMap::from([
360 /// // Set the root for fs, all operations will happen under this root.
361 /// //
362 /// // NOTE: the root must be absolute path.
363 /// ("root".to_string(), "/tmp".to_string()),
364 /// ]);
365 ///
366 /// // Build an `Operator` to start operating the storage.
367 /// let op: Operator = Operator::via_map(Scheme::Fs, map)?;
368 ///
369 /// Ok(())
370 /// }
371 /// ```
372 #[deprecated = "use via_iter instead"]
373 pub fn via_map(scheme: Scheme, map: HashMap<String, String>) -> Result<Operator> {
374 Self::via_iter(scheme, map)
375 }
376
377 /// Create a new layer with dynamic dispatch.
378 ///
379 /// Please note that `Layer` can modify internal contexts such as `HttpClient`
380 /// and `Runtime` for the operator. Therefore, it is recommended to add layers
381 /// before interacting with the storage. Adding or duplicating layers after
382 /// accessing the storage may result in unexpected behavior.
383 ///
384 /// # Notes
385 ///
386 /// `OperatorBuilder::layer()` is using static dispatch which is zero
387 /// cost. `Operator::layer()` is using dynamic dispatch which has a
388 /// bit runtime overhead with an extra vtable lookup and unable to
389 /// inline.
390 ///
391 /// It's always recommended to use `OperatorBuilder::layer()` instead.
392 ///
393 /// # Examples
394 ///
395 /// ```no_run
396 /// # use std::sync::Arc;
397 /// # use anyhow::Result;
398 /// use opendal::layers::LoggingLayer;
399 /// use opendal::services::Fs;
400 /// use opendal::Operator;
401 ///
402 /// # async fn test() -> Result<()> {
403 /// let op = Operator::new(Fs::default())?.finish();
404 /// let op = op.layer(LoggingLayer::default());
405 /// // All operations will go through the new_layer
406 /// let _ = op.read("test_file").await?;
407 /// # Ok(())
408 /// # }
409 /// ```
410 #[must_use]
411 pub fn layer<L: Layer<Accessor>>(self, layer: L) -> Self {
412 Self::from_inner(Arc::new(
413 TypeEraseLayer.layer(layer.layer(self.into_inner())),
414 ))
415 }
416}
417
418/// OperatorBuilder is a typed builder to build an Operator.
419///
420/// # Notes
421///
422/// OpenDAL uses static dispatch internally and only performs dynamic
423/// dispatch at the outmost type erase layer. OperatorBuilder is the only
424/// public API provided by OpenDAL come with generic parameters.
425///
426/// It's required to call `finish` after the operator built.
427///
428/// # Examples
429///
430/// For users who want to support many services, we can build a helper function like the following:
431///
432/// ```
433/// use std::collections::HashMap;
434///
435/// use opendal::layers::LoggingLayer;
436/// use opendal::layers::RetryLayer;
437/// use opendal::services;
438/// use opendal::Builder;
439/// use opendal::Operator;
440/// use opendal::Result;
441/// use opendal::Scheme;
442///
443/// fn init_service<B: Builder>(cfg: HashMap<String, String>) -> Result<Operator> {
444/// let op = Operator::from_map::<B>(cfg)?
445/// .layer(LoggingLayer::default())
446/// .layer(RetryLayer::new())
447/// .finish();
448///
449/// Ok(op)
450/// }
451///
452/// async fn init(scheme: Scheme, cfg: HashMap<String, String>) -> Result<()> {
453/// let _ = match scheme {
454/// Scheme::S3 => init_service::<services::S3>(cfg)?,
455/// Scheme::Fs => init_service::<services::Fs>(cfg)?,
456/// _ => todo!(),
457/// };
458///
459/// Ok(())
460/// }
461/// ```
462pub struct OperatorBuilder<A: Access> {
463 accessor: A,
464}
465
466impl<A: Access> OperatorBuilder<A> {
467 /// Create a new operator builder.
468 #[allow(clippy::new_ret_no_self)]
469 pub fn new(accessor: A) -> OperatorBuilder<impl Access> {
470 // Make sure error context layer has been attached.
471 OperatorBuilder { accessor }
472 .layer(ErrorContextLayer)
473 .layer(CompleteLayer)
474 .layer(CorrectnessCheckLayer)
475 }
476
477 /// Create a new layer with static dispatch.
478 ///
479 /// # Notes
480 ///
481 /// `OperatorBuilder::layer()` is using static dispatch which is zero
482 /// cost. `Operator::layer()` is using dynamic dispatch which has a
483 /// bit runtime overhead with an extra vtable lookup and unable to
484 /// inline.
485 ///
486 /// It's always recommended to use `OperatorBuilder::layer()` instead.
487 ///
488 /// # Examples
489 ///
490 /// ```no_run
491 /// # use std::sync::Arc;
492 /// # use anyhow::Result;
493 /// use opendal::layers::LoggingLayer;
494 /// use opendal::services::Fs;
495 /// use opendal::Operator;
496 ///
497 /// # async fn test() -> Result<()> {
498 /// let op = Operator::new(Fs::default())?
499 /// .layer(LoggingLayer::default())
500 /// .finish();
501 /// // All operations will go through the new_layer
502 /// let _ = op.read("test_file").await?;
503 /// # Ok(())
504 /// # }
505 /// ```
506 #[must_use]
507 pub fn layer<L: Layer<A>>(self, layer: L) -> OperatorBuilder<L::LayeredAccess> {
508 OperatorBuilder {
509 accessor: layer.layer(self.accessor),
510 }
511 }
512
513 /// Finish the building to construct an Operator.
514 pub fn finish(self) -> Operator {
515 let ob = self.layer(TypeEraseLayer);
516 Operator::from_inner(Arc::new(ob.accessor) as Accessor)
517 }
518}