opendal/services/azdls/
backend.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::fmt::Debug;
19use std::sync::Arc;
20
21use http::Response;
22use http::StatusCode;
23use log::debug;
24use reqsign::AzureStorageConfig;
25use reqsign::AzureStorageLoader;
26use reqsign::AzureStorageSigner;
27
28use super::AZDLS_SCHEME;
29use super::config::AzdlsConfig;
30use super::core::AzdlsCore;
31use super::core::DIRECTORY;
32use super::deleter::AzdlsDeleter;
33use super::error::parse_error;
34use super::lister::AzdlsLister;
35use super::writer::AzdlsWriter;
36use super::writer::AzdlsWriters;
37use crate::raw::*;
38use crate::*;
39
40impl From<AzureStorageConfig> for AzdlsConfig {
41    fn from(config: AzureStorageConfig) -> Self {
42        AzdlsConfig {
43            endpoint: config.endpoint,
44            account_name: config.account_name,
45            account_key: config.account_key,
46            client_secret: config.client_secret,
47            tenant_id: config.tenant_id,
48            client_id: config.client_id,
49            sas_token: config.sas_token,
50            authority_host: config.authority_host,
51            ..Default::default()
52        }
53    }
54}
55
56/// Azure Data Lake Storage Gen2 Support.
57#[doc = include_str!("docs.md")]
58#[derive(Default)]
59pub struct AzdlsBuilder {
60    pub(super) config: AzdlsConfig,
61
62    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
63    pub(super) http_client: Option<HttpClient>,
64}
65
66impl Debug for AzdlsBuilder {
67    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
68        f.debug_struct("AzdlsBuilder")
69            .field("config", &self.config)
70            .finish_non_exhaustive()
71    }
72}
73
74impl AzdlsBuilder {
75    /// Set root of this backend.
76    ///
77    /// All operations will happen under this root.
78    pub fn root(mut self, root: &str) -> Self {
79        self.config.root = if root.is_empty() {
80            None
81        } else {
82            Some(root.to_string())
83        };
84
85        self
86    }
87
88    /// Set filesystem name of this backend.
89    pub fn filesystem(mut self, filesystem: &str) -> Self {
90        self.config.filesystem = filesystem.to_string();
91
92        self
93    }
94
95    /// Set endpoint of this backend.
96    ///
97    /// Endpoint must be full uri, e.g.
98    ///
99    /// - Azblob: `https://accountname.blob.core.windows.net`
100    /// - Azurite: `http://127.0.0.1:10000/devstoreaccount1`
101    pub fn endpoint(mut self, endpoint: &str) -> Self {
102        if !endpoint.is_empty() {
103            // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
104            self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string());
105        }
106
107        self
108    }
109
110    /// Set account_name of this backend.
111    ///
112    /// - If account_name is set, we will take user's input first.
113    /// - If not, we will try to load it from environment.
114    pub fn account_name(mut self, account_name: &str) -> Self {
115        if !account_name.is_empty() {
116            self.config.account_name = Some(account_name.to_string());
117        }
118
119        self
120    }
121
122    /// Set account_key of this backend.
123    ///
124    /// - If account_key is set, we will take user's input first.
125    /// - If not, we will try to load it from environment.
126    pub fn account_key(mut self, account_key: &str) -> Self {
127        if !account_key.is_empty() {
128            self.config.account_key = Some(account_key.to_string());
129        }
130
131        self
132    }
133
134    /// Set client_secret of this backend.
135    ///
136    /// - If client_secret is set, we will take user's input first.
137    /// - If not, we will try to load it from environment.
138    /// - required for client_credentials authentication
139    pub fn client_secret(mut self, client_secret: &str) -> Self {
140        if !client_secret.is_empty() {
141            self.config.client_secret = Some(client_secret.to_string());
142        }
143
144        self
145    }
146
147    /// Set tenant_id of this backend.
148    ///
149    /// - If tenant_id is set, we will take user's input first.
150    /// - If not, we will try to load it from environment.
151    /// - required for client_credentials authentication
152    pub fn tenant_id(mut self, tenant_id: &str) -> Self {
153        if !tenant_id.is_empty() {
154            self.config.tenant_id = Some(tenant_id.to_string());
155        }
156
157        self
158    }
159
160    /// Set client_id of this backend.
161    ///
162    /// - If client_id is set, we will take user's input first.
163    /// - If not, we will try to load it from environment.
164    /// - required for client_credentials authentication
165    pub fn client_id(mut self, client_id: &str) -> Self {
166        if !client_id.is_empty() {
167            self.config.client_id = Some(client_id.to_string());
168        }
169
170        self
171    }
172
173    /// Set the sas_token of this backend.
174    pub fn sas_token(mut self, sas_token: &str) -> Self {
175        if !sas_token.is_empty() {
176            self.config.sas_token = Some(sas_token.to_string());
177        }
178
179        self
180    }
181
182    /// Set authority_host of this backend.
183    ///
184    /// - If authority_host is set, we will take user's input first.
185    /// - If not, we will try to load it from environment.
186    /// - default value: `https://login.microsoftonline.com`
187    pub fn authority_host(mut self, authority_host: &str) -> Self {
188        if !authority_host.is_empty() {
189            self.config.authority_host = Some(authority_host.to_string());
190        }
191
192        self
193    }
194
195    /// Specify the http client that used by this service.
196    ///
197    /// # Notes
198    ///
199    /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed
200    /// during minor updates.
201    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
202    #[allow(deprecated)]
203    pub fn http_client(mut self, client: HttpClient) -> Self {
204        self.http_client = Some(client);
205        self
206    }
207
208    /// Create a new `AzdlsBuilder` instance from an [Azure Storage connection string][1].
209    ///
210    /// [1]: https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string
211    ///
212    /// # Example
213    /// ```
214    /// use opendal::Builder;
215    /// use opendal::services::Azdls;
216    ///
217    /// let conn_str = "AccountName=example;DefaultEndpointsProtocol=https;EndpointSuffix=core.windows.net";
218    ///
219    /// let mut config = Azdls::from_connection_string(&conn_str)
220    ///     .unwrap()
221    ///     // Add additional configuration if needed
222    ///     .filesystem("myFilesystem")
223    ///     .client_id("myClientId")
224    ///     .client_secret("myClientSecret")
225    ///     .tenant_id("myTenantId")
226    ///     .build()
227    ///     .unwrap();
228    /// ```
229    pub fn from_connection_string(conn_str: &str) -> Result<Self> {
230        let config =
231            raw::azure_config_from_connection_string(conn_str, raw::AzureStorageService::Adls)?;
232
233        Ok(AzdlsConfig::from(config).into_builder())
234    }
235}
236
237impl Builder for AzdlsBuilder {
238    type Config = AzdlsConfig;
239
240    fn build(self) -> Result<impl Access> {
241        debug!("backend build started: {:?}", &self);
242
243        let root = normalize_root(&self.config.root.unwrap_or_default());
244        debug!("backend use root {root}");
245
246        // Handle endpoint, region and container name.
247        let filesystem = match self.config.filesystem.is_empty() {
248            false => Ok(&self.config.filesystem),
249            true => Err(Error::new(ErrorKind::ConfigInvalid, "filesystem is empty")
250                .with_operation("Builder::build")
251                .with_context("service", AZDLS_SCHEME)),
252        }?;
253        debug!("backend use filesystem {}", &filesystem);
254
255        let endpoint = match &self.config.endpoint {
256            Some(endpoint) => Ok(endpoint.clone().trim_end_matches('/').to_string()),
257            None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
258                .with_operation("Builder::build")
259                .with_context("service", AZDLS_SCHEME)),
260        }?;
261        debug!("backend use endpoint {}", &endpoint);
262
263        let config_loader = AzureStorageConfig {
264            account_name: self
265                .config
266                .account_name
267                .clone()
268                .or_else(|| raw::azure_account_name_from_endpoint(endpoint.as_str())),
269            account_key: self.config.account_key.clone(),
270            sas_token: self.config.sas_token,
271            client_id: self.config.client_id.clone(),
272            client_secret: self.config.client_secret.clone(),
273            tenant_id: self.config.tenant_id.clone(),
274            authority_host: self.config.authority_host.clone(),
275            ..Default::default()
276        };
277
278        let cred_loader = AzureStorageLoader::new(config_loader);
279        let signer = AzureStorageSigner::new();
280        Ok(AzdlsBackend {
281            core: Arc::new(AzdlsCore {
282                info: {
283                    let am = AccessorInfo::default();
284                    am.set_scheme(AZDLS_SCHEME)
285                        .set_root(&root)
286                        .set_name(filesystem)
287                        .set_native_capability(Capability {
288                            stat: true,
289
290                            read: true,
291
292                            write: true,
293                            write_can_append: true,
294                            write_with_if_none_match: true,
295                            write_with_if_not_exists: true,
296
297                            create_dir: true,
298                            delete: true,
299                            rename: true,
300
301                            list: true,
302
303                            shared: true,
304
305                            ..Default::default()
306                        });
307
308                    // allow deprecated api here for compatibility
309                    #[allow(deprecated)]
310                    if let Some(client) = self.http_client {
311                        am.update_http_client(|_| client);
312                    }
313
314                    am.into()
315                },
316                filesystem: self.config.filesystem.clone(),
317                root,
318                endpoint,
319                loader: cred_loader,
320                signer,
321            }),
322        })
323    }
324}
325
326/// Backend for azblob services.
327#[derive(Debug, Clone)]
328pub struct AzdlsBackend {
329    core: Arc<AzdlsCore>,
330}
331
332impl Access for AzdlsBackend {
333    type Reader = HttpBody;
334    type Writer = AzdlsWriters;
335    type Lister = oio::PageLister<AzdlsLister>;
336    type Deleter = oio::OneShotDeleter<AzdlsDeleter>;
337
338    fn info(&self) -> Arc<AccessorInfo> {
339        self.core.info.clone()
340    }
341
342    async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
343        let resp = self
344            .core
345            .azdls_create(path, DIRECTORY, &OpWrite::default())
346            .await?;
347
348        let status = resp.status();
349        match status {
350            StatusCode::CREATED | StatusCode::OK => Ok(RpCreateDir::default()),
351            _ => Err(parse_error(resp)),
352        }
353    }
354
355    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
356        // Stat root always returns a DIR.
357        // TODO: include metadata for the root (#4746)
358        if path == "/" {
359            return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
360        }
361
362        let metadata = self.core.azdls_stat_metadata(path).await?;
363        Ok(RpStat::new(metadata))
364    }
365
366    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
367        let resp = self.core.azdls_read(path, args.range()).await?;
368
369        let status = resp.status();
370        match status {
371            StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
372            _ => {
373                let (part, mut body) = resp.into_parts();
374                let buf = body.to_buffer().await?;
375                Err(parse_error(Response::from_parts(part, buf)))
376            }
377        }
378    }
379
380    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
381        let w = AzdlsWriter::new(self.core.clone(), args.clone(), path.to_string());
382        let w = if args.append() {
383            AzdlsWriters::Two(oio::AppendWriter::new(w))
384        } else {
385            AzdlsWriters::One(oio::OneShotWriter::new(w))
386        };
387        Ok((RpWrite::default(), w))
388    }
389
390    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
391        Ok((
392            RpDelete::default(),
393            oio::OneShotDeleter::new(AzdlsDeleter::new(self.core.clone())),
394        ))
395    }
396
397    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
398        let l = AzdlsLister::new(self.core.clone(), path.to_string(), args.limit());
399
400        Ok((RpList::default(), oio::PageLister::new(l)))
401    }
402
403    async fn rename(&self, from: &str, to: &str, _args: OpRename) -> Result<RpRename> {
404        if let Some(resp) = self.core.azdls_ensure_parent_path(to).await? {
405            let status = resp.status();
406            match status {
407                StatusCode::CREATED | StatusCode::CONFLICT => {}
408                _ => return Err(parse_error(resp)),
409            }
410        }
411
412        let resp = self.core.azdls_rename(from, to).await?;
413
414        let status = resp.status();
415
416        match status {
417            StatusCode::CREATED => Ok(RpRename::default()),
418            _ => Err(parse_error(resp)),
419        }
420    }
421}
opendal/services/azdls/backend.rs

opendal/services/azdls/
backend.rs
