opendal_service_azdls/
backend.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::fmt::Debug;
19use std::sync::Arc;
20
21use http::StatusCode;
22use log::debug;
23use reqsign_azure_storage::DefaultCredentialProvider;
24use reqsign_azure_storage::RequestSigner;
25use reqsign_azure_storage::StaticCredentialProvider;
26use reqsign_core::Context;
27use reqsign_core::Env as _;
28use reqsign_core::OsEnv;
29use reqsign_core::Signer;
30use reqsign_core::StaticEnv;
31use reqsign_file_read_tokio::TokioFileRead;
32
33use super::AZDLS_SCHEME;
34use super::config::AzdlsConfig;
35use super::core::AzdlsCore;
36use super::core::DIRECTORY;
37use super::core::parse_error;
38use super::deleter::AzdlsDeleter;
39use super::lister::AzdlsLister;
40use super::reader::*;
41use super::writer::AzdlsLazyPositionWriter;
42use super::writer::AzdlsWriter;
43use super::writer::AzdlsWriters;
44use opendal_core::raw::*;
45use opendal_core::*;
46use opendal_service_azure_common::{
47    AzureStorageConfig as AzureConnectionConfig, AzureStorageService,
48    azure_account_name_from_endpoint, azure_config_from_connection_string,
49};
50
51impl From<AzureConnectionConfig> for AzdlsConfig {
52    fn from(config: AzureConnectionConfig) -> Self {
53        AzdlsConfig {
54            endpoint: config.endpoint,
55            account_name: config.account_name,
56            account_key: config.account_key,
57            client_secret: config.client_secret,
58            tenant_id: config.tenant_id,
59            client_id: config.client_id,
60            sas_token: config.sas_token,
61            authority_host: config.authority_host,
62            ..Default::default()
63        }
64    }
65}
66
67/// Azure Data Lake Storage Gen2 Support.
68#[doc = include_str!("docs.md")]
69#[derive(Default)]
70pub struct AzdlsBuilder {
71    pub(super) config: AzdlsConfig,
72}
73
74impl Debug for AzdlsBuilder {
75    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
76        f.debug_struct("AzdlsBuilder")
77            .field("config", &self.config)
78            .finish_non_exhaustive()
79    }
80}
81
82impl AzdlsBuilder {
83    /// Set root of this backend.
84    ///
85    /// All operations will happen under this root.
86    pub fn root(mut self, root: &str) -> Self {
87        self.config.root = if root.is_empty() {
88            None
89        } else {
90            Some(root.to_string())
91        };
92
93        self
94    }
95
96    /// Set filesystem name of this backend.
97    pub fn filesystem(mut self, filesystem: &str) -> Self {
98        self.config.filesystem = filesystem.to_string();
99
100        self
101    }
102
103    /// Set endpoint of this backend.
104    ///
105    /// Endpoint must be full uri, e.g.
106    ///
107    /// - Azblob: `https://accountname.blob.core.windows.net`
108    /// - Azurite: `http://127.0.0.1:10000/devstoreaccount1`
109    pub fn endpoint(mut self, endpoint: &str) -> Self {
110        if !endpoint.is_empty() {
111            // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
112            self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string());
113        }
114
115        self
116    }
117
118    /// Set account_name of this backend.
119    ///
120    /// - If account_name is set, we will take user's input first.
121    /// - If not, we will try to load it from environment.
122    pub fn account_name(mut self, account_name: &str) -> Self {
123        if !account_name.is_empty() {
124            self.config.account_name = Some(account_name.to_string());
125        }
126
127        self
128    }
129
130    /// Set account_key of this backend.
131    ///
132    /// - If account_key is set, we will take user's input first.
133    /// - If not, we will try to load it from environment.
134    pub fn account_key(mut self, account_key: &str) -> Self {
135        if !account_key.is_empty() {
136            self.config.account_key = Some(account_key.to_string());
137        }
138
139        self
140    }
141
142    /// Set client_secret of this backend.
143    ///
144    /// - If client_secret is set, we will take user's input first.
145    /// - If not, we will try to load it from environment.
146    /// - required for client_credentials authentication
147    pub fn client_secret(mut self, client_secret: &str) -> Self {
148        if !client_secret.is_empty() {
149            self.config.client_secret = Some(client_secret.to_string());
150        }
151
152        self
153    }
154
155    /// Set tenant_id of this backend.
156    ///
157    /// - If tenant_id is set, we will take user's input first.
158    /// - If not, we will try to load it from environment.
159    /// - required for client_credentials authentication
160    pub fn tenant_id(mut self, tenant_id: &str) -> Self {
161        if !tenant_id.is_empty() {
162            self.config.tenant_id = Some(tenant_id.to_string());
163        }
164
165        self
166    }
167
168    /// Set client_id of this backend.
169    ///
170    /// - If client_id is set, we will take user's input first.
171    /// - If not, we will try to load it from environment.
172    /// - required for client_credentials authentication
173    pub fn client_id(mut self, client_id: &str) -> Self {
174        if !client_id.is_empty() {
175            self.config.client_id = Some(client_id.to_string());
176        }
177
178        self
179    }
180
181    /// Set the sas_token of this backend.
182    pub fn sas_token(mut self, sas_token: &str) -> Self {
183        if !sas_token.is_empty() {
184            self.config.sas_token = Some(sas_token.to_string());
185        }
186
187        self
188    }
189
190    /// Set authority_host of this backend.
191    ///
192    /// - If authority_host is set, we will take user's input first.
193    /// - If not, we will try to load it from environment.
194    /// - default value: `https://login.microsoftonline.com`
195    pub fn authority_host(mut self, authority_host: &str) -> Self {
196        if !authority_host.is_empty() {
197            self.config.authority_host = Some(authority_host.to_string());
198        }
199
200        self
201    }
202
203    /// Create a new `AzdlsBuilder` instance from an [Azure Storage connection string][1].
204    ///
205    /// [1]: https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string
206    ///
207    /// # Example
208    /// ```
209    /// use opendal_core::Builder;
210    /// use opendal_service_azdls::Azdls;
211    ///
212    /// let conn_str = "AccountName=example;DefaultEndpointsProtocol=https;EndpointSuffix=core.windows.net";
213    ///
214    /// let mut config = Azdls::from_connection_string(&conn_str)
215    ///     .unwrap()
216    ///     // Add additional configuration if needed
217    ///     .filesystem("myFilesystem")
218    ///     .client_id("myClientId")
219    ///     .client_secret("myClientSecret")
220    ///     .tenant_id("myTenantId")
221    ///     .build()
222    ///     .unwrap();
223    /// ```
224    pub fn from_connection_string(conn_str: &str) -> Result<Self> {
225        let config = azure_config_from_connection_string(conn_str, AzureStorageService::Adls)?;
226
227        Ok(AzdlsConfig::from(config).into_builder())
228    }
229
230    /// Enable or disable HNS (Hierarchical Namespace) for this backend.
231    pub fn enable_hns(mut self, enable: bool) -> Self {
232        self.config.enable_hns = enable;
233        self
234    }
235}
236
237impl Builder for AzdlsBuilder {
238    type Config = AzdlsConfig;
239
240    fn build(self) -> Result<impl Service> {
241        debug!("backend build started: {:?}", self);
242
243        let root = normalize_root(&self.config.root.unwrap_or_default());
244        debug!("backend use root {root}");
245
246        // Handle endpoint, region and container name.
247        let filesystem = match self.config.filesystem.is_empty() {
248            false => Ok(&self.config.filesystem),
249            true => Err(Error::new(ErrorKind::ConfigInvalid, "filesystem is empty")
250                .with_operation("Builder::build")
251                .with_context("service", AZDLS_SCHEME)),
252        }?;
253        debug!("backend use filesystem {}", filesystem);
254
255        let endpoint = match &self.config.endpoint {
256            Some(endpoint) => Ok(endpoint.clone().trim_end_matches('/').to_string()),
257            None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
258                .with_operation("Builder::build")
259                .with_context("service", AZDLS_SCHEME)),
260        }?;
261        debug!("backend use endpoint {}", endpoint);
262
263        let account_name = self
264            .config
265            .account_name
266            .clone()
267            .or_else(|| azure_account_name_from_endpoint(endpoint.as_str()));
268
269        let mut envs = std::collections::HashMap::new();
270
271        if let Some(v) = &account_name {
272            envs.insert("AZBLOB_ACCOUNT_NAME".to_string(), v.clone());
273            envs.insert("AZURE_STORAGE_ACCOUNT_NAME".to_string(), v.clone());
274        }
275        if let Some(v) = &self.config.account_key {
276            envs.insert("AZBLOB_ACCOUNT_KEY".to_string(), v.clone());
277            envs.insert("AZURE_STORAGE_ACCOUNT_KEY".to_string(), v.clone());
278        }
279        if let Some(v) = &self.config.sas_token {
280            envs.insert("AZURE_STORAGE_SAS_TOKEN".to_string(), v.clone());
281        }
282        if let Some(v) = &self.config.client_id {
283            envs.insert("AZURE_CLIENT_ID".to_string(), v.clone());
284        }
285        if let Some(v) = &self.config.client_secret {
286            envs.insert("AZURE_CLIENT_SECRET".to_string(), v.clone());
287        }
288        if let Some(v) = &self.config.tenant_id {
289            envs.insert("AZURE_TENANT_ID".to_string(), v.clone());
290        }
291        if let Some(v) = &self.config.authority_host {
292            envs.insert("AZURE_AUTHORITY_HOST".to_string(), v.clone());
293        }
294
295        let os_env = OsEnv;
296        let ctx = Context::new()
297            .with_file_read(TokioFileRead)
298            .with_env(StaticEnv {
299                home_dir: os_env.home_dir(),
300                envs,
301            });
302
303        let mut credential = DefaultCredentialProvider::new();
304
305        if let (Some(account_name), Some(account_key)) =
306            (account_name.as_deref(), self.config.account_key.as_deref())
307        {
308            credential = credential.push_front(StaticCredentialProvider::new_shared_key(
309                account_name,
310                account_key,
311            ));
312        }
313        if let Some(sas_token) = self.config.sas_token.as_deref() {
314            credential = credential.push_front(StaticCredentialProvider::new_sas_token(sas_token));
315        }
316
317        let sign_ctx = ctx;
318        let signer = Signer::new(sign_ctx.clone(), credential, RequestSigner::new());
319
320        let info = ServiceInfo::new(AZDLS_SCHEME, &root, filesystem);
321        let capability = Capability {
322            stat: true,
323
324            read: true,
325            read_with_if_match: true,
326            read_with_if_none_match: true,
327            read_with_if_modified_since: true,
328            read_with_if_unmodified_since: true,
329
330            write: true,
331            write_can_append: true,
332            write_can_multi: true,
333            write_with_if_none_match: true,
334            write_with_if_not_exists: true,
335            write_with_user_metadata: true,
336
337            create_dir: true,
338
339            delete: true,
340            delete_with_recursive: true,
341
342            rename: true,
343
344            list: true,
345
346            shared: true,
347
348            ..Default::default()
349        };
350
351        Ok(AzdlsBackend {
352            core: Arc::new(AzdlsCore {
353                info,
354                capability,
355                filesystem: self.config.filesystem.clone(),
356                root,
357                endpoint,
358                enable_hns: self.config.enable_hns,
359                signer,
360                sign_ctx,
361            }),
362        })
363    }
364}
365
366/// Backend for azblob services.
367#[derive(Debug, Clone)]
368pub struct AzdlsBackend {
369    pub(crate) core: Arc<AzdlsCore>,
370}
371
372impl Service for AzdlsBackend {
373    type Reader = oio::StreamReader<AzdlsReader>;
374    type Writer = AzdlsWriters;
375    type Lister = oio::PageLister<AzdlsLister>;
376    type Deleter = oio::OneShotDeleter<AzdlsDeleter>;
377    type Copier = ();
378
379    fn info(&self) -> ServiceInfo {
380        self.core.info.clone()
381    }
382
383    fn capability(&self) -> Capability {
384        self.core.capability
385    }
386
387    async fn create_dir(
388        &self,
389        ctx: &OperationContext,
390        path: &str,
391        _: OpCreateDir,
392    ) -> Result<RpCreateDir> {
393        let resp = self
394            .core
395            .azdls_create(ctx, path, DIRECTORY, &OpWrite::default())
396            .await?;
397
398        let status = resp.status();
399        match status {
400            StatusCode::CREATED | StatusCode::OK => Ok(RpCreateDir::default()),
401            _ => Err(parse_error(resp)),
402        }
403    }
404
405    async fn stat(&self, ctx: &OperationContext, path: &str, _: OpStat) -> Result<RpStat> {
406        // Stat root always returns a DIR.
407        // TODO: include metadata for the root (#4746)
408        if path == "/" {
409            return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
410        }
411
412        let metadata = self.core.azdls_stat_metadata(ctx, path).await?;
413        Ok(RpStat::new(metadata))
414    }
415    fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
416        let output: oio::StreamReader<AzdlsReader> = {
417            Ok(oio::StreamReader::new(AzdlsReader::new(
418                self.clone(),
419                ctx.clone(),
420                path,
421                args,
422            )))
423        }?;
424
425        Ok(output)
426    }
427
428    fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
429        let output: AzdlsWriters = {
430            if args.append() {
431                let w = AzdlsWriter::new(
432                    self.core.clone(),
433                    ctx.clone(),
434                    args.clone(),
435                    path.to_string(),
436                );
437                Ok(AzdlsWriters::Two(oio::AppendWriter::new(w)))
438            } else {
439                let w = AzdlsWriter::new(
440                    self.core.clone(),
441                    ctx.clone(),
442                    args.clone(),
443                    path.to_string(),
444                );
445                let w = oio::PositionWriter::new(
446                    ctx.executor().clone(),
447                    AzdlsLazyPositionWriter::new(w),
448                    args.concurrent(),
449                );
450                Ok(AzdlsWriters::One(w))
451            }
452        }?;
453
454        Ok(output)
455    }
456
457    fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
458        let output: oio::OneShotDeleter<AzdlsDeleter> = {
459            Ok(oio::OneShotDeleter::new(AzdlsDeleter::new(
460                self.core.clone(),
461                ctx.clone(),
462            )))
463        }?;
464
465        Ok(output)
466    }
467
468    fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
469        let output: oio::PageLister<AzdlsLister> = {
470            let l = AzdlsLister::new(
471                self.core.clone(),
472                ctx.clone(),
473                path.to_string(),
474                args.limit(),
475            );
476
477            Ok(oio::PageLister::new(l))
478        }?;
479
480        Ok(output)
481    }
482
483    fn copy(
484        &self,
485        _ctx: &OperationContext,
486        _from: &str,
487        _to: &str,
488        _args: OpCopy,
489        _opts: OpCopier,
490    ) -> Result<Self::Copier> {
491        Err(Error::new(
492            ErrorKind::Unsupported,
493            "operation is not supported",
494        ))
495    }
496
497    async fn rename(
498        &self,
499        ctx: &OperationContext,
500        from: &str,
501        to: &str,
502        _args: OpRename,
503    ) -> Result<RpRename> {
504        if let Some(resp) = self.core.azdls_ensure_parent_path(ctx, to).await? {
505            let status = resp.status();
506            match status {
507                StatusCode::CREATED | StatusCode::CONFLICT => {}
508                _ => return Err(parse_error(resp)),
509            }
510        }
511
512        let resp = self.core.azdls_rename(ctx, from, to).await?;
513
514        let status = resp.status();
515
516        match status {
517            StatusCode::CREATED => Ok(RpRename::default()),
518            _ => Err(parse_error(resp)),
519        }
520    }
521
522    async fn presign(
523        &self,
524        _ctx: &OperationContext,
525        _path: &str,
526        _args: OpPresign,
527    ) -> Result<RpPresign> {
528        Err(Error::new(
529            ErrorKind::Unsupported,
530            "operation is not supported",
531        ))
532    }
533}
opendal_service_azdls/backend.rs

opendal_service_azdls/
backend.rs
