opendal_service_azblob/
backend.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::fmt::Debug;
19use std::sync::Arc;
20
21use base64::Engine;
22use base64::prelude::BASE64_STANDARD;
23use http::StatusCode;
24use log::debug;
25use reqsign_azure_storage::Credential;
26use reqsign_azure_storage::DefaultCredentialProvider;
27use reqsign_azure_storage::RequestSigner;
28use reqsign_azure_storage::StaticCredentialProvider;
29use reqsign_core::Context;
30use reqsign_core::OsEnv;
31use reqsign_core::ProvideCredentialChain;
32use reqsign_core::Signer;
33use reqsign_file_read_tokio::TokioFileRead;
34use sha2::Digest;
35use sha2::Sha256;
36
37use super::AZBLOB_SCHEME;
38use super::config::AzblobConfig;
39use super::copier::AzblobCopiers;
40use super::copier::new_azblob_copier;
41use super::core::AzblobCore;
42use super::core::ErrorContext;
43use super::core::constants::AZBLOB_COPY_MAX_BLOCK_SIZE;
44use super::core::constants::AZBLOB_COPY_MIN_BLOCK_SIZE;
45use super::core::constants::X_MS_META_PREFIX;
46use super::core::constants::X_MS_VERSION_ID;
47use super::core::parse_error;
48use super::deleter::AzblobDeleter;
49use super::lister::AzblobLister;
50use super::reader::*;
51use super::writer::AzblobWriter;
52use super::writer::AzblobWriters;
53use opendal_core::raw::*;
54use opendal_core::*;
55use opendal_service_azure_common::{
56    AzureStorageConfig as AzureConnectionConfig, AzureStorageService,
57    azure_account_name_from_endpoint, azure_config_from_connection_string,
58};
59
60const AZBLOB_BATCH_LIMIT: usize = 256;
61
62impl From<AzureConnectionConfig> for AzblobConfig {
63    fn from(value: AzureConnectionConfig) -> Self {
64        Self {
65            endpoint: value.endpoint,
66            account_name: value.account_name,
67            account_key: value.account_key,
68            sas_token: value.sas_token,
69            ..Default::default()
70        }
71    }
72}
73
74#[doc = include_str!("docs.md")]
75#[derive(Default)]
76pub struct AzblobBuilder {
77    pub(super) config: AzblobConfig,
78    pub(super) credential_providers: Option<ProvideCredentialChain<Credential>>,
79}
80
81impl Debug for AzblobBuilder {
82    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
83        f.debug_struct("AzblobBuilder")
84            .field("config", &self.config)
85            .finish_non_exhaustive()
86    }
87}
88
89impl AzblobBuilder {
90    /// Set root of this backend.
91    ///
92    /// All operations will happen under this root.
93    pub fn root(mut self, root: &str) -> Self {
94        self.config.root = if root.is_empty() {
95            None
96        } else {
97            Some(root.to_string())
98        };
99
100        self
101    }
102
103    /// Set container name of this backend.
104    pub fn container(mut self, container: &str) -> Self {
105        self.config.container = container.to_string();
106
107        self
108    }
109
110    /// Set endpoint of this backend
111    ///
112    /// Endpoint must be full uri, e.g.
113    ///
114    /// - Azblob: `https://accountname.blob.core.windows.net`
115    /// - Azurite: `http://127.0.0.1:10000/devstoreaccount1`
116    pub fn endpoint(mut self, endpoint: &str) -> Self {
117        if !endpoint.is_empty() {
118            // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
119            self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string());
120        }
121
122        self
123    }
124
125    /// Set account_name of this backend.
126    ///
127    /// - If account_name is set, we will take user's input first.
128    /// - If not, we will try to load it from environment.
129    pub fn account_name(mut self, account_name: &str) -> Self {
130        if !account_name.is_empty() {
131            self.config.account_name = Some(account_name.to_string());
132        }
133
134        self
135    }
136
137    /// Set account_key of this backend.
138    ///
139    /// - If account_key is set, we will take user's input first.
140    /// - If not, we will try to load it from environment.
141    pub fn account_key(mut self, account_key: &str) -> Self {
142        if !account_key.is_empty() {
143            self.config.account_key = Some(account_key.to_string());
144        }
145
146        self
147    }
148
149    /// Set encryption_key of this backend.
150    ///
151    /// # Args
152    ///
153    /// `v`: Base64-encoded key that matches algorithm specified in `encryption_algorithm`.
154    ///
155    /// # Note
156    ///
157    /// This function is the low-level setting for SSE related features.
158    ///
159    /// SSE related options should be set carefully to make them works.
160    /// Please use `server_side_encryption_with_*` helpers if even possible.
161    pub fn encryption_key(mut self, v: &str) -> Self {
162        if !v.is_empty() {
163            self.config.encryption_key = Some(v.to_string());
164        }
165
166        self
167    }
168
169    /// Set encryption_key_sha256 of this backend.
170    ///
171    /// # Args
172    ///
173    /// `v`: Base64-encoded SHA256 digest of the key specified in encryption_key.
174    ///
175    /// # Note
176    ///
177    /// This function is the low-level setting for SSE related features.
178    ///
179    /// SSE related options should be set carefully to make them works.
180    /// Please use `server_side_encryption_with_*` helpers if even possible.
181    pub fn encryption_key_sha256(mut self, v: &str) -> Self {
182        if !v.is_empty() {
183            self.config.encryption_key_sha256 = Some(v.to_string());
184        }
185
186        self
187    }
188
189    /// Set encryption_algorithm of this backend.
190    ///
191    /// # Args
192    ///
193    /// `v`: server-side encryption algorithm. (Available values: `AES256`)
194    ///
195    /// # Note
196    ///
197    /// This function is the low-level setting for SSE related features.
198    ///
199    /// SSE related options should be set carefully to make them works.
200    /// Please use `server_side_encryption_with_*` helpers if even possible.
201    pub fn encryption_algorithm(mut self, v: &str) -> Self {
202        if !v.is_empty() {
203            self.config.encryption_algorithm = Some(v.to_string());
204        }
205
206        self
207    }
208
209    /// Enable server side encryption with customer key.
210    ///
211    /// As known as: CPK
212    ///
213    /// # Args
214    ///
215    /// `key`: Base64-encoded SHA256 digest of the key specified in encryption_key.
216    ///
217    /// # Note
218    ///
219    /// Function that helps the user to set the server-side customer-provided encryption key, the key's SHA256, and the algorithm.
220    /// See [Server-side encryption with customer-provided keys (CPK)](https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-customer-provided-keys)
221    /// for more info.
222    pub fn server_side_encryption_with_customer_key(mut self, key: &[u8]) -> Self {
223        // Only AES256 is supported for now
224        self.config.encryption_algorithm = Some("AES256".to_string());
225        self.config.encryption_key = Some(BASE64_STANDARD.encode(key));
226        let key_sha256 = Sha256::digest(key);
227        self.config.encryption_key_sha256 = Some(BASE64_STANDARD.encode(key_sha256));
228        self
229    }
230
231    /// Set sas_token of this backend.
232    ///
233    /// - If sas_token is set, we will take user's input first.
234    /// - If not, we will try to load it from environment.
235    ///
236    /// See [Grant limited access to Azure Storage resources using shared access signatures (SAS)](https://learn.microsoft.com/en-us/azure/storage/common/storage-sas-overview)
237    /// for more info.
238    pub fn sas_token(mut self, sas_token: &str) -> Self {
239        if !sas_token.is_empty() {
240            self.config.sas_token = Some(sas_token.to_string());
241        }
242
243        self
244    }
245
246    /// Replace the credential providers with a custom chain.
247    pub fn credential_provider_chain(mut self, chain: ProvideCredentialChain<Credential>) -> Self {
248        self.credential_providers = Some(chain);
249        self
250    }
251
252    /// Deprecated: Azblob delete batch capability is enabled by default with Azure Blob's 256-operation batch limit.
253    #[deprecated(
254        since = "0.57.0",
255        note = "Azblob delete batch capability is enabled by default with Azure Blob's 256-operation batch limit and this option is no longer needed."
256    )]
257    pub fn batch_max_operations(self, _batch_max_operations: usize) -> Self {
258        self
259    }
260
261    /// Skip signature will skip loading credentials and signing requests.
262    pub fn skip_signature(mut self) -> Self {
263        self.config.skip_signature = true;
264        self
265    }
266
267    /// from_connection_string will make a builder from connection string
268    ///
269    /// connection string looks like:
270    ///
271    /// ```txt
272    /// DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;
273    /// AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;
274    /// BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
275    /// QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;
276    /// TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;
277    /// ```
278    ///
279    /// Or
280    ///
281    /// ```txt
282    /// DefaultEndpointsProtocol=https;
283    /// AccountName=storagesample;
284    /// AccountKey=<account-key>;
285    /// EndpointSuffix=core.chinacloudapi.cn;
286    /// ```
287    ///
288    /// For reference: [Configure Azure Storage connection strings](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string)
289    ///
290    /// # Note
291    ///
292    /// Connection strings can only configure the endpoint, account name and
293    /// authentication information. Users still need to configure container name.
294    pub fn from_connection_string(conn: &str) -> Result<Self> {
295        let config = azure_config_from_connection_string(conn, AzureStorageService::Blob)?;
296
297        Ok(AzblobConfig::from(config).into_builder())
298    }
299}
300
301impl Builder for AzblobBuilder {
302    type Config = AzblobConfig;
303
304    fn build(self) -> Result<impl Service> {
305        debug!("backend build started: {:?}", self);
306
307        let root = normalize_root(&self.config.root.unwrap_or_default());
308        debug!("backend use root {root}");
309
310        // Handle endpoint, region and container name.
311        let container = match self.config.container.is_empty() {
312            false => Ok(&self.config.container),
313            true => Err(Error::new(ErrorKind::ConfigInvalid, "container is empty")
314                .with_operation("Builder::build")
315                .with_context("service", AZBLOB_SCHEME)),
316        }?;
317        debug!("backend use container {}", container);
318
319        let endpoint = match &self.config.endpoint {
320            Some(endpoint) => Ok(endpoint.clone()),
321            None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
322                .with_operation("Builder::build")
323                .with_context("service", AZBLOB_SCHEME)),
324        }?;
325        debug!("backend use endpoint {}", container);
326
327        let account_name = self
328            .config
329            .account_name
330            .clone()
331            .or_else(|| azure_account_name_from_endpoint(endpoint.as_str()));
332
333        if let Some(v) = &self.config.account_key {
334            // Validate that account_key can be decoded as base64
335            if let Err(e) = BASE64_STANDARD.decode(v) {
336                return Err(Error::new(
337                    ErrorKind::ConfigInvalid,
338                    format!("invalid account_key: cannot decode as base64: {e}"),
339                )
340                .with_operation("Builder::build")
341                .with_context("service", AZBLOB_SCHEME)
342                .with_context("key", "account_key"));
343            }
344        }
345
346        let encryption_key =
347            match &self.config.encryption_key {
348                None => None,
349                Some(v) => Some(build_header_value(v).map_err(|err| {
350                    err.with_context("key", "server_side_encryption_customer_key")
351                })?),
352            };
353
354        let encryption_key_sha256 = match &self.config.encryption_key_sha256 {
355            None => None,
356            Some(v) => Some(build_header_value(v).map_err(|err| {
357                err.with_context("key", "server_side_encryption_customer_key_sha256")
358            })?),
359        };
360
361        let encryption_algorithm = match &self.config.encryption_algorithm {
362            None => None,
363            Some(v) => {
364                if v == "AES256" {
365                    Some(build_header_value(v).map_err(|err| {
366                        err.with_context("key", "server_side_encryption_customer_algorithm")
367                    })?)
368                } else {
369                    return Err(Error::new(
370                        ErrorKind::ConfigInvalid,
371                        "encryption_algorithm value must be AES256",
372                    ));
373                }
374            }
375        };
376
377        let ctx = Context::new().with_file_read(TokioFileRead).with_env(OsEnv);
378
379        let mut credential_providers =
380            ProvideCredentialChain::new().push(DefaultCredentialProvider::new());
381
382        if let (Some(account_name), Some(account_key)) =
383            (account_name.as_deref(), self.config.account_key.as_deref())
384        {
385            credential_providers = credential_providers.push_front(
386                StaticCredentialProvider::new_shared_key(account_name, account_key),
387            );
388        }
389
390        if let Some(sas_token) = self.config.sas_token.as_deref() {
391            credential_providers =
392                credential_providers.push_front(StaticCredentialProvider::new_sas_token(sas_token));
393        }
394
395        if let Some(customized_credential_chain) = self.credential_providers {
396            credential_providers = customized_credential_chain;
397        }
398
399        let signer = Signer::new(
400            ctx,
401            credential_providers,
402            RequestSigner::new().with_service_sas_permissions("racwd"),
403        );
404
405        let info = ServiceInfo::new(AZBLOB_SCHEME, &root, container);
406        let capability = Capability {
407            stat: true,
408            stat_with_if_match: true,
409            stat_with_if_none_match: true,
410
411            read: true,
412
413            read_with_if_match: true,
414            read_with_if_none_match: true,
415            read_with_override_content_disposition: true,
416            read_with_if_modified_since: true,
417            read_with_if_unmodified_since: true,
418
419            write: true,
420            write_can_append: true,
421            write_can_empty: true,
422            write_can_multi: true,
423            write_with_cache_control: true,
424            write_with_content_type: true,
425            write_with_if_match: true,
426            write_with_if_not_exists: true,
427            write_with_if_none_match: true,
428            write_with_user_metadata: true,
429
430            delete: true,
431            delete_with_if_match: true,
432            delete_with_if_none_match: true,
433            delete_max_size: Some(AZBLOB_BATCH_LIMIT),
434
435            copy: true,
436            copy_with_if_not_exists: true,
437            copy_with_if_match: true,
438            copy_with_if_none_match: true,
439            copy_can_multi: true,
440            copy_multi_min_size: Some(AZBLOB_COPY_MIN_BLOCK_SIZE),
441            copy_multi_max_size: Some(AZBLOB_COPY_MAX_BLOCK_SIZE),
442
443            list: true,
444            list_with_recursive: true,
445
446            presign: self.config.sas_token.is_some(),
447            presign_stat: self.config.sas_token.is_some(),
448            presign_read: self.config.sas_token.is_some(),
449            presign_write: self.config.sas_token.is_some(),
450
451            shared: true,
452
453            ..Default::default()
454        };
455
456        Ok(AzblobBackend {
457            core: Arc::new(AzblobCore {
458                info,
459                capability,
460                root,
461                endpoint,
462                encryption_key,
463                encryption_key_sha256,
464                encryption_algorithm,
465                container: self.config.container.clone(),
466                skip_signature: self.config.skip_signature,
467                signer,
468            }),
469        })
470    }
471}
472
473/// Backend for azblob services.
474#[derive(Debug, Clone)]
475pub struct AzblobBackend {
476    pub(crate) core: Arc<AzblobCore>,
477}
478
479impl Service for AzblobBackend {
480    type Reader = oio::StreamReader<AzblobReader>;
481    type Writer = AzblobWriters;
482    type Lister = oio::PageLister<AzblobLister>;
483    type Deleter = oio::BatchDeleter<AzblobDeleter>;
484    type Copier = AzblobCopiers;
485    type Composer = ();
486
487    fn info(&self) -> ServiceInfo {
488        self.core.info.clone()
489    }
490
491    fn capability(&self) -> Capability {
492        self.core.capability
493    }
494
495    async fn create_dir(
496        &self,
497        _ctx: &OperationContext,
498        _path: &str,
499        _args: OpCreateDir,
500    ) -> Result<RpCreateDir> {
501        Err(Error::new(
502            ErrorKind::Unsupported,
503            "operation is not supported",
504        ))
505    }
506
507    async fn stat(&self, ctx: &OperationContext, path: &str, args: OpStat) -> Result<RpStat> {
508        let error_ctx = ErrorContext::new(ServiceOperation("GetBlobProperties"))
509            .with_caller_condition(args.is_conditional());
510        let resp = self
511            .core
512            .azblob_get_blob_properties(ctx, path, &args)
513            .await?;
514
515        let status = resp.status();
516
517        match status {
518            StatusCode::OK => {
519                let headers = resp.headers();
520                let mut meta = parse_into_metadata(path, headers)?.into_builder();
521                if let Some(version_id) = parse_header_to_str(headers, X_MS_VERSION_ID)? {
522                    meta.version(version_id);
523                }
524
525                let user_meta = parse_prefixed_headers(headers, X_MS_META_PREFIX);
526                if !user_meta.is_empty() {
527                    meta.user_metadata(user_meta);
528                }
529
530                Ok(RpStat::new(meta.build()))
531            }
532            _ => Err(parse_error(error_ctx, resp)),
533        }
534    }
535    fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
536        let output: oio::StreamReader<AzblobReader> = {
537            Ok(oio::StreamReader::new(AzblobReader::new(
538                self.clone(),
539                ctx.clone(),
540                path,
541                args,
542            )))
543        }?;
544
545        Ok(output)
546    }
547
548    fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
549        let output: AzblobWriters = {
550            let w = AzblobWriter::new(
551                self.core.clone(),
552                ctx.clone(),
553                args.clone(),
554                path.to_string(),
555            );
556            let w = if args.append() {
557                AzblobWriters::Two(oio::AppendWriter::new(w))
558            } else {
559                AzblobWriters::One(oio::BlockWriter::new(
560                    ctx.executor().clone(),
561                    w,
562                    args.concurrent(),
563                ))
564            };
565
566            Ok(w)
567        }?;
568
569        Ok(output)
570    }
571
572    fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
573        let output: oio::BatchDeleter<AzblobDeleter> = {
574            Ok(oio::BatchDeleter::new(
575                AzblobDeleter::new(self.core.clone(), ctx.clone()),
576                self.core.capability.delete_max_size,
577            ))
578        }?;
579
580        Ok(output)
581    }
582
583    fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
584        let output: oio::PageLister<AzblobLister> = {
585            let l = AzblobLister::new(
586                self.core.clone(),
587                ctx.clone(),
588                path.to_string(),
589                args.recursive(),
590                args.limit(),
591            );
592
593            Ok(oio::PageLister::new(l))
594        }?;
595
596        Ok(output)
597    }
598
599    fn copy(
600        &self,
601        ctx: &OperationContext,
602        from: &str,
603        to: &str,
604        args: OpCopy,
605    ) -> Result<Self::Copier> {
606        let output: AzblobCopiers = {
607            let copier = new_azblob_copier(self.core.clone(), ctx, from, to, args)?;
608            Ok(copier)
609        }?;
610
611        Ok(output)
612    }
613
614    async fn rename(
615        &self,
616        _ctx: &OperationContext,
617        _from: &str,
618        _to: &str,
619        _args: OpRename,
620    ) -> Result<RpRename> {
621        Err(Error::new(
622            ErrorKind::Unsupported,
623            "operation is not supported",
624        ))
625    }
626
627    async fn presign(
628        &self,
629        ctx: &OperationContext,
630        path: &str,
631        args: OpPresign,
632    ) -> Result<RpPresign> {
633        let req = match args.operation() {
634            PresignOperation::Stat(v) => self.core.azblob_head_blob_request(path, v),
635            PresignOperation::Read(range, v) => self.core.azblob_get_blob_request(path, *range, v),
636            PresignOperation::Write(v) => {
637                self.core
638                    .azblob_put_blob_request(path, None, v, Buffer::new())
639            }
640            PresignOperation::Delete(_) => Err(Error::new(
641                ErrorKind::Unsupported,
642                "operation is not supported",
643            )),
644            _ => Err(Error::new(
645                ErrorKind::Unsupported,
646                "presign operation is not supported",
647            )),
648        };
649
650        let req = req?;
651        let req = self.core.sign_query(ctx, req).await?;
652
653        let (parts, _) = req.into_parts();
654
655        Ok(RpPresign::new(PresignedRequest::new(
656            parts.method,
657            parts.uri,
658            parts.headers,
659        )))
660    }
661}
opendal_service_azblob/backend.rs

opendal_service_azblob/
backend.rs
