Skip to main content

opendal_service_cos/
backend.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::fmt::Debug;
19use std::sync::Arc;
20
21use bytes::Buf;
22use http::StatusCode;
23use http::Uri;
24use log::debug;
25use reqsign_core::Context;
26use reqsign_core::OsEnv;
27use reqsign_core::Signer;
28use reqsign_file_read_tokio::TokioFileRead;
29use reqsign_tencent_cos::DefaultCredentialProvider;
30use reqsign_tencent_cos::RequestSigner;
31use reqsign_tencent_cos::StaticCredentialProvider;
32
33use super::COS_SCHEME;
34use super::config::CosConfig;
35use super::core::parse_error;
36use super::core::*;
37use super::deleter::CosDeleter;
38use super::lister::CosLister;
39use super::lister::CosListers;
40use super::lister::CosObjectVersionsLister;
41use super::reader::*;
42use super::writer::CosWriter;
43use super::writer::CosWriters;
44use opendal_core::raw::*;
45use opendal_core::*;
46
47/// Tencent-Cloud COS services support.
48#[doc = include_str!("docs.md")]
49#[derive(Default)]
50pub struct CosBuilder {
51    pub(super) config: CosConfig,
52}
53
54impl Debug for CosBuilder {
55    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
56        f.debug_struct("CosBuilder")
57            .field("config", &self.config)
58            .finish_non_exhaustive()
59    }
60}
61
62impl CosBuilder {
63    /// Set root of this backend.
64    ///
65    /// All operations will happen under this root.
66    pub fn root(mut self, root: &str) -> Self {
67        self.config.root = if root.is_empty() {
68            None
69        } else {
70            Some(root.to_string())
71        };
72
73        self
74    }
75
76    /// Set endpoint of this backend.
77    ///
78    /// NOTE: no bucket or account id in endpoint, we will trim them if exists.
79    ///
80    /// # Examples
81    ///
82    /// - `https://cos.ap-singapore.myqcloud.com`
83    pub fn endpoint(mut self, endpoint: &str) -> Self {
84        if !endpoint.is_empty() {
85            self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string());
86        }
87
88        self
89    }
90
91    /// Set secret_id of this backend.
92    /// - If it is set, we will take user's input first.
93    /// - If not, we will try to load it from environment.
94    pub fn secret_id(mut self, secret_id: &str) -> Self {
95        if !secret_id.is_empty() {
96            self.config.secret_id = Some(secret_id.to_string());
97        }
98
99        self
100    }
101
102    /// Set secret_key of this backend.
103    /// - If it is set, we will take user's input first.
104    /// - If not, we will try to load it from environment.
105    pub fn secret_key(mut self, secret_key: &str) -> Self {
106        if !secret_key.is_empty() {
107            self.config.secret_key = Some(secret_key.to_string());
108        }
109
110        self
111    }
112
113    /// Set security_token (a.k.a. session token) of this backend.
114    ///
115    /// This is used when authenticating via Tencent Cloud STS temporary
116    /// credentials (e.g. obtained from `GetFederationToken` or
117    /// `AssumeRole`). When provided, it will be combined with `secret_id`
118    /// and `secret_key` to sign requests, and the `x-cos-security-token`
119    /// header will be attached automatically.
120    ///
121    /// - If this is set along with `secret_id` and `secret_key`, a static
122    ///   credential provider with the token will be used.
123    /// - If this is not set, the default credential chain in reqsign will
124    ///   try to load credentials (including the token) from environment
125    ///   variables such as `TENCENTCLOUD_TOKEN`,
126    ///   `TENCENTCLOUD_SECURITY_TOKEN`, and `QCLOUD_SECRET_TOKEN`
127    ///   (unless `disable_config_load` is enabled).
128    pub fn security_token(mut self, security_token: &str) -> Self {
129        if !security_token.is_empty() {
130            self.config.security_token = Some(security_token.to_string());
131        }
132
133        self
134    }
135
136    /// Set bucket of this backend.
137    /// The param is required.
138    pub fn bucket(mut self, bucket: &str) -> Self {
139        if !bucket.is_empty() {
140            self.config.bucket = Some(bucket.to_string());
141        }
142
143        self
144    }
145
146    /// Deprecated: COS versioning capability is enabled by default.
147    #[deprecated(
148        since = "0.57.0",
149        note = "COS versioning capability is enabled by default and this option is no longer needed."
150    )]
151    pub fn enable_versioning(self, _enabled: bool) -> Self {
152        self
153    }
154
155    /// Disable config load so that opendal will not load config from
156    /// environment.
157    ///
158    /// For examples:
159    ///
160    /// - envs like `TENCENTCLOUD_SECRET_ID`
161    pub fn disable_config_load(mut self) -> Self {
162        self.config.disable_config_load = true;
163        self
164    }
165}
166
167impl Builder for CosBuilder {
168    type Config = CosConfig;
169
170    fn build(self) -> Result<impl Service> {
171        debug!("backend build started: {:?}", self);
172
173        let root = normalize_root(&self.config.root.unwrap_or_default());
174        debug!("backend use root {root}");
175
176        let bucket = match &self.config.bucket {
177            Some(bucket) => Ok(bucket.to_string()),
178            None => Err(
179                Error::new(ErrorKind::ConfigInvalid, "The bucket is misconfigured")
180                    .with_context("service", COS_SCHEME),
181            ),
182        }?;
183        debug!("backend use bucket {}", bucket);
184
185        let uri = match &self.config.endpoint {
186            Some(endpoint) => endpoint.parse::<Uri>().map_err(|err| {
187                Error::new(ErrorKind::ConfigInvalid, "endpoint is invalid")
188                    .with_context("service", COS_SCHEME)
189                    .with_context("endpoint", endpoint)
190                    .set_source(err)
191            }),
192            None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
193                .with_context("service", COS_SCHEME)),
194        }?;
195
196        let endpoint = build_endpoint(&uri, &bucket)?;
197        debug!("backend use endpoint {}", endpoint);
198
199        let os_env = OsEnv;
200        let ctx = Context::new()
201            .with_file_read(TokioFileRead)
202            .with_env(os_env);
203
204        let mut credential = if self.config.disable_config_load {
205            DefaultCredentialProvider::builder()
206                .no_env()
207                .no_web_identity()
208                .build()
209        } else {
210            DefaultCredentialProvider::new()
211        };
212
213        if let (Some(secret_id), Some(secret_key)) = (
214            self.config.secret_id.as_deref(),
215            self.config.secret_key.as_deref(),
216        ) {
217            let static_provider = if let Some(token) = self.config.security_token.as_deref() {
218                StaticCredentialProvider::with_security_token(secret_id, secret_key, token)
219            } else {
220                StaticCredentialProvider::new(secret_id, secret_key)
221            };
222
223            credential = credential.push_front(static_provider);
224        }
225
226        let signer = Signer::new(ctx, credential, RequestSigner::new());
227
228        let info = ServiceInfo::new(COS_SCHEME, &root, &bucket);
229        let capability = Capability {
230            stat: true,
231            stat_with_if_match: true,
232            stat_with_if_none_match: true,
233            stat_with_version: true,
234
235            read: true,
236            read_with_suffix: true,
237
238            read_with_if_match: true,
239            read_with_if_none_match: true,
240            read_with_if_modified_since: true,
241            read_with_if_unmodified_since: true,
242            read_with_version: true,
243
244            write: true,
245            write_can_empty: true,
246            write_can_append: true,
247            write_can_multi: true,
248            write_with_content_type: true,
249            write_with_cache_control: true,
250            write_with_content_disposition: true,
251            write_with_if_not_exists: true,
252            copy_with_if_not_exists: true,
253            copy_with_source_version: true,
254            restore: true,
255            restore_with_version: true,
256            // The min multipart size of COS is 1 MiB.
257            //
258            // ref: <https://www.tencentcloud.com/document/product/436/14112>
259            write_multi_min_size: Some(1024 * 1024),
260            // The max multipart size of COS is 5 GiB.
261            //
262            // ref: <https://www.tencentcloud.com/document/product/436/14112>
263            write_multi_max_size: if cfg!(target_pointer_width = "64") {
264                Some(5 * 1024 * 1024 * 1024)
265            } else {
266                Some(usize::MAX)
267            },
268            write_with_user_metadata: true,
269
270            delete: true,
271            delete_with_version: true,
272            copy: true,
273
274            list: true,
275            list_with_recursive: true,
276            list_with_versions: true,
277            list_with_deleted: true,
278
279            presign: true,
280            presign_stat: true,
281            presign_read: true,
282            presign_write: true,
283
284            shared: true,
285
286            ..Default::default()
287        };
288
289        Ok(CosBackend {
290            core: Arc::new(CosCore {
291                info,
292                capability,
293                bucket: bucket.clone(),
294                root,
295                endpoint,
296                signer,
297            }),
298        })
299    }
300}
301
302/// Backend for Tencent-Cloud COS services.
303#[derive(Debug, Clone)]
304pub struct CosBackend {
305    pub(crate) core: Arc<CosCore>,
306}
307
308impl Service for CosBackend {
309    type Reader = oio::StreamReader<CosReader>;
310    type Writer = CosWriters;
311    type Lister = CosListers;
312    type Deleter = oio::OneShotDeleter<CosDeleter>;
313    type Copier = oio::OneShotCopier;
314    type Composer = ();
315
316    fn info(&self) -> ServiceInfo {
317        self.core.info.clone()
318    }
319
320    fn capability(&self) -> Capability {
321        self.core.capability
322    }
323
324    async fn create_dir(
325        &self,
326        _ctx: &OperationContext,
327        _path: &str,
328        _args: OpCreateDir,
329    ) -> Result<RpCreateDir> {
330        Err(Error::new(
331            ErrorKind::Unsupported,
332            "operation is not supported",
333        ))
334    }
335
336    async fn stat(&self, ctx: &OperationContext, path: &str, args: OpStat) -> Result<RpStat> {
337        let resp = self.core.cos_head_object(ctx, path, &args).await?;
338
339        let status = resp.status();
340
341        match status {
342            StatusCode::OK => {
343                let headers = resp.headers();
344                let mut meta = parse_into_metadata(path, headers)?.into_builder();
345
346                let user_meta = parse_prefixed_headers(headers, "x-cos-meta-");
347                if !user_meta.is_empty() {
348                    meta.user_metadata(user_meta);
349                }
350
351                if let Some(v) = parse_header_to_str(headers, constants::X_COS_VERSION_ID)?
352                    && v != "null"
353                {
354                    meta.version(v);
355                }
356
357                Ok(RpStat::new(meta.build()))
358            }
359            _ => Err(parse_error(
360                ErrorContext::new(ServiceOperation("HeadObject"))
361                    .with_caller_condition(args.is_conditional()),
362                resp,
363            )),
364        }
365    }
366    fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
367        let output: oio::StreamReader<CosReader> = {
368            Ok(oio::StreamReader::new(CosReader::new(
369                self.clone(),
370                ctx.clone(),
371                path,
372                args,
373            )))
374        }?;
375
376        Ok(output)
377    }
378
379    fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
380        let output: CosWriters = {
381            let writer = CosWriter::new(self.core.clone(), ctx.clone(), path, args.clone());
382
383            let w = if args.append() {
384                CosWriters::Two(oio::AppendWriter::new(writer))
385            } else {
386                CosWriters::One(oio::MultipartWriter::new(
387                    ctx.executor().clone(),
388                    writer,
389                    args.concurrent(),
390                ))
391            };
392
393            Ok(w)
394        }?;
395
396        Ok(output)
397    }
398
399    fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
400        let output: oio::OneShotDeleter<CosDeleter> = {
401            Ok(oio::OneShotDeleter::new(CosDeleter::new(
402                self.core.clone(),
403                ctx.clone(),
404            )))
405        }?;
406
407        Ok(output)
408    }
409
410    fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
411        let output: CosListers = {
412            let l = if args.versions() || args.deleted() {
413                TwoWays::Two(oio::PageLister::new(CosObjectVersionsLister::new(
414                    self.core.clone(),
415                    ctx.clone(),
416                    path,
417                    args,
418                )))
419            } else {
420                TwoWays::One(oio::PageLister::new(CosLister::new(
421                    self.core.clone(),
422                    ctx.clone(),
423                    path,
424                    args.recursive(),
425                    args.limit(),
426                )))
427            };
428
429            Ok(l)
430        }?;
431
432        Ok(output)
433    }
434
435    fn copy(
436        &self,
437        ctx: &OperationContext,
438        from: &str,
439        to: &str,
440        args: OpCopy,
441    ) -> Result<Self::Copier> {
442        let core = self.core.clone();
443        let ctx = ctx.clone();
444        let from = from.to_string();
445        let to = to.to_string();
446        Ok(oio::OneShotCopier::new(async move {
447            let source_size = match args.source_content_length_hint() {
448                Some(size) => size,
449                None => {
450                    let stat_args = options::StatOptions {
451                        version: args.source_version().map(str::to_owned),
452                        ..Default::default()
453                    }
454                    .into();
455                    let resp = core.cos_head_object(&ctx, &from, &stat_args).await?;
456                    match resp.status() {
457                        StatusCode::OK => {
458                            parse_into_metadata(&from, resp.headers())?.content_length()
459                        }
460                        _ => {
461                            return Err(parse_error(
462                                ErrorContext::new(ServiceOperation("HeadObject")),
463                                resp,
464                            ));
465                        }
466                    }
467                }
468            };
469            let resp = core.cos_copy_object(&ctx, &from, &to, &args).await?;
470
471            let status = resp.status();
472
473            match status {
474                StatusCode::OK => Ok(MetadataBuilder::file(source_size).build()),
475                _ => Err(parse_error(
476                    ErrorContext::new(ServiceOperation("CopyObject"))
477                        .with_if_not_exists(args.if_not_exists()),
478                    resp,
479                )),
480            }
481        }))
482    }
483
484    async fn rename(
485        &self,
486        _ctx: &OperationContext,
487        _from: &str,
488        _to: &str,
489        _args: OpRename,
490    ) -> Result<RpRename> {
491        Err(Error::new(
492            ErrorKind::Unsupported,
493            "operation is not supported",
494        ))
495    }
496
497    async fn restore(
498        &self,
499        ctx: &OperationContext,
500        path: &str,
501        args: OpRestore,
502    ) -> Result<RpRestore> {
503        if let Some(version) = args.version() {
504            let copy_args = OpCopy::from_options(
505                &self.capability(),
506                options::CopyOptions {
507                    source_version: Some(version.to_owned()),
508                    ..Default::default()
509                },
510            )?;
511            let resp = self
512                .core
513                .cos_copy_object(ctx, path, path, &copy_args)
514                .await?;
515            return match resp.status() {
516                StatusCode::OK => Ok(RpRestore::new()),
517                _ => Err(parse_error(
518                    ErrorContext::new(ServiceOperation("CopyObject")),
519                    resp,
520                )),
521            };
522        }
523
524        let resp = self
525            .core
526            .cos_list_object_versions(ctx, path, "", Some(1), "", "")
527            .await?;
528        if resp.status() != StatusCode::OK {
529            return Err(parse_error(
530                ErrorContext::new(ServiceOperation("ListObjectVersions")),
531                resp,
532            ));
533        }
534
535        let output: ListObjectVersionsOutput =
536            quick_xml::de::from_reader(resp.into_body().reader())
537                .map_err(new_xml_deserialize_error)
538                .map_err(Error::set_temporary)?;
539        let abs_path = build_abs_path(&self.core.root, path);
540
541        if output
542            .version
543            .iter()
544            .any(|version| version.key == abs_path && version.is_latest)
545        {
546            return Ok(RpRestore::new());
547        }
548
549        let Some(marker) = output
550            .delete_marker
551            .into_iter()
552            .find(|marker| marker.key == abs_path && marker.is_latest)
553        else {
554            return Err(Error::new(
555                ErrorKind::NotFound,
556                "no live object or current delete marker exists",
557            ));
558        };
559
560        let delete_args = OpDelete::from_options(
561            &self.capability(),
562            options::DeleteOptions {
563                version: Some(marker.version_id),
564                ..Default::default()
565            },
566        )?;
567        let resp = self.core.cos_delete_object(ctx, path, &delete_args).await?;
568        match resp.status() {
569            StatusCode::NO_CONTENT => {
570                // COS can still list a marker after it was deleted. Only a
571                // deletion of an existing marker returns this response header.
572                if resp
573                    .headers()
574                    .get("x-cos-delete-marker")
575                    .is_some_and(|v| v == "true")
576                {
577                    Ok(RpRestore::new())
578                } else {
579                    Err(Error::new(
580                        ErrorKind::Unexpected,
581                        "the listed delete marker no longer exists; retry after the version listing updates",
582                    )
583                    .set_temporary())
584                }
585            }
586            _ => Err(parse_error(
587                ErrorContext::new(ServiceOperation("DeleteObject")),
588                resp,
589            )),
590        }
591    }
592
593    async fn presign(
594        &self,
595        ctx: &OperationContext,
596        path: &str,
597        args: OpPresign,
598    ) -> Result<RpPresign> {
599        let req = match args.operation() {
600            PresignOperation::Stat(v) => self.core.cos_head_object_request(path, v),
601            PresignOperation::Read(range, v) => self.core.cos_get_object_request(path, *range, v),
602            PresignOperation::Write(v) => {
603                self.core
604                    .cos_put_object_request(path, None, v, Buffer::new())
605            }
606            PresignOperation::Delete(_) => Err(Error::new(
607                ErrorKind::Unsupported,
608                "operation is not supported",
609            )),
610            _ => Err(Error::new(
611                ErrorKind::Unsupported,
612                "operation is not supported",
613            )),
614        };
615        let req = req?;
616        let req = self.core.sign_query(ctx, req, args.expire()).await?;
617
618        // We don't need this request anymore, consume it directly.
619        let (parts, _) = req.into_parts();
620
621        Ok(RpPresign::new(PresignedRequest::new(
622            parts.method,
623            parts.uri,
624            parts.headers,
625        )))
626    }
627}
628
629/// Compose the request endpoint for a bucket, as `scheme://bucket.host[:port]`.
630///
631/// Extracted so it can be unit tested, mirroring `S3Builder::build_endpoint`.
632fn build_endpoint(uri: &Uri, bucket: &str) -> Result<String> {
633    let scheme = uri.scheme_str().unwrap_or("https");
634
635    let host = uri.host().ok_or_else(|| {
636        Error::new(ErrorKind::ConfigInvalid, "endpoint host is empty")
637            .with_context("service", COS_SCHEME)
638            .with_context("endpoint", uri.to_string())
639    })?;
640
641    // If the endpoint already carries the bucket as its leftmost label, don't add it twice.
642    let host = host.strip_prefix(&format!("{bucket}.")).unwrap_or(host);
643
644    // Keep the port. `Uri::host` omits it, so composing from the host alone silently sent every
645    // request to the scheme default.
646    Ok(match uri.port_u16() {
647        Some(port) => format!("{scheme}://{bucket}.{host}:{port}"),
648        None => format!("{scheme}://{bucket}.{host}"),
649    })
650}
651
652#[cfg(test)]
653mod tests {
654    use super::*;
655
656    fn endpoint_of(raw: &str, bucket: &str) -> String {
657        build_endpoint(&raw.parse::<Uri>().unwrap(), bucket).unwrap()
658    }
659
660    #[test]
661    fn build_endpoint_keeps_a_custom_port() {
662        assert_eq!(
663            endpoint_of(
664                "https://cos.internal.example.com:8443",
665                "examplebucket-1250000000"
666            ),
667            "https://examplebucket-1250000000.cos.internal.example.com:8443"
668        );
669    }
670
671    #[test]
672    fn build_endpoint_prefixes_the_bucket() {
673        assert_eq!(
674            endpoint_of(
675                "https://cos.ap-guangzhou.myqcloud.com",
676                "examplebucket-1250000000"
677            ),
678            "https://examplebucket-1250000000.cos.ap-guangzhou.myqcloud.com"
679        );
680    }
681
682    #[test]
683    fn build_endpoint_does_not_repeat_a_bucket_already_in_the_host() {
684        // The previous `replace("//{bucket}.", "//")` could never match, because Uri::host never
685        // contains "//", so this doubled the bucket label.
686        assert_eq!(
687            endpoint_of(
688                "https://examplebucket-1250000000.cos.ap-guangzhou.myqcloud.com",
689                "examplebucket-1250000000"
690            ),
691            "https://examplebucket-1250000000.cos.ap-guangzhou.myqcloud.com"
692        );
693    }
694
695    #[test]
696    fn build_endpoint_defaults_the_scheme_to_https() {
697        // A bare host parses with no scheme; "//host" parses with no host at all, which is why
698        // the missing-host case below is a real input rather than a contrived one.
699        assert_eq!(
700            endpoint_of("cos.ap-guangzhou.myqcloud.com", "b"),
701            "https://b.cos.ap-guangzhou.myqcloud.com"
702        );
703    }
704
705    #[test]
706    fn build_endpoint_reports_a_missing_host_instead_of_panicking() {
707        // Both of these parse to host = None. The previous code called .unwrap() on that.
708        for raw in ["/just/a/path", "//cos.ap-guangzhou.myqcloud.com"] {
709            let uri = raw.parse::<Uri>().unwrap();
710            assert!(
711                build_endpoint(&uri, "b").is_err(),
712                "expected an error for {raw}"
713            );
714        }
715    }
716}