opendal/services/gcs/
core.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19use std::fmt::Debug;
20use std::fmt::Formatter;
21use std::fmt::Write;
22use std::sync::Arc;
23use std::time::Duration;
24
25use backon::ExponentialBuilder;
26use backon::Retryable;
27use bytes::{Buf, Bytes};
28use http::header::CONTENT_ENCODING;
29use http::header::CONTENT_LENGTH;
30use http::header::CONTENT_TYPE;
31use http::header::HOST;
32use http::header::IF_MATCH;
33use http::header::IF_MODIFIED_SINCE;
34use http::header::IF_NONE_MATCH;
35use http::header::IF_UNMODIFIED_SINCE;
36use http::Request;
37use http::Response;
38use reqsign::GoogleCredential;
39use reqsign::GoogleCredentialLoader;
40use reqsign::GoogleSigner;
41use reqsign::GoogleToken;
42use reqsign::GoogleTokenLoader;
43use serde::Deserialize;
44use serde::Serialize;
45use std::sync::LazyLock;
46
47use super::uri::percent_encode_path;
48use crate::raw::*;
49use crate::*;
50use constants::*;
51
52pub mod constants {
53    pub const X_GOOG_ACL: &str = "x-goog-acl";
54    pub const X_GOOG_STORAGE_CLASS: &str = "x-goog-storage-class";
55    pub const X_GOOG_META_PREFIX: &str = "x-goog-meta-";
56}
57
58pub struct GcsCore {
59    pub info: Arc<AccessorInfo>,
60    pub endpoint: String,
61    pub bucket: String,
62    pub root: String,
63
64    pub signer: GoogleSigner,
65    pub token_loader: GoogleTokenLoader,
66    pub token: Option<String>,
67    pub scope: String,
68    pub credential_loader: GoogleCredentialLoader,
69
70    pub predefined_acl: Option<String>,
71    pub default_storage_class: Option<String>,
72
73    pub allow_anonymous: bool,
74}
75
76impl Debug for GcsCore {
77    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
78        let mut de = f.debug_struct("Backend");
79        de.field("endpoint", &self.endpoint)
80            .field("bucket", &self.bucket)
81            .field("root", &self.root)
82            .finish_non_exhaustive()
83    }
84}
85
86static BACKOFF: LazyLock<ExponentialBuilder> =
87    LazyLock::new(|| ExponentialBuilder::default().with_jitter());
88
89impl GcsCore {
90    async fn load_token(&self) -> Result<Option<GoogleToken>> {
91        if let Some(token) = &self.token {
92            return Ok(Some(GoogleToken::new(token, usize::MAX, &self.scope)));
93        }
94
95        let cred = { || self.token_loader.load() }
96            .retry(*BACKOFF)
97            .await
98            .map_err(new_request_credential_error)?;
99
100        if let Some(cred) = cred {
101            return Ok(Some(cred));
102        }
103
104        if self.allow_anonymous {
105            return Ok(None);
106        }
107
108        Err(Error::new(
109            ErrorKind::ConfigInvalid,
110            "no valid credential found",
111        ))
112    }
113
114    fn load_credential(&self) -> Result<Option<GoogleCredential>> {
115        let cred = self
116            .credential_loader
117            .load()
118            .map_err(new_request_credential_error)?;
119
120        if let Some(cred) = cred {
121            return Ok(Some(cred));
122        }
123
124        if self.allow_anonymous {
125            return Ok(None);
126        }
127
128        Err(Error::new(
129            ErrorKind::ConfigInvalid,
130            "no valid credential found",
131        ))
132    }
133
134    pub async fn sign<T>(&self, req: &mut Request<T>) -> Result<()> {
135        if let Some(cred) = self.load_token().await? {
136            self.signer
137                .sign(req, &cred)
138                .map_err(new_request_sign_error)?;
139        } else {
140            return Ok(());
141        }
142
143        // Always remove host header, let users' client to set it based on HTTP
144        // version.
145        //
146        // As discussed in <https://github.com/seanmonstar/reqwest/issues/1809>,
147        // google server could send RST_STREAM of PROTOCOL_ERROR if our request
148        // contains host header.
149        req.headers_mut().remove(HOST);
150
151        Ok(())
152    }
153
154    pub fn sign_query<T>(&self, req: &mut Request<T>, duration: Duration) -> Result<()> {
155        if let Some(cred) = self.load_credential()? {
156            self.signer
157                .sign_query(req, duration, &cred)
158                .map_err(new_request_sign_error)?;
159        } else {
160            return Ok(());
161        }
162
163        // Always remove host header, let users' client to set it based on HTTP
164        // version.
165        //
166        // As discussed in <https://github.com/seanmonstar/reqwest/issues/1809>,
167        // google server could send RST_STREAM of PROTOCOL_ERROR if our request
168        // contains host header.
169        req.headers_mut().remove(HOST);
170
171        Ok(())
172    }
173
174    #[inline]
175    pub async fn send(&self, req: Request<Buffer>) -> Result<Response<Buffer>> {
176        self.info.http_client().send(req).await
177    }
178}
179
180impl GcsCore {
181    pub fn gcs_get_object_request(
182        &self,
183        path: &str,
184        range: BytesRange,
185        args: &OpRead,
186    ) -> Result<Request<Buffer>> {
187        let p = build_abs_path(&self.root, path);
188
189        let url = format!(
190            "{}/storage/v1/b/{}/o/{}?alt=media",
191            self.endpoint,
192            self.bucket,
193            percent_encode_path(&p)
194        );
195
196        let mut req = Request::get(&url);
197
198        if let Some(if_match) = args.if_match() {
199            req = req.header(IF_MATCH, if_match);
200        }
201        if let Some(if_none_match) = args.if_none_match() {
202            req = req.header(IF_NONE_MATCH, if_none_match);
203        }
204        if !range.is_full() {
205            req = req.header(http::header::RANGE, range.to_header());
206        }
207
208        let req = req.extension(Operation::Read);
209
210        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
211
212        Ok(req)
213    }
214
215    // It's for presign operation. Gcs only supports query sign over XML API.
216    pub fn gcs_get_object_xml_request(&self, path: &str, args: &OpRead) -> Result<Request<Buffer>> {
217        let p = build_abs_path(&self.root, path);
218
219        let url = format!("{}/{}/{}", self.endpoint, self.bucket, p);
220
221        let mut req = Request::get(&url);
222
223        if let Some(if_match) = args.if_match() {
224            req = req.header(IF_MATCH, if_match);
225        }
226        if let Some(if_none_match) = args.if_none_match() {
227            req = req.header(IF_NONE_MATCH, if_none_match);
228        }
229
230        if let Some(if_modified_since) = args.if_modified_since() {
231            req = req.header(
232                IF_MODIFIED_SINCE,
233                format_datetime_into_http_date(if_modified_since),
234            );
235        }
236
237        if let Some(if_unmodified_since) = args.if_unmodified_since() {
238            req = req.header(
239                IF_UNMODIFIED_SINCE,
240                format_datetime_into_http_date(if_unmodified_since),
241            );
242        }
243
244        let req = req.extension(Operation::Read);
245
246        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
247
248        Ok(req)
249    }
250
251    pub async fn gcs_get_object(
252        &self,
253        path: &str,
254        range: BytesRange,
255        args: &OpRead,
256    ) -> Result<Response<HttpBody>> {
257        let mut req = self.gcs_get_object_request(path, range, args)?;
258
259        self.sign(&mut req).await?;
260        self.info.http_client().fetch(req).await
261    }
262
263    pub fn gcs_insert_object_request(
264        &self,
265        path: &str,
266        size: Option<u64>,
267        op: &OpWrite,
268        body: Buffer,
269    ) -> Result<Request<Buffer>> {
270        let p = build_abs_path(&self.root, path);
271
272        let request_metadata = InsertRequestMetadata {
273            storage_class: self.default_storage_class.as_deref(),
274            cache_control: op.cache_control(),
275            content_type: op.content_type(),
276            content_encoding: op.content_encoding(),
277            metadata: op.user_metadata(),
278        };
279
280        let mut url = format!(
281            "{}/upload/storage/v1/b/{}/o?uploadType={}&name={}",
282            self.endpoint,
283            self.bucket,
284            if request_metadata.is_empty() {
285                "media"
286            } else {
287                "multipart"
288            },
289            percent_encode_path(&p)
290        );
291
292        if let Some(acl) = &self.predefined_acl {
293            write!(&mut url, "&predefinedAcl={}", acl).unwrap();
294        }
295
296        // Makes the operation conditional on whether the object's current generation
297        // matches the given value. Setting to 0 makes the operation succeed only if
298        // there are no live versions of the object.
299        if op.if_not_exists() {
300            write!(&mut url, "&ifGenerationMatch=0").unwrap();
301        }
302
303        let mut req = Request::post(&url);
304
305        req = req.header(CONTENT_LENGTH, size.unwrap_or_default());
306
307        if request_metadata.is_empty() {
308            let req = req.extension(Operation::Write);
309            // If the metadata is empty, we do not set any `Content-Type` header,
310            // since if we had it in the `op.content_type()`, it would be already set in the
311            // `multipart` metadata body and this branch won't be executed.
312            let req = req.body(body).map_err(new_request_build_error)?;
313            Ok(req)
314        } else {
315            let mut multipart = Multipart::new();
316            let metadata_part = RelatedPart::new()
317                .header(
318                    CONTENT_TYPE,
319                    "application/json; charset=UTF-8".parse().unwrap(),
320                )
321                .content(
322                    serde_json::to_vec(&request_metadata)
323                        .expect("metadata serialization should succeed"),
324                );
325            multipart = multipart.part(metadata_part);
326
327            // Content-Type must be set, even if it is set in the metadata part
328            let content_type = op
329                .content_type()
330                .unwrap_or("application/octet-stream")
331                .parse()
332                .expect("Failed to parse content-type");
333            let media_part = RelatedPart::new()
334                .header(CONTENT_TYPE, content_type)
335                .content(body);
336            multipart = multipart.part(media_part);
337
338            let req = multipart.apply(Request::post(url).extension(Operation::Write))?;
339
340            Ok(req)
341        }
342    }
343
344    // It's for presign operation. Gcs only supports query sign over XML API.
345    pub fn gcs_insert_object_xml_request(
346        &self,
347        path: &str,
348        args: &OpWrite,
349        body: Buffer,
350    ) -> Result<Request<Buffer>> {
351        let p = build_abs_path(&self.root, path);
352
353        let url = format!("{}/{}/{}", self.endpoint, self.bucket, p);
354
355        let mut req = Request::put(&url);
356
357        if let Some(user_metadata) = args.user_metadata() {
358            for (key, value) in user_metadata {
359                req = req.header(format!("{X_GOOG_META_PREFIX}{key}"), value)
360            }
361        }
362
363        if let Some(content_type) = args.content_type() {
364            req = req.header(CONTENT_TYPE, content_type);
365        }
366
367        if let Some(content_encoding) = args.content_encoding() {
368            req = req.header(CONTENT_ENCODING, content_encoding);
369        }
370
371        if let Some(acl) = &self.predefined_acl {
372            req = req.header(X_GOOG_ACL, acl);
373        }
374
375        if let Some(storage_class) = &self.default_storage_class {
376            req = req.header(X_GOOG_STORAGE_CLASS, storage_class);
377        }
378
379        let req = req.extension(Operation::Write);
380
381        let req = req.body(body).map_err(new_request_build_error)?;
382
383        Ok(req)
384    }
385
386    pub fn gcs_head_object_request(&self, path: &str, args: &OpStat) -> Result<Request<Buffer>> {
387        let p = build_abs_path(&self.root, path);
388
389        let url = format!(
390            "{}/storage/v1/b/{}/o/{}",
391            self.endpoint,
392            self.bucket,
393            percent_encode_path(&p)
394        );
395
396        let mut req = Request::get(&url);
397
398        if let Some(if_none_match) = args.if_none_match() {
399            req = req.header(IF_NONE_MATCH, if_none_match);
400        }
401
402        if let Some(if_match) = args.if_match() {
403            req = req.header(IF_MATCH, if_match);
404        }
405
406        let req = req.extension(Operation::Stat);
407
408        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
409
410        Ok(req)
411    }
412
413    // It's for presign operation. Gcs only supports query sign over XML API.
414    pub fn gcs_head_object_xml_request(
415        &self,
416        path: &str,
417        args: &OpStat,
418    ) -> Result<Request<Buffer>> {
419        let p = build_abs_path(&self.root, path);
420
421        let url = format!("{}/{}/{}", self.endpoint, self.bucket, p);
422
423        let mut req = Request::head(&url);
424
425        if let Some(if_none_match) = args.if_none_match() {
426            req = req.header(IF_NONE_MATCH, if_none_match);
427        }
428
429        if let Some(if_match) = args.if_match() {
430            req = req.header(IF_MATCH, if_match);
431        }
432
433        let req = req.extension(Operation::Stat);
434
435        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
436
437        Ok(req)
438    }
439
440    pub async fn gcs_get_object_metadata(
441        &self,
442        path: &str,
443        args: &OpStat,
444    ) -> Result<Response<Buffer>> {
445        let mut req = self.gcs_head_object_request(path, args)?;
446
447        self.sign(&mut req).await?;
448
449        self.send(req).await
450    }
451
452    pub async fn gcs_delete_object(&self, path: &str) -> Result<Response<Buffer>> {
453        let mut req = self.gcs_delete_object_request(path)?;
454
455        self.sign(&mut req).await?;
456        self.send(req).await
457    }
458
459    pub fn gcs_delete_object_request(&self, path: &str) -> Result<Request<Buffer>> {
460        let p = build_abs_path(&self.root, path);
461
462        let url = format!(
463            "{}/storage/v1/b/{}/o/{}",
464            self.endpoint,
465            self.bucket,
466            percent_encode_path(&p)
467        );
468
469        Request::delete(&url)
470            .body(Buffer::new())
471            .map_err(new_request_build_error)
472    }
473
474    pub async fn gcs_delete_objects(&self, paths: Vec<String>) -> Result<Response<Buffer>> {
475        let uri = format!("{}/batch/storage/v1", self.endpoint);
476
477        let mut multipart = Multipart::new();
478
479        for (idx, path) in paths.iter().enumerate() {
480            let req = self.gcs_delete_object_request(path)?;
481
482            multipart = multipart.part(
483                MixedPart::from_request(req).part_header("content-id".parse().unwrap(), idx.into()),
484            );
485        }
486
487        let req = Request::post(uri).extension(Operation::Delete);
488        let mut req = multipart.apply(req)?;
489
490        self.sign(&mut req).await?;
491        self.send(req).await
492    }
493
494    pub async fn gcs_copy_object(&self, from: &str, to: &str) -> Result<Response<Buffer>> {
495        let source = build_abs_path(&self.root, from);
496        let dest = build_abs_path(&self.root, to);
497
498        let req_uri = format!(
499            "{}/storage/v1/b/{}/o/{}/copyTo/b/{}/o/{}",
500            self.endpoint,
501            self.bucket,
502            percent_encode_path(&source),
503            self.bucket,
504            percent_encode_path(&dest)
505        );
506
507        let mut req = Request::post(req_uri)
508            .header(CONTENT_LENGTH, 0)
509            .extension(Operation::Copy)
510            .body(Buffer::new())
511            .map_err(new_request_build_error)?;
512
513        self.sign(&mut req).await?;
514        self.send(req).await
515    }
516
517    pub async fn gcs_list_objects(
518        &self,
519        path: &str,
520        page_token: &str,
521        delimiter: &str,
522        limit: Option<usize>,
523        start_after: Option<String>,
524    ) -> Result<Response<Buffer>> {
525        let p = build_abs_path(&self.root, path);
526
527        let url = format!("{}/storage/v1/b/{}/o", self.endpoint, self.bucket,);
528
529        let mut url = QueryPairsWriter::new(&url);
530        url = url.push("prefix", &percent_encode_path(&p));
531
532        if !delimiter.is_empty() {
533            url = url.push("delimiter", delimiter);
534        }
535        if let Some(limit) = limit {
536            url = url.push("maxResults", &limit.to_string());
537        }
538        if let Some(start_after) = start_after {
539            let start_after = build_abs_path(&self.root, &start_after);
540            url = url.push("startOffset", &percent_encode_path(&start_after));
541        }
542
543        if !page_token.is_empty() {
544            // NOTE:
545            //
546            // GCS uses pageToken in request and nextPageToken in response
547            //
548            // Don't know how will those tokens be like so this part are copied
549            // directly from AWS S3 service.
550            url = url.push("pageToken", &percent_encode_path(page_token));
551        }
552
553        let mut req = Request::get(url.finish())
554            .extension(Operation::List)
555            .body(Buffer::new())
556            .map_err(new_request_build_error)?;
557
558        self.sign(&mut req).await?;
559
560        self.send(req).await
561    }
562
563    pub async fn gcs_initiate_multipart_upload(&self, path: &str) -> Result<Response<Buffer>> {
564        let p = build_abs_path(&self.root, path);
565
566        let url = format!("{}/{}/{}?uploads", self.endpoint, self.bucket, p);
567
568        let mut req = Request::post(&url)
569            .header(CONTENT_LENGTH, 0)
570            .extension(Operation::Write)
571            .body(Buffer::new())
572            .map_err(new_request_build_error)?;
573
574        self.sign(&mut req).await?;
575        self.send(req).await
576    }
577
578    pub async fn gcs_upload_part(
579        &self,
580        path: &str,
581        upload_id: &str,
582        part_number: usize,
583        size: u64,
584        body: Buffer,
585    ) -> Result<Response<Buffer>> {
586        let p = build_abs_path(&self.root, path);
587
588        let url = format!(
589            "{}/{}/{}?partNumber={}&uploadId={}",
590            self.endpoint,
591            self.bucket,
592            percent_encode_path(&p),
593            part_number,
594            percent_encode_path(upload_id)
595        );
596
597        let mut req = Request::put(&url);
598
599        req = req.header(CONTENT_LENGTH, size);
600
601        let req = req.extension(Operation::Write);
602
603        let mut req = req.body(body).map_err(new_request_build_error)?;
604
605        self.sign(&mut req).await?;
606        self.send(req).await
607    }
608
609    pub async fn gcs_complete_multipart_upload(
610        &self,
611        path: &str,
612        upload_id: &str,
613        parts: Vec<CompleteMultipartUploadRequestPart>,
614    ) -> Result<Response<Buffer>> {
615        let p = build_abs_path(&self.root, path);
616
617        let url = format!(
618            "{}/{}/{}?uploadId={}",
619            self.endpoint,
620            self.bucket,
621            percent_encode_path(&p),
622            percent_encode_path(upload_id)
623        );
624
625        let req = Request::post(&url);
626
627        let content = quick_xml::se::to_string(&CompleteMultipartUploadRequest { part: parts })
628            .map_err(new_xml_serialize_error)?;
629        // Make sure content length has been set to avoid post with chunked encoding.
630        let req = req.header(CONTENT_LENGTH, content.len());
631        // Set content-type to `application/xml` to avoid mixed with form post.
632        let req = req.header(CONTENT_TYPE, "application/xml");
633
634        let req = req.extension(Operation::Write);
635
636        let mut req = req
637            .body(Buffer::from(Bytes::from(content)))
638            .map_err(new_request_build_error)?;
639
640        self.sign(&mut req).await?;
641        self.send(req).await
642    }
643
644    pub async fn gcs_abort_multipart_upload(
645        &self,
646        path: &str,
647        upload_id: &str,
648    ) -> Result<Response<Buffer>> {
649        let p = build_abs_path(&self.root, path);
650
651        let url = format!(
652            "{}/{}/{}?uploadId={}",
653            self.endpoint,
654            self.bucket,
655            percent_encode_path(&p),
656            percent_encode_path(upload_id)
657        );
658
659        let mut req = Request::delete(&url)
660            .extension(Operation::Write)
661            .body(Buffer::new())
662            .map_err(new_request_build_error)?;
663        self.sign(&mut req).await?;
664        self.send(req).await
665    }
666
667    pub fn build_metadata_from_object_response(path: &str, data: Buffer) -> Result<Metadata> {
668        let meta: GetObjectJsonResponse =
669            serde_json::from_reader(data.reader()).map_err(new_json_deserialize_error)?;
670
671        let mut m = Metadata::new(EntryMode::from_path(path));
672
673        m.set_etag(&meta.etag);
674        m.set_content_md5(&meta.md5_hash);
675
676        let size = meta
677            .size
678            .parse::<u64>()
679            .map_err(|e| Error::new(ErrorKind::Unexpected, "parse u64").set_source(e))?;
680        m.set_content_length(size);
681        if !meta.content_type.is_empty() {
682            m.set_content_type(&meta.content_type);
683        }
684
685        if !meta.content_encoding.is_empty() {
686            m.set_content_encoding(&meta.content_encoding);
687        }
688
689        if !meta.cache_control.is_empty() {
690            m.set_cache_control(&meta.cache_control);
691        }
692
693        if !meta.content_disposition.is_empty() {
694            m.set_content_disposition(&meta.content_disposition);
695        }
696
697        if !meta.generation.is_empty() {
698            m.set_version(&meta.generation);
699        }
700
701        m.set_last_modified(parse_datetime_from_rfc3339(&meta.updated)?);
702
703        if !meta.metadata.is_empty() {
704            m.with_user_metadata(meta.metadata);
705        }
706
707        Ok(m)
708    }
709}
710
711#[derive(Debug, Serialize)]
712#[serde(default, rename_all = "camelCase")]
713pub struct InsertRequestMetadata<'a> {
714    #[serde(skip_serializing_if = "Option::is_none")]
715    content_type: Option<&'a str>,
716    #[serde(skip_serializing_if = "Option::is_none")]
717    content_encoding: Option<&'a str>,
718    #[serde(skip_serializing_if = "Option::is_none")]
719    storage_class: Option<&'a str>,
720    #[serde(skip_serializing_if = "Option::is_none")]
721    cache_control: Option<&'a str>,
722    #[serde(skip_serializing_if = "Option::is_none")]
723    metadata: Option<&'a HashMap<String, String>>,
724}
725
726impl InsertRequestMetadata<'_> {
727    pub fn is_empty(&self) -> bool {
728        self.content_type.is_none()
729            && self.content_encoding.is_none()
730            && self.storage_class.is_none()
731            && self.cache_control.is_none()
732            // We could also put content-encoding in the url parameters
733            && self.content_encoding.is_none()
734            && self.metadata.is_none()
735    }
736}
737/// Response JSON from GCS list objects API.
738///
739/// refer to https://cloud.google.com/storage/docs/json_api/v1/objects/list for details
740#[derive(Default, Debug, Deserialize)]
741#[serde(default, rename_all = "camelCase")]
742pub struct ListResponse {
743    /// The continuation token.
744    ///
745    /// If this is the last page of results, then no continuation token is returned.
746    pub next_page_token: Option<String>,
747    /// Object name prefixes for objects that matched the listing request
748    /// but were excluded from [items] because of a delimiter.
749    pub prefixes: Vec<String>,
750    /// The list of objects, ordered lexicographically by name.
751    pub items: Vec<ListResponseItem>,
752}
753
754#[derive(Default, Debug, Eq, PartialEq, Deserialize)]
755#[serde(default, rename_all = "camelCase")]
756pub struct ListResponseItem {
757    pub name: String,
758    pub size: String,
759    // metadata
760    pub etag: String,
761    pub md5_hash: String,
762    pub updated: String,
763    pub content_type: String,
764}
765
766/// Result of CreateMultipartUpload
767#[derive(Default, Debug, Deserialize)]
768#[serde(default, rename_all = "PascalCase")]
769pub struct InitiateMultipartUploadResult {
770    pub upload_id: String,
771}
772
773/// Request of CompleteMultipartUploadRequest
774#[derive(Default, Debug, Serialize)]
775#[serde(default, rename = "CompleteMultipartUpload", rename_all = "PascalCase")]
776pub struct CompleteMultipartUploadRequest {
777    pub part: Vec<CompleteMultipartUploadRequestPart>,
778}
779
780#[derive(Clone, Default, Debug, Serialize)]
781#[serde(default, rename_all = "PascalCase")]
782pub struct CompleteMultipartUploadRequestPart {
783    #[serde(rename = "PartNumber")]
784    pub part_number: usize,
785    #[serde(rename = "ETag")]
786    pub etag: String,
787}
788
789/// The raw json response returned by [`get`](https://cloud.google.com/storage/docs/json_api/v1/objects/get)
790#[derive(Debug, Default, Deserialize)]
791#[serde(default, rename_all = "camelCase")]
792struct GetObjectJsonResponse {
793    /// GCS will return size in string.
794    ///
795    /// For example: `"size": "56535"`
796    size: String,
797    /// etag is not quoted.
798    ///
799    /// For example: `"etag": "CKWasoTgyPkCEAE="`
800    etag: String,
801    /// RFC3339 styled datetime string.
802    ///
803    /// For example: `"updated": "2022-08-15T11:33:34.866Z"`
804    updated: String,
805    /// Content md5 hash
806    ///
807    /// For example: `"md5Hash": "fHcEH1vPwA6eTPqxuasXcg=="`
808    md5_hash: String,
809    /// Content type of this object.
810    ///
811    /// For example: `"contentType": "image/png",`
812    content_type: String,
813    /// Content encoding of this object
814    ///
815    /// For example: "contentEncoding": "br"
816    content_encoding: String,
817    /// Content disposition of this object
818    content_disposition: String,
819    /// Cache-Control directive for the object data.
820    cache_control: String,
821    /// Content generation of this object. Used for object versioning and soft delete.
822    generation: String,
823    /// Custom metadata of this object.
824    ///
825    /// For example: `"metadata" : { "my-key": "my-value" }`
826    metadata: HashMap<String, String>,
827}
828
829#[cfg(test)]
830mod tests {
831    use super::*;
832
833    #[test]
834    fn test_deserialize_get_object_json_response() {
835        let content = r#"{
836    "kind": "storage#object",
837    "id": "example/1.png/1660563214863653",
838    "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png",
839    "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media",
840    "name": "1.png",
841    "bucket": "example",
842    "generation": "1660563214863653",
843    "metageneration": "1",
844    "contentType": "image/png",
845    "contentEncoding": "br",
846    "contentDisposition": "attachment",
847    "cacheControl": "public, max-age=3600",
848    "storageClass": "STANDARD",
849    "size": "56535",
850    "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==",
851    "crc32c": "j/un9g==",
852    "etag": "CKWasoTgyPkCEAE=",
853    "timeCreated": "2022-08-15T11:33:34.866Z",
854    "updated": "2022-08-15T11:33:34.866Z",
855    "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z",
856    "metadata" : {
857        "location" : "everywhere"
858  }
859}"#;
860
861        let meta = GcsCore::build_metadata_from_object_response("1.png", content.into())
862            .expect("parse metadata should not fail");
863
864        assert_eq!(meta.content_length(), 56535);
865        assert_eq!(
866            meta.last_modified(),
867            Some(
868                parse_datetime_from_rfc3339("2022-08-15T11:33:34.866Z")
869                    .expect("parse date should not fail")
870            )
871        );
872        assert_eq!(meta.content_md5(), Some("fHcEH1vPwA6eTPqxuasXcg=="));
873        assert_eq!(meta.etag(), Some("CKWasoTgyPkCEAE="));
874        assert_eq!(meta.content_type(), Some("image/png"));
875        assert_eq!(meta.content_encoding(), Some("br"));
876        assert_eq!(meta.content_disposition(), Some("attachment"));
877        assert_eq!(meta.cache_control(), Some("public, max-age=3600"));
878        assert_eq!(meta.version(), Some("1660563214863653"));
879
880        let metadata = HashMap::from_iter([("location".to_string(), "everywhere".to_string())]);
881        assert_eq!(meta.user_metadata(), Some(&metadata));
882    }
883
884    #[test]
885    fn test_deserialize_list_response() {
886        let content = r#"
887    {
888  "kind": "storage#objects",
889  "prefixes": [
890    "dir/",
891    "test/"
892  ],
893  "items": [
894    {
895      "kind": "storage#object",
896      "id": "example/1.png/1660563214863653",
897      "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png",
898      "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media",
899      "name": "1.png",
900      "bucket": "example",
901      "generation": "1660563214863653",
902      "metageneration": "1",
903      "contentType": "image/png",
904      "storageClass": "STANDARD",
905      "size": "56535",
906      "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==",
907      "crc32c": "j/un9g==",
908      "etag": "CKWasoTgyPkCEAE=",
909      "timeCreated": "2022-08-15T11:33:34.866Z",
910      "updated": "2022-08-15T11:33:34.866Z",
911      "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z"
912    },
913    {
914      "kind": "storage#object",
915      "id": "example/2.png/1660563214883337",
916      "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/2.png",
917      "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media",
918      "name": "2.png",
919      "bucket": "example",
920      "generation": "1660563214883337",
921      "metageneration": "1",
922      "contentType": "image/png",
923      "storageClass": "STANDARD",
924      "size": "45506",
925      "md5Hash": "e6LsGusU7pFJZk+114NV1g==",
926      "crc32c": "L00QAg==",
927      "etag": "CIm0s4TgyPkCEAE=",
928      "timeCreated": "2022-08-15T11:33:34.886Z",
929      "updated": "2022-08-15T11:33:34.886Z",
930      "timeStorageClassUpdated": "2022-08-15T11:33:34.886Z"
931    }
932  ]
933}
934    "#;
935
936        let output: ListResponse =
937            serde_json::from_str(content).expect("JSON deserialize must succeed");
938        assert!(output.next_page_token.is_none());
939        assert_eq!(output.items.len(), 2);
940        assert_eq!(output.items[0].name, "1.png");
941        assert_eq!(output.items[0].size, "56535");
942        assert_eq!(output.items[0].md5_hash, "fHcEH1vPwA6eTPqxuasXcg==");
943        assert_eq!(output.items[0].etag, "CKWasoTgyPkCEAE=");
944        assert_eq!(output.items[0].updated, "2022-08-15T11:33:34.866Z");
945        assert_eq!(output.items[1].name, "2.png");
946        assert_eq!(output.items[1].size, "45506");
947        assert_eq!(output.items[1].md5_hash, "e6LsGusU7pFJZk+114NV1g==");
948        assert_eq!(output.items[1].etag, "CIm0s4TgyPkCEAE=");
949        assert_eq!(output.items[1].updated, "2022-08-15T11:33:34.886Z");
950        assert_eq!(output.items[1].content_type, "image/png");
951        assert_eq!(output.prefixes, vec!["dir/", "test/"])
952    }
953
954    #[test]
955    fn test_deserialize_list_response_with_next_page_token() {
956        let content = r#"
957    {
958  "kind": "storage#objects",
959  "prefixes": [
960    "dir/",
961    "test/"
962  ],
963  "nextPageToken": "CgYxMC5wbmc=",
964  "items": [
965    {
966      "kind": "storage#object",
967      "id": "example/1.png/1660563214863653",
968      "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png",
969      "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media",
970      "name": "1.png",
971      "bucket": "example",
972      "generation": "1660563214863653",
973      "metageneration": "1",
974      "contentType": "image/png",
975      "storageClass": "STANDARD",
976      "size": "56535",
977      "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==",
978      "crc32c": "j/un9g==",
979      "etag": "CKWasoTgyPkCEAE=",
980      "timeCreated": "2022-08-15T11:33:34.866Z",
981      "updated": "2022-08-15T11:33:34.866Z",
982      "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z"
983    },
984    {
985      "kind": "storage#object",
986      "id": "example/2.png/1660563214883337",
987      "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/2.png",
988      "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media",
989      "name": "2.png",
990      "bucket": "example",
991      "generation": "1660563214883337",
992      "metageneration": "1",
993      "contentType": "image/png",
994      "storageClass": "STANDARD",
995      "size": "45506",
996      "md5Hash": "e6LsGusU7pFJZk+114NV1g==",
997      "crc32c": "L00QAg==",
998      "etag": "CIm0s4TgyPkCEAE=",
999      "timeCreated": "2022-08-15T11:33:34.886Z",
1000      "updated": "2022-08-15T11:33:34.886Z",
1001      "timeStorageClassUpdated": "2022-08-15T11:33:34.886Z"
1002    }
1003  ]
1004}
1005    "#;
1006
1007        let output: ListResponse =
1008            serde_json::from_str(content).expect("JSON deserialize must succeed");
1009        assert_eq!(output.next_page_token, Some("CgYxMC5wbmc=".to_string()));
1010        assert_eq!(output.items.len(), 2);
1011        assert_eq!(output.items[0].name, "1.png");
1012        assert_eq!(output.items[0].size, "56535");
1013        assert_eq!(output.items[0].md5_hash, "fHcEH1vPwA6eTPqxuasXcg==");
1014        assert_eq!(output.items[0].etag, "CKWasoTgyPkCEAE=");
1015        assert_eq!(output.items[0].updated, "2022-08-15T11:33:34.866Z");
1016        assert_eq!(output.items[1].name, "2.png");
1017        assert_eq!(output.items[1].size, "45506");
1018        assert_eq!(output.items[1].md5_hash, "e6LsGusU7pFJZk+114NV1g==");
1019        assert_eq!(output.items[1].etag, "CIm0s4TgyPkCEAE=");
1020        assert_eq!(output.items[1].updated, "2022-08-15T11:33:34.886Z");
1021        assert_eq!(output.prefixes, vec!["dir/", "test/"])
1022    }
1023}
opendal/services/gcs/core.rs

opendal/services/gcs/
core.rs
