opendal/services/gcs/
core.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19use std::fmt::Debug;
20use std::fmt::Formatter;
21use std::fmt::Write;
22use std::sync::Arc;
23use std::time::Duration;
24
25use backon::ExponentialBuilder;
26use backon::Retryable;
27use bytes::{Buf, Bytes};
28use http::header::CONTENT_ENCODING;
29use http::header::CONTENT_LENGTH;
30use http::header::CONTENT_TYPE;
31use http::header::HOST;
32use http::header::IF_MATCH;
33use http::header::IF_MODIFIED_SINCE;
34use http::header::IF_NONE_MATCH;
35use http::header::IF_UNMODIFIED_SINCE;
36use http::Request;
37use http::Response;
38use reqsign::GoogleCredential;
39use reqsign::GoogleCredentialLoader;
40use reqsign::GoogleSigner;
41use reqsign::GoogleToken;
42use reqsign::GoogleTokenLoader;
43use serde::Deserialize;
44use serde::Serialize;
45use std::sync::LazyLock;
46
47use super::uri::percent_encode_path;
48use crate::raw::*;
49use crate::*;
50use constants::*;
51
52pub mod constants {
53    pub const X_GOOG_ACL: &str = "x-goog-acl";
54    pub const X_GOOG_STORAGE_CLASS: &str = "x-goog-storage-class";
55    pub const X_GOOG_META_PREFIX: &str = "x-goog-meta-";
56}
57
58pub struct GcsCore {
59    pub info: Arc<AccessorInfo>,
60    pub endpoint: String,
61    pub bucket: String,
62    pub root: String,
63
64    pub signer: GoogleSigner,
65    pub token_loader: GoogleTokenLoader,
66    pub token: Option<String>,
67    pub scope: String,
68    pub credential_loader: GoogleCredentialLoader,
69
70    pub predefined_acl: Option<String>,
71    pub default_storage_class: Option<String>,
72
73    pub allow_anonymous: bool,
74}
75
76impl Debug for GcsCore {
77    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
78        let mut de = f.debug_struct("Backend");
79        de.field("endpoint", &self.endpoint)
80            .field("bucket", &self.bucket)
81            .field("root", &self.root)
82            .finish_non_exhaustive()
83    }
84}
85
86static BACKOFF: LazyLock<ExponentialBuilder> =
87    LazyLock::new(|| ExponentialBuilder::default().with_jitter());
88
89impl GcsCore {
90    async fn load_token(&self) -> Result<Option<GoogleToken>> {
91        if let Some(token) = &self.token {
92            return Ok(Some(GoogleToken::new(token, usize::MAX, &self.scope)));
93        }
94
95        let cred = { || self.token_loader.load() }
96            .retry(*BACKOFF)
97            .await
98            .map_err(new_request_credential_error)?;
99
100        if let Some(cred) = cred {
101            return Ok(Some(cred));
102        }
103
104        if self.allow_anonymous {
105            return Ok(None);
106        }
107
108        Err(Error::new(
109            ErrorKind::ConfigInvalid,
110            "no valid credential found",
111        ))
112    }
113
114    fn load_credential(&self) -> Result<Option<GoogleCredential>> {
115        let cred = self
116            .credential_loader
117            .load()
118            .map_err(new_request_credential_error)?;
119
120        if let Some(cred) = cred {
121            return Ok(Some(cred));
122        }
123
124        if self.allow_anonymous {
125            return Ok(None);
126        }
127
128        Err(Error::new(
129            ErrorKind::ConfigInvalid,
130            "no valid credential found",
131        ))
132    }
133
134    pub async fn sign<T>(&self, req: &mut Request<T>) -> Result<()> {
135        if let Some(cred) = self.load_token().await? {
136            self.signer
137                .sign(req, &cred)
138                .map_err(new_request_sign_error)?;
139        } else {
140            return Ok(());
141        }
142
143        // Always remove host header, let users' client to set it based on HTTP
144        // version.
145        //
146        // As discussed in <https://github.com/seanmonstar/reqwest/issues/1809>,
147        // google server could send RST_STREAM of PROTOCOL_ERROR if our request
148        // contains host header.
149        req.headers_mut().remove(HOST);
150
151        Ok(())
152    }
153
154    pub fn sign_query<T>(&self, req: &mut Request<T>, duration: Duration) -> Result<()> {
155        if let Some(cred) = self.load_credential()? {
156            self.signer
157                .sign_query(req, duration, &cred)
158                .map_err(new_request_sign_error)?;
159        } else {
160            return Ok(());
161        }
162
163        // Always remove host header, let users' client to set it based on HTTP
164        // version.
165        //
166        // As discussed in <https://github.com/seanmonstar/reqwest/issues/1809>,
167        // google server could send RST_STREAM of PROTOCOL_ERROR if our request
168        // contains host header.
169        req.headers_mut().remove(HOST);
170
171        Ok(())
172    }
173
174    #[inline]
175    pub async fn send(&self, req: Request<Buffer>) -> Result<Response<Buffer>> {
176        self.info.http_client().send(req).await
177    }
178}
179
180impl GcsCore {
181    pub fn gcs_get_object_request(
182        &self,
183        path: &str,
184        range: BytesRange,
185        args: &OpRead,
186    ) -> Result<Request<Buffer>> {
187        let p = build_abs_path(&self.root, path);
188
189        let url = format!(
190            "{}/storage/v1/b/{}/o/{}?alt=media",
191            self.endpoint,
192            self.bucket,
193            percent_encode_path(&p)
194        );
195
196        let mut req = Request::get(&url);
197
198        if let Some(if_match) = args.if_match() {
199            req = req.header(IF_MATCH, if_match);
200        }
201        if let Some(if_none_match) = args.if_none_match() {
202            req = req.header(IF_NONE_MATCH, if_none_match);
203        }
204        if !range.is_full() {
205            req = req.header(http::header::RANGE, range.to_header());
206        }
207
208        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
209
210        Ok(req)
211    }
212
213    // It's for presign operation. Gcs only supports query sign over XML API.
214    pub fn gcs_get_object_xml_request(&self, path: &str, args: &OpRead) -> Result<Request<Buffer>> {
215        let p = build_abs_path(&self.root, path);
216
217        let url = format!("{}/{}/{}", self.endpoint, self.bucket, p);
218
219        let mut req = Request::get(&url);
220
221        if let Some(if_match) = args.if_match() {
222            req = req.header(IF_MATCH, if_match);
223        }
224        if let Some(if_none_match) = args.if_none_match() {
225            req = req.header(IF_NONE_MATCH, if_none_match);
226        }
227
228        if let Some(if_modified_since) = args.if_modified_since() {
229            req = req.header(
230                IF_MODIFIED_SINCE,
231                format_datetime_into_http_date(if_modified_since),
232            );
233        }
234
235        if let Some(if_unmodified_since) = args.if_unmodified_since() {
236            req = req.header(
237                IF_UNMODIFIED_SINCE,
238                format_datetime_into_http_date(if_unmodified_since),
239            );
240        }
241
242        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
243
244        Ok(req)
245    }
246
247    pub async fn gcs_get_object(
248        &self,
249        path: &str,
250        range: BytesRange,
251        args: &OpRead,
252    ) -> Result<Response<HttpBody>> {
253        let mut req = self.gcs_get_object_request(path, range, args)?;
254
255        self.sign(&mut req).await?;
256        self.info.http_client().fetch(req).await
257    }
258
259    pub fn gcs_insert_object_request(
260        &self,
261        path: &str,
262        size: Option<u64>,
263        op: &OpWrite,
264        body: Buffer,
265    ) -> Result<Request<Buffer>> {
266        let p = build_abs_path(&self.root, path);
267
268        let request_metadata = InsertRequestMetadata {
269            storage_class: self.default_storage_class.as_deref(),
270            cache_control: op.cache_control(),
271            content_type: op.content_type(),
272            content_encoding: op.content_encoding(),
273            metadata: op.user_metadata(),
274        };
275
276        let mut url = format!(
277            "{}/upload/storage/v1/b/{}/o?uploadType={}&name={}",
278            self.endpoint,
279            self.bucket,
280            if request_metadata.is_empty() {
281                "media"
282            } else {
283                "multipart"
284            },
285            percent_encode_path(&p)
286        );
287
288        if let Some(acl) = &self.predefined_acl {
289            write!(&mut url, "&predefinedAcl={}", acl).unwrap();
290        }
291
292        // Makes the operation conditional on whether the object's current generation
293        // matches the given value. Setting to 0 makes the operation succeed only if
294        // there are no live versions of the object.
295        if op.if_not_exists() {
296            write!(&mut url, "&ifGenerationMatch=0").unwrap();
297        }
298
299        let mut req = Request::post(&url);
300
301        req = req.header(CONTENT_LENGTH, size.unwrap_or_default());
302
303        if request_metadata.is_empty() {
304            // If the metadata is empty, we do not set any `Content-Type` header,
305            // since if we had it in the `op.content_type()`, it would be already set in the
306            // `multipart` metadata body and this branch won't be executed.
307            let req = req.body(body).map_err(new_request_build_error)?;
308            Ok(req)
309        } else {
310            let mut multipart = Multipart::new();
311            let metadata_part = RelatedPart::new()
312                .header(
313                    CONTENT_TYPE,
314                    "application/json; charset=UTF-8".parse().unwrap(),
315                )
316                .content(
317                    serde_json::to_vec(&request_metadata)
318                        .expect("metadata serialization should succeed"),
319                );
320            multipart = multipart.part(metadata_part);
321
322            // Content-Type must be set, even if it is set in the metadata part
323            let content_type = op
324                .content_type()
325                .unwrap_or("application/octet-stream")
326                .parse()
327                .expect("Failed to parse content-type");
328            let media_part = RelatedPart::new()
329                .header(CONTENT_TYPE, content_type)
330                .content(body);
331            multipart = multipart.part(media_part);
332
333            let req = multipart.apply(Request::post(url))?;
334            Ok(req)
335        }
336    }
337
338    // It's for presign operation. Gcs only supports query sign over XML API.
339    pub fn gcs_insert_object_xml_request(
340        &self,
341        path: &str,
342        args: &OpWrite,
343        body: Buffer,
344    ) -> Result<Request<Buffer>> {
345        let p = build_abs_path(&self.root, path);
346
347        let url = format!("{}/{}/{}", self.endpoint, self.bucket, p);
348
349        let mut req = Request::put(&url);
350
351        if let Some(user_metadata) = args.user_metadata() {
352            for (key, value) in user_metadata {
353                req = req.header(format!("{X_GOOG_META_PREFIX}{key}"), value)
354            }
355        }
356
357        if let Some(content_type) = args.content_type() {
358            req = req.header(CONTENT_TYPE, content_type);
359        }
360
361        if let Some(content_encoding) = args.content_encoding() {
362            req = req.header(CONTENT_ENCODING, content_encoding);
363        }
364
365        if let Some(acl) = &self.predefined_acl {
366            req = req.header(X_GOOG_ACL, acl);
367        }
368
369        if let Some(storage_class) = &self.default_storage_class {
370            req = req.header(X_GOOG_STORAGE_CLASS, storage_class);
371        }
372
373        let req = req.body(body).map_err(new_request_build_error)?;
374
375        Ok(req)
376    }
377
378    pub fn gcs_head_object_request(&self, path: &str, args: &OpStat) -> Result<Request<Buffer>> {
379        let p = build_abs_path(&self.root, path);
380
381        let url = format!(
382            "{}/storage/v1/b/{}/o/{}",
383            self.endpoint,
384            self.bucket,
385            percent_encode_path(&p)
386        );
387
388        let mut req = Request::get(&url);
389
390        if let Some(if_none_match) = args.if_none_match() {
391            req = req.header(IF_NONE_MATCH, if_none_match);
392        }
393
394        if let Some(if_match) = args.if_match() {
395            req = req.header(IF_MATCH, if_match);
396        }
397
398        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
399
400        Ok(req)
401    }
402
403    // It's for presign operation. Gcs only supports query sign over XML API.
404    pub fn gcs_head_object_xml_request(
405        &self,
406        path: &str,
407        args: &OpStat,
408    ) -> Result<Request<Buffer>> {
409        let p = build_abs_path(&self.root, path);
410
411        let url = format!("{}/{}/{}", self.endpoint, self.bucket, p);
412
413        let mut req = Request::head(&url);
414
415        if let Some(if_none_match) = args.if_none_match() {
416            req = req.header(IF_NONE_MATCH, if_none_match);
417        }
418
419        if let Some(if_match) = args.if_match() {
420            req = req.header(IF_MATCH, if_match);
421        }
422
423        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
424
425        Ok(req)
426    }
427
428    pub async fn gcs_get_object_metadata(
429        &self,
430        path: &str,
431        args: &OpStat,
432    ) -> Result<Response<Buffer>> {
433        let mut req = self.gcs_head_object_request(path, args)?;
434
435        self.sign(&mut req).await?;
436
437        self.send(req).await
438    }
439
440    pub async fn gcs_delete_object(&self, path: &str) -> Result<Response<Buffer>> {
441        let mut req = self.gcs_delete_object_request(path)?;
442
443        self.sign(&mut req).await?;
444        self.send(req).await
445    }
446
447    pub fn gcs_delete_object_request(&self, path: &str) -> Result<Request<Buffer>> {
448        let p = build_abs_path(&self.root, path);
449
450        let url = format!(
451            "{}/storage/v1/b/{}/o/{}",
452            self.endpoint,
453            self.bucket,
454            percent_encode_path(&p)
455        );
456
457        Request::delete(&url)
458            .body(Buffer::new())
459            .map_err(new_request_build_error)
460    }
461
462    pub async fn gcs_delete_objects(&self, paths: Vec<String>) -> Result<Response<Buffer>> {
463        let uri = format!("{}/batch/storage/v1", self.endpoint);
464
465        let mut multipart = Multipart::new();
466
467        for (idx, path) in paths.iter().enumerate() {
468            let req = self.gcs_delete_object_request(path)?;
469
470            multipart = multipart.part(
471                MixedPart::from_request(req).part_header("content-id".parse().unwrap(), idx.into()),
472            );
473        }
474
475        let req = Request::post(uri);
476        let mut req = multipart.apply(req)?;
477
478        self.sign(&mut req).await?;
479        self.send(req).await
480    }
481
482    pub async fn gcs_copy_object(&self, from: &str, to: &str) -> Result<Response<Buffer>> {
483        let source = build_abs_path(&self.root, from);
484        let dest = build_abs_path(&self.root, to);
485
486        let req_uri = format!(
487            "{}/storage/v1/b/{}/o/{}/copyTo/b/{}/o/{}",
488            self.endpoint,
489            self.bucket,
490            percent_encode_path(&source),
491            self.bucket,
492            percent_encode_path(&dest)
493        );
494
495        let mut req = Request::post(req_uri)
496            .header(CONTENT_LENGTH, 0)
497            .body(Buffer::new())
498            .map_err(new_request_build_error)?;
499
500        self.sign(&mut req).await?;
501        self.send(req).await
502    }
503
504    pub async fn gcs_list_objects(
505        &self,
506        path: &str,
507        page_token: &str,
508        delimiter: &str,
509        limit: Option<usize>,
510        start_after: Option<String>,
511    ) -> Result<Response<Buffer>> {
512        let p = build_abs_path(&self.root, path);
513
514        let mut url = format!(
515            "{}/storage/v1/b/{}/o?prefix={}",
516            self.endpoint,
517            self.bucket,
518            percent_encode_path(&p)
519        );
520        if !delimiter.is_empty() {
521            write!(url, "&delimiter={delimiter}").expect("write into string must succeed");
522        }
523        if let Some(limit) = limit {
524            write!(url, "&maxResults={limit}").expect("write into string must succeed");
525        }
526        if let Some(start_after) = start_after {
527            let start_after = build_abs_path(&self.root, &start_after);
528            write!(url, "&startOffset={}", percent_encode_path(&start_after))
529                .expect("write into string must succeed");
530        }
531
532        if !page_token.is_empty() {
533            // NOTE:
534            //
535            // GCS uses pageToken in request and nextPageToken in response
536            //
537            // Don't know how will those tokens be like so this part are copied
538            // directly from AWS S3 service.
539            write!(url, "&pageToken={}", percent_encode_path(page_token))
540                .expect("write into string must succeed");
541        }
542
543        let mut req = Request::get(&url)
544            .body(Buffer::new())
545            .map_err(new_request_build_error)?;
546
547        self.sign(&mut req).await?;
548
549        self.send(req).await
550    }
551
552    pub async fn gcs_initiate_multipart_upload(&self, path: &str) -> Result<Response<Buffer>> {
553        let p = build_abs_path(&self.root, path);
554
555        let url = format!("{}/{}/{}?uploads", self.endpoint, self.bucket, p);
556
557        let mut req = Request::post(&url)
558            .header(CONTENT_LENGTH, 0)
559            .body(Buffer::new())
560            .map_err(new_request_build_error)?;
561
562        self.sign(&mut req).await?;
563        self.send(req).await
564    }
565
566    pub async fn gcs_upload_part(
567        &self,
568        path: &str,
569        upload_id: &str,
570        part_number: usize,
571        size: u64,
572        body: Buffer,
573    ) -> Result<Response<Buffer>> {
574        let p = build_abs_path(&self.root, path);
575
576        let url = format!(
577            "{}/{}/{}?partNumber={}&uploadId={}",
578            self.endpoint,
579            self.bucket,
580            percent_encode_path(&p),
581            part_number,
582            percent_encode_path(upload_id)
583        );
584
585        let mut req = Request::put(&url);
586
587        req = req.header(CONTENT_LENGTH, size);
588
589        let mut req = req.body(body).map_err(new_request_build_error)?;
590
591        self.sign(&mut req).await?;
592        self.send(req).await
593    }
594
595    pub async fn gcs_complete_multipart_upload(
596        &self,
597        path: &str,
598        upload_id: &str,
599        parts: Vec<CompleteMultipartUploadRequestPart>,
600    ) -> Result<Response<Buffer>> {
601        let p = build_abs_path(&self.root, path);
602
603        let url = format!(
604            "{}/{}/{}?uploadId={}",
605            self.endpoint,
606            self.bucket,
607            percent_encode_path(&p),
608            percent_encode_path(upload_id)
609        );
610
611        let req = Request::post(&url);
612
613        let content = quick_xml::se::to_string(&CompleteMultipartUploadRequest { part: parts })
614            .map_err(new_xml_deserialize_error)?;
615        // Make sure content length has been set to avoid post with chunked encoding.
616        let req = req.header(CONTENT_LENGTH, content.len());
617        // Set content-type to `application/xml` to avoid mixed with form post.
618        let req = req.header(CONTENT_TYPE, "application/xml");
619
620        let mut req = req
621            .body(Buffer::from(Bytes::from(content)))
622            .map_err(new_request_build_error)?;
623
624        self.sign(&mut req).await?;
625        self.send(req).await
626    }
627
628    pub async fn gcs_abort_multipart_upload(
629        &self,
630        path: &str,
631        upload_id: &str,
632    ) -> Result<Response<Buffer>> {
633        let p = build_abs_path(&self.root, path);
634
635        let url = format!(
636            "{}/{}/{}?uploadId={}",
637            self.endpoint,
638            self.bucket,
639            percent_encode_path(&p),
640            percent_encode_path(upload_id)
641        );
642
643        let mut req = Request::delete(&url)
644            .body(Buffer::new())
645            .map_err(new_request_build_error)?;
646        self.sign(&mut req).await?;
647        self.send(req).await
648    }
649
650    pub fn build_metadata_from_object_response(path: &str, data: Buffer) -> Result<Metadata> {
651        let meta: GetObjectJsonResponse =
652            serde_json::from_reader(data.reader()).map_err(new_json_deserialize_error)?;
653
654        let mut m = Metadata::new(EntryMode::from_path(path));
655
656        m.set_etag(&meta.etag);
657        m.set_content_md5(&meta.md5_hash);
658
659        let size = meta
660            .size
661            .parse::<u64>()
662            .map_err(|e| Error::new(ErrorKind::Unexpected, "parse u64").set_source(e))?;
663        m.set_content_length(size);
664        if !meta.content_type.is_empty() {
665            m.set_content_type(&meta.content_type);
666        }
667
668        if !meta.content_encoding.is_empty() {
669            m.set_content_encoding(&meta.content_encoding);
670        }
671
672        if !meta.cache_control.is_empty() {
673            m.set_cache_control(&meta.cache_control);
674        }
675
676        if !meta.content_disposition.is_empty() {
677            m.set_content_disposition(&meta.content_disposition);
678        }
679
680        if !meta.generation.is_empty() {
681            m.set_version(&meta.generation);
682        }
683
684        m.set_last_modified(parse_datetime_from_rfc3339(&meta.updated)?);
685
686        if !meta.metadata.is_empty() {
687            m.with_user_metadata(meta.metadata);
688        }
689
690        Ok(m)
691    }
692}
693
694#[derive(Debug, Serialize)]
695#[serde(default, rename_all = "camelCase")]
696pub struct InsertRequestMetadata<'a> {
697    #[serde(skip_serializing_if = "Option::is_none")]
698    content_type: Option<&'a str>,
699    #[serde(skip_serializing_if = "Option::is_none")]
700    content_encoding: Option<&'a str>,
701    #[serde(skip_serializing_if = "Option::is_none")]
702    storage_class: Option<&'a str>,
703    #[serde(skip_serializing_if = "Option::is_none")]
704    cache_control: Option<&'a str>,
705    #[serde(skip_serializing_if = "Option::is_none")]
706    metadata: Option<&'a HashMap<String, String>>,
707}
708
709impl InsertRequestMetadata<'_> {
710    pub fn is_empty(&self) -> bool {
711        self.content_type.is_none()
712            && self.content_encoding.is_none()
713            && self.storage_class.is_none()
714            && self.cache_control.is_none()
715            // We could also put content-encoding in the url parameters
716            && self.content_encoding.is_none()
717            && self.metadata.is_none()
718    }
719}
720/// Response JSON from GCS list objects API.
721///
722/// refer to https://cloud.google.com/storage/docs/json_api/v1/objects/list for details
723#[derive(Default, Debug, Deserialize)]
724#[serde(default, rename_all = "camelCase")]
725pub struct ListResponse {
726    /// The continuation token.
727    ///
728    /// If this is the last page of results, then no continuation token is returned.
729    pub next_page_token: Option<String>,
730    /// Object name prefixes for objects that matched the listing request
731    /// but were excluded from [items] because of a delimiter.
732    pub prefixes: Vec<String>,
733    /// The list of objects, ordered lexicographically by name.
734    pub items: Vec<ListResponseItem>,
735}
736
737#[derive(Default, Debug, Eq, PartialEq, Deserialize)]
738#[serde(default, rename_all = "camelCase")]
739pub struct ListResponseItem {
740    pub name: String,
741    pub size: String,
742    // metadata
743    pub etag: String,
744    pub md5_hash: String,
745    pub updated: String,
746    pub content_type: String,
747}
748
749/// Result of CreateMultipartUpload
750#[derive(Default, Debug, Deserialize)]
751#[serde(default, rename_all = "PascalCase")]
752pub struct InitiateMultipartUploadResult {
753    pub upload_id: String,
754}
755
756/// Request of CompleteMultipartUploadRequest
757#[derive(Default, Debug, Serialize)]
758#[serde(default, rename = "CompleteMultipartUpload", rename_all = "PascalCase")]
759pub struct CompleteMultipartUploadRequest {
760    pub part: Vec<CompleteMultipartUploadRequestPart>,
761}
762
763#[derive(Clone, Default, Debug, Serialize)]
764#[serde(default, rename_all = "PascalCase")]
765pub struct CompleteMultipartUploadRequestPart {
766    #[serde(rename = "PartNumber")]
767    pub part_number: usize,
768    #[serde(rename = "ETag")]
769    pub etag: String,
770}
771
772/// The raw json response returned by [`get`](https://cloud.google.com/storage/docs/json_api/v1/objects/get)
773#[derive(Debug, Default, Deserialize)]
774#[serde(default, rename_all = "camelCase")]
775struct GetObjectJsonResponse {
776    /// GCS will return size in string.
777    ///
778    /// For example: `"size": "56535"`
779    size: String,
780    /// etag is not quoted.
781    ///
782    /// For example: `"etag": "CKWasoTgyPkCEAE="`
783    etag: String,
784    /// RFC3339 styled datetime string.
785    ///
786    /// For example: `"updated": "2022-08-15T11:33:34.866Z"`
787    updated: String,
788    /// Content md5 hash
789    ///
790    /// For example: `"md5Hash": "fHcEH1vPwA6eTPqxuasXcg=="`
791    md5_hash: String,
792    /// Content type of this object.
793    ///
794    /// For example: `"contentType": "image/png",`
795    content_type: String,
796    /// Content encoding of this object
797    ///
798    /// For example: "contentEncoding": "br"
799    content_encoding: String,
800    /// Content disposition of this object
801    content_disposition: String,
802    /// Cache-Control directive for the object data.
803    cache_control: String,
804    /// Content generation of this object. Used for object versioning and soft delete.
805    generation: String,
806    /// Custom metadata of this object.
807    ///
808    /// For example: `"metadata" : { "my-key": "my-value" }`
809    metadata: HashMap<String, String>,
810}
811
812#[cfg(test)]
813mod tests {
814    use super::*;
815
816    #[test]
817    fn test_deserialize_get_object_json_response() {
818        let content = r#"{
819    "kind": "storage#object",
820    "id": "example/1.png/1660563214863653",
821    "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png",
822    "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media",
823    "name": "1.png",
824    "bucket": "example",
825    "generation": "1660563214863653",
826    "metageneration": "1",
827    "contentType": "image/png",
828    "contentEncoding": "br",
829    "contentDisposition": "attachment",
830    "cacheControl": "public, max-age=3600",
831    "storageClass": "STANDARD",
832    "size": "56535",
833    "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==",
834    "crc32c": "j/un9g==",
835    "etag": "CKWasoTgyPkCEAE=",
836    "timeCreated": "2022-08-15T11:33:34.866Z",
837    "updated": "2022-08-15T11:33:34.866Z",
838    "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z",
839    "metadata" : {
840        "location" : "everywhere"
841  }
842}"#;
843
844        let meta = GcsCore::build_metadata_from_object_response("1.png", content.into())
845            .expect("parse metadata should not fail");
846
847        assert_eq!(meta.content_length(), 56535);
848        assert_eq!(
849            meta.last_modified(),
850            Some(
851                parse_datetime_from_rfc3339("2022-08-15T11:33:34.866Z")
852                    .expect("parse date should not fail")
853            )
854        );
855        assert_eq!(meta.content_md5(), Some("fHcEH1vPwA6eTPqxuasXcg=="));
856        assert_eq!(meta.etag(), Some("CKWasoTgyPkCEAE="));
857        assert_eq!(meta.content_type(), Some("image/png"));
858        assert_eq!(meta.content_encoding(), Some("br"));
859        assert_eq!(meta.content_disposition(), Some("attachment"));
860        assert_eq!(meta.cache_control(), Some("public, max-age=3600"));
861        assert_eq!(meta.version(), Some("1660563214863653"));
862
863        let metadata = HashMap::from_iter([("location".to_string(), "everywhere".to_string())]);
864        assert_eq!(meta.user_metadata(), Some(&metadata));
865    }
866
867    #[test]
868    fn test_deserialize_list_response() {
869        let content = r#"
870    {
871  "kind": "storage#objects",
872  "prefixes": [
873    "dir/",
874    "test/"
875  ],
876  "items": [
877    {
878      "kind": "storage#object",
879      "id": "example/1.png/1660563214863653",
880      "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png",
881      "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media",
882      "name": "1.png",
883      "bucket": "example",
884      "generation": "1660563214863653",
885      "metageneration": "1",
886      "contentType": "image/png",
887      "storageClass": "STANDARD",
888      "size": "56535",
889      "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==",
890      "crc32c": "j/un9g==",
891      "etag": "CKWasoTgyPkCEAE=",
892      "timeCreated": "2022-08-15T11:33:34.866Z",
893      "updated": "2022-08-15T11:33:34.866Z",
894      "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z"
895    },
896    {
897      "kind": "storage#object",
898      "id": "example/2.png/1660563214883337",
899      "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/2.png",
900      "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media",
901      "name": "2.png",
902      "bucket": "example",
903      "generation": "1660563214883337",
904      "metageneration": "1",
905      "contentType": "image/png",
906      "storageClass": "STANDARD",
907      "size": "45506",
908      "md5Hash": "e6LsGusU7pFJZk+114NV1g==",
909      "crc32c": "L00QAg==",
910      "etag": "CIm0s4TgyPkCEAE=",
911      "timeCreated": "2022-08-15T11:33:34.886Z",
912      "updated": "2022-08-15T11:33:34.886Z",
913      "timeStorageClassUpdated": "2022-08-15T11:33:34.886Z"
914    }
915  ]
916}
917    "#;
918
919        let output: ListResponse =
920            serde_json::from_str(content).expect("JSON deserialize must succeed");
921        assert!(output.next_page_token.is_none());
922        assert_eq!(output.items.len(), 2);
923        assert_eq!(output.items[0].name, "1.png");
924        assert_eq!(output.items[0].size, "56535");
925        assert_eq!(output.items[0].md5_hash, "fHcEH1vPwA6eTPqxuasXcg==");
926        assert_eq!(output.items[0].etag, "CKWasoTgyPkCEAE=");
927        assert_eq!(output.items[0].updated, "2022-08-15T11:33:34.866Z");
928        assert_eq!(output.items[1].name, "2.png");
929        assert_eq!(output.items[1].size, "45506");
930        assert_eq!(output.items[1].md5_hash, "e6LsGusU7pFJZk+114NV1g==");
931        assert_eq!(output.items[1].etag, "CIm0s4TgyPkCEAE=");
932        assert_eq!(output.items[1].updated, "2022-08-15T11:33:34.886Z");
933        assert_eq!(output.items[1].content_type, "image/png");
934        assert_eq!(output.prefixes, vec!["dir/", "test/"])
935    }
936
937    #[test]
938    fn test_deserialize_list_response_with_next_page_token() {
939        let content = r#"
940    {
941  "kind": "storage#objects",
942  "prefixes": [
943    "dir/",
944    "test/"
945  ],
946  "nextPageToken": "CgYxMC5wbmc=",
947  "items": [
948    {
949      "kind": "storage#object",
950      "id": "example/1.png/1660563214863653",
951      "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png",
952      "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media",
953      "name": "1.png",
954      "bucket": "example",
955      "generation": "1660563214863653",
956      "metageneration": "1",
957      "contentType": "image/png",
958      "storageClass": "STANDARD",
959      "size": "56535",
960      "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==",
961      "crc32c": "j/un9g==",
962      "etag": "CKWasoTgyPkCEAE=",
963      "timeCreated": "2022-08-15T11:33:34.866Z",
964      "updated": "2022-08-15T11:33:34.866Z",
965      "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z"
966    },
967    {
968      "kind": "storage#object",
969      "id": "example/2.png/1660563214883337",
970      "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/2.png",
971      "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media",
972      "name": "2.png",
973      "bucket": "example",
974      "generation": "1660563214883337",
975      "metageneration": "1",
976      "contentType": "image/png",
977      "storageClass": "STANDARD",
978      "size": "45506",
979      "md5Hash": "e6LsGusU7pFJZk+114NV1g==",
980      "crc32c": "L00QAg==",
981      "etag": "CIm0s4TgyPkCEAE=",
982      "timeCreated": "2022-08-15T11:33:34.886Z",
983      "updated": "2022-08-15T11:33:34.886Z",
984      "timeStorageClassUpdated": "2022-08-15T11:33:34.886Z"
985    }
986  ]
987}
988    "#;
989
990        let output: ListResponse =
991            serde_json::from_str(content).expect("JSON deserialize must succeed");
992        assert_eq!(output.next_page_token, Some("CgYxMC5wbmc=".to_string()));
993        assert_eq!(output.items.len(), 2);
994        assert_eq!(output.items[0].name, "1.png");
995        assert_eq!(output.items[0].size, "56535");
996        assert_eq!(output.items[0].md5_hash, "fHcEH1vPwA6eTPqxuasXcg==");
997        assert_eq!(output.items[0].etag, "CKWasoTgyPkCEAE=");
998        assert_eq!(output.items[0].updated, "2022-08-15T11:33:34.866Z");
999        assert_eq!(output.items[1].name, "2.png");
1000        assert_eq!(output.items[1].size, "45506");
1001        assert_eq!(output.items[1].md5_hash, "e6LsGusU7pFJZk+114NV1g==");
1002        assert_eq!(output.items[1].etag, "CIm0s4TgyPkCEAE=");
1003        assert_eq!(output.items[1].updated, "2022-08-15T11:33:34.886Z");
1004        assert_eq!(output.prefixes, vec!["dir/", "test/"])
1005    }
1006}
opendal/services/gcs/core.rs

opendal/services/gcs/
core.rs
