opendal/services/github/
core.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use base64::Engine;
19use bytes::Buf;
20use bytes::Bytes;
21use http::header;
22use http::request;
23use http::Request;
24use http::Response;
25use http::StatusCode;
26use serde::Deserialize;
27use serde::Serialize;
28use std::fmt::Debug;
29use std::fmt::Formatter;
30use std::sync::Arc;
31
32use super::error::parse_error;
33use crate::raw::*;
34use crate::*;
35
36/// Core of [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support.
37#[derive(Clone)]
38pub struct GithubCore {
39    pub info: Arc<AccessorInfo>,
40    /// The root of this core.
41    pub root: String,
42    /// Github access_token.
43    pub token: Option<String>,
44    /// Github repo owner.
45    pub owner: String,
46    /// Github repo name.
47    pub repo: String,
48}
49
50impl Debug for GithubCore {
51    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
52        f.debug_struct("Backend")
53            .field("root", &self.root)
54            .field("owner", &self.owner)
55            .field("repo", &self.repo)
56            .finish_non_exhaustive()
57    }
58}
59
60impl GithubCore {
61    #[inline]
62    pub async fn send(&self, req: Request<Buffer>) -> Result<Response<Buffer>> {
63        self.info.http_client().send(req).await
64    }
65
66    pub fn sign(&self, req: request::Builder) -> Result<request::Builder> {
67        let mut req = req
68            .header(header::USER_AGENT, format!("opendal-{}", VERSION))
69            .header("X-GitHub-Api-Version", "2022-11-28");
70
71        // Github access_token is optional.
72        if let Some(token) = &self.token {
73            req = req.header(
74                header::AUTHORIZATION,
75                format_authorization_by_bearer(token)?,
76            )
77        }
78
79        Ok(req)
80    }
81}
82
83impl GithubCore {
84    pub async fn get_file_sha(&self, path: &str) -> Result<Option<String>> {
85        // if the token is not set, we should not try to get the sha of the file.
86        if self.token.is_none() {
87            return Err(Error::new(
88                ErrorKind::PermissionDenied,
89                "Github access_token is not set",
90            ));
91        }
92
93        let resp = self.stat(path).await?;
94
95        match resp.status() {
96            StatusCode::OK => {
97                let body = resp.into_body();
98                let resp: Entry =
99                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
100
101                Ok(Some(resp.sha))
102            }
103            StatusCode::NOT_FOUND => Ok(None),
104            _ => Err(parse_error(resp)),
105        }
106    }
107
108    pub async fn stat(&self, path: &str) -> Result<Response<Buffer>> {
109        let path = build_abs_path(&self.root, path);
110
111        let url = format!(
112            "https://api.github.com/repos/{}/{}/contents/{}",
113            self.owner,
114            self.repo,
115            percent_encode_path(&path)
116        );
117
118        let req = Request::get(url);
119
120        let req = self.sign(req)?;
121
122        let req = req
123            .header("Accept", "application/vnd.github.object+json")
124            .body(Buffer::new())
125            .map_err(new_request_build_error)?;
126
127        self.send(req).await
128    }
129
130    pub async fn get(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
131        let path = build_abs_path(&self.root, path);
132
133        let url = format!(
134            "https://api.github.com/repos/{}/{}/contents/{}",
135            self.owner,
136            self.repo,
137            percent_encode_path(&path)
138        );
139
140        let req = Request::get(url);
141
142        let req = self.sign(req)?;
143
144        let req = req
145            .header(header::ACCEPT, "application/vnd.github.raw+json")
146            .header(header::RANGE, range.to_header())
147            .body(Buffer::new())
148            .map_err(new_request_build_error)?;
149
150        self.info.http_client().fetch(req).await
151    }
152
153    pub async fn upload(&self, path: &str, bs: Buffer) -> Result<Response<Buffer>> {
154        let sha = self.get_file_sha(path).await?;
155
156        let path = build_abs_path(&self.root, path);
157
158        let url = format!(
159            "https://api.github.com/repos/{}/{}/contents/{}",
160            self.owner,
161            self.repo,
162            percent_encode_path(&path)
163        );
164
165        let req = Request::put(url);
166
167        let req = self.sign(req)?;
168
169        let mut req_body = CreateOrUpdateContentsRequest {
170            message: format!("Write {} at {} via opendal", path, chrono::Local::now()),
171            content: base64::engine::general_purpose::STANDARD.encode(bs.to_bytes()),
172            sha: None,
173        };
174
175        if let Some(sha) = sha {
176            req_body.sha = Some(sha);
177        }
178
179        let req_body = serde_json::to_vec(&req_body).map_err(new_json_serialize_error)?;
180
181        let req = req
182            .header("Accept", "application/vnd.github+json")
183            .body(Buffer::from(req_body))
184            .map_err(new_request_build_error)?;
185
186        self.send(req).await
187    }
188
189    pub async fn delete(&self, path: &str) -> Result<()> {
190        // If path is a directory, we should delete path/.gitkeep
191        let formatted_path = format!("{}.gitkeep", path);
192        let p = if path.ends_with('/') {
193            formatted_path.as_str()
194        } else {
195            path
196        };
197
198        let Some(sha) = self.get_file_sha(p).await? else {
199            return Ok(());
200        };
201
202        let path = build_abs_path(&self.root, p);
203
204        let url = format!(
205            "https://api.github.com/repos/{}/{}/contents/{}",
206            self.owner,
207            self.repo,
208            percent_encode_path(&path)
209        );
210
211        let req = Request::delete(url);
212
213        let req = self.sign(req)?;
214
215        let req_body = DeleteContentsRequest {
216            message: format!("Delete {} at {} via opendal", path, chrono::Local::now()),
217            sha,
218        };
219
220        let req_body = serde_json::to_vec(&req_body).map_err(new_json_serialize_error)?;
221
222        let req = req
223            .header("Accept", "application/vnd.github.object+json")
224            .body(Buffer::from(Bytes::from(req_body)))
225            .map_err(new_request_build_error)?;
226
227        let resp = self.send(req).await?;
228
229        match resp.status() {
230            StatusCode::OK => Ok(()),
231            StatusCode::NOT_FOUND => Ok(()),
232            _ => Err(parse_error(resp)),
233        }
234    }
235
236    pub async fn list(&self, path: &str) -> Result<ListResponse> {
237        let path = build_abs_path(&self.root, path);
238
239        let url = format!(
240            "https://api.github.com/repos/{}/{}/contents/{}",
241            self.owner,
242            self.repo,
243            percent_encode_path(&path)
244        );
245
246        let req = Request::get(url);
247
248        let req = self.sign(req)?;
249
250        let req = req
251            .header("Accept", "application/vnd.github.object+json")
252            .body(Buffer::new())
253            .map_err(new_request_build_error)?;
254
255        let resp = self.send(req).await?;
256
257        match resp.status() {
258            StatusCode::OK => {
259                let body = resp.into_body();
260                let resp: ListResponse =
261                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
262
263                Ok(resp)
264            }
265            StatusCode::NOT_FOUND => Ok(ListResponse::default()),
266            _ => Err(parse_error(resp)),
267        }
268    }
269
270    /// We use git_url to call github's Tree based API.
271    pub async fn list_with_recursive(&self, git_url: &str) -> Result<Vec<Tree>> {
272        let url = format!("{}?recursive=true", git_url);
273
274        let req = Request::get(url);
275
276        let req = self.sign(req)?;
277
278        let req = req
279            .header("Accept", "application/vnd.github.object+json")
280            .body(Buffer::new())
281            .map_err(new_request_build_error)?;
282
283        let resp = self.send(req).await?;
284
285        match resp.status() {
286            StatusCode::OK => {
287                let body = resp.into_body();
288                let resp: ListTreeResponse =
289                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
290
291                Ok(resp.tree)
292            }
293            _ => Err(parse_error(resp)),
294        }
295    }
296}
297
298#[derive(Default, Debug, Clone, Serialize)]
299pub struct CreateOrUpdateContentsRequest {
300    pub message: String,
301    pub content: String,
302    pub sha: Option<String>,
303}
304
305#[derive(Default, Debug, Clone, Serialize)]
306pub struct DeleteContentsRequest {
307    pub message: String,
308    pub sha: String,
309}
310
311#[derive(Default, Debug, Clone, Deserialize)]
312pub struct ListTreeResponse {
313    pub tree: Vec<Tree>,
314}
315
316#[derive(Default, Debug, Clone, Deserialize)]
317pub struct Tree {
318    pub path: String,
319    #[serde(rename = "type")]
320    pub type_field: String,
321    pub size: Option<u64>,
322    pub sha: String,
323}
324
325#[derive(Default, Debug, Clone, Deserialize)]
326pub struct ListResponse {
327    pub git_url: String,
328    pub entries: Vec<Entry>,
329}
330
331#[derive(Default, Debug, Clone, Deserialize)]
332pub struct Entry {
333    pub path: String,
334    pub sha: String,
335    pub size: u64,
336    #[serde(rename = "type")]
337    pub type_field: String,
338}
opendal/services/github/core.rs

opendal/services/github/
core.rs
