opendal/services/github/
core.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use base64::Engine;
19use bytes::Buf;
20use bytes::Bytes;
21use http::header;
22use http::request;
23use http::Request;
24use http::Response;
25use http::StatusCode;
26use serde::Deserialize;
27use serde::Serialize;
28use std::fmt::Debug;
29use std::fmt::Formatter;
30use std::sync::Arc;
31
32use super::error::parse_error;
33use crate::raw::*;
34use crate::*;
35
36/// Core of [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support.
37#[derive(Clone)]
38pub struct GithubCore {
39    pub info: Arc<AccessorInfo>,
40    /// The root of this core.
41    pub root: String,
42    /// Github access_token.
43    pub token: Option<String>,
44    /// Github repo owner.
45    pub owner: String,
46    /// Github repo name.
47    pub repo: String,
48}
49
50impl Debug for GithubCore {
51    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
52        f.debug_struct("Backend")
53            .field("root", &self.root)
54            .field("owner", &self.owner)
55            .field("repo", &self.repo)
56            .finish_non_exhaustive()
57    }
58}
59
60impl GithubCore {
61    #[inline]
62    pub async fn send(&self, req: Request<Buffer>) -> Result<Response<Buffer>> {
63        self.info.http_client().send(req).await
64    }
65
66    pub fn sign(&self, req: request::Builder) -> Result<request::Builder> {
67        let mut req = req
68            .header(header::USER_AGENT, format!("opendal-{}", VERSION))
69            .header("X-GitHub-Api-Version", "2022-11-28");
70
71        // Github access_token is optional.
72        if let Some(token) = &self.token {
73            req = req.header(
74                header::AUTHORIZATION,
75                format_authorization_by_bearer(token)?,
76            )
77        }
78
79        Ok(req)
80    }
81}
82
83impl GithubCore {
84    pub async fn get_file_sha(&self, path: &str) -> Result<Option<String>> {
85        // if the token is not set, we should not try to get the sha of the file.
86        if self.token.is_none() {
87            return Err(Error::new(
88                ErrorKind::PermissionDenied,
89                "Github access_token is not set",
90            ));
91        }
92
93        let resp = self.stat(path).await?;
94
95        match resp.status() {
96            StatusCode::OK => {
97                let body = resp.into_body();
98                let resp: Entry =
99                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
100
101                Ok(Some(resp.sha))
102            }
103            StatusCode::NOT_FOUND => Ok(None),
104            _ => Err(parse_error(resp)),
105        }
106    }
107
108    pub async fn stat(&self, path: &str) -> Result<Response<Buffer>> {
109        let path = build_abs_path(&self.root, path);
110
111        let url = format!(
112            "https://api.github.com/repos/{}/{}/contents/{}",
113            self.owner,
114            self.repo,
115            percent_encode_path(&path)
116        );
117
118        let req = Request::get(url);
119
120        let req = req.extension(Operation::Stat);
121
122        let req = self.sign(req)?;
123
124        let req = req
125            .header("Accept", "application/vnd.github.object+json")
126            .body(Buffer::new())
127            .map_err(new_request_build_error)?;
128
129        self.send(req).await
130    }
131
132    pub async fn get(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
133        let path = build_abs_path(&self.root, path);
134
135        let url = format!(
136            "https://api.github.com/repos/{}/{}/contents/{}",
137            self.owner,
138            self.repo,
139            percent_encode_path(&path)
140        );
141
142        let req = Request::get(url);
143
144        let req = req.extension(Operation::Read);
145
146        let req = self.sign(req)?;
147
148        let req = req
149            .header(header::ACCEPT, "application/vnd.github.raw+json")
150            .header(header::RANGE, range.to_header())
151            .body(Buffer::new())
152            .map_err(new_request_build_error)?;
153
154        self.info.http_client().fetch(req).await
155    }
156
157    pub async fn upload(&self, path: &str, bs: Buffer) -> Result<Response<Buffer>> {
158        let sha = self.get_file_sha(path).await?;
159
160        let path = build_abs_path(&self.root, path);
161
162        let url = format!(
163            "https://api.github.com/repos/{}/{}/contents/{}",
164            self.owner,
165            self.repo,
166            percent_encode_path(&path)
167        );
168
169        let req = Request::put(url);
170
171        let req = req.extension(Operation::Write);
172
173        let req = self.sign(req)?;
174
175        let mut req_body = CreateOrUpdateContentsRequest {
176            message: format!("Write {} at {} via opendal", path, chrono::Local::now()),
177            content: base64::engine::general_purpose::STANDARD.encode(bs.to_bytes()),
178            sha: None,
179        };
180
181        if let Some(sha) = sha {
182            req_body.sha = Some(sha);
183        }
184
185        let req_body = serde_json::to_vec(&req_body).map_err(new_json_serialize_error)?;
186
187        let req = req
188            .header("Accept", "application/vnd.github+json")
189            .body(Buffer::from(req_body))
190            .map_err(new_request_build_error)?;
191
192        self.send(req).await
193    }
194
195    pub async fn delete(&self, path: &str) -> Result<()> {
196        // If path is a directory, we should delete path/.gitkeep
197        let formatted_path = format!("{}.gitkeep", path);
198        let p = if path.ends_with('/') {
199            formatted_path.as_str()
200        } else {
201            path
202        };
203
204        let Some(sha) = self.get_file_sha(p).await? else {
205            return Ok(());
206        };
207
208        let path = build_abs_path(&self.root, p);
209
210        let url = format!(
211            "https://api.github.com/repos/{}/{}/contents/{}",
212            self.owner,
213            self.repo,
214            percent_encode_path(&path)
215        );
216
217        let req = Request::delete(url);
218
219        let req = req.extension(Operation::Delete);
220
221        let req = self.sign(req)?;
222
223        let req_body = DeleteContentsRequest {
224            message: format!("Delete {} at {} via opendal", path, chrono::Local::now()),
225            sha,
226        };
227
228        let req_body = serde_json::to_vec(&req_body).map_err(new_json_serialize_error)?;
229
230        let req = req
231            .header("Accept", "application/vnd.github.object+json")
232            .body(Buffer::from(Bytes::from(req_body)))
233            .map_err(new_request_build_error)?;
234
235        let resp = self.send(req).await?;
236
237        match resp.status() {
238            StatusCode::OK => Ok(()),
239            StatusCode::NOT_FOUND => Ok(()),
240            _ => Err(parse_error(resp)),
241        }
242    }
243
244    pub async fn list(&self, path: &str) -> Result<ListResponse> {
245        let path = build_abs_path(&self.root, path);
246
247        let url = format!(
248            "https://api.github.com/repos/{}/{}/contents/{}",
249            self.owner,
250            self.repo,
251            percent_encode_path(&path)
252        );
253
254        let req = Request::get(url);
255
256        let req = req.extension(Operation::List);
257
258        let req = self.sign(req)?;
259
260        let req = req
261            .header("Accept", "application/vnd.github.object+json")
262            .body(Buffer::new())
263            .map_err(new_request_build_error)?;
264
265        let resp = self.send(req).await?;
266
267        match resp.status() {
268            StatusCode::OK => {
269                let body = resp.into_body();
270                let resp: ListResponse =
271                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
272
273                Ok(resp)
274            }
275            StatusCode::NOT_FOUND => Ok(ListResponse::default()),
276            _ => Err(parse_error(resp)),
277        }
278    }
279
280    /// We use git_url to call github's Tree based API.
281    pub async fn list_with_recursive(&self, git_url: &str) -> Result<Vec<Tree>> {
282        let url = format!("{}?recursive=true", git_url);
283
284        let req = Request::get(url);
285
286        let req = req.extension(Operation::List);
287
288        let req = self.sign(req)?;
289
290        let req = req
291            .header("Accept", "application/vnd.github.object+json")
292            .body(Buffer::new())
293            .map_err(new_request_build_error)?;
294
295        let resp = self.send(req).await?;
296
297        match resp.status() {
298            StatusCode::OK => {
299                let body = resp.into_body();
300                let resp: ListTreeResponse =
301                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
302
303                Ok(resp.tree)
304            }
305            _ => Err(parse_error(resp)),
306        }
307    }
308}
309
310#[derive(Default, Debug, Clone, Serialize)]
311pub struct CreateOrUpdateContentsRequest {
312    pub message: String,
313    pub content: String,
314    pub sha: Option<String>,
315}
316
317#[derive(Default, Debug, Clone, Serialize)]
318pub struct DeleteContentsRequest {
319    pub message: String,
320    pub sha: String,
321}
322
323#[derive(Default, Debug, Clone, Deserialize)]
324pub struct ListTreeResponse {
325    pub tree: Vec<Tree>,
326}
327
328#[derive(Default, Debug, Clone, Deserialize)]
329pub struct Tree {
330    pub path: String,
331    #[serde(rename = "type")]
332    pub type_field: String,
333    pub size: Option<u64>,
334    pub sha: String,
335}
336
337#[derive(Default, Debug, Clone, Deserialize)]
338pub struct ListResponse {
339    pub git_url: String,
340    pub entries: Vec<Entry>,
341}
342
343#[derive(Default, Debug, Clone, Deserialize)]
344pub struct Entry {
345    pub path: String,
346    pub sha: String,
347    pub size: u64,
348    #[serde(rename = "type")]
349    pub type_field: String,
350}
opendal/services/github/core.rs

opendal/services/github/
core.rs
