opendal/services/github/
core.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::fmt::Debug;
19use std::fmt::Formatter;
20use std::sync::Arc;
21
22use base64::Engine;
23use bytes::Buf;
24use bytes::Bytes;
25use http::header;
26use http::request;
27use http::Request;
28use http::Response;
29use http::StatusCode;
30use serde::Deserialize;
31use serde::Serialize;
32
33use super::error::parse_error;
34use crate::raw::*;
35use crate::*;
36
37/// Core of [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support.
38#[derive(Clone)]
39pub struct GithubCore {
40    pub info: Arc<AccessorInfo>,
41    /// The root of this core.
42    pub root: String,
43    /// Github access_token.
44    pub token: Option<String>,
45    /// Github repo owner.
46    pub owner: String,
47    /// Github repo name.
48    pub repo: String,
49}
50
51impl Debug for GithubCore {
52    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
53        f.debug_struct("Backend")
54            .field("root", &self.root)
55            .field("owner", &self.owner)
56            .field("repo", &self.repo)
57            .finish_non_exhaustive()
58    }
59}
60
61impl GithubCore {
62    #[inline]
63    pub async fn send(&self, req: Request<Buffer>) -> Result<Response<Buffer>> {
64        self.info.http_client().send(req).await
65    }
66
67    pub fn sign(&self, req: request::Builder) -> Result<request::Builder> {
68        let mut req = req
69            .header(header::USER_AGENT, format!("opendal-{VERSION}"))
70            .header("X-GitHub-Api-Version", "2022-11-28");
71
72        // Github access_token is optional.
73        if let Some(token) = &self.token {
74            req = req.header(
75                header::AUTHORIZATION,
76                format_authorization_by_bearer(token)?,
77            )
78        }
79
80        Ok(req)
81    }
82}
83
84impl GithubCore {
85    pub async fn get_file_sha(&self, path: &str) -> Result<Option<String>> {
86        // if the token is not set, we should not try to get the sha of the file.
87        if self.token.is_none() {
88            return Err(Error::new(
89                ErrorKind::PermissionDenied,
90                "Github access_token is not set",
91            ));
92        }
93
94        let resp = self.stat(path).await?;
95
96        match resp.status() {
97            StatusCode::OK => {
98                let body = resp.into_body();
99                let resp: Entry =
100                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
101
102                Ok(Some(resp.sha))
103            }
104            StatusCode::NOT_FOUND => Ok(None),
105            _ => Err(parse_error(resp)),
106        }
107    }
108
109    pub async fn stat(&self, path: &str) -> Result<Response<Buffer>> {
110        let path = build_abs_path(&self.root, path);
111
112        let url = format!(
113            "https://api.github.com/repos/{}/{}/contents/{}",
114            self.owner,
115            self.repo,
116            percent_encode_path(&path)
117        );
118
119        let req = Request::get(url);
120
121        let req = req.extension(Operation::Stat);
122
123        let req = self.sign(req)?;
124
125        let req = req
126            .header("Accept", "application/vnd.github.object+json")
127            .body(Buffer::new())
128            .map_err(new_request_build_error)?;
129
130        self.send(req).await
131    }
132
133    pub async fn get(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
134        let path = build_abs_path(&self.root, path);
135
136        let url = format!(
137            "https://api.github.com/repos/{}/{}/contents/{}",
138            self.owner,
139            self.repo,
140            percent_encode_path(&path)
141        );
142
143        let req = Request::get(url);
144
145        let req = req.extension(Operation::Read);
146
147        let req = self.sign(req)?;
148
149        let req = req
150            .header(header::ACCEPT, "application/vnd.github.raw+json")
151            .header(header::RANGE, range.to_header())
152            .body(Buffer::new())
153            .map_err(new_request_build_error)?;
154
155        self.info.http_client().fetch(req).await
156    }
157
158    pub async fn upload(&self, path: &str, bs: Buffer) -> Result<Response<Buffer>> {
159        let sha = self.get_file_sha(path).await?;
160
161        let path = build_abs_path(&self.root, path);
162
163        let url = format!(
164            "https://api.github.com/repos/{}/{}/contents/{}",
165            self.owner,
166            self.repo,
167            percent_encode_path(&path)
168        );
169
170        let req = Request::put(url);
171
172        let req = req.extension(Operation::Write);
173
174        let req = self.sign(req)?;
175
176        let mut req_body = CreateOrUpdateContentsRequest {
177            message: format!("Write {} at {} via opendal", path, chrono::Local::now()),
178            content: base64::engine::general_purpose::STANDARD.encode(bs.to_bytes()),
179            sha: None,
180        };
181
182        if let Some(sha) = sha {
183            req_body.sha = Some(sha);
184        }
185
186        let req_body = serde_json::to_vec(&req_body).map_err(new_json_serialize_error)?;
187
188        let req = req
189            .header("Accept", "application/vnd.github+json")
190            .body(Buffer::from(req_body))
191            .map_err(new_request_build_error)?;
192
193        self.send(req).await
194    }
195
196    pub async fn delete(&self, path: &str) -> Result<()> {
197        // If path is a directory, we should delete path/.gitkeep
198        let formatted_path = format!("{path}.gitkeep");
199        let p = if path.ends_with('/') {
200            formatted_path.as_str()
201        } else {
202            path
203        };
204
205        let Some(sha) = self.get_file_sha(p).await? else {
206            return Ok(());
207        };
208
209        let path = build_abs_path(&self.root, p);
210
211        let url = format!(
212            "https://api.github.com/repos/{}/{}/contents/{}",
213            self.owner,
214            self.repo,
215            percent_encode_path(&path)
216        );
217
218        let req = Request::delete(url);
219
220        let req = req.extension(Operation::Delete);
221
222        let req = self.sign(req)?;
223
224        let req_body = DeleteContentsRequest {
225            message: format!("Delete {} at {} via opendal", path, chrono::Local::now()),
226            sha,
227        };
228
229        let req_body = serde_json::to_vec(&req_body).map_err(new_json_serialize_error)?;
230
231        let req = req
232            .header("Accept", "application/vnd.github.object+json")
233            .body(Buffer::from(Bytes::from(req_body)))
234            .map_err(new_request_build_error)?;
235
236        let resp = self.send(req).await?;
237
238        match resp.status() {
239            StatusCode::OK => Ok(()),
240            StatusCode::NOT_FOUND => Ok(()),
241            _ => Err(parse_error(resp)),
242        }
243    }
244
245    pub async fn list(&self, path: &str) -> Result<ListResponse> {
246        let path = build_abs_path(&self.root, path);
247
248        let url = format!(
249            "https://api.github.com/repos/{}/{}/contents/{}",
250            self.owner,
251            self.repo,
252            percent_encode_path(&path)
253        );
254
255        let req = Request::get(url);
256
257        let req = req.extension(Operation::List);
258
259        let req = self.sign(req)?;
260
261        let req = req
262            .header("Accept", "application/vnd.github.object+json")
263            .body(Buffer::new())
264            .map_err(new_request_build_error)?;
265
266        let resp = self.send(req).await?;
267
268        match resp.status() {
269            StatusCode::OK => {
270                let body = resp.into_body();
271                let resp: ListResponse =
272                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
273
274                Ok(resp)
275            }
276            StatusCode::NOT_FOUND => Ok(ListResponse::default()),
277            _ => Err(parse_error(resp)),
278        }
279    }
280
281    /// We use git_url to call github's Tree based API.
282    pub async fn list_with_recursive(&self, git_url: &str) -> Result<Vec<Tree>> {
283        let url = format!("{git_url}?recursive=true");
284
285        let req = Request::get(url);
286
287        let req = req.extension(Operation::List);
288
289        let req = self.sign(req)?;
290
291        let req = req
292            .header("Accept", "application/vnd.github.object+json")
293            .body(Buffer::new())
294            .map_err(new_request_build_error)?;
295
296        let resp = self.send(req).await?;
297
298        match resp.status() {
299            StatusCode::OK => {
300                let body = resp.into_body();
301                let resp: ListTreeResponse =
302                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
303
304                Ok(resp.tree)
305            }
306            _ => Err(parse_error(resp)),
307        }
308    }
309}
310
311#[derive(Default, Debug, Clone, Serialize)]
312pub struct CreateOrUpdateContentsRequest {
313    pub message: String,
314    pub content: String,
315    pub sha: Option<String>,
316}
317
318#[derive(Default, Debug, Clone, Serialize)]
319pub struct DeleteContentsRequest {
320    pub message: String,
321    pub sha: String,
322}
323
324#[derive(Default, Debug, Clone, Deserialize)]
325pub struct ListTreeResponse {
326    pub tree: Vec<Tree>,
327}
328
329#[derive(Default, Debug, Clone, Deserialize)]
330pub struct Tree {
331    pub path: String,
332    #[serde(rename = "type")]
333    pub type_field: String,
334    pub size: Option<u64>,
335    pub sha: String,
336}
337
338#[derive(Default, Debug, Clone, Deserialize)]
339pub struct ListResponse {
340    pub git_url: String,
341    pub entries: Vec<Entry>,
342}
343
344#[derive(Default, Debug, Clone, Deserialize)]
345pub struct Entry {
346    pub path: String,
347    pub sha: String,
348    pub size: u64,
349    #[serde(rename = "type")]
350    pub type_field: String,
351}
opendal/services/github/core.rs

opendal/services/github/
core.rs
