opendal/services/ipfs/
core.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::fmt::Debug;
19use std::fmt::Formatter;
20use std::sync::Arc;
21
22use http::Request;
23use http::Response;
24use http::StatusCode;
25
26use super::error::parse_error;
27use crate::raw::*;
28use crate::*;
29
30pub struct IpfsCore {
31    pub info: Arc<AccessorInfo>,
32    pub endpoint: String,
33    pub root: String,
34}
35
36impl Debug for IpfsCore {
37    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
38        f.debug_struct("IpfsCore")
39            .field("endpoint", &self.endpoint)
40            .field("root", &self.root)
41            .finish()
42    }
43}
44
45impl IpfsCore {
46    pub async fn ipfs_get(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
47        let p = build_rooted_abs_path(&self.root, path);
48
49        let url = format!("{}{}", self.endpoint, percent_encode_path(&p));
50
51        let mut req = Request::get(&url);
52
53        if !range.is_full() {
54            req = req.header(http::header::RANGE, range.to_header());
55        }
56
57        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
58
59        self.info.http_client().fetch(req).await
60    }
61
62    pub async fn ipfs_head(&self, path: &str) -> Result<Response<Buffer>> {
63        let p = build_rooted_abs_path(&self.root, path);
64
65        let url = format!("{}{}", self.endpoint, percent_encode_path(&p));
66
67        let req = Request::head(&url);
68
69        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
70
71        self.info.http_client().send(req).await
72    }
73
74    pub async fn ipfs_list(&self, path: &str) -> Result<Response<Buffer>> {
75        let p = build_rooted_abs_path(&self.root, path);
76
77        let url = format!("{}{}", self.endpoint, percent_encode_path(&p));
78
79        let mut req = Request::get(&url);
80
81        // Use "application/vnd.ipld.raw" to disable IPLD codec deserialization
82        // OpenDAL will parse ipld data directly.
83        //
84        // ref: https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md
85        req = req.header(http::header::ACCEPT, "application/vnd.ipld.raw");
86
87        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
88
89        self.info.http_client().send(req).await
90    }
91
92    /// IPFS's stat behavior highly depends on its implementation.
93    ///
94    /// Based on IPFS [Path Gateway Specification](https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md),
95    /// response payload could be:
96    ///
97    /// > - UnixFS (implicit default)
98    /// >   - File
99    /// >     - Bytes representing file contents
100    /// >   - Directory
101    /// >     - Generated HTML with directory index
102    /// >     - When `index.html` is present, gateway can skip generating directory index and return it instead
103    /// > - Raw block (not this case)
104    /// > - CAR (not this case)
105    ///
106    /// When we HEAD a given path, we could have the following responses:
107    ///
108    /// - File
109    ///
110    /// ```http
111    /// :) curl -I https://ipfs.io/ipfs/QmPpCt1aYGb9JWJRmXRUnmJtVgeFFTJGzWFYEEX7bo9zGJ/normal_file
112    /// HTTP/1.1 200 Connection established
113    ///
114    /// HTTP/2 200
115    /// server: openresty
116    /// date: Thu, 08 Sep 2022 00:48:50 GMT
117    /// content-type: application/octet-stream
118    /// content-length: 262144
119    /// access-control-allow-methods: GET
120    /// cache-control: public, max-age=29030400, immutable
121    /// etag: "QmdP6teFTLSNVhT4W5jkhEuUBsjQ3xkp1GmRvDU6937Me1"
122    /// x-ipfs-gateway-host: ipfs-bank11-fr2
123    /// x-ipfs-path: /ipfs/QmPpCt1aYGb9JWJRmXRUnmJtVgeFFTJGzWFYEEX7bo9zGJ/normal_file
124    /// x-ipfs-roots: QmPpCt1aYGb9JWJRmXRUnmJtVgeFFTJGzWFYEEX7bo9zGJ,QmdP6teFTLSNVhT4W5jkhEuUBsjQ3xkp1GmRvDU6937Me1
125    /// x-ipfs-pop: ipfs-bank11-fr2
126    /// timing-allow-origin: *
127    /// x-ipfs-datasize: 262144
128    /// access-control-allow-origin: *
129    /// access-control-allow-methods: GET, POST, OPTIONS
130    /// access-control-allow-headers: X-Requested-With, Range, Content-Range, X-Chunked-Output, X-Stream-Output
131    /// access-control-expose-headers: Content-Range, X-Chunked-Output, X-Stream-Output
132    /// x-ipfs-lb-pop: gateway-bank1-fr2
133    /// strict-transport-security: max-age=31536000; includeSubDomains; preload
134    /// x-proxy-cache: MISS
135    /// accept-ranges: bytes
136    /// ```
137    ///
138    /// - Dir with generated index
139    ///
140    /// ```http
141    /// :( curl -I https://ipfs.io/ipfs/QmPpCt1aYGb9JWJRmXRUnmJtVgeFFTJGzWFYEEX7bo9zGJ/normal_dir
142    /// HTTP/1.1 200 Connection established
143    ///
144    /// HTTP/2 200
145    /// server: openresty
146    /// date: Wed, 07 Sep 2022 08:46:13 GMT
147    /// content-type: text/html
148    /// vary: Accept-Encoding
149    /// access-control-allow-methods: GET
150    /// etag: "DirIndex-2b567f6r5vvdg_CID-QmY44DyCDymRN1Qy7sGbupz1ysMkXTWomAQku5vBg7fRQW"
151    /// x-ipfs-gateway-host: ipfs-bank6-sg1
152    /// x-ipfs-path: /ipfs/QmPpCt1aYGb9JWJRmXRUnmJtVgeFFTJGzWFYEEX7bo9zGJ/normal_dir
153    /// x-ipfs-roots: QmPpCt1aYGb9JWJRmXRUnmJtVgeFFTJGzWFYEEX7bo9zGJ,QmY44DyCDymRN1Qy7sGbupz1ysMkXTWomAQku5vBg7fRQW
154    /// x-ipfs-pop: ipfs-bank6-sg1
155    /// timing-allow-origin: *
156    /// access-control-allow-origin: *
157    /// access-control-allow-methods: GET, POST, OPTIONS
158    /// access-control-allow-headers: X-Requested-With, Range, Content-Range, X-Chunked-Output, X-Stream-Output
159    /// access-control-expose-headers: Content-Range, X-Chunked-Output, X-Stream-Output
160    /// x-ipfs-lb-pop: gateway-bank3-sg1
161    /// strict-transport-security: max-age=31536000; includeSubDomains; preload
162    /// x-proxy-cache: MISS
163    /// ```
164    ///
165    /// - Dir with index.html
166    ///
167    /// ```http
168    /// :) curl -I http://127.0.0.1:8080/ipfs/QmVturFGV3z4WsP7cRV8Ci4avCdGWYXk2qBKvtAwFUp5Az
169    /// HTTP/1.1 302 Found
170    /// Access-Control-Allow-Headers: Content-Type
171    /// Access-Control-Allow-Headers: Range
172    /// Access-Control-Allow-Headers: User-Agent
173    /// Access-Control-Allow-Headers: X-Requested-With
174    /// Access-Control-Allow-Methods: GET
175    /// Access-Control-Allow-Origin: *
176    /// Access-Control-Expose-Headers: Content-Length
177    /// Access-Control-Expose-Headers: Content-Range
178    /// Access-Control-Expose-Headers: X-Chunked-Output
179    /// Access-Control-Expose-Headers: X-Ipfs-Path
180    /// Access-Control-Expose-Headers: X-Ipfs-Roots
181    /// Access-Control-Expose-Headers: X-Stream-Output
182    /// Content-Type: text/html; charset=utf-8
183    /// Location: /ipfs/QmVturFGV3z4WsP7cRV8Ci4avCdGWYXk2qBKvtAwFUp5Az/
184    /// X-Ipfs-Path: /ipfs/QmVturFGV3z4WsP7cRV8Ci4avCdGWYXk2qBKvtAwFUp5Az
185    /// X-Ipfs-Roots: QmVturFGV3z4WsP7cRV8Ci4avCdGWYXk2qBKvtAwFUp5Az
186    /// Date: Thu, 08 Sep 2022 00:52:29 GMT
187    /// ```
188    ///
189    /// In conclusion:
190    ///
191    /// - HTTP Status Code == 302 => directory
192    /// - HTTP Status Code == 200 && ETag starts with `"DirIndex` => directory
193    /// - HTTP Status Code == 200 && ETag not starts with `"DirIndex` => file
194    pub async fn ipfs_stat(&self, path: &str) -> Result<Metadata> {
195        // Stat root always returns a DIR.
196        if path == "/" {
197            return Ok(Metadata::new(EntryMode::DIR));
198        }
199
200        let resp = self.ipfs_head(path).await?;
201        let status = resp.status();
202
203        match status {
204            StatusCode::OK => {
205                let mut m = Metadata::new(EntryMode::Unknown);
206
207                if let Some(v) = parse_content_length(resp.headers())? {
208                    m.set_content_length(v);
209                }
210
211                if let Some(v) = parse_content_type(resp.headers())? {
212                    m.set_content_type(v);
213                }
214
215                if let Some(v) = parse_etag(resp.headers())? {
216                    m.set_etag(v);
217
218                    if v.starts_with("\"DirIndex") {
219                        m.set_mode(EntryMode::DIR);
220                    } else {
221                        m.set_mode(EntryMode::FILE);
222                    }
223                } else {
224                    // Some service will stream the output of DirIndex.
225                    // If we don't have an etag, it's highly to be a dir.
226                    m.set_mode(EntryMode::DIR);
227                }
228
229                if let Some(v) = parse_content_disposition(resp.headers())? {
230                    m.set_content_disposition(v);
231                }
232
233                Ok(m)
234            }
235            StatusCode::FOUND | StatusCode::MOVED_PERMANENTLY => Ok(Metadata::new(EntryMode::DIR)),
236            _ => Err(parse_error(resp)),
237        }
238    }
239}
opendal/services/ipfs/core.rs

opendal/services/ipfs/
core.rs
