opendal/services/huggingface/
lister.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::sync::Arc;
19
20use bytes::Buf;
21
22use super::core::HuggingfaceCore;
23use super::core::HuggingfaceStatus;
24use super::error::parse_error;
25use crate::raw::*;
26use crate::*;
27
28pub struct HuggingfaceLister {
29    core: Arc<HuggingfaceCore>,
30    path: String,
31    recursive: bool,
32}
33
34impl HuggingfaceLister {
35    pub fn new(core: Arc<HuggingfaceCore>, path: String, recursive: bool) -> Self {
36        Self {
37            core,
38            path,
39            recursive,
40        }
41    }
42}
43
44impl oio::PageList for HuggingfaceLister {
45    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
46        let response = self.core.hf_list(&self.path, self.recursive).await?;
47
48        let status_code = response.status();
49        if !status_code.is_success() {
50            let error = parse_error(response);
51            return Err(error);
52        }
53
54        let bytes = response.into_body();
55        let decoded_response: Vec<HuggingfaceStatus> =
56            serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
57
58        ctx.done = true;
59
60        for status in decoded_response {
61            let entry_type = match status.type_.as_str() {
62                "directory" => EntryMode::DIR,
63                "file" => EntryMode::FILE,
64                _ => EntryMode::Unknown,
65            };
66
67            let mut meta = Metadata::new(entry_type);
68
69            if let Some(commit_info) = status.last_commit.as_ref() {
70                meta.set_last_modified(parse_datetime_from_rfc3339(commit_info.date.as_str())?);
71            }
72
73            if entry_type == EntryMode::FILE {
74                meta.set_content_length(status.size);
75            }
76
77            let path = if entry_type == EntryMode::DIR {
78                format!("{}/", &status.path)
79            } else {
80                status.path.clone()
81            };
82
83            ctx.entries.push_back(oio::Entry::new(
84                &build_rel_path(&self.core.root, &path),
85                meta,
86            ));
87        }
88
89        Ok(())
90    }
91}
opendal/services/huggingface/lister.rs

opendal/services/huggingface/
lister.rs
