1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use bytes::Buf;
use http::StatusCode;

use super::backend::WebhdfsBackend;
use super::error::parse_error;
use super::message::*;
use crate::raw::*;
use crate::*;

pub struct WebhdfsLister {
    backend: WebhdfsBackend,
    path: String,
}

impl WebhdfsLister {
    pub fn new(backend: WebhdfsBackend, path: &str) -> Self {
        Self {
            backend,
            path: path.to_string(),
        }
    }
}

impl oio::PageList for WebhdfsLister {
    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
        let file_status = if self.backend.disable_list_batch {
            let resp = self.backend.webhdfs_list_status_request(&self.path).await?;
            match resp.status() {
                StatusCode::OK => {
                    ctx.done = true;
                    ctx.entries.push_back(oio::Entry::new(
                        format!("{}/", self.path).as_str(),
                        Metadata::new(EntryMode::DIR),
                    ));

                    let bs = resp.into_body();
                    serde_json::from_reader::<_, FileStatusesWrapper>(bs.reader())
                        .map_err(new_json_deserialize_error)?
                        .file_statuses
                        .file_status
                }
                StatusCode::NOT_FOUND => {
                    ctx.done = true;
                    return Ok(());
                }
                _ => return Err(parse_error(resp)),
            }
        } else {
            let resp = self
                .backend
                .webhdfs_list_status_batch_request(&self.path, &ctx.token)
                .await?;
            match resp.status() {
                StatusCode::OK => {
                    let bs = resp.into_body();
                    let res: DirectoryListingWrapper =
                        serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
                    let directory_listing = res.directory_listing;
                    let file_statuses = directory_listing.partial_listing.file_statuses.file_status;

                    if directory_listing.remaining_entries == 0 {
                        ctx.entries.push_back(oio::Entry::new(
                            format!("{}/", self.path).as_str(),
                            Metadata::new(EntryMode::DIR),
                        ));

                        ctx.done = true;
                    } else if !file_statuses.is_empty() {
                        ctx.token
                            .clone_from(&file_statuses.last().unwrap().path_suffix);
                    }

                    file_statuses
                }
                StatusCode::NOT_FOUND => {
                    ctx.done = true;
                    return Ok(());
                }
                _ => return Err(parse_error(resp)),
            }
        };

        for status in file_status {
            let mut path = if self.path.is_empty() {
                status.path_suffix.to_string()
            } else {
                format!("{}/{}", self.path, status.path_suffix)
            };

            let meta = match status.ty {
                FileStatusType::Directory => Metadata::new(EntryMode::DIR),
                FileStatusType::File => Metadata::new(EntryMode::FILE)
                    .with_content_length(status.length)
                    .with_last_modified(parse_datetime_from_from_timestamp_millis(
                        status.modification_time,
                    )?),
            };

            if meta.mode().is_file() {
                path = path.trim_end_matches('/').to_string();
            }
            if meta.mode().is_dir() {
                path += "/"
            }
            let entry = oio::Entry::new(&path, meta);
            ctx.entries.push_back(entry);
        }

        Ok(())
    }
}