opendal/services/hdfs/
reader.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::io::Read;
19
20use bytes::BytesMut;
21use futures::AsyncReadExt;
22use hdrs::AsyncFile;
23use hdrs::File;
24use tokio::io::ReadBuf;
25
26use crate::raw::*;
27use crate::*;
28
29pub struct HdfsReader<F> {
30    f: F,
31    read: usize,
32    size: usize,
33    buf_size: usize,
34    buf: BytesMut,
35}
36
37impl<F> HdfsReader<F> {
38    pub fn new(f: F, size: usize) -> Self {
39        Self {
40            f,
41            read: 0,
42            size,
43            // Use 2 MiB as default value.
44            buf_size: 2 * 1024 * 1024,
45            buf: BytesMut::new(),
46        }
47    }
48}
49
50impl oio::Read for HdfsReader<AsyncFile> {
51    async fn read(&mut self) -> Result<Buffer> {
52        if self.read >= self.size {
53            return Ok(Buffer::new());
54        }
55
56        let size = (self.size - self.read).min(self.buf_size);
57        self.buf.reserve(size);
58
59        let buf = &mut self.buf.spare_capacity_mut()[..size];
60        let mut read_buf: ReadBuf = ReadBuf::uninit(buf);
61
62        // SAFETY: Read at most `limit` bytes into `read_buf`.
63        unsafe {
64            read_buf.assume_init(size);
65        }
66
67        let n = self
68            .f
69            .read(read_buf.initialize_unfilled())
70            .await
71            .map_err(new_std_io_error)?;
72        read_buf.advance(n);
73        self.read += n;
74
75        // Safety: We make sure that bs contains `n` more bytes.
76        let filled = read_buf.filled().len();
77        unsafe { self.buf.set_len(filled) }
78
79        let frozen = self.buf.split().freeze();
80        Ok(Buffer::from(frozen))
81    }
82}
83
84impl oio::BlockingRead for HdfsReader<File> {
85    fn read(&mut self) -> Result<Buffer> {
86        if self.read >= self.size {
87            return Ok(Buffer::new());
88        }
89
90        let size = (self.size - self.read).min(self.buf_size);
91        self.buf.reserve(size);
92
93        let buf = &mut self.buf.spare_capacity_mut()[..size];
94        let mut read_buf: ReadBuf = ReadBuf::uninit(buf);
95
96        // SAFETY: Read at most `limit` bytes into `read_buf`.
97        unsafe {
98            read_buf.assume_init(size);
99        }
100
101        let n = self
102            .f
103            .read(read_buf.initialize_unfilled())
104            .map_err(new_std_io_error)?;
105        read_buf.advance(n);
106        self.read += n;
107
108        // Safety: We make sure that bs contains `n` more bytes.
109        let filled = read_buf.filled().len();
110        unsafe { self.buf.set_len(filled) }
111
112        let frozen = self.buf.split().freeze();
113        Ok(Buffer::from(frozen))
114    }
115}
opendal/services/hdfs/reader.rs

opendal/services/hdfs/
reader.rs
