opendal_service_hdfs/
backend.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::io;
19use std::sync::Arc;
20
21use log::debug;
22
23use super::HDFS_SCHEME;
24use super::config::HdfsConfig;
25use super::core::HdfsCore;
26use super::deleter::HdfsDeleter;
27use super::lister::HdfsLister;
28use super::reader::*;
29use opendal_core::raw::*;
30use opendal_core::*;
31
32#[doc = include_str!("docs.md")]
33#[derive(Debug, Default)]
34pub struct HdfsBuilder {
35    pub(super) config: HdfsConfig,
36}
37
38impl HdfsBuilder {
39    /// Set root of this backend.
40    ///
41    /// All operations will happen under this root.
42    pub fn root(mut self, root: &str) -> Self {
43        self.config.root = if root.is_empty() {
44            None
45        } else {
46            Some(root.to_string())
47        };
48
49        self
50    }
51
52    /// Set name_node of this backend.
53    ///
54    /// Valid format including:
55    ///
56    /// - `default`: using the default setting based on hadoop config.
57    /// - `hdfs://127.0.0.1:9000`: connect to hdfs cluster.
58    pub fn name_node(mut self, name_node: &str) -> Self {
59        if !name_node.is_empty() {
60            self.config.name_node = Some(name_node.to_string())
61        }
62
63        self
64    }
65
66    /// Set kerberos_ticket_cache_path of this backend
67    ///
68    /// This should be configured when kerberos is enabled.
69    pub fn kerberos_ticket_cache_path(mut self, kerberos_ticket_cache_path: &str) -> Self {
70        if !kerberos_ticket_cache_path.is_empty() {
71            self.config.kerberos_ticket_cache_path = Some(kerberos_ticket_cache_path.to_string())
72        }
73        self
74    }
75
76    /// Set user of this backend
77    pub fn user(mut self, user: &str) -> Self {
78        if !user.is_empty() {
79            self.config.user = Some(user.to_string())
80        }
81        self
82    }
83
84    /// Deprecated: HDFS append capability is enabled by default.
85    #[deprecated(
86        since = "0.57.0",
87        note = "HDFS append capability is enabled by default and this option is no longer needed."
88    )]
89    pub fn enable_append(self, _enable_append: bool) -> Self {
90        self
91    }
92
93    /// Set temp dir for atomic write.
94    ///
95    /// # Notes
96    ///
97    /// - When append is enabled, we will not use atomic write
98    ///   to avoid data loss and performance issue.
99    pub fn atomic_write_dir(mut self, dir: &str) -> Self {
100        self.config.atomic_write_dir = if dir.is_empty() {
101            None
102        } else {
103            Some(String::from(dir))
104        };
105        self
106    }
107}
108
109impl Builder for HdfsBuilder {
110    type Config = HdfsConfig;
111
112    fn build(self) -> Result<impl Service> {
113        debug!("backend build started: {:?}", self);
114
115        let name_node = match &self.config.name_node {
116            Some(v) => v,
117            None => {
118                return Err(Error::new(ErrorKind::ConfigInvalid, "name node is empty")
119                    .with_context("service", HDFS_SCHEME));
120            }
121        };
122
123        let root = normalize_root(&self.config.root.unwrap_or_default());
124        debug!("backend use root {root}");
125
126        let mut builder = hdrs::ClientBuilder::new(name_node);
127        if let Some(ticket_cache_path) = &self.config.kerberos_ticket_cache_path {
128            builder = builder.with_kerberos_ticket_cache_path(ticket_cache_path.as_str());
129        }
130        if let Some(user) = &self.config.user {
131            builder = builder.with_user(user.as_str());
132        }
133
134        let client = builder.connect().map_err(new_std_io_error)?;
135
136        // Create root dir if not exist.
137        if let Err(e) = client.metadata(&root)
138            && e.kind() == io::ErrorKind::NotFound
139        {
140            debug!("root {root} is not exist, creating now");
141
142            client.create_dir(&root).map_err(new_std_io_error)?
143        }
144
145        let atomic_write_dir = self.config.atomic_write_dir;
146
147        // If atomic write dir is not exist, we must create it.
148        if let Some(d) = &atomic_write_dir
149            && let Err(e) = client.metadata(d)
150            && e.kind() == io::ErrorKind::NotFound
151        {
152            client.create_dir(d).map_err(new_std_io_error)?
153        }
154
155        Ok(HdfsBackend {
156            core: Arc::new(HdfsCore {
157                info: ServiceInfo::new(HDFS_SCHEME, &root, ""),
158                capability: Capability {
159                    stat: true,
160
161                    read: true,
162
163                    write: true,
164                    write_can_append: true,
165
166                    create_dir: true,
167                    delete: true,
168                    delete_with_recursive: true,
169
170                    list: true,
171
172                    rename: true,
173                    rename_with_if_not_exists: true,
174
175                    shared: true,
176
177                    ..Default::default()
178                },
179                root,
180                atomic_write_dir,
181                client: Arc::new(client),
182            }),
183        })
184    }
185}
186
187/// Backend for hdfs services.
188#[derive(Debug, Clone)]
189pub struct HdfsBackend {
190    pub(crate) core: Arc<HdfsCore>,
191}
192
193impl Service for HdfsBackend {
194    type Reader = oio::PositionReader<HdfsReader>;
195    type Writer = HdfsLazyWriter;
196    type Lister = Option<HdfsLister>;
197    type Deleter = oio::OneShotDeleter<HdfsDeleter>;
198    type Copier = ();
199
200    fn info(&self) -> ServiceInfo {
201        self.core.info.clone()
202    }
203
204    fn capability(&self) -> Capability {
205        self.core.capability
206    }
207
208    async fn create_dir(
209        &self,
210        _ctx: &OperationContext,
211        path: &str,
212        _: OpCreateDir,
213    ) -> Result<RpCreateDir> {
214        self.core.hdfs_create_dir(path)?;
215        Ok(RpCreateDir::default())
216    }
217
218    async fn stat(&self, _ctx: &OperationContext, path: &str, _: OpStat) -> Result<RpStat> {
219        let m = self.core.hdfs_stat(path)?;
220        Ok(RpStat::new(m))
221    }
222    fn read(&self, _ctx: &OperationContext, path: &str, _: OpRead) -> Result<Self::Reader> {
223        Ok(oio::PositionReader::new(HdfsReader::new(
224            self.core.clone(),
225            path,
226        )))
227    }
228
229    fn write(&self, _ctx: &OperationContext, path: &str, op: OpWrite) -> Result<Self::Writer> {
230        Ok(HdfsLazyWriter::new(self.core.clone(), path, op))
231    }
232
233    fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
234        let output: oio::OneShotDeleter<HdfsDeleter> = {
235            Ok(oio::OneShotDeleter::new(HdfsDeleter::new(Arc::clone(
236                &self.core,
237            ))))
238        }?;
239
240        Ok(output)
241    }
242
243    fn list(&self, _ctx: &OperationContext, path: &str, _: OpList) -> Result<Self::Lister> {
244        let output: Option<HdfsLister> = {
245            match self.core.hdfs_list(path)? {
246                Some(f) => {
247                    let rd = HdfsLister::new(&self.core.root, f, path);
248                    Ok(Some(rd))
249                }
250                None => Ok(None),
251            }
252        }?;
253
254        Ok(output)
255    }
256
257    fn copy(
258        &self,
259        _ctx: &OperationContext,
260        _from: &str,
261        _to: &str,
262        _args: OpCopy,
263        _opts: OpCopier,
264    ) -> Result<Self::Copier> {
265        Err(Error::new(
266            ErrorKind::Unsupported,
267            "operation is not supported",
268        ))
269    }
270
271    async fn rename(
272        &self,
273        _ctx: &OperationContext,
274        from: &str,
275        to: &str,
276        args: OpRename,
277    ) -> Result<RpRename> {
278        self.core.hdfs_rename(from, to, &args)?;
279        Ok(RpRename::new())
280    }
281
282    async fn presign(
283        &self,
284        _ctx: &OperationContext,
285        _path: &str,
286        _args: OpPresign,
287    ) -> Result<RpPresign> {
288        Err(Error::new(
289            ErrorKind::Unsupported,
290            "operation is not supported",
291        ))
292    }
293}
opendal_service_hdfs/backend.rs

opendal_service_hdfs/
backend.rs
