hdfs的JAVA必会操作
2024-10-09 17:04:13
hdfs的必会操作
创建目录
//创建目录
public static void mkdir(String filePath) throws URISyntaxException, IOException, InterruptedException{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path path= new Path(filePath);
fs.mkdirs(path);
System.out.println("目录创建成功:"+filePath);
fs.close();
}
创建文件
//创建文件
public static void createFile(String remoteFilePath)throws URISyntaxException, IOException, InterruptedException {
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path remotePath = new Path(remoteFilePath);
FSDataOutputStream outputStream = fs.create(remotePath);
outputStream.close();
System.out.println("文件创建成功!!"+remoteFilePath);
}
删除文件
//删除文件
public static void deleteFile(String filePath) throws URISyntaxException, IOException, InterruptedException{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path path= new Path(filePath);
if(fs.deleteOnExit(path)) {
System.out.println("文件删除成功:"+filePath);
} else {
System.out.println("文件删除失败:"+filePath);
}
fs.close();
}
移动文件到本地
// 移动文件到本地
public static void moveToLocalFile(String remoteFilePath, String localFilePath) throws IOException, InterruptedException, URISyntaxException {
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path remotePath = new Path(remoteFilePath);
Path localPath = new Path(localFilePath);
fs.moveToLocalFile(remotePath, localPath);
}
显示文件里的内容
//显示文件内容
public static void cat(String file)throws URISyntaxException, IOException, InterruptedException
{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path filePath = new Path(file);
if(fs.exists(filePath)) {
FSDataInputStream in = fs.open(filePath);
BufferedReader br = new BufferedReader(new InputStreamReader(in));
String content = null;
while((content = br.readLine()) != null) {
System.out.println(content);
}
br.close();
fs.close();
}else {
System.out.println("file "+filePath+ "doesn't exist.");
}
fs.close();
}
移动hdfs的文件
//移动hdfs中的文件
public static void moveFile(String srcPath, String dirPath) throws URISyntaxException, IOException, InterruptedException {
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
if(fs.exists(new Path(dirPath))) {
System.out.println("文件被占用。");
return;
}
if(fs.rename(new Path(srcPath), new Path(dirPath))) {
System.out.println("文件移动成功。");
} else {
System.out.println("文件移动失败。");
}
}
上传与下载
//将本地文件上传到hdfs
public static void copyFromLocalFile(String localPath,String srcPath) throws URISyntaxException, IOException, InterruptedException
{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
fs.copyFromLocalFile(new Path(localPath),new Path(srcPath));
System.out.println("上传成功!!!");
fs.close();
} //将hdfs的文件下载到本地
public static void downFromHdfs(String src,String dst)throws URISyntaxException, IOException, InterruptedException
{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path dstPath=new Path(dst);
//第一个参数为:是否删除原文件,源文件,目的文件,是否使用本地文件系统
fs.copyToLocalFile(false,new Path(src),dstPath,true);
fs.close();
System.out.println("文件下载成功已存放到"+dst);
}
追加内容到文件结尾
//追加到结尾
public static void addContentToTail(String filePath, String content, boolean head) throws IOException, InterruptedException, URISyntaxException {
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path path= new Path(filePath);
FSDataOutputStream ops = fs.append(path);
ops.write(content.getBytes());
if (!head) {
System.out.println("内容以追加到结尾。");
}
ops.close();
fs.close();
}
追加本地文件内容到hdfs文件结尾
// 追加本地文件内容到目的文件结尾
public static void addFileToTail(String localFilePath, String remoteFilePath) throws IOException, InterruptedException, URISyntaxException{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path remotePath = new Path(remoteFilePath);
// 创建一个本地文件(需要追加的文件)读入流
FileInputStream inps = new FileInputStream(localFilePath);
// 创建一个文件输出留,输出的内容追加到文件末尾
FSDataOutputStream ops = fs.append(remotePath);
byte[] buffer = new byte[1024];
int read = -1;
while((read = inps.read(buffer)) > 0) {
ops.write(buffer, 0, read);
}
ops.close();
inps.close();
fs.close();
}
追加内容到hdfs文件开头
// 追加内容到文件开头
public static void addContentToHead(String filePath, String content) throws IOException, InterruptedException, URISyntaxException {
// 创建一个临时本地文件
String localFilePath = new File("").getCanonicalPath()+"\\web\\text\\xlf.txt" ;
// 将要追加的文件移动到本地
moveToLocalFile(filePath, localFilePath);
// 创建一个新的HDFS文件(空的)
createFile(filePath);
appendContentToTail(content,filePath, true);
addFileToTail(localFilePath, filePath);
System.out.println("内容以追加到开头。");
}
判断hdfs中文件是否存在
public static void existandcreat(String path) throws URISyntaxException, IOException, InterruptedException {
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path path1=new Path(path);
if(fs.exists(path1))
{
System.out.println("存在!!");
}
else
{
FSDataOutputStream ops=fs.create(path1);
ops.close();
fs.close();
}
}
递归查看目录下的文件信息
/*查看当前目录下的文件信息*/
public static void printfile(String file)throws URISyntaxException, IOException, InterruptedException
{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
FileStatus[] statuses=fs.listStatus(new Path(file));
for(FileStatus s:statuses)
{
System.out.println("读写权限:"+s.getPermission()+"; 文件大小:"+s.getBlockSize()+"; 文件路径:"
+s.getPath()+"; 文件创建时间:"+s.getModificationTime());
}
fs.close();
} /*递归查看目录下的文件信息*/
public static void prinfileInfo(String file)throws URISyntaxException, IOException, InterruptedException
{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path path= new Path(file);
RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(path, true);
while(iterator.hasNext()) {
FileStatus s = iterator.next();
System.out.println("读写权限:"+s.getPermission()+"; 文件大小:"+s.getBlockSize()+"; 文件路径:"
+s.getPath()+"; 文件创建时间:"+s.getModificationTime());
}
fs.close();
}
最新文章
- h5嵌入视频遇到的bug及总结
- HTML补充
- hdu 1502 Regular Words
- nodejs解决找不到express命令的问题
- SQL Server(五)——常用函数
- 浅谈可扩展性框架:MEF
- uGUI练习(七) Drag And Drop
- rop框架中@ServiceMethod注解属性
- C#委托的详细使用
- 安装CDH
- SRM 620 D2L3: RandomGraph, dp
- golang 浮点数 取精度的效率对比
- Spring Security(三十五):Part III. Testing
- Ubuntu允许root远程登录配置
- 信用评分卡 (part 5 of 7)
- 整合Flask中的目录结构
- 微信小游戏跳一跳简单手动外挂(基于adb 和 python)
- Minimum Integer CodeForces - 1101A (思维+公式)
- javascript 面向对象之路.2 - 小蜜蜂
- Android之socket多线程(二)