hdfs的必会操作

创建目录

//创建目录
public static void mkdir(String filePath) throws URISyntaxException, IOException, InterruptedException{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path path= new Path(filePath);
fs.mkdirs(path);
System.out.println("目录创建成功:"+filePath);
fs.close();
}

创建文件

//创建文件
public static void createFile(String remoteFilePath)throws URISyntaxException, IOException, InterruptedException {
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path remotePath = new Path(remoteFilePath);
FSDataOutputStream outputStream = fs.create(remotePath);
outputStream.close();
System.out.println("文件创建成功!!"+remoteFilePath);
}

删除文件

//删除文件
public static void deleteFile(String filePath) throws URISyntaxException, IOException, InterruptedException{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path path= new Path(filePath);
if(fs.deleteOnExit(path)) {
System.out.println("文件删除成功:"+filePath);
} else {
System.out.println("文件删除失败:"+filePath);
}
fs.close();
}

移动文件到本地

 // 移动文件到本地
public static void moveToLocalFile(String remoteFilePath, String localFilePath) throws IOException, InterruptedException, URISyntaxException {
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path remotePath = new Path(remoteFilePath);
Path localPath = new Path(localFilePath);
fs.moveToLocalFile(remotePath, localPath);
}

显示文件里的内容

//显示文件内容
public static void cat(String file)throws URISyntaxException, IOException, InterruptedException
{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path filePath = new Path(file);
if(fs.exists(filePath)) {
FSDataInputStream in = fs.open(filePath);
BufferedReader br = new BufferedReader(new InputStreamReader(in));
String content = null;
while((content = br.readLine()) != null) {
System.out.println(content);
}
br.close();
fs.close();
}else {
System.out.println("file "+filePath+ "doesn't exist.");
}
fs.close();
}

移动hdfs的文件

//移动hdfs中的文件
public static void moveFile(String srcPath, String dirPath) throws URISyntaxException, IOException, InterruptedException {
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
if(fs.exists(new Path(dirPath))) {
System.out.println("文件被占用。");
return;
}
if(fs.rename(new Path(srcPath), new Path(dirPath))) {
System.out.println("文件移动成功。");
} else {
System.out.println("文件移动失败。");
}
}

上传与下载

//将本地文件上传到hdfs
public static void copyFromLocalFile(String localPath,String srcPath) throws URISyntaxException, IOException, InterruptedException
{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
fs.copyFromLocalFile(new Path(localPath),new Path(srcPath));
System.out.println("上传成功!!!");
fs.close();
} //将hdfs的文件下载到本地
public static void downFromHdfs(String src,String dst)throws URISyntaxException, IOException, InterruptedException
{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path dstPath=new Path(dst);
//第一个参数为:是否删除原文件,源文件,目的文件,是否使用本地文件系统
fs.copyToLocalFile(false,new Path(src),dstPath,true);
fs.close();
System.out.println("文件下载成功已存放到"+dst);
}

追加内容到文件结尾

//追加到结尾
public static void addContentToTail(String filePath, String content, boolean head) throws IOException, InterruptedException, URISyntaxException {
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path path= new Path(filePath);
FSDataOutputStream ops = fs.append(path);
ops.write(content.getBytes());
if (!head) {
System.out.println("内容以追加到结尾。");
}
ops.close();
fs.close();
}

追加本地文件内容到hdfs文件结尾

 // 追加本地文件内容到目的文件结尾
public static void addFileToTail(String localFilePath, String remoteFilePath) throws IOException, InterruptedException, URISyntaxException{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path remotePath = new Path(remoteFilePath);
// 创建一个本地文件(需要追加的文件)读入流
FileInputStream inps = new FileInputStream(localFilePath);
// 创建一个文件输出留,输出的内容追加到文件末尾
FSDataOutputStream ops = fs.append(remotePath);
byte[] buffer = new byte[1024];
int read = -1;
while((read = inps.read(buffer)) > 0) {
ops.write(buffer, 0, read);
}
ops.close();
inps.close();
fs.close();
}

追加内容到hdfs文件开头

// 追加内容到文件开头
public static void addContentToHead(String filePath, String content) throws IOException, InterruptedException, URISyntaxException {
// 创建一个临时本地文件
String localFilePath = new File("").getCanonicalPath()+"\\web\\text\\xlf.txt" ;
// 将要追加的文件移动到本地
moveToLocalFile(filePath, localFilePath);
// 创建一个新的HDFS文件(空的)
createFile(filePath);
appendContentToTail(content,filePath, true);
addFileToTail(localFilePath, filePath);
System.out.println("内容以追加到开头。");
}

判断hdfs中文件是否存在

public  static void existandcreat(String path) throws URISyntaxException, IOException, InterruptedException {
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path path1=new Path(path);
if(fs.exists(path1))
{
System.out.println("存在!!");
}
else
{
FSDataOutputStream ops=fs.create(path1);
ops.close();
fs.close();
}
}

递归查看目录下的文件信息

/*查看当前目录下的文件信息*/
public static void printfile(String file)throws URISyntaxException, IOException, InterruptedException
{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
FileStatus[] statuses=fs.listStatus(new Path(file));
for(FileStatus s:statuses)
{
System.out.println("读写权限:"+s.getPermission()+"; 文件大小:"+s.getBlockSize()+"; 文件路径:"
+s.getPath()+"; 文件创建时间:"+s.getModificationTime());
}
fs.close();
} /*递归查看目录下的文件信息*/
public static void prinfileInfo(String file)throws URISyntaxException, IOException, InterruptedException
{
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop101:8020"), new Configuration(), "root");
Path path= new Path(file);
RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(path, true);
while(iterator.hasNext()) {
FileStatus s = iterator.next();
System.out.println("读写权限:"+s.getPermission()+"; 文件大小:"+s.getBlockSize()+"; 文件路径:"
+s.getPath()+"; 文件创建时间:"+s.getModificationTime());
}
fs.close();
}

最新文章

  1. h5嵌入视频遇到的bug及总结
  2. HTML补充
  3. hdu 1502 Regular Words
  4. nodejs解决找不到express命令的问题
  5. SQL Server(五)——常用函数
  6. 浅谈可扩展性框架:MEF
  7. uGUI练习(七) Drag And Drop
  8. rop框架中@ServiceMethod注解属性
  9. C#委托的详细使用
  10. 安装CDH
  11. SRM 620 D2L3: RandomGraph, dp
  12. golang 浮点数 取精度的效率对比
  13. Spring Security(三十五):Part III. Testing
  14. Ubuntu允许root远程登录配置
  15. 信用评分卡 (part 5 of 7)
  16. 整合Flask中的目录结构
  17. 微信小游戏跳一跳简单手动外挂(基于adb 和 python)
  18. Minimum Integer CodeForces - 1101A (思维+公式)
  19. javascript 面向对象之路.2 - 小蜜蜂
  20. Android之socket多线程(二)

热门文章

  1. 使用Java7提供的WatchService给目录添加新建文件监控
  2. jmeter连接redis取数据
  3. html基础:DOM操作
  4. C# 读取 ttf字体文件里的 Unicode
  5. [剑指Offer]56-数组中数字出现的次数(位运算)
  6. [Java核心技术]第九章-集合(Java集合框架、具体的集合、映射)
  7. JavaScript 流程控制-分支
  8. 详解volatile关键字和原子引用
  9. hystrix熔断器之配置
  10. Hadoop演进与Hadoop生态