hadoop之HDFS核心类Filesystem的使用
2024-09-06 16:46:39
1.导入jar包，要使用hadoop的HDFS就要导入hadoop-2.7.7\share\hadoop\common下的3个jar包和lib下的依赖包、hadoop-2.7.7\share\hadoop\hdfs下的3个jar包和lib下的依赖包
2.-ls 查看目录下的所有文件和文件夹
@Test
public void listStatus() {
Configuration conf = new Configuration();
//使用hdfs的fs功能，客户端就会访问core-site.xml配置文件
//这里是设置core-site.xml中的属性fs.defaultFS和属性值hdfs://192.168.xx.xx:9000
//注意写自己的ip地址
conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
try {
FileSystem fileSystem = FileSystem.get(conf);
//查看的路径
FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
for(int i = 0; i < listStatus.length; ++i)
{ String dpath = listStatus[i].getPath().toString();
System.out.println(dpath);
}
fileSystem.close();
} catch (IOException e) {
e.printStackTrace();
}
}
3.-lsr 或者 -ls -R 递归查看
@Test
public void lsrtest() {
// 要递归遍历的路径
lsr("/");
}
public static List<String> lsr(String path) {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
try {
FileSystem fileSystem = FileSystem.get(conf);
FileStatus[] listStatus = fileSystem.listStatus(new Path(path));
for(int i = 0; i < listStatus.length; ++i)
{
String abpath = listStatus[i].getPath().toString();
System.out.println(abpath);
boolean directory = listStatus[i].isDirectory();
if (directory) {
lsr(abpath);
}
}
fileSystem.close();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
4.-mkdir 创建文件夹
@Test
public void mkdir() {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
try {
FileSystem fileSystem = FileSystem.get(conf);
boolean mkdirs = fileSystem.mkdirs(new Path("/lyx02/lyx002/lyx0002"));
System.out.println(mkdirs?"创建成功":"创建失败");
}catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
5.-touchz 创建文件
@Test
public void createNewFile() {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
try {
FileSystem fileSystem = FileSystem.get(conf);
boolean createNewFile = fileSystem.createNewFile(new Path("/lyx02/lyx002/22.txt"));
System.out.println(createNewFile?"创建成功":"创建失败");
}catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
6.-put 下载hdfs下的文件到主机
@Test
public void put() {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
try {
FileSystem fileSystem = FileSystem.get(conf);
//hdfs下的路径
FSDataInputStream in = fileSystem.open(new Path("/1.txt"));
FileOutputStream destFile = new FileOutputStream(new File("D:\\111.txt"));
BufferedOutputStream out = new BufferedOutputStream(destFile);
int count = -1;
byte[]buffer = new byte[1024 *8];
while((count=in.read(buffer))!=-1) {
out.write(buffer,0,count);
}
in.close();
out.close();
}catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
7.-get 上传主机文件到hdfs
@Test
public void get() {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
try {
FileSystem fileSystem = FileSystem.get(conf);
FSDataOutputStream out = fileSystem.create(new Path("/555.txt"));
FileInputStream srcFile = new FileInputStream(new File("D:\\111.txt"));
BufferedInputStream in = new BufferedInputStream(srcFile);
int count = -1;
byte[]buffer = new byte[1024 *8];
while((count=in.read(buffer))!=-1) {
out.write(buffer,0,count);
}
}catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
8.-copyFromLocalFile 同get
@Test
public void copyFromLocalFile() {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
try {
FileSystem fileSystem = FileSystem.get(conf);
fileSystem.copyFromLocalFile(new Path("D:\\111.txt"), new Path("/666.txt"));
}catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
9.-copyToLocalFile 同put
@Test
public void copyToLocalFile() {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
try {
FileSystem fileSystem = FileSystem.get(conf);
fileSystem.copyToLocalFile(false,new Path("/666.txt"),new Path("D:\\666.txt"),true);
}catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
最新文章
- 第2月第5天 arc invocation getReturnValue
- javascript中比较数字大小
- CSS3动画属性animation的用法
- 【CF】148D Bag of mice
- 关闭/开启 ubuntu 自动更新提示
- EHCACHE采用分布需要注意的地方
- microsoft azure Media Services 媒体服务解决方案
- Spark源码编译
- 从微信推送看Android Service的创建和销毁
- 根据SimpleScheduleBuilder配置不同的SimpleTrigger触发器
- Windows Phone
- VC实现卡拉OK字幕叠加
- HDU4992 求所有原根
- ruby中如何调用与局部变量同名的私有方法
- 深度辨析 Python 的 eval() 与 exec()
- Linux发展历程
- <meta http-equiv="X-UA-Compatible" content="ie=edge">的意思
- (转)每天一个linux命令（21）：find命令之xargs
- BZOJ 3170 松鼠聚会(切比雪夫距离转曼哈顿距离)
- 手机安全卫士-——Splash总结
热门文章
- Java容器解析系列(14) IdentityHashMap详解
- Web前端-HTML、CSS、JS
- Git详解之服务部署
- 线程池技术之：ThreadPoolExecutor 源码解析
- Leetcode 题目整理-3 Palindrome Number & Roman to Integer
- Java8新特性一点通 | 回顾功能接口Functional Interface
- linux系统CentOS7中find命令使用
- 在cmd中启动tomcat
- linux下的 $ ./configure $ sudo make && sudo make install
- 机器学习（ML）七之模型选择、欠拟合和过拟合