搭建hadoop java开发环境
2024-10-21 11:50:16
package hadoopDemo; import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator; public class Aa { public static void main(String[] args) throws Exception{
Aa a=new Aa(); }
public void upload(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
//拿到一个文件系统与客户端一个实例
FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),
new Configuration(),"root");
filesys.copyFromLocalFile(new Path(src), new Path(dst));
filesys.close();
}
public void download(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
FileSystem fileSystem = FileSystem.get(new URI("hdfs://linux1:9000"), new Configuration(), "root");
fileSystem.copyToLocalFile(new Path(src), new Path(dst));
fileSystem.close();
} public void getAllFile() throws IOException, InterruptedException, URISyntaxException{
Configuration cfg=new Configuration();
FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
RemoteIterator<LocatedFileStatus> listFiles=filesys.listFiles(new Path("/"), true); while(listFiles.hasNext()){
LocatedFileStatus next2=listFiles.next();
String name=next2.getPath().getName();
System.out.println(next2.getPath()+" "+name); }
} public void mkdir(String dir) throws IOException, InterruptedException, URISyntaxException{
Configuration cfg=new Configuration();
FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
filesys.mkdirs(new Path(dir));
filesys.close();
}
public void mv(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
Configuration cfg=new Configuration();
// cfg.set("fs.defaultFs", "hdfs://linux1:9000");
FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
filesys.moveFromLocalFile(new Path(src), new Path(dst));
filesys.close();
}
public void delete(String path) throws IOException, InterruptedException, URISyntaxException{
Configuration cfg=new Configuration();
FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
filesys.delete(new Path(path), true);
filesys.close();
}
}
1.解压hadoop-2.6.4.tar.gz
将此bin文件夹与hadoop-2.6.4文件夹中的bin文件夹合并
将此bin文件夹中的hadoop.dll文件拷贝到C:\Windows\System32目录中
配置windows环境变量
控制面板——>系统——>更改设置——>高级——>环境变量
新建变量:HADOOP_HOME,路径:解压文件夹位置
Path后添加:;%HADOOP_HOME%/bin;%HADOOP_HOME%/sbin
测试生效:
cmd输入hadoop
保险起见可以重启电脑
2.创建user library
3.添加jar包
添加完jar包以后进行简单测试:
package hadoopDemo; import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator; public class Aa { public static void main(String[] args) throws Exception{
Aa a=new Aa();
a.mv("e:/c.txt", "/aaa"); a.getAllFile();
a.delete("/aaa");
a.delete("/b.txt");
}
public void upload(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
//拿到一个文件系统与客户端一个实例
FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),
new Configuration(),"root");
filesys.copyFromLocalFile(new Path(src), new Path(dst));
filesys.close();
}
public void download(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
FileSystem fileSystem = FileSystem.get(new URI("hdfs://linux1:9000"), new Configuration(), "root");
fileSystem.copyToLocalFile(new Path(src), new Path(dst));
fileSystem.close();
} public void getAllFile() throws IOException, InterruptedException, URISyntaxException{
Configuration cfg=new Configuration();
FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
RemoteIterator<LocatedFileStatus> listFiles=filesys.listFiles(new Path("/"), true); while(listFiles.hasNext()){
LocatedFileStatus next2=listFiles.next();
String name=next2.getPath().getName();
System.out.println(next2.getPath()+" "+name); }
} public void mkdir(String dir) throws IOException, InterruptedException, URISyntaxException{
Configuration cfg=new Configuration();
FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
filesys.mkdirs(new Path(dir));
filesys.close();
}
public void mv(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
Configuration cfg=new Configuration();
// cfg.set("fs.defaultFs", "hdfs://linux1:9000");
FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
filesys.moveFromLocalFile(new Path(src), new Path(dst));
filesys.close();
}
public void delete(String path) throws IOException, InterruptedException, URISyntaxException{
Configuration cfg=new Configuration();
FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
filesys.delete(new Path(path), true);
filesys.close();
}
}
最新文章
- [转]ASP.NET Core 之 Identity 入门(一)
- java设计模式--简单工厂模式
- Android成长日记-Noification实现状态栏通知
- EntityFramework tt模板
- hdu 2256 好神奇的矩阵!
- linux下修改IP信息
- 基于DDD的现代ASP.NET开发框架--ABP系列之2、ABP入门教程
- [转]WIBKIT技术资料
- 《Node.js In Action》笔记之流程控制
- Android于fragment_main.xml文件问题组件收购
- PHP 6:PHP 基本数据类型
- [LeetCode] Largest Palindrome Product 最大回文串乘积
- JavaScript进阶(八)JS实现图片预览并导入服务器功能
- spring mvc+redis实现微信小程序登录
- 【原创】大叔经验分享(17)编程实践对比Java vs Scala
- 登录界面之Axure原型制作
- hdu 5183(Hash处理区间问题)
- [TJOI2018]智力竞赛【网络流】
- springboot项目logback配置文件示例
- 让docker容器开机启动