1 客户端环境准备

根据自己电脑的操作系统拷贝对应的编译后的hadoop jar包到非中文路径

配置HADOOP_HOME的环境变量,并且在path中配置hadoop的bin

重启电脑

2. HdfsClientDemo

创建一个Maven项目,在pom.xml中导入相应的依赖,导入失败的话,试一试Reimport

<dependencies>

        <dependency>

            <groupId>junit</groupId>

            <artifactId>junit</artifactId>

            <version>RELEASE</version>

        </dependency>

        <dependency>

            <groupId>org.apache.logging.log4j</groupId>

            <artifactId>log4j-core</artifactId>

            <version>2.8.2</version>

        </dependency>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-common</artifactId>

            <version>2.7.2</version>

        </dependency>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-client</artifactId>

            <version>2.7.2</version>

        </dependency>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-hdfs</artifactId>

            <version>2.7.2</version>

        </dependency>

        <dependency>

            <groupId>jdk.tools</groupId>

            <artifactId>jdk.tools</artifactId>

            <version>1.8</version>

            <scope>system</scope>

            <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>

        </dependency>

</dependencies>

在src/main/resources目录下创建log4j.properties文件

log4j.rootLogger=INFO, stdout

log4j.appender.stdout=org.apache.log4j.ConsoleAppender

log4j.appender.stdout.layout=org.apache.log4j.PatternLayout

log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n

log4j.appender.logfile=org.apache.log4j.FileAppender

log4j.appender.logfile.File=target/spring.log

log4j.appender.logfile.layout=org.apache.log4j.PatternLayout

log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

创建包和HdfsClientDemo类

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.junit.After;

import org.junit.Before;

import org.junit.Test;

import java.net.URI;

public class HdfsClient {

    private static final String HADOOP_URI = "hdfs://hadoop100:9000";

    private Configuration configuration;

    private FileSystem fileSystem;

    @Before

    public void before() throws Exception {

        //configuration 对象相对应的就是 hadoop的那些配置文件,比如修改当前客户端上传文件的备份数量为1

        //configuration.set("replication","1");

        configuration = new Configuration();

        fileSystem = FileSystem.get(new URI(HADOOP_URI),configuration,"nty");

    }

    @After

    public void after() throws Exception {

        fileSystem.close();

    }

    /**

     * 创建目录

     */

    @Test

    public void mkdir() throws Exception {

        fileSystem.mkdirs(new Path("/client_test"));

    }

    /**

     * 上传文件

     */

    @Test

    public void upload() throws Exception {

        fileSystem.copyFromLocalFile(new Path("d:\\Hadoop_test\\test1.txt"), new Path("/client_test"));

    }

    /**

     * 下载文件

     */

    @Test

    public void download() throws Exception {

        fileSystem.copyToLocalFile(new Path("/client_test/test1.txt"), new Path("d:\\Hadoop_test\\test1_1.txt"));

    }

    /**

     * 删除文件

     *

     */

    @Test

    public void delete() throws Exception {

        fileSystem.delete(new Path("/output"),true);

    }

    /**

     * 重命名

     */

    @Test

    public void rename() throws Exception {

        fileSystem.rename(new Path("/input"), new Path("/input_rename"));

    }

    /**

     * 文件遍历

     */

    @Test

    public void liststatus() throws Exception {

        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));

        for(FileStatus fs : fileStatuses){

            System.out.println(fs.isDirectory() ? (fs.getPath().getName() + " is directory") : (fs.getPath().getName() + " is file"));

        }

    }

}

巴特西

Hadoop(7)-HDFS客户端的API操作

1 客户端环境准备

2. HdfsClientDemo

最新文章

热门文章