分布式文件系统HDFS

一.启动hdfs

在sbin目录下 ./start-dfs.sh jps检测进程是否开启

查看进程方式一：jps检测进程是否开启方式二：http://192.168.198.10:50070

发现进程有误，在logs目录下寻找log文件。

浏览器打不开：查看防火墙状态：sudo firewall-cmd --state

关闭防火墙: sudo systemctl stop firewalld.service

start-dfs.sh = hadoop-daemons.sh start namenode

hadoop-daemons.sh start datanode

hadoop-daemons.sh start secondarynamenode

`二.命令行操作`

hadoop常用命令：
hadoop fs -ls /
hadoop fs -put 从本地复制到hdfs
hadoop fs -copyFromLocal
hadoop fs -moveFromLocal
hadoop fs -cat
hadoop fs -text
hadoop fs -get 复制到本地
hadoop fs -mkdir
hadoop fs -mv 移动/改名
hadoop fs -getmerge
hadoop fs -rm
hadoop fs -rmdir
hadoop fs -rm -r

HDFS存储扩展：

put: 1file ==> 1...n block ==> 存放在不同的节点上的

get: 去nn上查找这个file对应的元数据信息

`三.常用API`

Configuration configuration = new Configuration();

FileSystem filesystem = FileSystem.get(newURI("hdfs://hadoop000:8020"),configuration,"hadoop");

fileSystem.mkdirs(new Path("/hdfsapi/test"));//创建文件夹

FSDataInputStream in = fileSystem.open(new Path("/README.txt"));
IOUtils.copyBytes(in,System.out,1024);//读文件

FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/a.txt"));
out.writeUTF("hello world！！！！");//写文件
out.flush();
out.close();

Path oldPath = new Path("/hdfsapi/test/b.txt");
Path newPath = new Path("/hdfsapi/test/c.txt");
boolean result = fileSystem.rename(oldPath, newPath);
System.out.println(result);//更改文件名

Path src = new Path("/Users/rocky/data/hello.txt");
Path dst = new Path("/hdfsapi/test/");
fileSystem.copyFromLocalFile(src,dst);//本地复制到hdfs

 /**
     * 拷贝大文件到HDFS文件系统：带进度
     */
    @Test
    public void copyFromLocalBigFile() throws Exception {

        InputStream in = new BufferedInputStream(new FileInputStream(new File("/Users/rocky/tmp/software/jdk-8u91-linux-x64.tar.gz")));

        FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/jdk.tgz"),
                new Progressable() {
                    public void progress() {
                        System.out.print(".");
                    }
                });

        IOUtils.copyBytes(in, out ,4096);

    }

/**
     * 查看目标文件夹下的所有文件
     */
    @Test
    public void listFiles() throws Exception {
        FileStatus[] statuses = fileSystem.listStatus(new Path("/hdfsapi/test"));

        for(FileStatus file : statuses) {
            String isDir = file.isDirectory() ? "文件夹" : "文件";
            String permission = file.getPermission().toString();
            short replication = file.getReplication();
            long length = file.getLen();
            String path = file.getPath().toString();


            System.out.println(isDir + "\t" + permission
                    + "\t" + replication + "\t" + length
                    + "\t" + path
            );
        }

    }

 /**
     * 递归查看目标文件夹下的所有文件
     */
    @Test
    public void listFilesRecursive() throws Exception {

        RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/hdfsapi/test"), true);

        while (files.hasNext()) {
            LocatedFileStatus file = files.next();
            String isDir = file.isDirectory() ? "文件夹" : "文件";
            String permission = file.getPermission().toString();
            short replication = file.getReplication();
            long length = file.getLen();
            String path = file.getPath().toString();


            System.out.println(isDir + "\t" + permission
                    + "\t" + replication + "\t" + length
                    + "\t" + path
            );
        }
    }

/**
     * 查看文件块信息
     */
    @Test
    public void getFileBlockLocations() throws Exception {

        FileStatus fileStatus = fileSystem.getFileStatus(new Path("/hdfsapi/test/jdk.tgz"));
        BlockLocation[] blocks = fileSystem.getFileBlockLocations(fileStatus,0,fileStatus.getLen());

        for(BlockLocation block : blocks) {

            for(String name: block.getNames()) {
                System.out.println(name +" : " + block.getOffset() + " : " + block.getLength() + " : " + block.getHosts());
            }
        }
    }

来源：CSDN

作者：程研板

链接：https://blog.csdn.net/qq_38258720/article/details/103844893

标签

HDFS

分布式文件系统

Hadoop