hadoop，
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
// windows cmd操作hadoop见最下面
public class hadtest {
// hadoop fs的配置文件
static Configuration conf = new Configuration(true);
static FileSystem fs =null;
static String fileStoragePath="/testfile";
static {
// 指定hadoop fs的地址 hdfs详细配置：https://blog.csdn.net/cuitaixiong/article/details/51591410
conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
conf.set("fs.defaultFS", "hdfs://127.0.0.1:9000");
try {
fs = FileSystem.get(conf);
if(!fs.exists(new Path(fileStoragePath))){
fs.mkdirs(new Path(fileStoragePath));
}
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 判断路径是否存在
*
* @param conf
* @param path
* @return
* @throws IOException
*/
public static boolean exits(String path) throws IOException {
return fs.exists(new Path(path));
}
/**
* 创建文件
*
* @param conf
* @param filePath
* @param contents
* @throws IOException
*/
public static void createFile(String filePath, byte[] contents)
throws IOException {
Path path = new Path(filePath);
FSDataOutputStream outputStream = fs.create(path);
outputStream.write(contents);
outputStream.close();
fs.close();
}
/**
* 创建文件
*
* @param conf
* @param filePath
* @param fileContent
* @throws IOException
*/
public static void createFile(String filePath, String fileContent)
throws IOException {
createFile(filePath, fileContent.getBytes());
}
/**
* @param conf
* @param localFilePath
* @param remoteFilePath
* @throws IOException
*/
public static void copyFromLocalFile(String localFilePath,
String remoteFilePath) throws IOException {
FileSystem fs = FileSystem.get(conf);
Path localPath = new Path(localFilePath);
Path remotePath = new Path(remoteFilePath);
fs.copyFromLocalFile(false, true, localPath, remotePath);
fs.close();
}
/**
* 删除目录或文件
*
* @param conf
* @param remoteFilePath
* @param recursive
* @return
* @throws IOException
*/
public static boolean deleteFile(String remoteFilePath, boolean recursive)
throws IOException {
FileSystem fs = FileSystem.get(conf);
boolean result = fs.delete(new Path(remoteFilePath), recursive);
fs.close();
return result;
}
/**
* 删除目录或文件(如果有子目录,则级联删除)
*
* @param conf
* @param remoteFilePath
* @return
* @throws IOException
*/
public static boolean deleteFile(String remoteFilePath) throws IOException {
return deleteFile(remoteFilePath, true);
}
/**
* 文件重命名
*
* @param conf
* @param oldFileName
* @param newFileName
* @return
* @throws IOException
*/
public static boolean renameFile(String oldFileName, String newFileName)
throws IOException {
FileSystem fs = FileSystem.get(conf);
Path oldPath = new Path(oldFileName);
Path newPath = new Path(newFileName);
boolean result = fs.rename(oldPath, newPath);
fs.close();
return result;
}
/**
* 创建目录
*
* @param conf
* @param dirName
* @return
* @throws IOException
*/
public static boolean createDirectory(String dirName) throws IOException {
FileSystem fs = FileSystem.get(conf);
Path dir = new Path(dirName);
boolean result = false;
if (!fs.exists(dir)) {
result = fs.mkdirs(dir);
}
fs.close();
return result;
}
/**
* 列出指定路径下的所有文件(不包含目录)
*
* @param conf
* @param basePath
* @param recursive
*/
public static RemoteIterator<LocatedFileStatus> listFiles(String basePath,
boolean recursive) throws IOException {
RemoteIterator<LocatedFileStatus> fileStatusRemoteIterator = fs
.listFiles(new Path(basePath), recursive);
return fileStatusRemoteIterator;
}
/**
* 列出指定目录下的文件\子目录信息（非递归）
*
* @param conf
* @param dirPath
* @return
* @throws IOException
*/
public static FileStatus[] listStatus(String dirPath) throws IOException {
FileStatus[] fileStatuses = fs.listStatus(new Path(dirPath));
return fileStatuses;
}
/**
* 读取文件内容
*
* @param conf
* @param filePath
* @return
* @throws IOException
*/
public static byte[] readFile(String filePath) throws IOException {
byte[] fileContent = null;
Path path = new Path(filePath);
if (fs.exists(path)) {
InputStream inputStream = null;
ByteArrayOutputStream outputStream = null;
try {
inputStream = fs.open(path);
outputStream = new ByteArrayOutputStream(
inputStream.available());
IOUtils.copyBytes(inputStream, outputStream, conf);
fileContent = outputStream.toByteArray();
} finally {
IOUtils.closeStream(inputStream);
IOUtils.closeStream(outputStream);
}
}
return fileContent;
}
/**
* 下载 hdfs上的文件
*
* @param conf
* @param uri
* @param remote
* @param local
* @throws IOException
*/
public static void download(String remote, String local) throws IOException {
Path path = new Path(remote);
fs.copyToLocalFile(path, new Path(local));
System.out.println("download: from" + remote + " to " + local);
fs.close();
}
public static void main(String[] args) throws Exception {
// System.out.println(hadtest.exits("/testfile"));
// hadtest.createFile("eee.txt","23132123123123213".getBytes());
RemoteIterator<LocatedFileStatus> listIterator=hadtest.listFiles("/",true);
while (listIterator.hasNext()) {
LocatedFileStatus te=listIterator.next();
System.out.println(te.getPath());
}
System.out.println(fs.getHomeDirectory());
// FileStatus[] dFileStatus=hadtest.listStatus("/");
// System.out.println(dFileStatus.length);
// System.out.println(hadtest.readFile("eee.txt"));
// hadtest.download("eee.txt","eee1.txt");
}
// 1:进入到hadoop的bin下面。
// 2:hadoop namenode -format 打开NameNode(HDFS服务器)
// 3:hadoop fs -ls 查看文件
//如果出现 ls: `.': No such file or directory 则 hadoop fs -ls /
// 4:hadoop fs -mkdir /user/input 创建目录
// 5：hadoop fs -put /home/file.txt /user/input 插入文件
// 6：hadoop fs -cat /user/output/outfile 查看文件内容
// 7：hadoop fs -get /user/output/ /home/hadoop_tp/ 取得文件
// 8：stop-dfs.sh 关闭
}