Skip to content

Commit

Permalink
Merge pull request #252 from cmssnu/jyang-duplicate-filestatus
Browse files Browse the repository at this point in the history 
Fix the bug in HDFS-Surf FileStatus merging
  • Loading branch information
@bchocho
bchocho committed May 19, 2015
2 parents 3af119c + f7c1649 commit 2b6a53c
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 89 deletions.
5 changes: 1 addition & 4 deletions reef-inmemory/src/main/java/org/apache/reef/inmemory/driver/metatree/MetaTree.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,12 +86,9 @@ public FileMetaStatus getFileMetaStatus(final String path) throws IOException {
final Entry entry = getEntryInTree(path);
if (entry != null) {
// If found, return Surf's FileMetaStatus
LOG.log(Level.INFO, "getfile surf: " + path);
return fileMetaStatusFactory.newFileMetaStatus(path, entry);
} else {
// If not found, return Base's FileMetaStatus
LOG.log(Level.INFO, "getfile base: " + baseFileMetaStatus.toString());

return baseFileMetaStatus;
}
} finally {
Expand Down Expand Up @@ -120,7 +117,7 @@ public List<FileMetaStatus> listFileMetaStatus(final String path) throws IOExcep
if (entry.isDirectory()) {
if (((DirectoryEntry) entry).getChildren().size() > 0) {
for (final Entry childEntry : ((DirectoryEntry) entry).getChildren()) {
final String childPath = path + "/" + childEntry.getName();
final String childPath = (entry == ROOT ? "/" + childEntry.getName() : path + "/" + childEntry.getName());
surfFileMetaStatusList.add(fileMetaStatusFactory.newFileMetaStatus(childPath, childEntry));
}
} else {
Expand Down
30 changes: 30 additions & 0 deletions reef-inmemory/src/test/java/org/apache/reef/inmemory/client/SurfFSDirectoryITCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.util.logging.Logger;

import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;

/**
* Test Surf's directory-related operations
Expand Down Expand Up @@ -198,6 +199,35 @@ public void testDelete() throws IOException {
}
}

/**
* Test set/get of working directory
*/
@Test
public void testWorkingDirectory() throws IOException {
final Path newWorkingDir = new Path("/workingdir");
final Path originalWorkingDir = surfFs.getWorkingDirectory();

// Set new working directory
surfFs.setWorkingDirectory(newWorkingDir);
assertEquals(newWorkingDir, surfFs.getWorkingDirectory());

// Restore for other tests
surfFs.setWorkingDirectory(originalWorkingDir);
assertEquals(originalWorkingDir, surfFs.getWorkingDirectory());

// Test relative file path
final Path file = new Path("file");
final FSDataOutputStream stream = surfFs.create(file);
stream.close();
assertEquals(ITUtils.getTestDir()+"/"+file, surfFs.getFileStatus(file).getPath().toUri().getPath());
final FileStatus[] fileStatuses = surfFs.listStatus(file);
assertEquals(1, fileStatuses.length);
assertEquals(ITUtils.getTestDir() + "/" + file, fileStatuses[0].getPath().toUri().getPath());

// Clean up to avoid conflicts with other tests
assertTrue("delete() must return true, as the file should be found", surfFs.delete(file, true));
}

private void testGetFileStatusOnDir(final String path) throws IOException {
final FileStatus fileStatus = surfFs.getFileStatus(new Path(path));
assertEquals(path, fileStatus.getPath().toUri().getPath());
Expand Down
131 changes: 46 additions & 85 deletions ...inmemory/client/SurfFSMetadataITCase.java → ...memory/client/SurfFSFileStatusITCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,18 @@
import static org.junit.Assert.assertTrue;

/**
* Tests for SurfFS methods that deal with FileMeta.
* Tests SurfFS methods that query FileStatus.
*/
public final class SurfFSMetadataITCase {
private static final Logger LOG = Logger.getLogger(SurfFSMetadataITCase.class.getName());
public final class SurfFSFileStatusITCase {
private static final Logger LOG = Logger.getLogger(SurfFSFileStatusITCase.class.getName());

private static FileSystem baseFs;
private static SurfFS surfFs;
private static final byte[] b = new byte[]{(byte)1, (byte)2, (byte)3, (byte)4, (byte)5, (byte)6, (byte)7, (byte)8};

private static final String TESTDIR = ITUtils.getTestDir();
private static final String ROOTDIR = "/";
private static final String TESTFILE = "README.md";
private static final String ABSPATH = TESTDIR+"/"+TESTFILE;
private static final String ABSPATH = ROOTDIR +TESTFILE;

private static final String SURF = "surf";
private static final String SURF_ADDRESS = "localhost:18000";
Expand All @@ -49,105 +50,83 @@ public static void setUpClass() throws IOException {
hdfsConfig.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);

baseFs = ITUtils.getHdfs(hdfsConfig);
baseFs.mkdirs(new Path(TESTDIR));

surfLauncher.launch(baseFs);

final Configuration conf = new Configuration();
conf.set(SurfFS.BASE_FS_ADDRESS_KEY, baseFs.getUri().toString());
surfFs = new SurfFS();
surfFs.initialize(URI.create(SURF + "://" + SURF_ADDRESS), conf);

final FSDataOutputStream stream = surfFs.create(new Path(ABSPATH));
stream.writeUTF("Hello Readme");
stream.write(b);
stream.close();
}

/**
* Remove all directories.
* Clean up
*/
@AfterClass
public static void tearDownClass() {
try {
baseFs.delete(new Path(TESTDIR), true); // TODO: Delete when SurfFs.delete is implemented
// surfFs.delete(new Path(TESTDIR), true); TODO: Enable when SurfFs.delete is implemented
surfFs.delete(new Path(ROOTDIR), true);
} catch (IOException e) {
LOG.log(Level.SEVERE, "Failed to delete " + TESTDIR, e);
LOG.log(Level.SEVERE, "Failed to delete " + ROOTDIR, e);
}
surfLauncher.close();
}

/**
* Test ls on directory
*/
@Test
public void testDirectoryListStatus() throws IOException {
FileStatus[] statuses = surfFs.listStatus(new Path(SURF, SURF_ADDRESS, TESTDIR));
for (FileStatus status : statuses) {
URI uri = status.getPath().toUri();
assertEquals(SURF, uri.getScheme());
assertEquals(SURF_ADDRESS, uri.getAuthority());
assertEquals(ABSPATH, uri.getPath());

assertEquals(status, surfFs.getFileStatus(status.getPath()));
}
}

/**
* Test ls using a path including surf:// scheme and address
*/
@Test
public void testFullPathListStatus() throws IOException {
Path filePath = new Path(SURF, SURF_ADDRESS, ABSPATH);
FileStatus fileStatus = surfFs.getFileStatus(filePath);

FileStatus[] statuses = surfFs.listStatus(filePath);
for (FileStatus status : statuses) {
URI uri = status.getPath().toUri();
assertEquals(SURF, uri.getScheme());
assertEquals(SURF_ADDRESS, uri.getAuthority());
assertEquals(ABSPATH, uri.getPath());

assertEquals(fileStatus, status);
}
testListStatusOfFile(new Path(SURF, SURF_ADDRESS, ABSPATH));
testListStatusOfDir(new Path(SURF, SURF_ADDRESS, ROOTDIR));
}

/**
* Test ls using an absolute path
*/
@Test
public void testAbsPathListStatus() throws IOException {
Path filePath = new Path(ABSPATH);
FileStatus fileStatus = surfFs.getFileStatus(filePath);

FileStatus[] statuses = surfFs.listStatus(filePath);
for (FileStatus status : statuses) {
URI uri = status.getPath().toUri();
assertEquals(SURF, uri.getScheme());
assertEquals(SURF_ADDRESS, uri.getAuthority());
assertEquals(ABSPATH, uri.getPath());
testListStatusOfFile(new Path(ABSPATH));
testListStatusOfDir(new Path(ROOTDIR));
}

assertEquals(fileStatus, status);
private void testListStatusOfDir(final Path dirPath) throws IOException {
final FileStatus[] statuses = surfFs.listStatus(dirPath);
boolean tested = false;
for (final FileStatus fileStatusLS : statuses) {
// Since there can be remaining directories from other tests, we only check the file we created in this test
if (fileStatusLS.getPath().toUri().getPath().equals(ABSPATH)) {
testFileStatusOfTestFile(fileStatusLS);
tested = true;
}
}
assertTrue("A filestatus whose path is ABSPATH must be tested", tested);
}

/**
* Test ls using a relative path
*/
@Test
public void testRelPathListStatus() throws IOException {
Path filePath = new Path(TESTFILE);
FileStatus fileStatus = surfFs.getFileStatus(filePath);

FileStatus[] statuses = surfFs.listStatus(filePath);
for (FileStatus status : statuses) {
URI uri = status.getPath().toUri();
assertEquals(SURF, uri.getScheme());
assertEquals(SURF_ADDRESS, uri.getAuthority());
assertEquals(ABSPATH, uri.getPath());

assertEquals(fileStatus, status);
private void testListStatusOfFile(final Path filePath) throws IOException {
final FileStatus fileStatusGFS = surfFs.getFileStatus(filePath);
testFileStatusOfTestFile(fileStatusGFS);
boolean tested = false;

final FileStatus[] statuses = surfFs.listStatus(filePath);
for (final FileStatus fileStatusLS : statuses) {
// Since there can be remaining directories from other tests, we only check the file we created in this test
if (fileStatusLS.getPath().toUri().getPath().equals(ABSPATH)) {
assertEquals(fileStatusGFS.getPath(), fileStatusLS.getPath());
testFileStatusOfTestFile(fileStatusLS);
tested = true;
}
}
assertTrue("A filestatus whose path is ABSPATH must be tested", tested);
}

private void testFileStatusOfTestFile(final FileStatus fileStatus) throws IOException {
assertEquals(b.length, fileStatus.getLen());
final URI uri = fileStatus.getPath().toUri();
assertEquals(SURF, uri.getScheme());
assertEquals(SURF_ADDRESS, uri.getAuthority());
}

/**
Expand Down Expand Up @@ -181,22 +160,4 @@ public void testSurfPathToHdfs() {
assertEquals(baseFs.getUri().getAuthority(), hdfsUri.getAuthority());
assertEquals(ABSPATH, hdfsUri.getPath());
}

/**
* Test set/get of working directory
*/
@Test
public void testWorkingDirectory() throws IOException {
Path path = new Path("/user/otheruser");
baseFs.mkdirs(path);

Path original = surfFs.getWorkingDirectory();

surfFs.setWorkingDirectory(path);
assertEquals(path, surfFs.getWorkingDirectory());

// Restore for other tests
surfFs.setWorkingDirectory(original);
assertEquals(original, surfFs.getWorkingDirectory());
}
}

0 comments on commit 2b6a53c

Please sign in to comment.