[fix.] segfault when create_extfs() failed.

doc/docs/introduction/how-to-build.md

@@ -107,7 +107,7 @@ The examples and test code are built together.
 # Install additional dependencies
 dnf install epel-release
 dnf config-manager --set-enabled powertools
-dnf install gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel
+dnf install gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel nasm
 
 # Build examples and test code
 cmake -B build -D PHOTON_BUILD_TESTING=ON
@@ -125,7 +125,7 @@ ctest
 
 ```bash
 # Install additional dependencies
-apt install libgtest-dev libgmock-dev libgflags-dev libfuse-dev libgsasl7-dev
+apt install libgtest-dev libgmock-dev libgflags-dev libfuse-dev libgsasl7-dev nasm
 
 # Build examples and test code
 cmake -B build -D PHOTON_BUILD_TESTING=ON

fs/extfs/extfs.cpp

@@ -1298,6 +1298,9 @@ class ExtFileSystem : public photon::fs::IFileSystem, public photon::fs::IFileSy
             extfs_io_manager = new_io_manager(_image_file);
         }
         fs = do_ext2fs_open(extfs_io_manager);
+        if (fs == nullptr) {
+            return;
+        }
         memset(fs->reserved, 0, sizeof(fs->reserved));
         auto reserved = reinterpret_cast<std::uintptr_t *>(fs->reserved);
         reserved[0] = reinterpret_cast<std::uintptr_t>(this);
@@ -1505,7 +1508,11 @@ int ExtFile::flush_buffer() {
 
 photon::fs::IFileSystem *new_extfs(photon::fs::IFile *file, bool buffer) {
     auto extfs = new ExtFileSystem(file, buffer);
-    return extfs->fs ? extfs : nullptr;
+    if (extfs->fs == nullptr) {
+        delete extfs;
+        return nullptr;
+    }
+    return extfs;
 }
 
 }

fs/extfs/test/test.cpp

@@ -16,6 +16,7 @@ limitations under the License.
 #include "../extfs.h"
 #include <fcntl.h>
 #include <dirent.h>
+#include <unistd.h>
 #include <utime.h>
 #include <sys/sysmacros.h>
 #include <sys/time.h>
@@ -28,6 +29,7 @@ limitations under the License.
 #include <photon/common/alog-stdstring.h>
 #include <photon/common/enumerable.h>
 #include "../../../test/gtest.h"
+#include "photon/common/utility.h"
 
 #define FILE_SIZE (2 * 1024 * 1024)
 
@@ -992,6 +994,19 @@ TEST_F(ExtfsTest, Xattr) {
     EXPECT_EQ(ENOENT, errno);
 }
 
+
+TEST_F(ExtfsTest, InvalidFs) {
+    std::string rootfs = "/tmp/invalid_fs.img";
+    auto file = photon::fs::open_localfile_adaptor(rootfs.c_str(), O_CREAT|O_TRUNC|O_RDWR, 0666);
+    EXPECT_NE(nullptr, file);
+    DEFER({
+        delete file;
+        ::unlink(rootfs.c_str());
+    });
+    auto err_fs = photon::fs::new_extfs(file, false);
+    EXPECT_EQ(err_fs, nullptr);
+}
+
 int main(int argc, char **argv) {
     photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_DEFAULT);
     DEFER(photon::fini(););