#167 Removing S3 policy service.

philterd · Nov 26, 2024 · f9e5537 · f9e5537
1 parent 09de3b8
commit f9e5537
Show file tree

Hide file tree

Showing 11 changed files with 65 additions and 458 deletions.
diff --git a/phileas-core/pom.xml b/phileas-core/pom.xml
@@ -119,12 +119,6 @@
 			<version>${junit.version}</version>
 			<scope>test</scope>
 		</dependency>
-		<dependency>
-			<groupId>io.findify</groupId>
-			<artifactId>s3mock_2.12</artifactId>
-			<version>0.2.6</version>
-			<scope>test</scope>
-		</dependency>
 		<dependency>
 			<groupId>ai.grakn</groupId>
 			<artifactId>redis-mock</artifactId>

diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/PhileasFilterService.java b/phileas-core/src/main/java/ai/philterd/phileas/services/PhileasFilterService.java
@@ -31,17 +31,30 @@
 import ai.philterd.phileas.model.responses.BinaryDocumentFilterResponse;
 import ai.philterd.phileas.model.responses.FilterResponse;
 import ai.philterd.phileas.model.serializers.PlaceholderDeserializer;
-import ai.philterd.phileas.model.services.*;
+import ai.philterd.phileas.model.services.AlertService;
+import ai.philterd.phileas.model.services.AnonymizationCacheService;
+import ai.philterd.phileas.model.services.Classification;
+import ai.philterd.phileas.model.services.DocumentProcessor;
+import ai.philterd.phileas.model.services.FilterService;
+import ai.philterd.phileas.model.services.MetricsService;
+import ai.philterd.phileas.model.services.PolicyService;
+import ai.philterd.phileas.model.services.PostFilter;
+import ai.philterd.phileas.model.services.Redacter;
+import ai.philterd.phileas.model.services.SentimentDetector;
+import ai.philterd.phileas.model.services.SplitService;
 import ai.philterd.phileas.processors.unstructured.UnstructuredDocumentProcessor;
 import ai.philterd.phileas.service.ai.sentiment.OpenNLPSentimentDetector;
 import ai.philterd.phileas.services.alerts.AlertServiceFactory;
 import ai.philterd.phileas.services.anonymization.cache.AnonymizationCacheServiceFactory;
 import ai.philterd.phileas.services.disambiguation.VectorBasedSpanDisambiguationService;
 import ai.philterd.phileas.services.metrics.NoOpMetricsService;
 import ai.philterd.phileas.services.policies.LocalPolicyService;
-import ai.philterd.phileas.services.policies.S3PolicyService;
 import ai.philterd.phileas.services.policies.utils.PolicyUtils;
-import ai.philterd.phileas.services.postfilters.*;
+import ai.philterd.phileas.services.postfilters.IgnoredPatternsFilter;
+import ai.philterd.phileas.services.postfilters.IgnoredTermsFilter;
+import ai.philterd.phileas.services.postfilters.TrailingNewLinePostFilter;
+import ai.philterd.phileas.services.postfilters.TrailingPeriodPostFilter;
+import ai.philterd.phileas.services.postfilters.TrailingSpacePostFilter;
 import ai.philterd.phileas.services.split.SplitFactory;
 import ai.philterd.services.pdf.PdfRedacter;
 import ai.philterd.services.pdf.PdfTextExtractor;
@@ -55,7 +68,15 @@
 import org.apache.logging.log4j.Logger;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 
 public class PhileasFilterService implements FilterService {
@@ -86,12 +107,10 @@ public PhileasFilterService(final PhileasConfiguration phileasConfiguration, fin
         this.filterCache = new ConcurrentHashMap<>();
 
         // Configure the deserialization.
-        final GsonBuilder gsonBuilder = new GsonBuilder();
-        gsonBuilder.registerTypeAdapter(String.class, new PlaceholderDeserializer());
-        final Gson gson = gsonBuilder.create();
+        final Gson gson = new GsonBuilder().registerTypeAdapter(String.class, new PlaceholderDeserializer()).create();
 
         // Set the policy services.
-        this.policyService = buildPolicyService(phileasConfiguration);
+        this.policyService = new LocalPolicyService(phileasConfiguration);
         this.policyUtils = new PolicyUtils(policyService, gson);
 
         // Set the anonymization cache service.
@@ -362,28 +381,6 @@ private List<BoundingBox> getBoundingBoxes(final Policy policy, final MimeType m
 
     }
 
-    private PolicyService buildPolicyService(final PhileasConfiguration phileasConfiguration) throws IOException {
-
-        final PolicyService policyService;
-        final String s3Bucket = phileasConfiguration.policiesS3Bucket();
-
-        // If an S3 bucket is provided then instantiate an S3PolicyService.
-        if(StringUtils.isNotEmpty(s3Bucket)) {
-
-            LOGGER.info("Initializing configuration for policies S3 bucket.");
-            policyService = new S3PolicyService(phileasConfiguration, false);
-
-        } else {
-
-            LOGGER.info("Using local storage for policies.");
-            policyService = new LocalPolicyService(phileasConfiguration);
-
-        }
-
-        return policyService;
-
-    }
-
     private List<PostFilter> getPostFiltersForPolicy(final Policy policy) throws IOException {
 
         LOGGER.debug("Reloading policies.");

diff --git a/...ervices/registry/S3PolicyServiceTest.java → ...ices/policies/LocalPolicyServiceTest.java b/...ervices/registry/S3PolicyServiceTest.java → ...ices/policies/LocalPolicyServiceTest.java
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package ai.philterd.test.phileas.services.registry;
+package ai.philterd.test.phileas.services.policies;
 
 import ai.philterd.phileas.model.configuration.PhileasConfiguration;
 import ai.philterd.phileas.model.policy.Identifiers;
@@ -23,129 +23,49 @@
 import ai.philterd.phileas.model.policy.filters.strategies.rules.AgeFilterStrategy;
 import ai.philterd.phileas.model.policy.filters.strategies.rules.CreditCardFilterStrategy;
 import ai.philterd.phileas.model.services.PolicyService;
-import ai.philterd.phileas.services.policies.S3PolicyService;
+import ai.philterd.phileas.services.policies.LocalPolicyService;
 import com.google.gson.Gson;
-import io.findify.s3mock.S3Mock;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import redis.embedded.RedisServer;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
 import static org.junit.jupiter.api.Assumptions.assumeFalse;
 
-public class S3PolicyServiceTest {
+public class LocalPolicyServiceTest {
 
-    private static final Logger LOGGER = LogManager.getLogger(S3PolicyServiceTest.class);
+    private static final Logger LOGGER = LogManager.getLogger(LocalPolicyServiceTest.class);
 
-    private Gson gson = new Gson();
-    private RedisServer redisServer;
-    private S3Mock api;
-
-    private static boolean isExternalRedis = false;
+    private final Gson gson = new Gson();
 
     private PhileasConfiguration getConfiguration() throws IOException {
 
-        final Properties properties = new Properties();
-
-        final String redisHost = System.getenv("PHILTER_REDIS_HOST");
-        final String redisPort = System.getenv("PHILTER_REDIS_PORT");
-        final String redisSsl = System.getenv("PHILTER_REDIS_SSL");
-        final String redisToken = System.getenv("PHILTER_REDIS_AUTH_TOKEN");
-        final String redisClustered = System.getenv("PHILTER_REDIS_CLUSTERED");
-
-        properties.setProperty("filter.policies", "s3");
-        properties.setProperty("filter.policies.s3.bucket", "policies");
-        properties.setProperty("filter.policies.s3.prefix", "test/");
-        properties.setProperty("cache.redis.enabled", "true");
-
-        if(StringUtils.isNotEmpty(redisHost)) {
-
-            LOGGER.info("Using redis host: {}", redisHost);
+        final String tempDirectory = Files.createTempDirectory("phileas-policies").toFile().getAbsolutePath();
 
-            properties.setProperty("cache.redis.host", redisHost);
-            properties.setProperty("cache.redis.port", redisPort);
-            properties.setProperty("cache.redis.ssl", redisSsl);
-            properties.setProperty("cache.redis.auth.token", redisToken);
-            properties.setProperty("cache.redis.cluster", "true");
-            properties.setProperty("cache.redis.cluster", redisClustered);
-
-        } else {
-
-            LOGGER.info("Using local redis host.");
-
-            properties.setProperty("cache.redis.host", "localhost");
-            properties.setProperty("cache.redis.port", "31000");
-            properties.setProperty("cache.redis.ssl", "false");
-            properties.setProperty("cache.redis.auth.token", "");
-            properties.setProperty("cache.redis.cluster", "false");
-
-        }
+        final Properties properties = new Properties();
+        properties.setProperty("filter.policies.directory", tempDirectory);
+        properties.setProperty("cache.redis.enabled", "false");
 
         return new PhileasConfiguration(properties);
 
     }
 
-    @BeforeAll
-    public static void beforeClass() {
-
-        assumeFalse(System.getProperty("os.name").toLowerCase().startsWith("win"));
-
-        final String redisHost = System.getenv("PHILTER_REDIS_HOST");
-
-        if(StringUtils.isNotEmpty(redisHost)) {
-            isExternalRedis = true;
-        }
-
-    }
-
-    @BeforeEach
-    public void before() throws IOException {
-
-        LOGGER.info("Starting S3 emulator.");
-
-        final Path temporaryDirectory = Files.createTempDirectory("s3mock");
-
-        api = new S3Mock.Builder().withPort(8001).withFileBackend(temporaryDirectory.toFile().getAbsolutePath()).build();
-        api.start();
-
-        if(!isExternalRedis) {
-            redisServer = RedisServer.builder().setting("bind 127.0.0.1").port(31000).build();
-            redisServer.start();
-        }
-
-    }
-
-    @AfterEach
-    public void after() {
-
-        api.shutdown();
-
-        if(!isExternalRedis) {
-            redisServer.stop();
-        }
-
-    }
-
     @Test
     public void list() throws IOException {
 
-        final PolicyService policyService = new S3PolicyService(getConfiguration(), true);
+        final PolicyService policyService = new LocalPolicyService(getConfiguration());
 
         policyService.save(gson.toJson(getPolicy("name1")));
         policyService.save(gson.toJson(getPolicy("name2")));
+
         final List<String> names = policyService.get();
 
         LOGGER.info("Found {} policies", names.size());
@@ -159,7 +79,7 @@ public void list() throws IOException {
     @Test
     public void getAll() throws IOException {
 
-        final PolicyService policyService = new S3PolicyService(getConfiguration(), true);
+        final PolicyService policyService = new LocalPolicyService(getConfiguration());
 
         policyService.save(gson.toJson(getPolicy("name1")));
         policyService.save(gson.toJson(getPolicy("name2")));
@@ -181,7 +101,7 @@ public void save() throws IOException {
 
         final String policy = gson.toJson(getPolicy(name));
 
-        final PolicyService policyService = new S3PolicyService(getConfiguration(), true);
+        final PolicyService policyService = new LocalPolicyService(getConfiguration());
 
         policyService.save(policy);
 
@@ -199,7 +119,7 @@ public void get() throws IOException {
 
         final String policy = gson.toJson(getPolicy(name));
 
-        final PolicyService policyService = new S3PolicyService(getConfiguration(), true);
+        final PolicyService policyService = new LocalPolicyService(getConfiguration());
 
         policyService.save(policy);
 
@@ -215,7 +135,7 @@ public void delete() throws IOException {
         final String name = "default";
         final String policy = gson.toJson(getPolicy(name));
 
-        final PolicyService policyService = new S3PolicyService(getConfiguration(), true);
+        final PolicyService policyService = new LocalPolicyService(getConfiguration());
 
         policyService.save(policy);
 
@@ -225,6 +145,25 @@ public void delete() throws IOException {
 
     }
 
+    @Test
+    public void deleteOutsidePath() throws IOException {
+
+        final File tempFile = File.createTempFile("phileas-", "-temp");
+
+        Assertions.assertTrue(Files.exists(tempFile.toPath()));
+        LOGGER.info("Temp file: {}", tempFile.getAbsolutePath());
+
+        final String name = "../" + tempFile.getName();
+        LOGGER.info("Policy name: {}", name);
+
+        final PolicyService policyService = new LocalPolicyService(getConfiguration());
+
+        policyService.delete(name);
+
+        Assertions.assertTrue(Files.exists(tempFile.toPath()));
+
+    }
+
     private Policy getPolicy(String name) {
 
         AgeFilterStrategy ageFilterStrategy = new AgeFilterStrategy();

diff --git a/...eas-model/src/main/java/ai/philterd/phileas/model/configuration/PhileasConfiguration.java b/...eas-model/src/main/java/ai/philterd/phileas/model/configuration/PhileasConfiguration.java
@@ -117,18 +117,6 @@ public String policiesDirectory() {
         return getProperty("filter.policies.directory", "./policies/");
     }
 
-    public String policiesS3Bucket() {
-        return getProperty("filter.policies.s3.bucket", "");
-    }
-
-    public String policiesS3Prefix() {
-        return getProperty("filter.policies.s3.prefix", "");
-    }
-
-    public String policiesS3Region() {
-        return getProperty("filter.policies.s3.region", "us-east-1");
-    }
-
     private String getProperty(final String property, final String defaultValue) {
 
         final String environmentVariableValue = getEnvironmentVariable(property);

diff --git a/...-alerts/src/test/java/ai/philterd/test/phileas/services/alerts/RedisAlertServiceTest.java b/...-alerts/src/test/java/ai/philterd/test/phileas/services/alerts/RedisAlertServiceTest.java
@@ -53,10 +53,6 @@ private PhileasConfiguration getConfiguration() throws IOException {
         final String redisToken = System.getenv("PHILTER_REDIS_AUTH_TOKEN");
         final String redisClustered = System.getenv("PHILTER_REDIS_CLUSTERED");
 
-        properties.setProperty("filter.policies", "s3");
-        properties.setProperty("filter.policies.s3.bucket", "policies");
-        properties.setProperty("filter.policies.s3.prefix", "/");
-
         properties.setProperty("cache.redis.enabled", "true");
 
         if(StringUtils.isNotEmpty(redisHost)) {

diff --git a/...hilterd/test/phileas/services/anonymization/cache/RedisAnonymizationCacheServiceTest.java b/...hilterd/test/phileas/services/anonymization/cache/RedisAnonymizationCacheServiceTest.java
@@ -45,10 +45,6 @@ private PhileasConfiguration getConfiguration() throws IOException {
         final String redisToken = System.getenv("PHILTER_REDIS_AUTH_TOKEN");
         final String redisClustered = System.getenv("PHILTER_REDIS_CLUSTERED");
 
-        properties.setProperty("filter.policies", "s3");
-        properties.setProperty("filter.policies.s3.bucket", "policies");
-        properties.setProperty("filter.policies.s3.prefix", "/");
-
         properties.setProperty("cache.redis.enabled", "true");
 
         if(StringUtils.isNotEmpty(redisHost)) {

diff --git a/phileas-services/phileas-services-policies/pom.xml b/phileas-services/phileas-services-policies/pom.xml
@@ -14,11 +14,6 @@
       <artifactId>phileas-model</artifactId>
       <version>${project.version}</version>
     </dependency>
-    <dependency>
-      <groupId>com.amazonaws</groupId>
-      <artifactId>aws-java-sdk-s3</artifactId>
-      <version>${aws.version}</version>
-    </dependency>
     <dependency>
       <groupId>org.json</groupId>
       <artifactId>json</artifactId>

diff --git a/...ices-policies/src/main/java/ai/philterd/phileas/services/policies/LocalPolicyService.java b/...ices-policies/src/main/java/ai/philterd/phileas/services/policies/LocalPolicyService.java
@@ -160,6 +160,8 @@ public void delete(String policyName) throws IOException {
 
         final File file = new File(policiesDirectory, policyName + JSON_EXTENSION);
 
+        LOGGER.info("Deleting policy at: {}", file.getAbsolutePath());
+
         if(file.exists()) {
 
             if(!file.delete()) {