From 2e3222456e32a885d5c6ae23bd3451bd5e5ea729 Mon Sep 17 00:00:00 2001 From: JAM <272807+JessieAMorris@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:44:07 -0700 Subject: [PATCH 1/2] Add truncation filter strategy generally --- .../docs/filter_policies/filter_strategies.md | 12 +++- .../strategies/AbstractFilterStrategy.java | 46 ++++++++++++++ .../strategies/StandardFilterStrategy.java | 14 +++++ .../strategies/ai/PhEyeFilterStrategy.java | 14 +++++ .../strategies/rules/DateFilterStrategy.java | 14 +++++ .../rules/ZipCodeFilterStrategy.java | 20 +++--- .../AbstractFilterStrategyTest.java | 62 +++++++++++++++++++ .../rules/ZipCodeFilterStrategyTest.java | 28 +++++++++ 8 files changed, 197 insertions(+), 13 deletions(-) diff --git a/docs/docs/filter_policies/filter_strategies.md b/docs/docs/filter_policies/filter_strategies.md index 69052f831..508310a08 100644 --- a/docs/docs/filter_policies/filter_strategies.md +++ b/docs/docs/filter_policies/filter_strategies.md @@ -208,9 +208,15 @@ An example policy using the `STATIC_REPLACE` filter strategy: ### The `TRUNCATE` Filter Strategy {id="truncate"} -Available only to zip codes, this strategy allows for truncating zip codes to only a select number of digits. Specify `truncateDigits` to set the desired number of leading digits to leave. For example, if `truncateDigits` is 2, the zip code 90210 will be truncated to `90***`. - -The TRUNCATE filter strategy is available only to the zip code filter. An example policy using the `TRUNCATE` filter strategy: +This strategy allows for truncating tokens to only a select number of digits. Specify `truncateDigits` +to set the desired number of digits to leave. For example, if `truncateDigits` is 4, the +string `4111111111111111` will be truncated to `4111************`. `truncateDirection` can be set to +`LEADING` (the default) which leaves N leading digits or `TRAILING` which leaves N trailing digits. +`truncateCharacter` can be overwritten (defaults to `*`) to change the character that is used for the +replacement. + +The `TRUNCATE` filter has special behavior for the zip code filter. For zip codes the Zip will always be truncated +to 5 digits long. For example, `truncateDigits=2` and a token of `90210-0110` will result in `90***`. ``` { diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java index 5bebd23df..76ef9275a 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java @@ -44,6 +44,9 @@ public abstract class AbstractFilterStrategy { public static final String LAST_4 = "LAST_4"; public static final String MASK = "MASK"; public static final String SAME = AbstractFilterStrategy.SAME; + public static final String TRUNCATE = "TRUNCATE"; + public static final String LEADING = "LEADING"; + public static final String TRAILING = "TRAILING"; // NER Person's name strategies public static final String ABBREVIATE = "ABBREVIATE"; @@ -107,6 +110,18 @@ public abstract class AbstractFilterStrategy { @Expose protected String maskLength = SAME; + @SerializedName("truncateDigits") + @Expose + protected Integer truncateDigits; + + @SerializedName("truncateCharacter") + @Expose + protected String truncateCharacter = "*"; + + @SerializedName("truncateDirection") + @Expose + protected String truncateDirection = LEADING; + @SerializedName("condition") @Expose protected String condition = ""; @@ -365,6 +380,37 @@ public String getMaskLength() { return maskLength; } + public Integer getTruncateDigits() { + return truncateDigits; + } + + public void setTruncateDigits(Integer truncateDigits) { + + // Make sure it is a valid value. + if(truncateDigits >= 1) { + this.truncateDigits = truncateDigits; + } else { + throw new IllegalArgumentException("Truncate length must be greater than 1"); + } + + } + + public String getTruncateCharacter() { + return truncateCharacter; + } + + public void setTruncateCharacter(String truncateCharacter) { + this.truncateCharacter = truncateCharacter; + } + + public String getTruncateDirection() { + return truncateDirection; + } + + public void setTruncateDirection(String truncateDirection) { + this.truncateDirection = truncateDirection; + } + public void setConditions(String condition) { this.condition = condition; } diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java index 8efc828e0..ebc48f77a 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java @@ -54,6 +54,20 @@ public Replacement getStandardReplacement(String label, String context, String d replacement = maskCharacter.repeat(characters); + } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) { + + int truncateLength = getValueOrDefault(truncateDigits, 4); + + if (truncateLength < 1) { + truncateLength = 1; + } + + if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) { + replacement = token.substring(0, truncateLength) + StringUtils.repeat(truncateCharacter, token.length() - truncateLength); + } else { + replacement = StringUtils.repeat(truncateCharacter, token.length() - truncateLength) + token.substring(token.length() - truncateLength); + } + } else if(StringUtils.equalsIgnoreCase(strategy, RANDOM_REPLACE)) { // Default to document scope. diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java index eb2904a83..0a3e359ee 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java @@ -156,6 +156,20 @@ public Replacement getReplacement(String label, String context, String documentI replacement = maskCharacter.repeat(characters); + } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) { + + int truncateLength = getValueOrDefault(truncateDigits, 4); + + if (truncateLength < 1) { + truncateLength = 1; + } + + if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) { + replacement = token.substring(0, truncateLength) + StringUtils.repeat(truncateCharacter, token.length() - truncateLength); + } else { + replacement = StringUtils.repeat(truncateCharacter, token.length() - truncateLength) + token.substring(token.length() - truncateLength); + } + } else if(StringUtils.equalsIgnoreCase(strategy, RANDOM_REPLACE)) { // Default to document scope. diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java index 0bd49124d..7e1b1134d 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java @@ -174,6 +174,20 @@ public Replacement getReplacement(String label, String context, String documentI replacement = maskCharacter.repeat(characters); + } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) { + + int truncateLength = getValueOrDefault(truncateDigits, 4); + + if (truncateLength < 1) { + truncateLength = 1; + } + + if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) { + replacement = token.substring(0, truncateLength) + StringUtils.repeat(truncateCharacter, token.length() - truncateLength); + } else { + replacement = StringUtils.repeat(truncateCharacter, token.length() - truncateLength) + token.substring(token.length() - truncateLength); + } + } else if(StringUtils.equalsIgnoreCase(strategy, RANDOM_REPLACE)) { // Default to document scope. diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java index 4d0c0fa76..b2619bafd 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java @@ -15,9 +15,6 @@ */ package ai.philterd.phileas.model.policy.filters.strategies.rules; -import ai.philterd.phileas.model.policy.Policy; -import com.google.gson.annotations.Expose; -import com.google.gson.annotations.SerializedName; import ai.philterd.phileas.model.conditions.ParsedCondition; import ai.philterd.phileas.model.conditions.ParserListener; import ai.philterd.phileas.model.enums.FilterType; @@ -28,6 +25,7 @@ import ai.philterd.phileas.model.objects.Replacement; import ai.philterd.phileas.model.policy.Crypto; import ai.philterd.phileas.model.policy.FPE; +import ai.philterd.phileas.model.policy.Policy; import ai.philterd.phileas.model.policy.filters.strategies.AbstractFilterStrategy; import ai.philterd.phileas.model.services.AnonymizationService; import ai.philterd.phileas.model.utils.Encryption; @@ -62,10 +60,6 @@ public FilterType getFilterType() { return filterType; } - @SerializedName("truncateDigits") - @Expose - private Integer truncateDigits; - @Override public boolean evaluateCondition(Policy policy, String context, String documentId, String token, String[] window, String condition, double confidence, Map attributes) { @@ -177,7 +171,7 @@ public Replacement getReplacement(String label, String context, String documentI characters = Integer.parseInt(maskLength); } - if(characters < 1) { + if (characters < 1) { characters = 5; } @@ -200,8 +194,14 @@ public Replacement getReplacement(String label, String context, String documentI } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) { - final int truncateLength = getValueOrDefault(truncateDigits, 2); - replacement = token.substring(0, truncateDigits) + StringUtils.repeat("*", Math.min(token.length() - truncateLength, 5 - truncateDigits)); + if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) { + final int truncateLength = getValueOrDefault(truncateDigits, 2); + replacement = token.substring(0, truncateDigits) + StringUtils.repeat(truncateCharacter, Math.min(token.length() - truncateLength, 5 - truncateDigits)); + } else { + final int truncateLength = getValueOrDefault(truncateDigits, 2); + replacement = StringUtils.repeat(truncateCharacter, Math.min(token.length() - truncateLength, 5 - truncateDigits)) + token.substring(Math.min(token.length() - truncateLength, 5 - truncateDigits), 5); + } + } else if(StringUtils.equalsIgnoreCase(strategy, ZERO_LEADING)) { diff --git a/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java b/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java index c2b097ab0..ffe197d6c 100644 --- a/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java +++ b/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java @@ -292,6 +292,68 @@ public void replacementWithMaskCharacterForSetLengthWithNegativeLength() throws } + @Test + public void truncate1() throws Exception { + + final AnonymizationService anonymizationService = Mockito.mock(AnonymizationService.class); + final AnonymizationCacheService anonymizationCacheService = Mockito.mock(AnonymizationCacheService.class); + + when(anonymizationService.getAnonymizationCacheService()).thenReturn(anonymizationCacheService); + + final AbstractFilterStrategy strategy = getFilterStrategy(); + strategy.setStrategy(AbstractFilterStrategy.TRUNCATE); + strategy.setTruncateDigits(2); + + final String token = "12345"; + final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null); + + Assertions.assertEquals(replacement.getReplacement(), "12***"); + Assertions.assertEquals(replacement.getReplacement().length(), 5); + + } + + @Test + public void truncate2() throws Exception { + + final AnonymizationService anonymizationService = Mockito.mock(AnonymizationService.class); + final AnonymizationCacheService anonymizationCacheService = Mockito.mock(AnonymizationCacheService.class); + + when(anonymizationService.getAnonymizationCacheService()).thenReturn(anonymizationCacheService); + + final AbstractFilterStrategy strategy = getFilterStrategy(); + strategy.setStrategy(AbstractFilterStrategy.TRUNCATE); + strategy.setTruncateDirection(AbstractFilterStrategy.LEADING); + strategy.setTruncateDigits(2); + + final String token = "12345"; + final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null); + + Assertions.assertEquals(replacement.getReplacement(), "12***"); + Assertions.assertEquals(replacement.getReplacement().length(), 5); + + } + + @Test + public void truncate3() throws Exception { + + final AnonymizationService anonymizationService = Mockito.mock(AnonymizationService.class); + final AnonymizationCacheService anonymizationCacheService = Mockito.mock(AnonymizationCacheService.class); + + when(anonymizationService.getAnonymizationCacheService()).thenReturn(anonymizationCacheService); + + final AbstractFilterStrategy strategy = getFilterStrategy(); + strategy.setStrategy(AbstractFilterStrategy.TRUNCATE); + strategy.setTruncateDirection(AbstractFilterStrategy.TRAILING); + strategy.setTruncateDigits(4); + + final String token = "4111111111111111"; + final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null); + + Assertions.assertEquals(replacement.getReplacement(), "************1111"); + Assertions.assertEquals(replacement.getReplacement().length(), 16); + + } + @Test public void evaluateCondition1() throws IOException { diff --git a/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java b/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java index cf1219c60..bfcb4ef8f 100644 --- a/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java +++ b/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java @@ -177,6 +177,24 @@ public void truncateTo1() throws Exception { } + @Test + public void truncateTo1Trailing() throws Exception { + + ZipCodeFilterStrategy strategy = new ZipCodeFilterStrategy(); + strategy.setStrategy(ZipCodeFilterStrategy.TRUNCATE); + strategy.setTruncateDigits(1); + strategy.setTruncateDirection(AbstractFilterStrategy.TRAILING); + + AnonymizationService anonymizationService = Mockito.mock(AnonymizationService.class); + + final Replacement replacement = strategy.getReplacement("name", "context", "documentid", "90210-0110", WINDOW, new Crypto(), new FPE(), anonymizationService, null); + + LOGGER.info(replacement); + + Assertions.assertEquals("****0", replacement.getReplacement()); + + } + @Test public void zeroLeading1() throws Exception { @@ -193,4 +211,14 @@ public void zeroLeading1() throws Exception { } + // Override the standard truncate tests since zip has a different truncate behavior + @Test + public void truncate1() throws Exception {} + + @Test + public void truncate2() throws Exception {} + + @Test + public void truncate3() throws Exception {} + } From 2ad0bccd4a7c63297bfb97ab3e7aed333b1d9f99 Mon Sep 17 00:00:00 2001 From: JAM <272807+JessieAMorris@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:46:02 -0700 Subject: [PATCH 2/2] Update truncateDigits argument to truncateLeaveCharacters --- .../docs/filter_policies/filter_strategies.md | 8 ++--- .../filters/common_filters/zip-codes.md | 18 +++++------ .../filter_policies/sample_filter_policies.md | 2 +- .../strategies/AbstractFilterStrategy.java | 22 ++++++++++---- .../strategies/StandardFilterStrategy.java | 10 +++---- .../strategies/ai/PhEyeFilterStrategy.java | 10 +++---- .../strategies/rules/DateFilterStrategy.java | 10 +++---- .../rules/ZipCodeFilterStrategy.java | 22 ++++++++------ .../AbstractFilterStrategyTest.java | 30 +++++++++++++++---- .../rules/ZipCodeFilterStrategyTest.java | 21 +++++++++++++ 10 files changed, 104 insertions(+), 49 deletions(-) diff --git a/docs/docs/filter_policies/filter_strategies.md b/docs/docs/filter_policies/filter_strategies.md index 508310a08..3304b1629 100644 --- a/docs/docs/filter_policies/filter_strategies.md +++ b/docs/docs/filter_policies/filter_strategies.md @@ -208,15 +208,15 @@ An example policy using the `STATIC_REPLACE` filter strategy: ### The `TRUNCATE` Filter Strategy {id="truncate"} -This strategy allows for truncating tokens to only a select number of digits. Specify `truncateDigits` -to set the desired number of digits to leave. For example, if `truncateDigits` is 4, the +This strategy allows for truncating tokens to only a select number of digits. Specify `truncateLeaveCharacters` +to set the desired number of digits to leave. For example, if `truncateLeaveCharacters` is 4, the string `4111111111111111` will be truncated to `4111************`. `truncateDirection` can be set to `LEADING` (the default) which leaves N leading digits or `TRAILING` which leaves N trailing digits. `truncateCharacter` can be overwritten (defaults to `*`) to change the character that is used for the replacement. The `TRUNCATE` filter has special behavior for the zip code filter. For zip codes the Zip will always be truncated -to 5 digits long. For example, `truncateDigits=2` and a token of `90210-0110` will result in `90***`. +to 5 digits long. For example, `truncateLeaveCharacters=2` and a token of `90210-0110` will result in `90***`. ``` { @@ -226,7 +226,7 @@ to 5 digits long. For example, `truncateDigits=2` and a token of `90210-0110` wi "zipCodeFilterStrategies": [ { "strategy": "TRUNCATE", - "truncateDigits": 3 + "truncateLeaveCharacters": 3 } ] } diff --git a/docs/docs/filter_policies/filters/common_filters/zip-codes.md b/docs/docs/filter_policies/filters/common_filters/zip-codes.md index 6ca527a79..de2b42661 100644 --- a/docs/docs/filter_policies/filters/common_filters/zip-codes.md +++ b/docs/docs/filter_policies/filters/common_filters/zip-codes.md @@ -21,15 +21,15 @@ This filter has no required parameters. The filter may have zero or more filter strategies. When no filter strategy is given the default strategy of `REDACT` is used. When multiple filter strategies are given the filter strategies will be applied in order as they are listed. See [Filter Strategies](#filter-strategies) for details. -| Strategy | Description | -| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | -| `REDACT` | Replace the sensitive text with a placeholder. | -| `RANDOM_REPLACE` | Replace the sensitive text with a similar, random value. | -| `STATIC_REPLACE` | Replace the sensitive text with a given value. | -| `CRYPTO_REPLACE` | Replace the sensitive text with its encrypted value. | -| `HASH_SHA256_REPLACE` | Replace the sensitive text with its SHA256 hash value. | -| `TRUNCATE` | Replace the sensitive text by removing the last `x` digits. (Set the number of digits using the `truncateDigits` parameter of the filter strategy.) | -| `ZERO_LEADING` | Replace the sensitive text by zeroing the first 3 digits. | +| Strategy | Description | +| --------------------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `REDACT` | Replace the sensitive text with a placeholder. | +| `RANDOM_REPLACE` | Replace the sensitive text with a similar, random value. | +| `STATIC_REPLACE` | Replace the sensitive text with a given value. | +| `CRYPTO_REPLACE` | Replace the sensitive text with its encrypted value. | +| `HASH_SHA256_REPLACE` | Replace the sensitive text with its SHA256 hash value. | +| `TRUNCATE` | Replace the sensitive text by removing everything except `x` characters. (Set the number of characters to leave using the `truncateLeaveCharacters` parameter of the filter strategy.) | +| `ZERO_LEADING` | Replace the sensitive text by zeroing the first 3 digits. | ### Conditions diff --git a/docs/docs/filter_policies/sample_filter_policies.md b/docs/docs/filter_policies/sample_filter_policies.md index c2a8f4216..5ac50c874 100644 --- a/docs/docs/filter_policies/sample_filter_policies.md +++ b/docs/docs/filter_policies/sample_filter_policies.md @@ -129,7 +129,7 @@ This policy finds ZIP codes starting with `90` and truncates the zip code to jus { "condition": "token startswith \"90\"", "strategy": "TRUNCATE", - "truncateDigits": 2 + "truncateLeaveCharacters": 2 } ] } diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java index 76ef9275a..547cbaba3 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java @@ -112,8 +112,13 @@ public abstract class AbstractFilterStrategy { @SerializedName("truncateDigits") @Expose + @Deprecated protected Integer truncateDigits; + @SerializedName("truncateLeaveCharacters") + @Expose + protected Integer truncateLeaveCharacters; + @SerializedName("truncateCharacter") @Expose protected String truncateCharacter = "*"; @@ -380,21 +385,26 @@ public String getMaskLength() { return maskLength; } - public Integer getTruncateDigits() { - return truncateDigits; + @Deprecated + public void setTruncateDigits(Integer truncateDigits) { + setTruncateLeaveCharacters(truncateDigits); } - public void setTruncateDigits(Integer truncateDigits) { + public void setTruncateLeaveCharacters(Integer truncateLeaveCharacters) { // Make sure it is a valid value. - if(truncateDigits >= 1) { - this.truncateDigits = truncateDigits; + if(truncateLeaveCharacters >= 1) { + this.truncateLeaveCharacters = truncateLeaveCharacters; } else { - throw new IllegalArgumentException("Truncate length must be greater than 1"); + throw new IllegalArgumentException("Truncate leave characters must be greater than or equal to 1"); } } + public Integer getTruncateLeaveCharacters() { + return truncateLeaveCharacters; + } + public String getTruncateCharacter() { return truncateCharacter; } diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java index ebc48f77a..eea8c8980 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java @@ -56,16 +56,16 @@ public Replacement getStandardReplacement(String label, String context, String d } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) { - int truncateLength = getValueOrDefault(truncateDigits, 4); + int leaveCharacters = getValueOrDefault(getValueOrDefault(truncateLeaveCharacters, truncateDigits), 4); - if (truncateLength < 1) { - truncateLength = 1; + if (leaveCharacters < 1) { + leaveCharacters = 1; } if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) { - replacement = token.substring(0, truncateLength) + StringUtils.repeat(truncateCharacter, token.length() - truncateLength); + replacement = token.substring(0, leaveCharacters) + StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters); } else { - replacement = StringUtils.repeat(truncateCharacter, token.length() - truncateLength) + token.substring(token.length() - truncateLength); + replacement = StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters) + token.substring(token.length() - leaveCharacters); } } else if(StringUtils.equalsIgnoreCase(strategy, RANDOM_REPLACE)) { diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java index 0a3e359ee..62891bb4e 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java @@ -158,16 +158,16 @@ public Replacement getReplacement(String label, String context, String documentI } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) { - int truncateLength = getValueOrDefault(truncateDigits, 4); + int leaveCharacters = getValueOrDefault(getValueOrDefault(truncateDigits, truncateLeaveCharacters), 4); - if (truncateLength < 1) { - truncateLength = 1; + if (leaveCharacters < 1) { + leaveCharacters = 1; } if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) { - replacement = token.substring(0, truncateLength) + StringUtils.repeat(truncateCharacter, token.length() - truncateLength); + replacement = token.substring(0, leaveCharacters) + StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters); } else { - replacement = StringUtils.repeat(truncateCharacter, token.length() - truncateLength) + token.substring(token.length() - truncateLength); + replacement = StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters) + token.substring(token.length() - leaveCharacters); } } else if(StringUtils.equalsIgnoreCase(strategy, RANDOM_REPLACE)) { diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java index 7e1b1134d..8bbca2c8f 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java @@ -176,16 +176,16 @@ public Replacement getReplacement(String label, String context, String documentI } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) { - int truncateLength = getValueOrDefault(truncateDigits, 4); + int leaveCharacters = getValueOrDefault(getValueOrDefault(truncateDigits, truncateLeaveCharacters), 4); - if (truncateLength < 1) { - truncateLength = 1; + if (leaveCharacters < 1) { + leaveCharacters = 1; } if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) { - replacement = token.substring(0, truncateLength) + StringUtils.repeat(truncateCharacter, token.length() - truncateLength); + replacement = token.substring(0, leaveCharacters) + StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters); } else { - replacement = StringUtils.repeat(truncateCharacter, token.length() - truncateLength) + token.substring(token.length() - truncateLength); + replacement = StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters) + token.substring(token.length() - leaveCharacters); } } else if(StringUtils.equalsIgnoreCase(strategy, RANDOM_REPLACE)) { diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java index b2619bafd..dea22b57e 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java @@ -194,12 +194,16 @@ public Replacement getReplacement(String label, String context, String documentI } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) { + int leaveCharacters = getValueOrDefault(getValueOrDefault(truncateDigits, truncateLeaveCharacters), 4); + + if (leaveCharacters < 1) { + leaveCharacters = 1; + } + if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) { - final int truncateLength = getValueOrDefault(truncateDigits, 2); - replacement = token.substring(0, truncateDigits) + StringUtils.repeat(truncateCharacter, Math.min(token.length() - truncateLength, 5 - truncateDigits)); + replacement = token.substring(0, leaveCharacters) + StringUtils.repeat(truncateCharacter, Math.min(token.length() - leaveCharacters, 5 - leaveCharacters)); } else { - final int truncateLength = getValueOrDefault(truncateDigits, 2); - replacement = StringUtils.repeat(truncateCharacter, Math.min(token.length() - truncateLength, 5 - truncateDigits)) + token.substring(Math.min(token.length() - truncateLength, 5 - truncateDigits), 5); + replacement = StringUtils.repeat(truncateCharacter, Math.min(token.length() - leaveCharacters, 5 - leaveCharacters)) + token.substring(Math.min(token.length() - leaveCharacters, 5 - leaveCharacters), 5); } @@ -230,15 +234,15 @@ public Replacement getReplacement(String label, String context, String documentI } - public Integer getTruncateDigits() { - return truncateDigits; + public void setTruncateDigits(Integer truncateDigits) { + setTruncateLeaveCharacters(truncateDigits); } - public void setTruncateDigits(Integer truncateDigits) { + public void setTruncateLeaveCharacters(Integer truncateLeaveCharacters) { // Make sure it is a valid value. - if(truncateDigits >= 1 && truncateDigits <= 4) { - this.truncateDigits = truncateDigits; + if(truncateLeaveCharacters >= 1 && truncateLeaveCharacters <= 4) { + this.truncateLeaveCharacters = truncateLeaveCharacters; } else { throw new IllegalArgumentException("Truncate length must be between 1 and 4, inclusive."); } diff --git a/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java b/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java index ffe197d6c..44549a385 100644 --- a/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java +++ b/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java @@ -302,12 +302,12 @@ public void truncate1() throws Exception { final AbstractFilterStrategy strategy = getFilterStrategy(); strategy.setStrategy(AbstractFilterStrategy.TRUNCATE); - strategy.setTruncateDigits(2); + strategy.setTruncateDigits(1); final String token = "12345"; final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null); - Assertions.assertEquals(replacement.getReplacement(), "12***"); + Assertions.assertEquals(replacement.getReplacement(), "1****"); Assertions.assertEquals(replacement.getReplacement().length(), 5); } @@ -320,10 +320,30 @@ public void truncate2() throws Exception { when(anonymizationService.getAnonymizationCacheService()).thenReturn(anonymizationCacheService); + final AbstractFilterStrategy strategy = getFilterStrategy(); + strategy.setStrategy(AbstractFilterStrategy.TRUNCATE); + strategy.setTruncateLeaveCharacters(4); + + final String token = "12345"; + final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null); + + Assertions.assertEquals(replacement.getReplacement(), "1234*"); + Assertions.assertEquals(replacement.getReplacement().length(), 5); + + } + + @Test + public void truncate3() throws Exception { + + final AnonymizationService anonymizationService = Mockito.mock(AnonymizationService.class); + final AnonymizationCacheService anonymizationCacheService = Mockito.mock(AnonymizationCacheService.class); + + when(anonymizationService.getAnonymizationCacheService()).thenReturn(anonymizationCacheService); + final AbstractFilterStrategy strategy = getFilterStrategy(); strategy.setStrategy(AbstractFilterStrategy.TRUNCATE); strategy.setTruncateDirection(AbstractFilterStrategy.LEADING); - strategy.setTruncateDigits(2); + strategy.setTruncateLeaveCharacters(2); final String token = "12345"; final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null); @@ -334,7 +354,7 @@ public void truncate2() throws Exception { } @Test - public void truncate3() throws Exception { + public void truncate4() throws Exception { final AnonymizationService anonymizationService = Mockito.mock(AnonymizationService.class); final AnonymizationCacheService anonymizationCacheService = Mockito.mock(AnonymizationCacheService.class); @@ -344,7 +364,7 @@ public void truncate3() throws Exception { final AbstractFilterStrategy strategy = getFilterStrategy(); strategy.setStrategy(AbstractFilterStrategy.TRUNCATE); strategy.setTruncateDirection(AbstractFilterStrategy.TRAILING); - strategy.setTruncateDigits(4); + strategy.setTruncateLeaveCharacters(4); final String token = "4111111111111111"; final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null); diff --git a/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java b/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java index bfcb4ef8f..3b46a2581 100644 --- a/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java +++ b/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java @@ -195,6 +195,24 @@ public void truncateTo1Trailing() throws Exception { } + @Test + public void truncateTo1Leave() throws Exception { + + ZipCodeFilterStrategy strategy = new ZipCodeFilterStrategy(); + strategy.setStrategy(ZipCodeFilterStrategy.TRUNCATE); + strategy.setTruncateLeaveCharacters(1); + strategy.setTruncateDirection(AbstractFilterStrategy.TRAILING); + + AnonymizationService anonymizationService = Mockito.mock(AnonymizationService.class); + + final Replacement replacement = strategy.getReplacement("name", "context", "documentid", "90210-0110", WINDOW, new Crypto(), new FPE(), anonymizationService, null); + + LOGGER.info(replacement); + + Assertions.assertEquals("****0", replacement.getReplacement()); + + } + @Test public void zeroLeading1() throws Exception { @@ -221,4 +239,7 @@ public void truncate2() throws Exception {} @Test public void truncate3() throws Exception {} + @Test + public void truncate4() throws Exception {} + }