removed removeslack check as levensthein defunct now

ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java

@@ -318,8 +318,6 @@ public class Datahandler {
         SentimentValueCache sentimentCacheStr = sentimentCachingMap.getOrDefault(strCollection, null);
         List<SimilarityMatrix> smxReturnList = new ArrayList();
         List<String> randomIterationComparision = new ArrayList();
-        int iteratecap = strCacheLocal.size() > prefix_size ? strCacheLocal.size() - prefix_size : strCollection.size() - 1;
-        int iterator = ThreadLocalRandom.current().nextInt(0, iteratecap);
         if (strCacheLocal.size() < prefix_size)
         {
             for (String colStr : strCollection)
@@ -327,6 +325,9 @@ public class Datahandler {
                 strCacheLocal.put(strCacheLocal.size(), colStr);
             }
         }
+        int iteratecap = strCacheLocal.size() > prefix_size ? strCacheLocal.size() - prefix_size : strCacheLocal.size() - 1;
+        int iterator = ThreadLocalRandom.current().nextInt(0, iteratecap);
+        
         int iterated = 0;
         for (String str1 : strCacheLocal.values()) {
             if (iterated >= iterator && iterated < iterator + prefix_size) {
@@ -404,7 +405,7 @@ public class Datahandler {
     }
 
     private ConcurrentMap<Integer, String> removeNonSensicalStrings(ConcurrentMap<Integer, String> strmap) {
-        final ConcurrentMap<Integer, String> strCacheLocal = stringCache.size() < 150 ? strmap : stringCache;
+        final ConcurrentMap<Integer, String> strCacheLocal = stringCache;
         final ConcurrentMap<String, Annotation> localJMWEMap = getMultipleJMWEAnnotation(strmap.values());
         final ConcurrentMap<String, Annotation> localPipelineAnnotation = getMultiplePipelineAnnotation(strmap.values());
         final ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation = getMultiplePipelineSentimentAnnotation(strmap.values());
@@ -416,12 +417,9 @@ public class Datahandler {
         if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
             ConcurrentMap<Integer, String> str = MessageResponseHandler.getStr();
             System.out.println("str size: " + str.size());
-            str = cutContent(str, hlStatsMsg);
             str = filterContent(str);
-            str = removeSlacks(str);
-            //System.out.println("finished removeSlacks \n" + str.size() + "\n");
             str = removeNonSensicalStrings(str);
-            System.out.println("removeNonSensicalStrings str size POST: " + str.size() + "\n");
+            //System.out.println("removeNonSensicalStrings str size POST: " + str.size() + "\n");
             str = annotationCacheUpdate(str);
             System.out.println("annotationCacheUpdate str size POST: " + str.size() + "\n");
             ConcurrentMap<Integer, String> strf = str;
@@ -622,20 +620,6 @@ public class Datahandler {
         return 0.0;
     }
 
-    public static ConcurrentMap<Integer, String> cutContent(ConcurrentMap<Integer, String> str, boolean hlStatsMsg) {
-        ConcurrentMap<Integer, String> returnlist = new MapMaker().concurrencyLevel(2).makeMap();
-        str.values().forEach(str1 -> {
-            int iend = str1.indexOf("content: ");
-            if (iend != -1) {
-                String trs = str1.substring(iend + 9);
-                returnlist.put(returnlist.size() + 1, trs.substring(0, trs.length() - 1));
-            } else if (hlStatsMsg) {
-                returnlist.put(returnlist.size() + 1, str1);
-            }
-        });
-        return returnlist;
-    }
-
     public static ConcurrentMap<Integer, String> filterContent(ConcurrentMap<Integer, String> str) {
         ConcurrentMap<Integer, String> strlistreturn = new MapMaker().concurrencyLevel(2).makeMap();
         str.values().forEach(str1 -> {
@@ -739,38 +723,13 @@ public class Datahandler {
                 }
                 str1 = str1.trim();
                 if (str1.length() > 2 && (!str1.startsWith("!"))) {
-                    strlistreturn.put(strlistreturn.size() + 1, str1);
+                    strlistreturn.put(strlistreturn.size(), str1);
                 }
             }
         });
         return strlistreturn;
     }
 
-    private ConcurrentMap<Integer, String> removeSlacks(ConcurrentMap<Integer, String> str) {
-        ConcurrentMap<Integer, String> strreturn = new MapMaker().concurrencyLevel(2).makeMap();
-        if (stringCache.isEmpty()) {
-            return str;
-        }
-        Collection<String> values = stringCache.values();
-        str.values().forEach(str1 -> {
-            boolean tooclosematch = false;
-            for (String strVals : values) {
-                LevenshteinDistance leven = new LevenshteinDistance(strVals, str1);
-                double Distance = leven.computeLevenshteinDistance();
-                Double maxpermittedDistance = 2.5;
-                if (Distance < maxpermittedDistance) {
-                    tooclosematch = true;
-                    break;
-                }
-            }
-            if (!tooclosematch) {
-                strreturn.put(strreturn.size() + 1, str1);
-                //System.out.println("adding strreturn str1: " + str1 + "\n");
-            }
-        });
-        return strreturn;
-    }
-
     private ConcurrentMap<Integer, String> annotationCacheUpdate(ConcurrentMap<Integer, String> strmap) {
         ConcurrentMap<String, Annotation> jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strmap.values());
         for (Entry<String, Annotation> jmweitr : jmweAnnotation.entrySet()) {

ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java

@@ -46,7 +46,6 @@ public class MessageResponseHandler {
     public static String selectReponseMessage(String toString, String personName) throws CustomError {
         ConcurrentMap<Integer, String> str1 = new MapMaker().concurrencyLevel(6).makeMap();
         str1.put(str1.size() + 1, toString);
-        str1 = Datahandler.cutContent(str1, false);
         String strreturn = "";
         for (String str : str1.values()) {
             if (!str.isEmpty()) {

ArtificialAutism/target/maven-archiver/pom.properties

@@ -1,5 +1,5 @@
 #Generated by Maven
-#Sun Nov 01 22:49:28 CET 2020
+#Sun Nov 01 23:59:26 CET 2020
 version=1.0
 groupId=com.mycompany
 artifactId=ArtificialAutism