diff --git a/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java index 2f19939b..c928088c 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java @@ -318,8 +318,6 @@ public class Datahandler { SentimentValueCache sentimentCacheStr = sentimentCachingMap.getOrDefault(strCollection, null); List smxReturnList = new ArrayList(); List randomIterationComparision = new ArrayList(); - int iteratecap = strCacheLocal.size() > prefix_size ? strCacheLocal.size() - prefix_size : strCollection.size() - 1; - int iterator = ThreadLocalRandom.current().nextInt(0, iteratecap); if (strCacheLocal.size() < prefix_size) { for (String colStr : strCollection) @@ -327,6 +325,9 @@ public class Datahandler { strCacheLocal.put(strCacheLocal.size(), colStr); } } + int iteratecap = strCacheLocal.size() > prefix_size ? strCacheLocal.size() - prefix_size : strCacheLocal.size() - 1; + int iterator = ThreadLocalRandom.current().nextInt(0, iteratecap); + int iterated = 0; for (String str1 : strCacheLocal.values()) { if (iterated >= iterator && iterated < iterator + prefix_size) { @@ -404,7 +405,7 @@ public class Datahandler { } private ConcurrentMap removeNonSensicalStrings(ConcurrentMap strmap) { - final ConcurrentMap strCacheLocal = stringCache.size() < 150 ? strmap : stringCache; + final ConcurrentMap strCacheLocal = stringCache; final ConcurrentMap localJMWEMap = getMultipleJMWEAnnotation(strmap.values()); final ConcurrentMap localPipelineAnnotation = getMultiplePipelineAnnotation(strmap.values()); final ConcurrentMap localPipelineSentimentAnnotation = getMultiplePipelineSentimentAnnotation(strmap.values()); @@ -416,12 +417,9 @@ public class Datahandler { if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) { ConcurrentMap str = MessageResponseHandler.getStr(); System.out.println("str size: " + str.size()); - str = cutContent(str, hlStatsMsg); str = filterContent(str); - str = removeSlacks(str); - //System.out.println("finished removeSlacks \n" + str.size() + "\n"); str = removeNonSensicalStrings(str); - System.out.println("removeNonSensicalStrings str size POST: " + str.size() + "\n"); + //System.out.println("removeNonSensicalStrings str size POST: " + str.size() + "\n"); str = annotationCacheUpdate(str); System.out.println("annotationCacheUpdate str size POST: " + str.size() + "\n"); ConcurrentMap strf = str; @@ -622,20 +620,6 @@ public class Datahandler { return 0.0; } - public static ConcurrentMap cutContent(ConcurrentMap str, boolean hlStatsMsg) { - ConcurrentMap returnlist = new MapMaker().concurrencyLevel(2).makeMap(); - str.values().forEach(str1 -> { - int iend = str1.indexOf("content: "); - if (iend != -1) { - String trs = str1.substring(iend + 9); - returnlist.put(returnlist.size() + 1, trs.substring(0, trs.length() - 1)); - } else if (hlStatsMsg) { - returnlist.put(returnlist.size() + 1, str1); - } - }); - return returnlist; - } - public static ConcurrentMap filterContent(ConcurrentMap str) { ConcurrentMap strlistreturn = new MapMaker().concurrencyLevel(2).makeMap(); str.values().forEach(str1 -> { @@ -739,38 +723,13 @@ public class Datahandler { } str1 = str1.trim(); if (str1.length() > 2 && (!str1.startsWith("!"))) { - strlistreturn.put(strlistreturn.size() + 1, str1); + strlistreturn.put(strlistreturn.size(), str1); } } }); return strlistreturn; } - private ConcurrentMap removeSlacks(ConcurrentMap str) { - ConcurrentMap strreturn = new MapMaker().concurrencyLevel(2).makeMap(); - if (stringCache.isEmpty()) { - return str; - } - Collection values = stringCache.values(); - str.values().forEach(str1 -> { - boolean tooclosematch = false; - for (String strVals : values) { - LevenshteinDistance leven = new LevenshteinDistance(strVals, str1); - double Distance = leven.computeLevenshteinDistance(); - Double maxpermittedDistance = 2.5; - if (Distance < maxpermittedDistance) { - tooclosematch = true; - break; - } - } - if (!tooclosematch) { - strreturn.put(strreturn.size() + 1, str1); - //System.out.println("adding strreturn str1: " + str1 + "\n"); - } - }); - return strreturn; - } - private ConcurrentMap annotationCacheUpdate(ConcurrentMap strmap) { ConcurrentMap jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strmap.values()); for (Entry jmweitr : jmweAnnotation.entrySet()) { diff --git a/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java b/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java index a4c1fb12..a498e057 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java @@ -46,7 +46,6 @@ public class MessageResponseHandler { public static String selectReponseMessage(String toString, String personName) throws CustomError { ConcurrentMap str1 = new MapMaker().concurrencyLevel(6).makeMap(); str1.put(str1.size() + 1, toString); - str1 = Datahandler.cutContent(str1, false); String strreturn = ""; for (String str : str1.values()) { if (!str.isEmpty()) { diff --git a/ArtificialAutism/target/ArtificialAutism-1.0.jar b/ArtificialAutism/target/ArtificialAutism-1.0.jar index fe7ab02f..17239d60 100644 Binary files a/ArtificialAutism/target/ArtificialAutism-1.0.jar and b/ArtificialAutism/target/ArtificialAutism-1.0.jar differ diff --git a/ArtificialAutism/target/classes/FunctionLayer/Datahandler$AnnotationCollector.class b/ArtificialAutism/target/classes/FunctionLayer/Datahandler$AnnotationCollector.class index 90f03d64..d1a6e514 100644 Binary files a/ArtificialAutism/target/classes/FunctionLayer/Datahandler$AnnotationCollector.class and b/ArtificialAutism/target/classes/FunctionLayer/Datahandler$AnnotationCollector.class differ diff --git a/ArtificialAutism/target/classes/FunctionLayer/Datahandler.class b/ArtificialAutism/target/classes/FunctionLayer/Datahandler.class index 72d42e75..1d7f54a0 100644 Binary files a/ArtificialAutism/target/classes/FunctionLayer/Datahandler.class and b/ArtificialAutism/target/classes/FunctionLayer/Datahandler.class differ diff --git a/ArtificialAutism/target/classes/FunctionLayer/MessageResponseHandler.class b/ArtificialAutism/target/classes/FunctionLayer/MessageResponseHandler.class index d198e3f0..02b28d2a 100644 Binary files a/ArtificialAutism/target/classes/FunctionLayer/MessageResponseHandler.class and b/ArtificialAutism/target/classes/FunctionLayer/MessageResponseHandler.class differ diff --git a/ArtificialAutism/target/maven-archiver/pom.properties b/ArtificialAutism/target/maven-archiver/pom.properties index e474df4b..0177e827 100644 --- a/ArtificialAutism/target/maven-archiver/pom.properties +++ b/ArtificialAutism/target/maven-archiver/pom.properties @@ -1,5 +1,5 @@ #Generated by Maven -#Sun Nov 01 22:49:28 CET 2020 +#Sun Nov 01 23:59:26 CET 2020 version=1.0 groupId=com.mycompany artifactId=ArtificialAutism