diff --git a/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java index d239f202..5b477ac4 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; @@ -68,6 +69,7 @@ public class Datahandler { private static ConcurrentMap coreDocumentAnnotationCache; private static ConcurrentMap conversationMatchMap; private static ConcurrentMap conversationUserMatchMap; + private static final ConcurrentMap locateFaultySentences = new MapMaker().concurrencyLevel(6).makeMap(); private static final ConcurrentMap mapUdate = new MapMaker().concurrencyLevel(4).makeMap(); private final static ConcurrentMap strmapreturn = new MapMaker().concurrencyLevel(4).makeMap(); private LinkedHashMap> lHMSMX = new LinkedHashMap(); @@ -231,7 +233,7 @@ public class Datahandler { hlStatsMessages.put(str, hlStatsMessages.size()); } } - int capacity = 10000; + int capacity = 15000; hlStatsMessages.keySet().forEach(str -> { if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) { String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null); @@ -276,18 +278,18 @@ public class Datahandler { } } - private final static void futuresReturnOverallEvaluation(ConcurrentMap> futures, String str) { - - for (final Future future : futures.values()) { - final SimilarityMatrix getSMX = retrieveFutureSMX(future); - if (handleRetrievedSMX(getSMX, str)) { + private final static void futuresReturnOverallEvaluation(ConcurrentMap> entries, String str) { + for (Entry> entrySet : entries.entrySet()) { + String transmittedStr = entrySet.getKey(); + final SimilarityMatrix getSMX = retrieveFutureSMX(entrySet.getValue()); + if (handleRetrievedSMX(getSMX, str, transmittedStr)) { break; } } sentenceRelationMap.put(str, mapUdate); } - private static final boolean handleRetrievedSMX(SimilarityMatrix getSMX, String str) { + private static final boolean handleRetrievedSMX(SimilarityMatrix getSMX, String str, String transmittedStr) { final int relationCap = 20; if (getSMX != null) { final Double scoreRelationNewMsgToRecentMsg = getSMX.getDistance(); @@ -302,21 +304,32 @@ public class Datahandler { return true; } } else if (scoreRelationNewMsgToRecentMsg <= -6000.0) { - negativeRelationCounter += 5; + negativeRelationCounter += 1; } } else { - negativeRelationCounter += 2; + Integer getCounts = locateFaultySentences.getOrDefault(transmittedStr, null); + if (getCounts == null) { + locateFaultySentences.put(transmittedStr, 0); + } else { + locateFaultySentences.put(transmittedStr, getCounts + 1); + } } - if (negativeRelationCounter >= relationCap * 3) { + if (negativeRelationCounter >= relationCap) { return true; } return false; } - private final static ConcurrentMap> StrComparringNoSentenceRelationMap(ConcurrentMap strCacheLocal, - String str, ConcurrentMap localJMWEMap, ConcurrentMap localPipelineAnnotation, - ConcurrentMap localPipelineSentimentAnnotation, ConcurrentMap localCoreDocumentMap, int strmapSize) { - final ConcurrentMap> futures = new MapMaker().concurrencyLevel(4).makeMap(); + private final static Entry>, ConcurrentMap> StrComparringNoSentenceRelationMap(ConcurrentMap strCacheLocal, String str, ConcurrentMap localJMWEMap, ConcurrentMap localPipelineAnnotation, ConcurrentMap localPipelineSentimentAnnotation, + ConcurrentMap localCoreDocumentMap) { + final ConcurrentMap> futures = new MapMaker().concurrencyLevel(4).makeMap(); + strCacheLocal.values().removeIf(e -> { + Integer getFaultyOccurences = locateFaultySentences.getOrDefault(e, null); + if (getFaultyOccurences == null || getFaultyOccurences <= 20) { + return false; + } + return true; + }); for (String str1 : strCacheLocal.values()) { if (!str.equals(str1)) { final SimilarityMatrix SMX = new SimilarityMatrix(str, str1); @@ -332,10 +345,12 @@ public class Datahandler { pipelineAnnotationCache.get(str1), localPipelineSentimentAnnotation.get(str), pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1)); } - futures.put(futures.size() + 1, executor.submit(worker)); + futures.put(str1, executor.submit(worker)); } } - return futures; + Map.Entry>, ConcurrentMap> entryReturn + = new AbstractMap.SimpleEntry(futures, strCacheLocal); + return entryReturn; } private static ConcurrentMap stringIteratorComparator(ConcurrentMap strmap, @@ -344,13 +359,20 @@ public class Datahandler { ConcurrentMap localCoreDocumentMap) { int i = 0; for (String str : strmap.values()) { - positiveRelationCounter = 0; - negativeRelationCounter = 0; - final ConcurrentMap> futures = StrComparringNoSentenceRelationMap(strCacheLocal, str, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, strmap.size()); - futuresReturnOverallEvaluation(futures, str); - System.out.println("overall advancement of future iterator: " + i + "\nstrmap size: " + strmap.values().size() + "\nstrmapreturn size: " + strmapreturn.size() - + "\n"); - i++; + Integer getFaultyOccurences = locateFaultySentences.getOrDefault(str, null); + if (getFaultyOccurences == null || getFaultyOccurences <= 20) { + positiveRelationCounter = 0; + negativeRelationCounter = 0; + Entry>, ConcurrentMap> strRelationMap + = StrComparringNoSentenceRelationMap(strCacheLocal, str, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, + localCoreDocumentMap); + strCacheLocal = strRelationMap.getValue(); + final ConcurrentMap> futures = strRelationMap.getKey(); + futuresReturnOverallEvaluation(futures, str); + System.out.println("overall advancement of future iterator: " + i + "\nstrmap size: " + strmap.values().size() + "\nstrmapreturn size: " + strmapreturn.size() + + "\nlocateFaultySentences size: " + locateFaultySentences.size() + "\n"); + i++; + } } return strmapreturn; }