UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

timeout caching collection

Browse Source
This commit is contained in:
jenzur 2019-05-12 19:06:22 +02:00
parent 810321a520
commit 5ccadbffbd
1 changed files with 45 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java
View File

@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Properties; import java.util.Properties;
import java.util.Set;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
@ -68,6 +69,7 @@ public class Datahandler {
private static ConcurrentMap<String, CoreDocument> coreDocumentAnnotationCache; private static ConcurrentMap<String, CoreDocument> coreDocumentAnnotationCache;
private static ConcurrentMap<String, Integer> conversationMatchMap; private static ConcurrentMap<String, Integer> conversationMatchMap;
private static ConcurrentMap<String, Integer> conversationUserMatchMap; private static ConcurrentMap<String, Integer> conversationUserMatchMap;
private static final ConcurrentMap<String, Integer> locateFaultySentences = new MapMaker().concurrencyLevel(6).makeMap();
private static final ConcurrentMap<String, Double> mapUdate = new MapMaker().concurrencyLevel(4).makeMap(); private static final ConcurrentMap<String, Double> mapUdate = new MapMaker().concurrencyLevel(4).makeMap();
private final static ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(4).makeMap(); private final static ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(4).makeMap();
private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap(); private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap();
@ -231,7 +233,7 @@ public class Datahandler {
hlStatsMessages.put(str, hlStatsMessages.size()); hlStatsMessages.put(str, hlStatsMessages.size());
} }
} }
int capacity = 10000; int capacity = 15000;
hlStatsMessages.keySet().forEach(str -> { hlStatsMessages.keySet().forEach(str -> {
if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) { if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) {
String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null); String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null);
@ -276,18 +278,18 @@ public class Datahandler {
} }
} }
private final static void futuresReturnOverallEvaluation(ConcurrentMap<Integer, Future<SimilarityMatrix>> futures, String str) { private final static void futuresReturnOverallEvaluation(ConcurrentMap<String, Future<SimilarityMatrix>> entries, String str) {
for (Entry<String, Future<SimilarityMatrix>> entrySet : entries.entrySet()) {
for (final Future future : futures.values()) { String transmittedStr = entrySet.getKey();
final SimilarityMatrix getSMX = retrieveFutureSMX(future); final SimilarityMatrix getSMX = retrieveFutureSMX(entrySet.getValue());
if (handleRetrievedSMX(getSMX, str)) { if (handleRetrievedSMX(getSMX, str, transmittedStr)) {
break; break;
} }
} }
sentenceRelationMap.put(str, mapUdate); sentenceRelationMap.put(str, mapUdate);
} }
private static final boolean handleRetrievedSMX(SimilarityMatrix getSMX, String str) { private static final boolean handleRetrievedSMX(SimilarityMatrix getSMX, String str, String transmittedStr) {
final int relationCap = 20; final int relationCap = 20;
if (getSMX != null) { if (getSMX != null) {
final Double scoreRelationNewMsgToRecentMsg = getSMX.getDistance(); final Double scoreRelationNewMsgToRecentMsg = getSMX.getDistance();
@ -302,21 +304,32 @@ public class Datahandler {
return true; return true;
} }
} else if (scoreRelationNewMsgToRecentMsg <= -6000.0) { } else if (scoreRelationNewMsgToRecentMsg <= -6000.0) {
negativeRelationCounter += 5; negativeRelationCounter += 1;
} }
} else { } else {
negativeRelationCounter += 2; Integer getCounts = locateFaultySentences.getOrDefault(transmittedStr, null);
if (getCounts == null) {
locateFaultySentences.put(transmittedStr, 0);
} else {
locateFaultySentences.put(transmittedStr, getCounts + 1);
}
} }
if (negativeRelationCounter >= relationCap * 3) { if (negativeRelationCounter >= relationCap) {
return true; return true;
} }
return false; return false;
} }
private final static ConcurrentMap<Integer, Future<SimilarityMatrix>> StrComparringNoSentenceRelationMap(ConcurrentMap<Integer, String> strCacheLocal, private final static Entry<ConcurrentMap<String, Future<SimilarityMatrix>>, ConcurrentMap<Integer, String>> StrComparringNoSentenceRelationMap(ConcurrentMap<Integer, String> strCacheLocal, String str, ConcurrentMap<String, Annotation> localJMWEMap, ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation,
String str, ConcurrentMap<String, Annotation> localJMWEMap, ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, CoreDocument> localCoreDocumentMap) {
ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation, ConcurrentMap<String, CoreDocument> localCoreDocumentMap, int strmapSize) { final ConcurrentMap<String, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(4).makeMap();
final ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(4).makeMap(); strCacheLocal.values().removeIf(e -> {
Integer getFaultyOccurences = locateFaultySentences.getOrDefault(e, null);
if (getFaultyOccurences == null || getFaultyOccurences <= 20) {
return false;
}
return true;
});
for (String str1 : strCacheLocal.values()) { for (String str1 : strCacheLocal.values()) {
if (!str.equals(str1)) { if (!str.equals(str1)) {
final SimilarityMatrix SMX = new SimilarityMatrix(str, str1); final SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
@ -332,10 +345,12 @@ public class Datahandler {
pipelineAnnotationCache.get(str1), localPipelineSentimentAnnotation.get(str), pipelineAnnotationCache.get(str1), localPipelineSentimentAnnotation.get(str),
pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1)); pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1));
} }
futures.put(futures.size() + 1, executor.submit(worker)); futures.put(str1, executor.submit(worker));
} }
} }
return futures; Map.Entry<ConcurrentMap<String, Future<SimilarityMatrix>>, ConcurrentMap<Integer, String>> entryReturn
= new AbstractMap.SimpleEntry(futures, strCacheLocal);
return entryReturn;
} }
private static ConcurrentMap<Integer, String> stringIteratorComparator(ConcurrentMap<Integer, String> strmap, private static ConcurrentMap<Integer, String> stringIteratorComparator(ConcurrentMap<Integer, String> strmap,
@ -344,13 +359,20 @@ public class Datahandler {
ConcurrentMap<String, CoreDocument> localCoreDocumentMap) { ConcurrentMap<String, CoreDocument> localCoreDocumentMap) {
int i = 0; int i = 0;
for (String str : strmap.values()) { for (String str : strmap.values()) {
positiveRelationCounter = 0; Integer getFaultyOccurences = locateFaultySentences.getOrDefault(str, null);
negativeRelationCounter = 0; if (getFaultyOccurences == null || getFaultyOccurences <= 20) {
final ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = StrComparringNoSentenceRelationMap(strCacheLocal, str, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, strmap.size()); positiveRelationCounter = 0;
futuresReturnOverallEvaluation(futures, str); negativeRelationCounter = 0;
System.out.println("overall advancement of future iterator: " + i + "\nstrmap size: " + strmap.values().size() + "\nstrmapreturn size: " + strmapreturn.size() Entry<ConcurrentMap<String, Future<SimilarityMatrix>>, ConcurrentMap<Integer, String>> strRelationMap
+ "\n"); = StrComparringNoSentenceRelationMap(strCacheLocal, str, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation,
i++; localCoreDocumentMap);
strCacheLocal = strRelationMap.getValue();
final ConcurrentMap<String, Future<SimilarityMatrix>> futures = strRelationMap.getKey();
futuresReturnOverallEvaluation(futures, str);
System.out.println("overall advancement of future iterator: " + i + "\nstrmap size: " + strmap.values().size() + "\nstrmapreturn size: " + strmapreturn.size()
+ "\nlocateFaultySentences size: " + locateFaultySentences.size() + "\n");
i++;
}
} }
return strmapreturn; return strmapreturn;
} }