UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

timeout caching collection

Browse Source
This commit is contained in:
jenzur 2019-05-12 19:06:22 +02:00
parent 810321a520
commit 5ccadbffbd
1 changed files with 45 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java
View File

@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
@ -68,6 +69,7 @@ public class Datahandler {
private static ConcurrentMap<String, CoreDocument> coreDocumentAnnotationCache;
private static ConcurrentMap<String, Integer> conversationMatchMap;
private static ConcurrentMap<String, Integer> conversationUserMatchMap;
private static final ConcurrentMap<String, Integer> locateFaultySentences = new MapMaker().concurrencyLevel(6).makeMap();
private static final ConcurrentMap<String, Double> mapUdate = new MapMaker().concurrencyLevel(4).makeMap();
private final static ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(4).makeMap();
private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap();
@ -231,7 +233,7 @@ public class Datahandler {
hlStatsMessages.put(str, hlStatsMessages.size());
}
}
int capacity = 10000;
int capacity = 15000;
hlStatsMessages.keySet().forEach(str -> {
if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) {
String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null);
@ -276,18 +278,18 @@ public class Datahandler {
}
}
private final static void futuresReturnOverallEvaluation(ConcurrentMap<Integer, Future<SimilarityMatrix>> futures, String str) {
for (final Future future : futures.values()) {
final SimilarityMatrix getSMX = retrieveFutureSMX(future);
if (handleRetrievedSMX(getSMX, str)) {
private final static void futuresReturnOverallEvaluation(ConcurrentMap<String, Future<SimilarityMatrix>> entries, String str) {
for (Entry<String, Future<SimilarityMatrix>> entrySet : entries.entrySet()) {
String transmittedStr = entrySet.getKey();
final SimilarityMatrix getSMX = retrieveFutureSMX(entrySet.getValue());
if (handleRetrievedSMX(getSMX, str, transmittedStr)) {
break;
}
}
sentenceRelationMap.put(str, mapUdate);
}
private static final boolean handleRetrievedSMX(SimilarityMatrix getSMX, String str) {
private static final boolean handleRetrievedSMX(SimilarityMatrix getSMX, String str, String transmittedStr) {
final int relationCap = 20;
if (getSMX != null) {
final Double scoreRelationNewMsgToRecentMsg = getSMX.getDistance();
@ -302,21 +304,32 @@ public class Datahandler {
return true;
}
} else if (scoreRelationNewMsgToRecentMsg <= -6000.0) {
negativeRelationCounter += 5;
negativeRelationCounter += 1;
}
} else {
negativeRelationCounter += 2;
Integer getCounts = locateFaultySentences.getOrDefault(transmittedStr, null);
if (getCounts == null) {
locateFaultySentences.put(transmittedStr, 0);
} else {
locateFaultySentences.put(transmittedStr, getCounts + 1);
}
}
if (negativeRelationCounter >= relationCap * 3) {
if (negativeRelationCounter >= relationCap) {
return true;
}
return false;
}
private final static ConcurrentMap<Integer, Future<SimilarityMatrix>> StrComparringNoSentenceRelationMap(ConcurrentMap<Integer, String> strCacheLocal,
String str, ConcurrentMap<String, Annotation> localJMWEMap, ConcurrentMap<String, Annotation> localPipelineAnnotation,
ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation, ConcurrentMap<String, CoreDocument> localCoreDocumentMap, int strmapSize) {
final ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(4).makeMap();
private final static Entry<ConcurrentMap<String, Future<SimilarityMatrix>>, ConcurrentMap<Integer, String>> StrComparringNoSentenceRelationMap(ConcurrentMap<Integer, String> strCacheLocal, String str, ConcurrentMap<String, Annotation> localJMWEMap, ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation,
ConcurrentMap<String, CoreDocument> localCoreDocumentMap) {
final ConcurrentMap<String, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(4).makeMap();
strCacheLocal.values().removeIf(e -> {
Integer getFaultyOccurences = locateFaultySentences.getOrDefault(e, null);
if (getFaultyOccurences == null || getFaultyOccurences <= 20) {
return false;
}
return true;
});
for (String str1 : strCacheLocal.values()) {
if (!str.equals(str1)) {
final SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
@ -332,10 +345,12 @@ public class Datahandler {
pipelineAnnotationCache.get(str1), localPipelineSentimentAnnotation.get(str),
pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1));
}
futures.put(futures.size() + 1, executor.submit(worker));
futures.put(str1, executor.submit(worker));
}
}
return futures;
Map.Entry<ConcurrentMap<String, Future<SimilarityMatrix>>, ConcurrentMap<Integer, String>> entryReturn
= new AbstractMap.SimpleEntry(futures, strCacheLocal);
return entryReturn;
}
private static ConcurrentMap<Integer, String> stringIteratorComparator(ConcurrentMap<Integer, String> strmap,
@ -344,13 +359,20 @@ public class Datahandler {
ConcurrentMap<String, CoreDocument> localCoreDocumentMap) {
int i = 0;
for (String str : strmap.values()) {
positiveRelationCounter = 0;
negativeRelationCounter = 0;
final ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = StrComparringNoSentenceRelationMap(strCacheLocal, str, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, strmap.size());
futuresReturnOverallEvaluation(futures, str);
System.out.println("overall advancement of future iterator: " + i + "\nstrmap size: " + strmap.values().size() + "\nstrmapreturn size: " + strmapreturn.size()
+ "\n");
i++;
Integer getFaultyOccurences = locateFaultySentences.getOrDefault(str, null);
if (getFaultyOccurences == null || getFaultyOccurences <= 20) {
positiveRelationCounter = 0;
negativeRelationCounter = 0;
Entry<ConcurrentMap<String, Future<SimilarityMatrix>>, ConcurrentMap<Integer, String>> strRelationMap
= StrComparringNoSentenceRelationMap(strCacheLocal, str, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation,
localCoreDocumentMap);
strCacheLocal = strRelationMap.getValue();
final ConcurrentMap<String, Future<SimilarityMatrix>> futures = strRelationMap.getKey();
futuresReturnOverallEvaluation(futures, str);
System.out.println("overall advancement of future iterator: " + i + "\nstrmap size: " + strmap.values().size() + "\nstrmapreturn size: " + strmapreturn.size()
+ "\nlocateFaultySentences size: " + locateFaultySentences.size() + "\n");
i++;
}
}
return strmapreturn;
}