UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

simplifying which should hopefully help with trimming bad sentences out

Browse Source
This commit is contained in:
jenzur 2019-05-11 20:33:45 +02:00
parent b1aa449b59
commit 810321a520
2 changed files with 50 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java
View File

@ -56,7 +56,7 @@ public class Datahandler {
private static Annotation strAnnoSentiment;
private static Annotation strAnnoJMWE;
private static CoreDocument coreDoc;
private static ConcurrentMap<String, ConcurrentMap<String, Double>> sentenceRelationMap;
private static final ConcurrentMap<String, ConcurrentMap<String, Double>> sentenceRelationMap = new MapMaker().concurrencyLevel(4).makeMap();
private static volatile Double preRelationCounters = 0.0;
private static volatile Double preRelationUserCounters = 0.0;
private static final ConcurrentMap<Integer, String> stringCache = new MapMaker().concurrencyLevel(2).makeMap();
@ -68,6 +68,8 @@ public class Datahandler {
private static ConcurrentMap<String, CoreDocument> coreDocumentAnnotationCache;
private static ConcurrentMap<String, Integer> conversationMatchMap;
private static ConcurrentMap<String, Integer> conversationUserMatchMap;
private static final ConcurrentMap<String, Double> mapUdate = new MapMaker().concurrencyLevel(4).makeMap();
private final static ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(4).makeMap();
private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap();
private final Stopwatch stopwatch;
private static final ForkJoinPool executor = instantiateExecutor();
@ -94,13 +96,12 @@ public class Datahandler {
public Datahandler() {
this.stopwatch = Stopwatch.createUnstarted();
this.jmweAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap();
this.pipelineAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap();
this.pipelineSentimentAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap();
this.coreDocumentAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap();
this.conversationMatchMap = new MapMaker().concurrencyLevel(2).makeMap();
this.sentenceRelationMap = new MapMaker().concurrencyLevel(2).makeMap();
this.conversationUserMatchMap = new MapMaker().concurrencyLevel(2).makeMap();
this.jmweAnnotationCache = new MapMaker().concurrencyLevel(3).makeMap();
this.pipelineAnnotationCache = new MapMaker().concurrencyLevel(4).makeMap();
this.pipelineSentimentAnnotationCache = new MapMaker().concurrencyLevel(4).makeMap();
this.coreDocumentAnnotationCache = new MapMaker().concurrencyLevel(5).makeMap();
this.conversationMatchMap = new MapMaker().concurrencyLevel(4).makeMap();
this.conversationUserMatchMap = new MapMaker().concurrencyLevel(4).makeMap();
}
private static StanfordCoreNLP getPipeLineSetUp() {
@ -230,7 +231,7 @@ public class Datahandler {
hlStatsMessages.put(str, hlStatsMessages.size());
}
}
int capacity = 35000;
int capacity = 10000;
hlStatsMessages.keySet().forEach(str -> {
if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) {
String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null);
@ -275,58 +276,41 @@ public class Datahandler {
}
}
private final static ConcurrentMap<Integer, String> cachedReturnEvaluations(ConcurrentMap<String, Double> getMap, ConcurrentMap<Integer, String> strmapreturn, String str) {
List<Integer> dummy = new ArrayList();
int relationCap = 20;
getMap.entrySet().parallelStream().forEach(mapValues -> {
if (!strmapreturn.values().contains(str) && dummy.isEmpty()) {
Double scoreRelationNewMsgToRecentMsg = mapValues.getValue();
System.out.println("strmapreturn size: " + strmapreturn.size() + "\nscoreRelationNewMsgToRecentMsg: "
+ scoreRelationNewMsgToRecentMsg + "\n");
if (scoreRelationNewMsgToRecentMsg >= 4100.0) {
positiveRelationCounter++;
if (positiveRelationCounter > relationCap) {
strmapreturn.put(strmapreturn.size() + 1, str);
}
} else if (scoreRelationNewMsgToRecentMsg <= -5000.0) {
negativeRelationCounter++;
if (negativeRelationCounter > relationCap * 2) {
dummy.add(1);
}
}
private final static void futuresReturnOverallEvaluation(ConcurrentMap<Integer, Future<SimilarityMatrix>> futures, String str) {
for (final Future future : futures.values()) {
final SimilarityMatrix getSMX = retrieveFutureSMX(future);
if (handleRetrievedSMX(getSMX, str)) {
break;
}
});
return strmapreturn;
}
sentenceRelationMap.put(str, mapUdate);
}
private final static ConcurrentMap<Integer, String> futuresReturnOverallEvaluation(ConcurrentMap<Integer, Future<SimilarityMatrix>> futures, ConcurrentMap<Integer, String> strmapreturn, String str) {
int relationCap = 20;
final ConcurrentMap<String, Double> mapUdate = new MapMaker().concurrencyLevel(2).makeMap();
final List<Integer> dummy = new ArrayList();
futures.values().parallelStream().forEach(future -> {
if (!strmapreturn.values().contains(str) && dummy.isEmpty()) {
final SimilarityMatrix getSMX = retrieveFutureSMX(future);
if (getSMX != null) {
Double scoreRelationNewMsgToRecentMsg = getSMX.getDistance();
System.out.println("strmapreturn size: " + strmapreturn.size() + "\nscoreRelationNewMsgToRecentMsg: "
+ scoreRelationNewMsgToRecentMsg + "\n");
mapUdate.put(getSMX.getSecondaryString(), scoreRelationNewMsgToRecentMsg);
if (scoreRelationNewMsgToRecentMsg >= 4100.0) {
positiveRelationCounter++;
if (positiveRelationCounter > relationCap) {
strmapreturn.put(strmapreturn.size() + 1, str);
}
} else if (scoreRelationNewMsgToRecentMsg <= -5000.0) {
negativeRelationCounter++;
if (negativeRelationCounter > relationCap * 2) {
dummy.add(1);
}
}
private static final boolean handleRetrievedSMX(SimilarityMatrix getSMX, String str) {
final int relationCap = 20;
if (getSMX != null) {
final Double scoreRelationNewMsgToRecentMsg = getSMX.getDistance();
System.out.println("scoreRelationNewMsgToRecentMsg: " + scoreRelationNewMsgToRecentMsg + "\n");
mapUdate.put(getSMX.getSecondaryString(), scoreRelationNewMsgToRecentMsg);
System.out.println("getSMX primary: " + getSMX.getPrimaryString() + "\ngetSMX secodary: " + getSMX.getSecondaryString() + "\n");
if (scoreRelationNewMsgToRecentMsg >= 200.0) {
positiveRelationCounter++;
if (positiveRelationCounter >= relationCap) {
System.out.println("added to strmapreturn str: " + str + "\n");
strmapreturn.put(strmapreturn.size() + 1, str);
return true;
}
} else if (scoreRelationNewMsgToRecentMsg <= -6000.0) {
negativeRelationCounter += 5;
}
});
sentenceRelationMap.put(str, mapUdate);
return strmapreturn;
} else {
negativeRelationCounter += 2;
}
if (negativeRelationCounter >= relationCap * 3) {
return true;
}
return false;
}
private final static ConcurrentMap<Integer, Future<SimilarityMatrix>> StrComparringNoSentenceRelationMap(ConcurrentMap<Integer, String> strCacheLocal,
@ -349,32 +333,29 @@ public class Datahandler {
pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1));
}
futures.put(futures.size() + 1, executor.submit(worker));
System.out.println("futures.size(): " + futures.size() + "\nstrmap.values().size(): " + strmapSize + "\n");
}
}
return futures;
}
private final static ConcurrentMap<Integer, String> stringIteratorComparator(ConcurrentMap<Integer, String> strmap,
private static ConcurrentMap<Integer, String> stringIteratorComparator(ConcurrentMap<Integer, String> strmap,
ConcurrentMap<Integer, String> strCacheLocal, ConcurrentMap<String, Annotation> localJMWEMap,
ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation,
ConcurrentMap<String, CoreDocument> localCoreDocumentMap) {
ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(4).makeMap();
int i = 0;
for (String str : strmap.values()) {
final ConcurrentMap<String, Double> getMap = sentenceRelationMap.get(str);
positiveRelationCounter = 0;
negativeRelationCounter = 0;
if (getMap == null) {
final ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = StrComparringNoSentenceRelationMap(strCacheLocal, str, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, strmap.size());
strmapreturn = futuresReturnOverallEvaluation(futures, strmapreturn, str);
} else {
strmapreturn = cachedReturnEvaluations(getMap, strmapreturn, str);
}
final ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = StrComparringNoSentenceRelationMap(strCacheLocal, str, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, strmap.size());
futuresReturnOverallEvaluation(futures, str);
System.out.println("overall advancement of future iterator: " + i + "\nstrmap size: " + strmap.values().size() + "\nstrmapreturn size: " + strmapreturn.size()
+ "\n");
i++;
}
return strmapreturn;
}
private static ConcurrentMap<Integer, String> removeNonSensicalStrings(ConcurrentMap<Integer, String> strmap) {
private static final ConcurrentMap<Integer, String> removeNonSensicalStrings(ConcurrentMap<Integer, String> strmap) {
final ConcurrentMap<Integer, String> strCacheLocal = stringCache.size() < 150 ? strmap : stringCache;
final ConcurrentMap<String, Annotation> localJMWEMap = getMultipleJMWEAnnotation(strmap.values());
final ConcurrentMap<String, Annotation> localPipelineAnnotation = getMultiplePipelineAnnotation(strmap.values());
@ -384,7 +365,7 @@ public class Datahandler {
return stringIteratorComparator(strmap, strCacheLocal, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap);
}
private static SimilarityMatrix retrieveFutureSMX(Future<SimilarityMatrix> future) {
private final static SimilarityMatrix retrieveFutureSMX(Future<SimilarityMatrix> future) {
try {
return future.get(5, TimeUnit.SECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException ex) {

2
ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java
View File

@ -1116,7 +1116,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
} catch (Exception ex) {
//System.out.println("SENTIMENT stacktrace Overall catch: " + ex.getMessage() + "\n");
}
System.out.println("Final current score: " + score + "\nSentence 1: " + str + "\nSentence 2: " + str1 + "\n");
//System.out.println("Final current score: " + score + "\nSentence 1: " + str + "\nSentence 2: " + str1 + "\n");
smxParam.setDistance(score);
return smxParam;
}