UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

simplifying which should hopefully help with trimming bad sentences out

Browse Source
This commit is contained in:
jenzur 2019-05-11 20:33:45 +02:00
parent b1aa449b59
commit 810321a520
2 changed files with 50 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java
View File

@ -56,7 +56,7 @@ public class Datahandler {
private static Annotation strAnnoSentiment; private static Annotation strAnnoSentiment;
private static Annotation strAnnoJMWE; private static Annotation strAnnoJMWE;
private static CoreDocument coreDoc; private static CoreDocument coreDoc;
private static ConcurrentMap<String, ConcurrentMap<String, Double>> sentenceRelationMap; private static final ConcurrentMap<String, ConcurrentMap<String, Double>> sentenceRelationMap = new MapMaker().concurrencyLevel(4).makeMap();
private static volatile Double preRelationCounters = 0.0; private static volatile Double preRelationCounters = 0.0;
private static volatile Double preRelationUserCounters = 0.0; private static volatile Double preRelationUserCounters = 0.0;
private static final ConcurrentMap<Integer, String> stringCache = new MapMaker().concurrencyLevel(2).makeMap(); private static final ConcurrentMap<Integer, String> stringCache = new MapMaker().concurrencyLevel(2).makeMap();
@ -68,6 +68,8 @@ public class Datahandler {
private static ConcurrentMap<String, CoreDocument> coreDocumentAnnotationCache; private static ConcurrentMap<String, CoreDocument> coreDocumentAnnotationCache;
private static ConcurrentMap<String, Integer> conversationMatchMap; private static ConcurrentMap<String, Integer> conversationMatchMap;
private static ConcurrentMap<String, Integer> conversationUserMatchMap; private static ConcurrentMap<String, Integer> conversationUserMatchMap;
private static final ConcurrentMap<String, Double> mapUdate = new MapMaker().concurrencyLevel(4).makeMap();
private final static ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(4).makeMap();
private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap(); private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap();
private final Stopwatch stopwatch; private final Stopwatch stopwatch;
private static final ForkJoinPool executor = instantiateExecutor(); private static final ForkJoinPool executor = instantiateExecutor();
@ -94,13 +96,12 @@ public class Datahandler {
public Datahandler() { public Datahandler() {
this.stopwatch = Stopwatch.createUnstarted(); this.stopwatch = Stopwatch.createUnstarted();
this.jmweAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap(); this.jmweAnnotationCache = new MapMaker().concurrencyLevel(3).makeMap();
this.pipelineAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap(); this.pipelineAnnotationCache = new MapMaker().concurrencyLevel(4).makeMap();
this.pipelineSentimentAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap(); this.pipelineSentimentAnnotationCache = new MapMaker().concurrencyLevel(4).makeMap();
this.coreDocumentAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap(); this.coreDocumentAnnotationCache = new MapMaker().concurrencyLevel(5).makeMap();
this.conversationMatchMap = new MapMaker().concurrencyLevel(2).makeMap(); this.conversationMatchMap = new MapMaker().concurrencyLevel(4).makeMap();
this.sentenceRelationMap = new MapMaker().concurrencyLevel(2).makeMap(); this.conversationUserMatchMap = new MapMaker().concurrencyLevel(4).makeMap();
this.conversationUserMatchMap = new MapMaker().concurrencyLevel(2).makeMap();
} }
private static StanfordCoreNLP getPipeLineSetUp() { private static StanfordCoreNLP getPipeLineSetUp() {
@ -230,7 +231,7 @@ public class Datahandler {
hlStatsMessages.put(str, hlStatsMessages.size()); hlStatsMessages.put(str, hlStatsMessages.size());
} }
} }
int capacity = 35000; int capacity = 10000;
hlStatsMessages.keySet().forEach(str -> { hlStatsMessages.keySet().forEach(str -> {
if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) { if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) {
String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null); String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null);
@ -275,58 +276,41 @@ public class Datahandler {
} }
} }
private final static ConcurrentMap<Integer, String> cachedReturnEvaluations(ConcurrentMap<String, Double> getMap, ConcurrentMap<Integer, String> strmapreturn, String str) { private final static void futuresReturnOverallEvaluation(ConcurrentMap<Integer, Future<SimilarityMatrix>> futures, String str) {
List<Integer> dummy = new ArrayList();
int relationCap = 20; for (final Future future : futures.values()) {
getMap.entrySet().parallelStream().forEach(mapValues -> { final SimilarityMatrix getSMX = retrieveFutureSMX(future);
if (!strmapreturn.values().contains(str) && dummy.isEmpty()) { if (handleRetrievedSMX(getSMX, str)) {
Double scoreRelationNewMsgToRecentMsg = mapValues.getValue(); break;
System.out.println("strmapreturn size: " + strmapreturn.size() + "\nscoreRelationNewMsgToRecentMsg: "
+ scoreRelationNewMsgToRecentMsg + "\n");
if (scoreRelationNewMsgToRecentMsg >= 4100.0) {
positiveRelationCounter++;
if (positiveRelationCounter > relationCap) {
strmapreturn.put(strmapreturn.size() + 1, str);
}
} else if (scoreRelationNewMsgToRecentMsg <= -5000.0) {
negativeRelationCounter++;
if (negativeRelationCounter > relationCap * 2) {
dummy.add(1);
}
}
} }
}); }
return strmapreturn; sentenceRelationMap.put(str, mapUdate);
} }
private final static ConcurrentMap<Integer, String> futuresReturnOverallEvaluation(ConcurrentMap<Integer, Future<SimilarityMatrix>> futures, ConcurrentMap<Integer, String> strmapreturn, String str) { private static final boolean handleRetrievedSMX(SimilarityMatrix getSMX, String str) {
int relationCap = 20; final int relationCap = 20;
final ConcurrentMap<String, Double> mapUdate = new MapMaker().concurrencyLevel(2).makeMap(); if (getSMX != null) {
final List<Integer> dummy = new ArrayList(); final Double scoreRelationNewMsgToRecentMsg = getSMX.getDistance();
futures.values().parallelStream().forEach(future -> { System.out.println("scoreRelationNewMsgToRecentMsg: " + scoreRelationNewMsgToRecentMsg + "\n");
if (!strmapreturn.values().contains(str) && dummy.isEmpty()) { mapUdate.put(getSMX.getSecondaryString(), scoreRelationNewMsgToRecentMsg);
final SimilarityMatrix getSMX = retrieveFutureSMX(future); System.out.println("getSMX primary: " + getSMX.getPrimaryString() + "\ngetSMX secodary: " + getSMX.getSecondaryString() + "\n");
if (getSMX != null) { if (scoreRelationNewMsgToRecentMsg >= 200.0) {
Double scoreRelationNewMsgToRecentMsg = getSMX.getDistance(); positiveRelationCounter++;
System.out.println("strmapreturn size: " + strmapreturn.size() + "\nscoreRelationNewMsgToRecentMsg: " if (positiveRelationCounter >= relationCap) {
+ scoreRelationNewMsgToRecentMsg + "\n"); System.out.println("added to strmapreturn str: " + str + "\n");
mapUdate.put(getSMX.getSecondaryString(), scoreRelationNewMsgToRecentMsg); strmapreturn.put(strmapreturn.size() + 1, str);
if (scoreRelationNewMsgToRecentMsg >= 4100.0) { return true;
positiveRelationCounter++;
if (positiveRelationCounter > relationCap) {
strmapreturn.put(strmapreturn.size() + 1, str);
}
} else if (scoreRelationNewMsgToRecentMsg <= -5000.0) {
negativeRelationCounter++;
if (negativeRelationCounter > relationCap * 2) {
dummy.add(1);
}
}
} }
} else if (scoreRelationNewMsgToRecentMsg <= -6000.0) {
negativeRelationCounter += 5;
} }
}); } else {
sentenceRelationMap.put(str, mapUdate); negativeRelationCounter += 2;
return strmapreturn; }
if (negativeRelationCounter >= relationCap * 3) {
return true;
}
return false;
} }
private final static ConcurrentMap<Integer, Future<SimilarityMatrix>> StrComparringNoSentenceRelationMap(ConcurrentMap<Integer, String> strCacheLocal, private final static ConcurrentMap<Integer, Future<SimilarityMatrix>> StrComparringNoSentenceRelationMap(ConcurrentMap<Integer, String> strCacheLocal,
@ -349,32 +333,29 @@ public class Datahandler {
pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1)); pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1));
} }
futures.put(futures.size() + 1, executor.submit(worker)); futures.put(futures.size() + 1, executor.submit(worker));
System.out.println("futures.size(): " + futures.size() + "\nstrmap.values().size(): " + strmapSize + "\n");
} }
} }
return futures; return futures;
} }
private final static ConcurrentMap<Integer, String> stringIteratorComparator(ConcurrentMap<Integer, String> strmap, private static ConcurrentMap<Integer, String> stringIteratorComparator(ConcurrentMap<Integer, String> strmap,
ConcurrentMap<Integer, String> strCacheLocal, ConcurrentMap<String, Annotation> localJMWEMap, ConcurrentMap<Integer, String> strCacheLocal, ConcurrentMap<String, Annotation> localJMWEMap,
ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation, ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation,
ConcurrentMap<String, CoreDocument> localCoreDocumentMap) { ConcurrentMap<String, CoreDocument> localCoreDocumentMap) {
ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(4).makeMap(); int i = 0;
for (String str : strmap.values()) { for (String str : strmap.values()) {
final ConcurrentMap<String, Double> getMap = sentenceRelationMap.get(str);
positiveRelationCounter = 0; positiveRelationCounter = 0;
negativeRelationCounter = 0; negativeRelationCounter = 0;
if (getMap == null) { final ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = StrComparringNoSentenceRelationMap(strCacheLocal, str, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, strmap.size());
final ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = StrComparringNoSentenceRelationMap(strCacheLocal, str, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, strmap.size()); futuresReturnOverallEvaluation(futures, str);
strmapreturn = futuresReturnOverallEvaluation(futures, strmapreturn, str); System.out.println("overall advancement of future iterator: " + i + "\nstrmap size: " + strmap.values().size() + "\nstrmapreturn size: " + strmapreturn.size()
} else { + "\n");
strmapreturn = cachedReturnEvaluations(getMap, strmapreturn, str); i++;
}
} }
return strmapreturn; return strmapreturn;
} }
private static ConcurrentMap<Integer, String> removeNonSensicalStrings(ConcurrentMap<Integer, String> strmap) { private static final ConcurrentMap<Integer, String> removeNonSensicalStrings(ConcurrentMap<Integer, String> strmap) {
final ConcurrentMap<Integer, String> strCacheLocal = stringCache.size() < 150 ? strmap : stringCache; final ConcurrentMap<Integer, String> strCacheLocal = stringCache.size() < 150 ? strmap : stringCache;
final ConcurrentMap<String, Annotation> localJMWEMap = getMultipleJMWEAnnotation(strmap.values()); final ConcurrentMap<String, Annotation> localJMWEMap = getMultipleJMWEAnnotation(strmap.values());
final ConcurrentMap<String, Annotation> localPipelineAnnotation = getMultiplePipelineAnnotation(strmap.values()); final ConcurrentMap<String, Annotation> localPipelineAnnotation = getMultiplePipelineAnnotation(strmap.values());
@ -384,7 +365,7 @@ public class Datahandler {
return stringIteratorComparator(strmap, strCacheLocal, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap); return stringIteratorComparator(strmap, strCacheLocal, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap);
} }
private static SimilarityMatrix retrieveFutureSMX(Future<SimilarityMatrix> future) { private final static SimilarityMatrix retrieveFutureSMX(Future<SimilarityMatrix> future) {
try { try {
return future.get(5, TimeUnit.SECONDS); return future.get(5, TimeUnit.SECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException ex) { } catch (InterruptedException | ExecutionException | TimeoutException ex) {

2
ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java
View File

@ -1116,7 +1116,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
} catch (Exception ex) { } catch (Exception ex) {
//System.out.println("SENTIMENT stacktrace Overall catch: " + ex.getMessage() + "\n"); //System.out.println("SENTIMENT stacktrace Overall catch: " + ex.getMessage() + "\n");
} }
System.out.println("Final current score: " + score + "\nSentence 1: " + str + "\nSentence 2: " + str1 + "\n"); //System.out.println("Final current score: " + score + "\nSentence 1: " + str + "\nSentence 2: " + str1 + "\n");
smxParam.setDistance(score); smxParam.setDistance(score);
return smxParam; return smxParam;
} }