UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

reduced requirements for passing adaption, saves more data now again

Browse Source
This commit is contained in:
jenzur 2020-04-15 18:46:51 +02:00
parent 6a9b76a0c2
commit 9f38949b08
1 changed files with 28 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java
View File

@ -220,7 +220,7 @@ public class Datahandler {
hlStatsMessages.put(str, hlStatsMessages.size()); hlStatsMessages.put(str, hlStatsMessages.size());
} }
} }
int capacity = 1250; int capacity = 600;
hlStatsMessages.keySet().forEach(str -> { hlStatsMessages.keySet().forEach(str -> {
if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) { if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) {
String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null); String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null);
@ -273,20 +273,13 @@ public class Datahandler {
private ConcurrentMap<Integer, String> futuresReturnOverallEvaluation(List<SimilarityMatrix> similarityMatrixes) { private ConcurrentMap<Integer, String> futuresReturnOverallEvaluation(List<SimilarityMatrix> similarityMatrixes) {
ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(6).makeMap(); ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(6).makeMap();
if (!similarityMatrixes.isEmpty()) { if (!similarityMatrixes.isEmpty()) {
String newPrimary = similarityMatrixes.get(0).getPrimaryString();
int evaluationCap = 500;
int iterator = 0; int iterator = 0;
Double caphelper = 0.0;
for (SimilarityMatrix SMX : similarityMatrixes) { for (SimilarityMatrix SMX : similarityMatrixes) {
final Double scoreRelationNewMsgToRecentMsg = SMX.getDistance(); final Double scoreRelationNewMsgToRecentMsg = SMX.getDistance();
if (scoreRelationNewMsgToRecentMsg > 0.0) { if (scoreRelationNewMsgToRecentMsg > 0.0) {
caphelper += scoreRelationNewMsgToRecentMsg;
}
if (caphelper > evaluationCap) {
strmapreturn = addSMXToMapReturn(strmapreturn, SMX); strmapreturn = addSMXToMapReturn(strmapreturn, SMX);
caphelper = 0.0;
} }
//System.out.println("similarityMatrixes size: " + similarityMatrixes.size() + "\niterator: " + iterator); //System.out.println("scoreRelationNewMsgToRecentMsg: " + scoreRelationNewMsgToRecentMsg + "\niterator: " + iterator);
iterator++; iterator++;
} }
} }
@ -312,8 +305,11 @@ public class Datahandler {
private List<SimilarityMatrix> StrComparringNoSentenceRelationMap( private List<SimilarityMatrix> StrComparringNoSentenceRelationMap(
ConcurrentMap<Integer, String> strCacheLocal, Collection<String> strCollection, ConcurrentMap<String, Annotation> localJMWEMap, ConcurrentMap<Integer, String> strCacheLocal, Collection<String> strCollection, ConcurrentMap<String, Annotation> localJMWEMap,
ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation, ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation,
ConcurrentMap<String, CoreDocument> localCoreDocumentMap, CompletionService<SimilarityMatrix> ecs, int index) { ConcurrentMap<String, CoreDocument> localCoreDocumentMap) {
int prefix_size = 125; ExecutorService threadPool = Executors.newCachedThreadPool();
CompletionService<SimilarityMatrix> ecs = new ExecutorCompletionService<>(threadPool);
int index = 0;
int prefix_size = 150;
SentimentValueCache sentimentCacheStr = sentimentCachingMap.getOrDefault(strCollection, null); SentimentValueCache sentimentCacheStr = sentimentCachingMap.getOrDefault(strCollection, null);
List<SimilarityMatrix> smxReturnList = new ArrayList(); List<SimilarityMatrix> smxReturnList = new ArrayList();
List<String> randomIterationComparision = new ArrayList(); List<String> randomIterationComparision = new ArrayList();
@ -348,6 +344,21 @@ public class Datahandler {
} }
ecs.submit(worker); ecs.submit(worker);
index++; index++;
if (index % 1000 == 0 && index > 0) {
for (int i = 0; i < index; i++) {
try {
Future<SimilarityMatrix> take = ecs.take();
SimilarityMatrix smx = take.get();
if (smx != null) {
smxReturnList.add(smx);
}
} catch (InterruptedException | ExecutionException ex) {
//
}
}
index = 0;
//System.out.println("smxReturnList size iterating ECS.take(): " + smxReturnList.size());
}
} }
} }
} }
@ -356,23 +367,14 @@ public class Datahandler {
Future<SimilarityMatrix> take = ecs.take(); Future<SimilarityMatrix> take = ecs.take();
SimilarityMatrix smx = take.get(); SimilarityMatrix smx = take.get();
if (smx != null) { if (smx != null) {
boolean present = false; smxReturnList.add(smx);
for (SimilarityMatrix str_smx : smxReturnList) {
if (str_smx.getPrimaryString().equals(smx.getPrimaryString())) {
present = true;
break;
}
}
if (!present) {
smxReturnList.add(smx);
}
} }
} catch (InterruptedException | ExecutionException ex) { } catch (InterruptedException | ExecutionException ex) {
// //
} }
} }
index = 0; //System.out.println("smxReturnList size: " + smxReturnList.size());
System.out.println("smxReturnList size: " + smxReturnList.size()); threadPool.shutdown();
return smxReturnList; return smxReturnList;
} }
@ -380,17 +382,13 @@ public class Datahandler {
ConcurrentMap<Integer, String> strCacheLocal, ConcurrentMap<String, Annotation> localJMWEMap, ConcurrentMap<Integer, String> strCacheLocal, ConcurrentMap<String, Annotation> localJMWEMap,
ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation, ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation,
ConcurrentMap<String, CoreDocument> localCoreDocumentMap) { ConcurrentMap<String, CoreDocument> localCoreDocumentMap) {
ExecutorService threadPool = Executors.newCachedThreadPool();
CompletionService<SimilarityMatrix> ecs = new ExecutorCompletionService<>(threadPool);
int index = 0;
//System.out.println("strmap siuze: " + strmap.size()); //System.out.println("strmap siuze: " + strmap.size());
List<SimilarityMatrix> StrComparringNoSentenceRelationMap = StrComparringNoSentenceRelationMap(strCacheLocal, strmap.values(), List<SimilarityMatrix> StrComparringNoSentenceRelationMap = StrComparringNoSentenceRelationMap(strCacheLocal, strmap.values(),
localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, ecs, index); localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap);
threadPool.shutdown(); System.out.println("StrComparringNoSentenceRelationMap size: " + StrComparringNoSentenceRelationMap.size());
//System.out.println("StrComparringNoSentenceRelationMap size: " + StrComparringNoSentenceRelationMap.size());
Collections.sort(StrComparringNoSentenceRelationMap, (e1, e2) -> e1.getPrimaryString().compareTo(e2.getPrimaryString())); Collections.sort(StrComparringNoSentenceRelationMap, (e1, e2) -> e1.getPrimaryString().compareTo(e2.getPrimaryString()));
ConcurrentMap<Integer, String> strmapreturn = futuresReturnOverallEvaluation(StrComparringNoSentenceRelationMap); ConcurrentMap<Integer, String> strmapreturn = futuresReturnOverallEvaluation(StrComparringNoSentenceRelationMap);
System.out.println("strmapreturn size: " + strmapreturn.size()); //System.out.println("strmapreturn size: " + strmapreturn.size());
return strmapreturn; return strmapreturn;
} }