diff --git a/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java index c22981d1..429c68e3 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java @@ -220,7 +220,7 @@ public class Datahandler { hlStatsMessages.put(str, hlStatsMessages.size()); } } - int capacity = 1250; + int capacity = 600; hlStatsMessages.keySet().forEach(str -> { if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) { String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null); @@ -273,20 +273,13 @@ public class Datahandler { private ConcurrentMap futuresReturnOverallEvaluation(List similarityMatrixes) { ConcurrentMap strmapreturn = new MapMaker().concurrencyLevel(6).makeMap(); if (!similarityMatrixes.isEmpty()) { - String newPrimary = similarityMatrixes.get(0).getPrimaryString(); - int evaluationCap = 500; int iterator = 0; - Double caphelper = 0.0; for (SimilarityMatrix SMX : similarityMatrixes) { final Double scoreRelationNewMsgToRecentMsg = SMX.getDistance(); if (scoreRelationNewMsgToRecentMsg > 0.0) { - caphelper += scoreRelationNewMsgToRecentMsg; - } - if (caphelper > evaluationCap) { strmapreturn = addSMXToMapReturn(strmapreturn, SMX); - caphelper = 0.0; } - //System.out.println("similarityMatrixes size: " + similarityMatrixes.size() + "\niterator: " + iterator); + //System.out.println("scoreRelationNewMsgToRecentMsg: " + scoreRelationNewMsgToRecentMsg + "\niterator: " + iterator); iterator++; } } @@ -312,8 +305,11 @@ public class Datahandler { private List StrComparringNoSentenceRelationMap( ConcurrentMap strCacheLocal, Collection strCollection, ConcurrentMap localJMWEMap, ConcurrentMap localPipelineAnnotation, ConcurrentMap localPipelineSentimentAnnotation, - ConcurrentMap localCoreDocumentMap, CompletionService ecs, int index) { - int prefix_size = 125; + ConcurrentMap localCoreDocumentMap) { + ExecutorService threadPool = Executors.newCachedThreadPool(); + CompletionService ecs = new ExecutorCompletionService<>(threadPool); + int index = 0; + int prefix_size = 150; SentimentValueCache sentimentCacheStr = sentimentCachingMap.getOrDefault(strCollection, null); List smxReturnList = new ArrayList(); List randomIterationComparision = new ArrayList(); @@ -348,6 +344,21 @@ public class Datahandler { } ecs.submit(worker); index++; + if (index % 1000 == 0 && index > 0) { + for (int i = 0; i < index; i++) { + try { + Future take = ecs.take(); + SimilarityMatrix smx = take.get(); + if (smx != null) { + smxReturnList.add(smx); + } + } catch (InterruptedException | ExecutionException ex) { + // + } + } + index = 0; + //System.out.println("smxReturnList size iterating ECS.take(): " + smxReturnList.size()); + } } } } @@ -356,23 +367,14 @@ public class Datahandler { Future take = ecs.take(); SimilarityMatrix smx = take.get(); if (smx != null) { - boolean present = false; - for (SimilarityMatrix str_smx : smxReturnList) { - if (str_smx.getPrimaryString().equals(smx.getPrimaryString())) { - present = true; - break; - } - } - if (!present) { - smxReturnList.add(smx); - } + smxReturnList.add(smx); } } catch (InterruptedException | ExecutionException ex) { // } } - index = 0; - System.out.println("smxReturnList size: " + smxReturnList.size()); + //System.out.println("smxReturnList size: " + smxReturnList.size()); + threadPool.shutdown(); return smxReturnList; } @@ -380,17 +382,13 @@ public class Datahandler { ConcurrentMap strCacheLocal, ConcurrentMap localJMWEMap, ConcurrentMap localPipelineAnnotation, ConcurrentMap localPipelineSentimentAnnotation, ConcurrentMap localCoreDocumentMap) { - ExecutorService threadPool = Executors.newCachedThreadPool(); - CompletionService ecs = new ExecutorCompletionService<>(threadPool); - int index = 0; //System.out.println("strmap siuze: " + strmap.size()); List StrComparringNoSentenceRelationMap = StrComparringNoSentenceRelationMap(strCacheLocal, strmap.values(), - localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, ecs, index); - threadPool.shutdown(); - //System.out.println("StrComparringNoSentenceRelationMap size: " + StrComparringNoSentenceRelationMap.size()); + localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap); + System.out.println("StrComparringNoSentenceRelationMap size: " + StrComparringNoSentenceRelationMap.size()); Collections.sort(StrComparringNoSentenceRelationMap, (e1, e2) -> e1.getPrimaryString().compareTo(e2.getPrimaryString())); ConcurrentMap strmapreturn = futuresReturnOverallEvaluation(StrComparringNoSentenceRelationMap); - System.out.println("strmapreturn size: " + strmapreturn.size()); + //System.out.println("strmapreturn size: " + strmapreturn.size()); return strmapreturn; }