From abb033a9b802915ae9a4a2b5a43b6fdc8771bd38 Mon Sep 17 00:00:00 2001 From: jenzur Date: Sat, 21 Mar 2020 18:43:46 +0100 Subject: [PATCH] no more random numbers by names and no more repetitive data, good and end now i guess --- .../main/java/FunctionLayer/Datahandler.java | 98 ++++++++----------- .../src/main/java/FunctionLayer/DoStuff.java | 9 +- .../FunctionLayer/MessageResponseHandler.java | 15 +-- .../PresentationLayer/DiscordHandler.java | 5 +- 4 files changed, 52 insertions(+), 75 deletions(-) diff --git a/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java index b1823897..bfb8c644 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java @@ -67,8 +67,7 @@ public class Datahandler { private static Annotation strAnnoSentiment; private static Annotation strAnnoJMWE; private static CoreDocument coreDoc; - private static final ConcurrentMap stringCache = new MapMaker().concurrencyLevel(2).makeMap(); - private static int RelationCounter = 0; + private static final ConcurrentMap stringCache = new MapMaker().concurrencyLevel(6).makeMap(); private static ConcurrentMap pipelineAnnotationCache; private static ConcurrentMap pipelineSentimentAnnotationCache; private static ConcurrentMap jmweAnnotationCache; @@ -221,7 +220,7 @@ public class Datahandler { hlStatsMessages.put(str, hlStatsMessages.size()); } } - int capacity = 500; + int capacity = 1250; hlStatsMessages.keySet().forEach(str -> { if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) { String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null); @@ -271,29 +270,16 @@ public class Datahandler { } } - private ConcurrentMap futuresReturnOverallEvaluation(List similarityMatrixes, - ConcurrentMap strmapreturn) { + private ConcurrentMap futuresReturnOverallEvaluation(List similarityMatrixes) { + ConcurrentMap strmapreturn = new MapMaker().concurrencyLevel(6).makeMap(); if (!similarityMatrixes.isEmpty()) { String newPrimary = similarityMatrixes.get(0).getPrimaryString(); - int evaluationCap = 50000; - boolean hitCap = false; + int evaluationCap = 500; int iterator = 0; for (SimilarityMatrix SMX : similarityMatrixes) { - if (!newPrimary.equals(SMX.getPrimaryString())) { - newPrimary = SMX.getPrimaryString(); + final Double scoreRelationNewMsgToRecentMsg = SMX.getDistance(); + if (scoreRelationNewMsgToRecentMsg > evaluationCap) { strmapreturn = addSMXToMapReturn(strmapreturn, SMX); - hitCap = false; - } - if (!hitCap) { - final Double scoreRelationNewMsgToRecentMsg = SMX.getDistance(); - RelationCounter += scoreRelationNewMsgToRecentMsg > 500 ? scoreRelationNewMsgToRecentMsg * 2 : scoreRelationNewMsgToRecentMsg; - if (RelationCounter > evaluationCap) { - strmapreturn = addSMXToMapReturn(strmapreturn, SMX); - hitCap = true; - } else if (RelationCounter < evaluationCap * -1) { - addSMXToMapReturn(strmapreturn, SMX); - hitCap = true; - } } //System.out.println("similarityMatrixes size: " + similarityMatrixes.size() + "\niterator: " + iterator); iterator++; @@ -303,26 +289,23 @@ public class Datahandler { } private ConcurrentMap addSMXToMapReturn(ConcurrentMap strmapreturn, SimilarityMatrix SMX) { - //System.out.println("RelationCounter cap: " + RelationCounter); - boolean related = RelationCounter > 500; - if (related) { + if (!strmapreturn.containsValue(SMX.getPrimaryString())) { strmapreturn.put(strmapreturn.size(), SMX.getPrimaryString()); String transmittedStr = SMX.getSecondaryString(); SentimentValueCache cacheValue1 = SMX.getCacheValue1(); SentimentValueCache cacheValue2 = SMX.getCacheValue2(); if (cacheValue1 != null && !sentimentCachingMap.keySet().contains(SMX.getPrimaryString())) { - sentimentCachingMap.put(SMX.getPrimaryString(), SMX.getCacheValue1()); + sentimentCachingMap.put(SMX.getSecondaryString(), SMX.getCacheValue1()); } if (cacheValue2 != null && !sentimentCachingMap.keySet().contains(transmittedStr)) { sentimentCachingMap.put(transmittedStr, SMX.getCacheValue2()); } } - RelationCounter = 0; return strmapreturn; } private List StrComparringNoSentenceRelationMap( - ConcurrentMap strCacheLocal, String str, ConcurrentMap localJMWEMap, + ConcurrentMap strCacheLocal, Collection str, ConcurrentMap localJMWEMap, ConcurrentMap localPipelineAnnotation, ConcurrentMap localPipelineSentimentAnnotation, ConcurrentMap localCoreDocumentMap, CompletionService ecs, int index) { int prefix_size = 125; @@ -342,35 +325,40 @@ public class Datahandler { iterated++; } for (String str1 : randomIterationComparision) { - if (!str.equals(str1)) { - SimilarityMatrix SMXInit = new SimilarityMatrix(str, str1); - SentimentValueCache sentimentCacheStr1 = sentimentCachingMap.getOrDefault(str1, null); - Callable worker; - if (stringCache.size() < prefix_size) { - worker = new SentimentAnalyzerTest(str, str1, SMXInit, - localJMWEMap.get(str), localJMWEMap.get(str1), localPipelineAnnotation.get(str), - localPipelineAnnotation.get(str1), localPipelineSentimentAnnotation.get(str), - localPipelineSentimentAnnotation.get(str1), localCoreDocumentMap.get(str), localCoreDocumentMap.get(str1), sentimentCacheStr, sentimentCacheStr1); - } else { - worker = new SentimentAnalyzerTest(str, str1, SMXInit, - localJMWEMap.get(str), jmweAnnotationCache.get(str1), localPipelineAnnotation.get(str), - pipelineAnnotationCache.get(str1), localPipelineSentimentAnnotation.get(str), - pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1), sentimentCacheStr, sentimentCacheStr1); + for (String str2 : str) { + if (!str2.equals(str1)) { + SimilarityMatrix SMXInit = new SimilarityMatrix(str2, str1); + SentimentValueCache sentimentCacheStr1 = sentimentCachingMap.getOrDefault(str1, null); + Callable worker; + if (stringCache.size() < prefix_size) { + worker = new SentimentAnalyzerTest(str2, str1, SMXInit, + localJMWEMap.get(str), localJMWEMap.get(str1), localPipelineAnnotation.get(str), + localPipelineAnnotation.get(str1), localPipelineSentimentAnnotation.get(str), + localPipelineSentimentAnnotation.get(str1), localCoreDocumentMap.get(str), localCoreDocumentMap.get(str1), sentimentCacheStr, sentimentCacheStr1); + } else { + worker = new SentimentAnalyzerTest(str2, str1, SMXInit, + localJMWEMap.get(str), jmweAnnotationCache.get(str1), localPipelineAnnotation.get(str), + pipelineAnnotationCache.get(str1), localPipelineSentimentAnnotation.get(str), + pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1), sentimentCacheStr, sentimentCacheStr1); + } + ecs.submit(worker); + index++; } - ecs.submit(worker); - index++; } } for (int i = 0; i < index; i++) { try { Future take = ecs.take(); SimilarityMatrix smx = take.get(); - smxReturnList.add(smx); + if (smx != null && !smxReturnList.contains(smx)) { + smxReturnList.add(smx); + } } catch (InterruptedException | ExecutionException ex) { // } } index = 0; + System.out.println("smxReturnList size: " + smxReturnList.size()); return smxReturnList; } @@ -378,21 +366,17 @@ public class Datahandler { ConcurrentMap strCacheLocal, ConcurrentMap localJMWEMap, ConcurrentMap localPipelineAnnotation, ConcurrentMap localPipelineSentimentAnnotation, ConcurrentMap localCoreDocumentMap) { - ConcurrentMap strmapreturn = new MapMaker().concurrencyLevel(6).makeMap(); - List strSenseRelationMap = new ArrayList(); ExecutorService threadPool = Executors.newCachedThreadPool(); CompletionService ecs = new ExecutorCompletionService<>(threadPool); int index = 0; - strmap.values().forEach(str -> { - List localNoSentenceRelationList = StrComparringNoSentenceRelationMap(strCacheLocal, str, - localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, ecs, index); - for (SimilarityMatrix SMX : localNoSentenceRelationList) { - strSenseRelationMap.add(SMX); - } - }); + //System.out.println("strmap siuze: " + strmap.size()); + List StrComparringNoSentenceRelationMap = StrComparringNoSentenceRelationMap(strCacheLocal, strmap.values(), + localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, ecs, index); threadPool.shutdown(); - Collections.sort(strSenseRelationMap, (e1, e2) -> e1.getPrimaryString().compareTo(e2.getPrimaryString())); - strmapreturn = futuresReturnOverallEvaluation(strSenseRelationMap, strmapreturn); + //System.out.println("StrComparringNoSentenceRelationMap size: " + StrComparringNoSentenceRelationMap.size()); + Collections.sort(StrComparringNoSentenceRelationMap, (e1, e2) -> e1.getPrimaryString().compareTo(e2.getPrimaryString())); + ConcurrentMap strmapreturn = futuresReturnOverallEvaluation(StrComparringNoSentenceRelationMap); + //System.out.println("strmapreturn size: " + strmapreturn.size()); return strmapreturn; } @@ -408,13 +392,15 @@ public class Datahandler { public synchronized void checkIfUpdateStrings(boolean hlStatsMsg) throws CustomError { if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) { ConcurrentMap str = MessageResponseHandler.getStr(); + System.out.println("str size: " + str.size()); str = cutContent(str, hlStatsMsg); str = filterContent(str); str = removeSlacks(str); //System.out.println("finished removeSlacks \n" + str.size() + "\n"); str = removeNonSensicalStrings(str); + System.out.println("removeNonSensicalStrings str size POST: " + str.size() + "\n"); str = annotationCacheUpdate(str); - //System.out.println("annotationCacheUpdate str size POST: " + str.size() + "\n"); + System.out.println("annotationCacheUpdate str size POST: " + str.size() + "\n"); ConcurrentMap strf = str; if (!stringCache.isEmpty()) { new Thread(() -> { diff --git a/ArtificialAutism/src/main/java/FunctionLayer/DoStuff.java b/ArtificialAutism/src/main/java/FunctionLayer/DoStuff.java index 24c635bc..903e4eb3 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/DoStuff.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/DoStuff.java @@ -20,16 +20,13 @@ import org.javacord.api.event.message.MessageCreateEvent; */ public class DoStuff { - public volatile boolean occupied = false; + public static boolean occupied = false; - public boolean isOccupied() { + public static boolean isOccupied() { return occupied; } - public DoStuff() { - } - - public void doStuff(MessageCreateEvent event, DiscordApi api) { + public static void doStuff(MessageCreateEvent event, DiscordApi api) { if (!event.getMessage().getAuthor().isYourself()) { occupied = true; String strtest = event.getServerTextChannel().get().getCategory().toString(); diff --git a/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java b/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java index 68362239..a4c1fb12 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java @@ -78,18 +78,13 @@ public class MessageResponseHandler { boolean isMatched = matcher.matches(); if (!emText.equals(personName) && !isMatched) { for (CoreEntityMention emLastMsg : pipelineCoreDcoumentLastMsg.entityMentions()) { - if (!emText.equals(emLastMsg.text())) { - str = strreturn.replaceFirst(emText, emLastMsg.text()); + if (!emText.equals(emLastMsg.text()) && !Character.isDigit(emLastMsg.text().trim().charAt(0))) { + //System.out.println("emLastMsg.text(): " + emLastMsg.text()); + str = strreturn.substring(0, strreturn.indexOf(emText)) + " " + + emLastMsg + " " + strreturn.substring(strreturn.indexOf(emText)); } } - try { - //System.out.println("personName: " + personName + " str: " + str); - str = str.replace(str.substring(str.lastIndexOf(emText), str.length()), personName); - //System.out.println("str after: " + str); - } catch (Exception e) { - //System.out.println("failed replacing: " + e.getLocalizedMessage() + "\n"); - str += " " + personName; - } + str += " " + personName; return str; } } diff --git a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java index 6eaceca2..5f69859f 100644 --- a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java +++ b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java @@ -48,10 +48,9 @@ public class DiscordHandler { Datahandler.instance.updateStringCache(); String token = "NTI5NzAxNTk5NjAyMjc4NDAx.Dw0vDg.7-aMjVWdQMYPl8qVNyvTCPS5F_A"; DiscordApi api = new DiscordApiBuilder().setToken(token).login().join(); - DoStuff dostuff = new DoStuff(); api.addMessageCreateListener(event -> { - if (!dostuff.isOccupied()) { - dostuff.doStuff(event, api); + if (!FunctionLayer.DoStuff.isOccupied()) { + FunctionLayer.DoStuff.doStuff(event, api); } }); }