From d8d415cbe055c11b35d502fc090bea795ccb0ccb Mon Sep 17 00:00:00 2001 From: jenzur Date: Mon, 4 Mar 2019 23:26:15 +0100 Subject: [PATCH] should reduce idleness of calculations. Once stringcache is large enough and stored calculations are widespread enogh accross the stringcache it should effectly give response messages instead of doing manual calculations each time, also replaced arraylists with concurrentmaps --- .../java/FunctionLayer/MYSQLDatahandler.java | 152 +++++++++--------- .../StanfordParser/SentimentAnalyzerTest.java | 57 ++++--- .../PresentationLayer/DiscordHandler.java | 2 +- 3 files changed, 108 insertions(+), 103 deletions(-) diff --git a/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java b/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java index 1f2e3988..d797135a 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.StringReader; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -58,7 +59,7 @@ public class MYSQLDatahandler { private volatile boolean refreshMatrixFromDB; private final ConcurrentMap stringCache; private LinkedHashMap> lHMSMX = new LinkedHashMap(); - private List multiprocessCalculations = new ArrayList(); + private ConcurrentMap multiprocessCalculations; private List updatedRows = new ArrayList(); private final Stopwatch stopwatch; private final Stopwatch stopwatch1; @@ -78,6 +79,7 @@ public class MYSQLDatahandler { private static AbstractSequenceClassifier classifier; private static StanfordCoreNLP pipeline; private static StanfordCoreNLP pipelineSentiment; + private static volatile boolean permitted = false; public static AbstractSequenceClassifier getClassifier() { return classifier; @@ -91,6 +93,7 @@ public class MYSQLDatahandler { this.stopwatch = Stopwatch.createUnstarted(); this.stopwatch1 = Stopwatch.createStarted(); this.stringCache = new MapMaker().concurrencyLevel(2).makeMap(); + this.multiprocessCalculations = new MapMaker().concurrencyLevel(2).makeMap(); } public static void shiftReduceParserInitiate() { @@ -108,11 +111,17 @@ public class MYSQLDatahandler { props.setProperty("annotators", "tokenize,ssplit,pos,lemma,parse"); props.setProperty("parse.model", modelPath); props.setProperty("parse.maxlen", "100"); + props.setProperty("tokenize.maxlen", "100"); + props.setProperty("ssplit.maxlen", "100"); + props.setProperty("lemma.maxlen", "100"); props.setProperty("parse.binaryTrees", "true"); propsSentiment.setProperty("annotators", "tokenize, ssplit, parse, sentiment"); propsSentiment.setProperty("parse.model", lexParserEnglishRNN); propsSentiment.setProperty("sentiment.model", sentimentModel); + propsSentiment.setProperty("sentiment.maxlen", "100"); propsSentiment.setProperty("parse.maxlen", "100"); + propsSentiment.setProperty("tokenize.maxlen", "100"); + propsSentiment.setProperty("ssplit.maxlen", "100"); // set up pipeline pipeline = new StanfordCoreNLP(props); pipelineSentiment = new StanfordCoreNLP(propsSentiment); @@ -164,7 +173,7 @@ public class MYSQLDatahandler { public synchronized void checkIfUpdateMatrixes() { refreshMatrixFromDB = false; int calculationBoundaries = 6; - int updateBadgesInteger = 65; + int updateBadgesInteger = 80; while (lHMSMX.size() < (stringCache.values().size() * stringCache.values().size()) - stringCache.values().size()) { if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) { refreshMatrixFromDB = true; @@ -174,7 +183,7 @@ public class MYSQLDatahandler { } } if (stringCache.values().size() > 10 && !refreshMatrixFromDB) { - if (multiprocessCalculations.size() <= (calculationBoundaries * calculationBoundaries)) { + if (threadCounter < 25) { threadCounter++; ConcurrentMap stringCachelocal = stringCache; List updateLocal = updatedRows; @@ -205,7 +214,7 @@ public class MYSQLDatahandler { while (beginindex + ij < temp) { String get = stringCachelocal.get(beginindex + ij); strIndexNavigator.put(ij, get); - multiprocessCalculations.add(get); + multiprocessCalculations.put(multiprocessCalculations.size() + 1, get); ij++; } new Thread(() -> { @@ -230,7 +239,7 @@ public class MYSQLDatahandler { strIndexNavigatorL.values().forEach((str) -> { randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { boolean present = false; - if (multiprocessCalculations.contains(str1)) { + if (multiprocessCalculations.values().contains(str1)) { present = true; } else if (LHMSMXLocal.containsKey(str)) { LinkedHashMap orDefault = LHMSMXLocal.get(str); @@ -281,77 +290,68 @@ public class MYSQLDatahandler { }).start(); }). start(); - } else { - if (threadCounter == 0) { - threadCounter++; - new Thread(() -> { - LinkedHashMap> LHMSMXLocal = lHMSMX; - ConcurrentMap strList = stringCache; - ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap randomStrList = new MapMaker().concurrencyLevel(2).makeMap(); - int indexes = updateBadgesInteger; - if (indexes >= strList.size()) { - indexes = strList.size() - 1; - } - int beginindexes = new Random().nextInt((strList.size()) - indexes); - int ij1 = 0; - while (beginindexes + ij1 < beginindexes + indexes) { - String get1 = strList.get(beginindexes + ij1); - randomStrList.put(ij1, get1); - ij1++; - } - ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); - ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); - multiprocessCalculations.forEach((str) -> { - randomStrList.values().forEach((str1) -> { - boolean present = false; - if (LHMSMXLocal.containsKey(str)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str); - if (orDefault.containsKey(str1)) { - present = true; - } - } else if (LHMSMXLocal.containsKey(str1)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str1); - if (orDefault.containsKey(str)) { - present = true; - } + } else if (!permitted) { + permitted = true; + new Thread(() -> { + ConcurrentMap stringCachelocal = stringCache; + ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); + ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + Collection values = multiprocessCalculations.values(); + LinkedHashMap> LHMSMXLocal = lHMSMX; + values.forEach((str) -> { + stringCachelocal.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { + boolean present = false; + if (LHMSMXLocal.containsKey(str)) { + LinkedHashMap orDefault = LHMSMXLocal.get(str); + if (orDefault.containsKey(str1)) { + present = true; } - if (!present) { - SimilarityMatrix SMX = new SimilarityMatrix(str, str1); - Callable worker = new SentimentAnalyzerTest(str, str1, SMX); - futures.put(futures.size() + 1, executor.submit(worker)); + } else if (LHMSMXLocal.containsKey(str1)) { + LinkedHashMap orDefault = LHMSMXLocal.get(str1); + if (orDefault.containsKey(str)) { + present = true; } - }); - }); - executor.shutdown(); - try { - for (Future future : futures.values()) { - SimilarityMatrix SMX = future.get(); - LinkedHashMap get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); - if (get == null) { - get = new LinkedHashMap(); - } - get.put(SMX.getSecondaryString(), SMX.getDistance()); - lHMSMX.put(SMX.getPrimaryString(), get); - matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); } - } catch (InterruptedException | ExecutionException ex) { - Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); + if (!present) { + SimilarityMatrix SMX = new SimilarityMatrix(str, str1); + Callable worker = new SentimentAnalyzerTest(str, str1, SMX); + futures.put(futures.size() + 1, executor.submit(worker)); + } + }); + }); + executor.shutdown(); + try { + System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); + for (Future future : futures.values()) { + SimilarityMatrix SMX = future.get(); + LinkedHashMap get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); + if (get == null) { + get = new LinkedHashMap(); + } + get.put(SMX.getSecondaryString(), SMX.getDistance()); + lHMSMX.put(SMX.getPrimaryString(), get); + matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); } + } catch (InterruptedException | ExecutionException ex) { + Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); + } + new Thread(() -> { try { + multiprocessCalculations = new MapMaker().concurrencyLevel(2).makeMap(); + permitted = false; if (!matrixUpdateList.isEmpty()) { DataMapper.insertSementicMatrixes(matrixUpdateList); System.out.println("finished datamapper semetic insert"); } + threadCounter--; + System.out.println("\nthreadCounter: " + threadCounter + "\n"); } catch (CustomError ex) { Logger.getLogger(MYSQLDatahandler.class .getName()).log(Level.SEVERE, null, ex); } - multiprocessCalculations = new ArrayList(); - updatedRows = new ArrayList(); - threadCounter--; }).start(); - } + }).start(); } } } @@ -464,14 +464,14 @@ public class MYSQLDatahandler { public String mostSimilar(String toBeCompared, ConcurrentMap concurrentStrings) { int minDistance = 8; String similar = ""; - List> futures = new ArrayList(); + ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); concurrentStrings.values().stream().map((str) -> new LevenshteinDistance(str, toBeCompared, new DistanceObject())).forEachOrdered((worker) -> { - futures.add(executor.submit(worker)); + futures.put(futures.size() + 1, executor.submit(worker)); }); executor.shutdown(); try { - for (Future future : futures) { + for (Future future : futures.values()) { DistanceObject d = future.get(); try { if (d.getSentence() != null && d.getDistance() != null) { @@ -598,42 +598,42 @@ public class MYSQLDatahandler { ConcurrentMap strreturn = new MapMaker().concurrencyLevel(2).makeMap(); for (String str1 : str.values()) { int counter = 0; - List TGWList = new ArrayList(); + ConcurrentMap TGWList = new MapMaker().concurrencyLevel(2).makeMap(); DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1)); for (List sentence : tokenizer) { List tagged1 = tagger.tagSentence(sentence); Tree tree = model.apply(tagged1); taggedWords = tree.taggedYield(); for (TaggedWord TGW : taggedWords) { - if (!TGWList.contains(TGW.tag()) && !TGW.tag().equals(":") && !TGW.word().equals(TGW.tag())) { - TGWList.add(TGW.tag()); + if (!TGWList.values().contains(TGW.tag()) && !TGW.tag().equals(":") && !TGW.word().equals(TGW.tag())) { + TGWList.put(TGWList.size() + 1, TGW.tag()); counter++; } if (counter > 3) { int addCounter = 0; - List wordList = new ArrayList(); + ConcurrentMap wordList = new MapMaker().concurrencyLevel(2).makeMap(); for (Word lab : tree.yieldWords()) { if (lab != null && lab.word() != null) { //System.out.println("lab: " + lab + " \n"); - if (!wordList.contains(lab) && !lab.equals(":")) { - wordList.add(lab); + if (!wordList.values().contains(lab) && lab.value() != null && !lab.value().equals(":")) { + wordList.put(wordList.size() + 1, lab); addCounter++; } } } if (addCounter > 3) { addCounter = 0; - List HWlist = new ArrayList(); + ConcurrentMap HWlist = new MapMaker().concurrencyLevel(2).makeMap(); for (HasWord HW : tree.yieldHasWord()) { - if (HW != null && HW.word() != null && !HWlist.contains(HW)) { - //System.out.println("HasWord: " + HW + "\n"); + if (HW != null && HW.word() != null && !HWlist.values().contains(HW)) { addCounter++; - HWlist.add(HW); + HWlist.put(HWlist.size() + 1, HW); } } if (addCounter > 3) { boolean tooclosematch = false; - for (String strVals : stringCache.values()) { + Collection values = stringCache.values(); + for (String strVals : values) { LevenshteinDistance leven = new LevenshteinDistance(strVals, str1); double Distance = leven.computeLevenshteinDistance(); int maxpermittedDistance = 2; diff --git a/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java b/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java index 8e369c35..d6c2458b 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java @@ -3,6 +3,7 @@ package FunctionLayer.StanfordParser; import FunctionLayer.LevenshteinDistance; import FunctionLayer.MYSQLDatahandler; import FunctionLayer.SimilarityMatrix; +import com.google.common.collect.MapMaker; import edu.stanford.nlp.ie.AbstractSequenceClassifier; import edu.stanford.nlp.ling.CoreAnnotations; import edu.stanford.nlp.ling.CoreLabel; @@ -32,6 +33,7 @@ import java.util.Collection; import java.util.List; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; import org.ejml.simple.SimpleMatrix; @@ -89,48 +91,50 @@ public class SentimentAnalyzerTest implements Callable { int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter; overValue *= 16; score -= overValue; - List tgwlistIndex = new ArrayList(); + ConcurrentMap tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap(); taggedwordlist1.forEach((TGWList) -> { TGWList.forEach((TaggedWord) -> { - if (!tgwlistIndex.contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) { - tgwlistIndex.add(TaggedWord.tag()); + if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) { + tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag()); } }); }); + taggedwordlist1.clear(); AtomicInteger runCount = new AtomicInteger(0); taggedwordlist2.forEach((TGWList) -> { TGWList.forEach((TaggedWord) -> { - if (tgwlistIndex.contains(TaggedWord.tag())) { - tgwlistIndex.remove(TaggedWord.tag()); + if (tgwlistIndex.values().contains(TaggedWord.tag())) { + tgwlistIndex.values().remove(TaggedWord.tag()); runCount.getAndIncrement(); } }); }); + tgwlistIndex.clear(); + taggedwordlist2.clear(); score += runCount.get() * 64; Annotation annotation = new Annotation(str1); pipeline.annotate(annotation); - List sentenceConstituencyParseList = new ArrayList(); + ConcurrentMap sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap(); for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) { Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); - sentenceConstituencyParseList.add(sentenceConstituencyParse); + sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse); } Annotation annotation1 = new Annotation(str); pipeline.annotate(annotation1); - List nerList = new ArrayList(); for (CoreMap sentence : annotation1.get(CoreAnnotations.SentencesAnnotation.class)) { Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse); Collection allTypedDependencies = gs.allTypedDependencies(); - List filerTreeContent = new ArrayList(); - for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList) { + ConcurrentMap filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap(); + for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) { Set inT1notT2 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1); Set inT2notT1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse); - List constiLabels = new ArrayList(); + ConcurrentMap constiLabels = new MapMaker().concurrencyLevel(2).makeMap(); for (Constituent consti : inT1notT2) { for (Constituent consti1 : inT2notT1) { - if (consti.value().equals(consti1.value()) && !constiLabels.contains(consti.value())) { - score += 64; //256 - constiLabels.add(consti.value()); + if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) { + score += 64; + constiLabels.put(constiLabels.size(), consti.value()); } } } @@ -163,8 +167,8 @@ public class SentimentAnalyzerTest implements Callable { AtomicInteger runCount1 = new AtomicInteger(0); sentenceConstituencyParse.taggedLabeledYield().forEach((LBW) -> { sentenceConstituencyParse1.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma()) - && !filerTreeContent.contains(LBW.lemma()))).map((_item) -> { - filerTreeContent.add(LBW.lemma()); + && !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> { + filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma()); return _item; }).forEachOrdered((_item) -> { runCount1.getAndIncrement(); @@ -173,30 +177,31 @@ public class SentimentAnalyzerTest implements Callable { score += runCount1.get() * 1500; } } + sentenceConstituencyParseList.clear(); Annotation annotationSentiment1 = pipelineSentiment.process(str); - List simpleSMXlist = new ArrayList(); - List simpleSMXlistVector = new ArrayList(); - List sentiment1 = new ArrayList(); - List sentiment2 = new ArrayList(); + ConcurrentMap simpleSMXlist = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap simpleSMXlistVector = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap sentiment1 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap sentiment2 = new MapMaker().concurrencyLevel(2).makeMap(); for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) { Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); - sentiment1.add(RNNCoreAnnotations.getPredictedClass(tree)); + sentiment1.put(sentiment1.size(), RNNCoreAnnotations.getPredictedClass(tree)); SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); - simpleSMXlist.add(predictions); - simpleSMXlistVector.add(nodeVector); + simpleSMXlist.put(simpleSMXlist.size(), predictions); + simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector); } annotationSentiment1 = pipelineSentiment.process(str1); for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) { Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); - sentiment2.add(RNNCoreAnnotations.getPredictedClass(tree)); + sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree)); SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); - score = simpleSMXlist.stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> { + score = simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> { subtracter *= 25; return subtracter; }).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator - _item); - for (SimpleMatrix simpleSMX : simpleSMXlistVector) { + for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) { double dot = nodeVector.dot(simpleSMX); double elementSum = nodeVector.kron(simpleSMX).elementSum(); elementSum = Math.round(elementSum * 100.0) / 100.0; diff --git a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java index 2f811438..5964cff1 100644 --- a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java +++ b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java @@ -82,7 +82,7 @@ public class DiscordHandler { MessageResponseHandler.getMessage(strresult); try { MYSQLDatahandler.instance.checkIfUpdateStrings(); - //MYSQLDatahandler.instance.checkIfUpdateMatrixes(); + MYSQLDatahandler.instance.checkIfUpdateMatrixes(); } catch (CustomError ex) { Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex); }