diff --git a/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java b/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java index 1f2e3988..d797135a 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.StringReader; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -58,7 +59,7 @@ public class MYSQLDatahandler { private volatile boolean refreshMatrixFromDB; private final ConcurrentMap stringCache; private LinkedHashMap> lHMSMX = new LinkedHashMap(); - private List multiprocessCalculations = new ArrayList(); + private ConcurrentMap multiprocessCalculations; private List updatedRows = new ArrayList(); private final Stopwatch stopwatch; private final Stopwatch stopwatch1; @@ -78,6 +79,7 @@ public class MYSQLDatahandler { private static AbstractSequenceClassifier classifier; private static StanfordCoreNLP pipeline; private static StanfordCoreNLP pipelineSentiment; + private static volatile boolean permitted = false; public static AbstractSequenceClassifier getClassifier() { return classifier; @@ -91,6 +93,7 @@ public class MYSQLDatahandler { this.stopwatch = Stopwatch.createUnstarted(); this.stopwatch1 = Stopwatch.createStarted(); this.stringCache = new MapMaker().concurrencyLevel(2).makeMap(); + this.multiprocessCalculations = new MapMaker().concurrencyLevel(2).makeMap(); } public static void shiftReduceParserInitiate() { @@ -108,11 +111,17 @@ public class MYSQLDatahandler { props.setProperty("annotators", "tokenize,ssplit,pos,lemma,parse"); props.setProperty("parse.model", modelPath); props.setProperty("parse.maxlen", "100"); + props.setProperty("tokenize.maxlen", "100"); + props.setProperty("ssplit.maxlen", "100"); + props.setProperty("lemma.maxlen", "100"); props.setProperty("parse.binaryTrees", "true"); propsSentiment.setProperty("annotators", "tokenize, ssplit, parse, sentiment"); propsSentiment.setProperty("parse.model", lexParserEnglishRNN); propsSentiment.setProperty("sentiment.model", sentimentModel); + propsSentiment.setProperty("sentiment.maxlen", "100"); propsSentiment.setProperty("parse.maxlen", "100"); + propsSentiment.setProperty("tokenize.maxlen", "100"); + propsSentiment.setProperty("ssplit.maxlen", "100"); // set up pipeline pipeline = new StanfordCoreNLP(props); pipelineSentiment = new StanfordCoreNLP(propsSentiment); @@ -164,7 +173,7 @@ public class MYSQLDatahandler { public synchronized void checkIfUpdateMatrixes() { refreshMatrixFromDB = false; int calculationBoundaries = 6; - int updateBadgesInteger = 65; + int updateBadgesInteger = 80; while (lHMSMX.size() < (stringCache.values().size() * stringCache.values().size()) - stringCache.values().size()) { if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) { refreshMatrixFromDB = true; @@ -174,7 +183,7 @@ public class MYSQLDatahandler { } } if (stringCache.values().size() > 10 && !refreshMatrixFromDB) { - if (multiprocessCalculations.size() <= (calculationBoundaries * calculationBoundaries)) { + if (threadCounter < 25) { threadCounter++; ConcurrentMap stringCachelocal = stringCache; List updateLocal = updatedRows; @@ -205,7 +214,7 @@ public class MYSQLDatahandler { while (beginindex + ij < temp) { String get = stringCachelocal.get(beginindex + ij); strIndexNavigator.put(ij, get); - multiprocessCalculations.add(get); + multiprocessCalculations.put(multiprocessCalculations.size() + 1, get); ij++; } new Thread(() -> { @@ -230,7 +239,7 @@ public class MYSQLDatahandler { strIndexNavigatorL.values().forEach((str) -> { randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { boolean present = false; - if (multiprocessCalculations.contains(str1)) { + if (multiprocessCalculations.values().contains(str1)) { present = true; } else if (LHMSMXLocal.containsKey(str)) { LinkedHashMap orDefault = LHMSMXLocal.get(str); @@ -281,77 +290,68 @@ public class MYSQLDatahandler { }).start(); }). start(); - } else { - if (threadCounter == 0) { - threadCounter++; - new Thread(() -> { - LinkedHashMap> LHMSMXLocal = lHMSMX; - ConcurrentMap strList = stringCache; - ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap randomStrList = new MapMaker().concurrencyLevel(2).makeMap(); - int indexes = updateBadgesInteger; - if (indexes >= strList.size()) { - indexes = strList.size() - 1; - } - int beginindexes = new Random().nextInt((strList.size()) - indexes); - int ij1 = 0; - while (beginindexes + ij1 < beginindexes + indexes) { - String get1 = strList.get(beginindexes + ij1); - randomStrList.put(ij1, get1); - ij1++; - } - ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); - ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); - multiprocessCalculations.forEach((str) -> { - randomStrList.values().forEach((str1) -> { - boolean present = false; - if (LHMSMXLocal.containsKey(str)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str); - if (orDefault.containsKey(str1)) { - present = true; - } - } else if (LHMSMXLocal.containsKey(str1)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str1); - if (orDefault.containsKey(str)) { - present = true; - } + } else if (!permitted) { + permitted = true; + new Thread(() -> { + ConcurrentMap stringCachelocal = stringCache; + ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); + ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + Collection values = multiprocessCalculations.values(); + LinkedHashMap> LHMSMXLocal = lHMSMX; + values.forEach((str) -> { + stringCachelocal.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { + boolean present = false; + if (LHMSMXLocal.containsKey(str)) { + LinkedHashMap orDefault = LHMSMXLocal.get(str); + if (orDefault.containsKey(str1)) { + present = true; } - if (!present) { - SimilarityMatrix SMX = new SimilarityMatrix(str, str1); - Callable worker = new SentimentAnalyzerTest(str, str1, SMX); - futures.put(futures.size() + 1, executor.submit(worker)); + } else if (LHMSMXLocal.containsKey(str1)) { + LinkedHashMap orDefault = LHMSMXLocal.get(str1); + if (orDefault.containsKey(str)) { + present = true; } - }); - }); - executor.shutdown(); - try { - for (Future future : futures.values()) { - SimilarityMatrix SMX = future.get(); - LinkedHashMap get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); - if (get == null) { - get = new LinkedHashMap(); - } - get.put(SMX.getSecondaryString(), SMX.getDistance()); - lHMSMX.put(SMX.getPrimaryString(), get); - matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); } - } catch (InterruptedException | ExecutionException ex) { - Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); + if (!present) { + SimilarityMatrix SMX = new SimilarityMatrix(str, str1); + Callable worker = new SentimentAnalyzerTest(str, str1, SMX); + futures.put(futures.size() + 1, executor.submit(worker)); + } + }); + }); + executor.shutdown(); + try { + System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); + for (Future future : futures.values()) { + SimilarityMatrix SMX = future.get(); + LinkedHashMap get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); + if (get == null) { + get = new LinkedHashMap(); + } + get.put(SMX.getSecondaryString(), SMX.getDistance()); + lHMSMX.put(SMX.getPrimaryString(), get); + matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); } + } catch (InterruptedException | ExecutionException ex) { + Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); + } + new Thread(() -> { try { + multiprocessCalculations = new MapMaker().concurrencyLevel(2).makeMap(); + permitted = false; if (!matrixUpdateList.isEmpty()) { DataMapper.insertSementicMatrixes(matrixUpdateList); System.out.println("finished datamapper semetic insert"); } + threadCounter--; + System.out.println("\nthreadCounter: " + threadCounter + "\n"); } catch (CustomError ex) { Logger.getLogger(MYSQLDatahandler.class .getName()).log(Level.SEVERE, null, ex); } - multiprocessCalculations = new ArrayList(); - updatedRows = new ArrayList(); - threadCounter--; }).start(); - } + }).start(); } } } @@ -464,14 +464,14 @@ public class MYSQLDatahandler { public String mostSimilar(String toBeCompared, ConcurrentMap concurrentStrings) { int minDistance = 8; String similar = ""; - List> futures = new ArrayList(); + ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); concurrentStrings.values().stream().map((str) -> new LevenshteinDistance(str, toBeCompared, new DistanceObject())).forEachOrdered((worker) -> { - futures.add(executor.submit(worker)); + futures.put(futures.size() + 1, executor.submit(worker)); }); executor.shutdown(); try { - for (Future future : futures) { + for (Future future : futures.values()) { DistanceObject d = future.get(); try { if (d.getSentence() != null && d.getDistance() != null) { @@ -598,42 +598,42 @@ public class MYSQLDatahandler { ConcurrentMap strreturn = new MapMaker().concurrencyLevel(2).makeMap(); for (String str1 : str.values()) { int counter = 0; - List TGWList = new ArrayList(); + ConcurrentMap TGWList = new MapMaker().concurrencyLevel(2).makeMap(); DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1)); for (List sentence : tokenizer) { List tagged1 = tagger.tagSentence(sentence); Tree tree = model.apply(tagged1); taggedWords = tree.taggedYield(); for (TaggedWord TGW : taggedWords) { - if (!TGWList.contains(TGW.tag()) && !TGW.tag().equals(":") && !TGW.word().equals(TGW.tag())) { - TGWList.add(TGW.tag()); + if (!TGWList.values().contains(TGW.tag()) && !TGW.tag().equals(":") && !TGW.word().equals(TGW.tag())) { + TGWList.put(TGWList.size() + 1, TGW.tag()); counter++; } if (counter > 3) { int addCounter = 0; - List wordList = new ArrayList(); + ConcurrentMap wordList = new MapMaker().concurrencyLevel(2).makeMap(); for (Word lab : tree.yieldWords()) { if (lab != null && lab.word() != null) { //System.out.println("lab: " + lab + " \n"); - if (!wordList.contains(lab) && !lab.equals(":")) { - wordList.add(lab); + if (!wordList.values().contains(lab) && lab.value() != null && !lab.value().equals(":")) { + wordList.put(wordList.size() + 1, lab); addCounter++; } } } if (addCounter > 3) { addCounter = 0; - List HWlist = new ArrayList(); + ConcurrentMap HWlist = new MapMaker().concurrencyLevel(2).makeMap(); for (HasWord HW : tree.yieldHasWord()) { - if (HW != null && HW.word() != null && !HWlist.contains(HW)) { - //System.out.println("HasWord: " + HW + "\n"); + if (HW != null && HW.word() != null && !HWlist.values().contains(HW)) { addCounter++; - HWlist.add(HW); + HWlist.put(HWlist.size() + 1, HW); } } if (addCounter > 3) { boolean tooclosematch = false; - for (String strVals : stringCache.values()) { + Collection values = stringCache.values(); + for (String strVals : values) { LevenshteinDistance leven = new LevenshteinDistance(strVals, str1); double Distance = leven.computeLevenshteinDistance(); int maxpermittedDistance = 2; diff --git a/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java b/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java index 8e369c35..d6c2458b 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java @@ -3,6 +3,7 @@ package FunctionLayer.StanfordParser; import FunctionLayer.LevenshteinDistance; import FunctionLayer.MYSQLDatahandler; import FunctionLayer.SimilarityMatrix; +import com.google.common.collect.MapMaker; import edu.stanford.nlp.ie.AbstractSequenceClassifier; import edu.stanford.nlp.ling.CoreAnnotations; import edu.stanford.nlp.ling.CoreLabel; @@ -32,6 +33,7 @@ import java.util.Collection; import java.util.List; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; import org.ejml.simple.SimpleMatrix; @@ -89,48 +91,50 @@ public class SentimentAnalyzerTest implements Callable { int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter; overValue *= 16; score -= overValue; - List tgwlistIndex = new ArrayList(); + ConcurrentMap tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap(); taggedwordlist1.forEach((TGWList) -> { TGWList.forEach((TaggedWord) -> { - if (!tgwlistIndex.contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) { - tgwlistIndex.add(TaggedWord.tag()); + if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) { + tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag()); } }); }); + taggedwordlist1.clear(); AtomicInteger runCount = new AtomicInteger(0); taggedwordlist2.forEach((TGWList) -> { TGWList.forEach((TaggedWord) -> { - if (tgwlistIndex.contains(TaggedWord.tag())) { - tgwlistIndex.remove(TaggedWord.tag()); + if (tgwlistIndex.values().contains(TaggedWord.tag())) { + tgwlistIndex.values().remove(TaggedWord.tag()); runCount.getAndIncrement(); } }); }); + tgwlistIndex.clear(); + taggedwordlist2.clear(); score += runCount.get() * 64; Annotation annotation = new Annotation(str1); pipeline.annotate(annotation); - List sentenceConstituencyParseList = new ArrayList(); + ConcurrentMap sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap(); for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) { Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); - sentenceConstituencyParseList.add(sentenceConstituencyParse); + sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse); } Annotation annotation1 = new Annotation(str); pipeline.annotate(annotation1); - List nerList = new ArrayList(); for (CoreMap sentence : annotation1.get(CoreAnnotations.SentencesAnnotation.class)) { Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse); Collection allTypedDependencies = gs.allTypedDependencies(); - List filerTreeContent = new ArrayList(); - for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList) { + ConcurrentMap filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap(); + for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) { Set inT1notT2 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1); Set inT2notT1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse); - List constiLabels = new ArrayList(); + ConcurrentMap constiLabels = new MapMaker().concurrencyLevel(2).makeMap(); for (Constituent consti : inT1notT2) { for (Constituent consti1 : inT2notT1) { - if (consti.value().equals(consti1.value()) && !constiLabels.contains(consti.value())) { - score += 64; //256 - constiLabels.add(consti.value()); + if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) { + score += 64; + constiLabels.put(constiLabels.size(), consti.value()); } } } @@ -163,8 +167,8 @@ public class SentimentAnalyzerTest implements Callable { AtomicInteger runCount1 = new AtomicInteger(0); sentenceConstituencyParse.taggedLabeledYield().forEach((LBW) -> { sentenceConstituencyParse1.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma()) - && !filerTreeContent.contains(LBW.lemma()))).map((_item) -> { - filerTreeContent.add(LBW.lemma()); + && !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> { + filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma()); return _item; }).forEachOrdered((_item) -> { runCount1.getAndIncrement(); @@ -173,30 +177,31 @@ public class SentimentAnalyzerTest implements Callable { score += runCount1.get() * 1500; } } + sentenceConstituencyParseList.clear(); Annotation annotationSentiment1 = pipelineSentiment.process(str); - List simpleSMXlist = new ArrayList(); - List simpleSMXlistVector = new ArrayList(); - List sentiment1 = new ArrayList(); - List sentiment2 = new ArrayList(); + ConcurrentMap simpleSMXlist = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap simpleSMXlistVector = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap sentiment1 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap sentiment2 = new MapMaker().concurrencyLevel(2).makeMap(); for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) { Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); - sentiment1.add(RNNCoreAnnotations.getPredictedClass(tree)); + sentiment1.put(sentiment1.size(), RNNCoreAnnotations.getPredictedClass(tree)); SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); - simpleSMXlist.add(predictions); - simpleSMXlistVector.add(nodeVector); + simpleSMXlist.put(simpleSMXlist.size(), predictions); + simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector); } annotationSentiment1 = pipelineSentiment.process(str1); for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) { Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); - sentiment2.add(RNNCoreAnnotations.getPredictedClass(tree)); + sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree)); SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); - score = simpleSMXlist.stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> { + score = simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> { subtracter *= 25; return subtracter; }).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator - _item); - for (SimpleMatrix simpleSMX : simpleSMXlistVector) { + for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) { double dot = nodeVector.dot(simpleSMX); double elementSum = nodeVector.kron(simpleSMX).elementSum(); elementSum = Math.round(elementSum * 100.0) / 100.0; diff --git a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java index 2f811438..5964cff1 100644 --- a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java +++ b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java @@ -82,7 +82,7 @@ public class DiscordHandler { MessageResponseHandler.getMessage(strresult); try { MYSQLDatahandler.instance.checkIfUpdateStrings(); - //MYSQLDatahandler.instance.checkIfUpdateMatrixes(); + MYSQLDatahandler.instance.checkIfUpdateMatrixes(); } catch (CustomError ex) { Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex); }