UNLOZE/projects-jenz
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

should reduce idleness of calculations. Once stringcache is large enough and stored calculations are widespread enogh accross the stringcache it should effectly give response messages instead of doing manual calculations each time, also replaced arraylists with concurrentmaps

Browse Source
This commit is contained in:
jenzur
2019-03-04 23:26:15 +01:00
parent aff2ddae5b
commit d8d415cbe0
3 changed files with 108 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java
+31 -26
View File
@@ -3,6 +3,7 @@ package FunctionLayer.StanfordParser;
import FunctionLayer.LevenshteinDistance;
import FunctionLayer.MYSQLDatahandler;
import FunctionLayer.SimilarityMatrix;
import com.google.common.collect.MapMaker;
import edu.stanford.nlp.ie.AbstractSequenceClassifier;
import edu.stanford.nlp.ling.CoreAnnotations;
import edu.stanford.nlp.ling.CoreLabel;
@@ -32,6 +33,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.ejml.simple.SimpleMatrix;
@@ -89,48 +91,50 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter;
overValue *= 16;
score -= overValue;
List<String> tgwlistIndex = new ArrayList();
ConcurrentMap<Integer, String> tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap();
taggedwordlist1.forEach((TGWList) -> {
TGWList.forEach((TaggedWord) -> {
if (!tgwlistIndex.contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) {
tgwlistIndex.add(TaggedWord.tag());
if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) {
tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag());
}
});
});
taggedwordlist1.clear();
AtomicInteger runCount = new AtomicInteger(0);
taggedwordlist2.forEach((TGWList) -> {
TGWList.forEach((TaggedWord) -> {
if (tgwlistIndex.contains(TaggedWord.tag())) {
tgwlistIndex.remove(TaggedWord.tag());
if (tgwlistIndex.values().contains(TaggedWord.tag())) {
tgwlistIndex.values().remove(TaggedWord.tag());
runCount.getAndIncrement();
}
});
});
tgwlistIndex.clear();
taggedwordlist2.clear();
score += runCount.get() * 64;
Annotation annotation = new Annotation(str1);
pipeline.annotate(annotation);
List<Tree> sentenceConstituencyParseList = new ArrayList();
ConcurrentMap<Integer, Tree> sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap();
for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) {
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
sentenceConstituencyParseList.add(sentenceConstituencyParse);
sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse);
}
Annotation annotation1 = new Annotation(str);
pipeline.annotate(annotation1);
List<String> nerList = new ArrayList();
for (CoreMap sentence : annotation1.get(CoreAnnotations.SentencesAnnotation.class)) {
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse);
Collection<TypedDependency> allTypedDependencies = gs.allTypedDependencies();
List<String> filerTreeContent = new ArrayList();
for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList) {
ConcurrentMap<Integer, String> filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap();
for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) {
Set<Constituent> inT1notT2 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1);
Set<Constituent> inT2notT1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse);
List<String> constiLabels = new ArrayList();
ConcurrentMap<Integer, String> constiLabels = new MapMaker().concurrencyLevel(2).makeMap();
for (Constituent consti : inT1notT2) {
for (Constituent consti1 : inT2notT1) {
if (consti.value().equals(consti1.value()) && !constiLabels.contains(consti.value())) {
score += 64; //256
constiLabels.add(consti.value());
if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) {
score += 64;
constiLabels.put(constiLabels.size(), consti.value());
}
}
}
@@ -163,8 +167,8 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
AtomicInteger runCount1 = new AtomicInteger(0);
sentenceConstituencyParse.taggedLabeledYield().forEach((LBW) -> {
sentenceConstituencyParse1.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma())
&& !filerTreeContent.contains(LBW.lemma()))).map((_item) -> {
filerTreeContent.add(LBW.lemma());
&& !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> {
filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma());
return _item;
}).forEachOrdered((_item) -> {
runCount1.getAndIncrement();
@@ -173,30 +177,31 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
score += runCount1.get() * 1500;
}
}
sentenceConstituencyParseList.clear();
Annotation annotationSentiment1 = pipelineSentiment.process(str);
List<SimpleMatrix> simpleSMXlist = new ArrayList();
List<SimpleMatrix> simpleSMXlistVector = new ArrayList();
List<Integer> sentiment1 = new ArrayList();
List<Integer> sentiment2 = new ArrayList();
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlistVector = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Integer> sentiment1 = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Integer> sentiment2 = new MapMaker().concurrencyLevel(2).makeMap();
for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) {
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
sentiment1.add(RNNCoreAnnotations.getPredictedClass(tree));
sentiment1.put(sentiment1.size(), RNNCoreAnnotations.getPredictedClass(tree));
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
simpleSMXlist.add(predictions);
simpleSMXlistVector.add(nodeVector);
simpleSMXlist.put(simpleSMXlist.size(), predictions);
simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector);
}
annotationSentiment1 = pipelineSentiment.process(str1);
for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) {
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
sentiment2.add(RNNCoreAnnotations.getPredictedClass(tree));
sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree));
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
score = simpleSMXlist.stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> {
score = simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> {
subtracter *= 25;
return subtracter;
}).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator - _item);
for (SimpleMatrix simpleSMX : simpleSMXlistVector) {
for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) {
double dot = nodeVector.dot(simpleSMX);
double elementSum = nodeVector.kron(simpleSMX).elementSum();
elementSum = Math.round(elementSum * 100.0) / 100.0;