annotation caching with instance safety for jmwe, changed entry slightly, multithreaded loading in beginning, parralel streaming the annotations, forkjoinpool should handle futures better now with try catches in sementicsanalyzer
This commit is contained in:
jenzur
+334
-322
@@ -1,7 +1,7 @@
|
||||
package FunctionLayer.StanfordParser;
|
||||
|
||||
import FunctionLayer.LevenshteinDistance;
|
||||
import FunctionLayer.MYSQLDatahandler;
|
||||
import FunctionLayer.Datahandler;
|
||||
import FunctionLayer.SimilarityMatrix;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import edu.mit.jmwe.data.IMWE;
|
||||
@@ -73,12 +73,12 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
this.str = str;
|
||||
this.str1 = str1;
|
||||
this.smxParam = smxParam;
|
||||
this.model = MYSQLDatahandler.getModel();
|
||||
this.tagger = MYSQLDatahandler.getTagger();
|
||||
this.pipeline = MYSQLDatahandler.getPipeline();
|
||||
this.pipelineSentiment = MYSQLDatahandler.getPipelineSentiment();
|
||||
this.gsf = MYSQLDatahandler.getGsf();
|
||||
this.classifier = MYSQLDatahandler.getClassifier();
|
||||
this.model = Datahandler.getModel();
|
||||
this.tagger = Datahandler.getTagger();
|
||||
this.pipeline = Datahandler.getPipeline();
|
||||
this.pipelineSentiment = Datahandler.getPipelineSentiment();
|
||||
this.gsf = Datahandler.getGsf();
|
||||
this.classifier = Datahandler.getClassifier();
|
||||
this.jmweStrAnnotation1 = str1Annotation;
|
||||
this.jmweStrAnnotation2 = str2Annotation;
|
||||
this.pipelineAnnotation1 = strPipeline1;
|
||||
@@ -90,332 +90,344 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
@Override
|
||||
public SimilarityMatrix call() {
|
||||
Double score = -100.0;
|
||||
List<List<TaggedWord>> taggedwordlist1 = new ArrayList();
|
||||
List<List<TaggedWord>> taggedwordlist2 = new ArrayList();
|
||||
DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1));
|
||||
for (List<HasWord> sentence : tokenizer) {
|
||||
taggedwordlist1.add(model.apply(tagger.tagSentence(sentence)).taggedYield());
|
||||
}
|
||||
tokenizer = new DocumentPreprocessor(new StringReader(str));
|
||||
for (List<HasWord> sentence : tokenizer) {
|
||||
taggedwordlist2.add(model.apply(tagger.tagSentence(sentence)).taggedYield());
|
||||
}
|
||||
int counter = 0;
|
||||
int counter1 = 0;
|
||||
counter = taggedwordlist2.stream().map((taggedlist2) -> taggedlist2.size()).reduce(counter, Integer::sum);
|
||||
counter1 = taggedwordlist1.stream().map((taggedlist1) -> taggedlist1.size()).reduce(counter1, Integer::sum);
|
||||
int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter;
|
||||
overValue *= 16;
|
||||
score -= overValue;
|
||||
ConcurrentMap<Integer, String> tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
taggedwordlist1.forEach((TGWList) -> {
|
||||
TGWList.forEach((TaggedWord) -> {
|
||||
if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) {
|
||||
tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag());
|
||||
}
|
||||
});
|
||||
});
|
||||
taggedwordlist1.clear();
|
||||
AtomicInteger runCount = new AtomicInteger(0);
|
||||
taggedwordlist2.forEach((TGWList) -> {
|
||||
TGWList.forEach((TaggedWord) -> {
|
||||
if (tgwlistIndex.values().contains(TaggedWord.tag())) {
|
||||
tgwlistIndex.values().remove(TaggedWord.tag());
|
||||
runCount.getAndIncrement();
|
||||
}
|
||||
});
|
||||
});
|
||||
tgwlistIndex.clear();
|
||||
taggedwordlist2.clear();
|
||||
score += runCount.get() * 64;
|
||||
ConcurrentMap<Integer, Tree> sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (CoreMap sentence : pipelineAnnotation1.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
||||
sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse);
|
||||
}
|
||||
for (CoreMap sentence : pipelineAnnotation2.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
||||
GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse);
|
||||
Collection<TypedDependency> allTypedDependencies = gs.allTypedDependencies();
|
||||
ConcurrentMap<Integer, String> filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) {
|
||||
Set<Constituent> inT1notT2 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1);
|
||||
Set<Constituent> inT2notT1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse);
|
||||
ConcurrentMap<Integer, String> constiLabels = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (Constituent consti : inT1notT2) {
|
||||
for (Constituent consti1 : inT2notT1) {
|
||||
if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) {
|
||||
score += 64;
|
||||
constiLabels.put(constiLabels.size(), consti.value());
|
||||
}
|
||||
try {
|
||||
List<List<TaggedWord>> taggedwordlist1 = new ArrayList();
|
||||
List<List<TaggedWord>> taggedwordlist2 = new ArrayList();
|
||||
DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1));
|
||||
for (List<HasWord> sentence : tokenizer) {
|
||||
taggedwordlist1.add(model.apply(tagger.tagSentence(sentence)).taggedYield());
|
||||
}
|
||||
tokenizer = new DocumentPreprocessor(new StringReader(str));
|
||||
for (List<HasWord> sentence : tokenizer) {
|
||||
taggedwordlist2.add(model.apply(tagger.tagSentence(sentence)).taggedYield());
|
||||
}
|
||||
int counter = 0;
|
||||
int counter1 = 0;
|
||||
counter = taggedwordlist2.stream().map((taggedlist2) -> taggedlist2.size()).reduce(counter, Integer::sum);
|
||||
counter1 = taggedwordlist1.stream().map((taggedlist1) -> taggedlist1.size()).reduce(counter1, Integer::sum);
|
||||
int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter;
|
||||
overValue *= 16;
|
||||
score -= overValue;
|
||||
ConcurrentMap<Integer, String> tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
taggedwordlist1.forEach((TGWList) -> {
|
||||
TGWList.forEach((TaggedWord) -> {
|
||||
if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) {
|
||||
tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag());
|
||||
}
|
||||
}
|
||||
GrammaticalStructure gs1 = gsf.newGrammaticalStructure(sentenceConstituencyParse1);
|
||||
Collection<TypedDependency> allTypedDependencies1 = gs1.allTypedDependencies();
|
||||
for (TypedDependency TDY1 : allTypedDependencies1) {
|
||||
IndexedWord dep = TDY1.dep();
|
||||
IndexedWord gov = TDY1.gov();
|
||||
GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep);
|
||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 900;
|
||||
}
|
||||
GrammaticalRelation reln = TDY1.reln();
|
||||
if (reln.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 256;
|
||||
}
|
||||
}
|
||||
for (TypedDependency TDY : allTypedDependencies) {
|
||||
IndexedWord dep = TDY.dep();
|
||||
IndexedWord gov = TDY.gov();
|
||||
GrammaticalRelation grammaticalRelation = gs1.getGrammaticalRelation(gov, dep);
|
||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 900;
|
||||
}
|
||||
GrammaticalRelation reln = TDY.reln();
|
||||
if (reln.isApplicable(sentenceConstituencyParse1)) {
|
||||
score += 256;
|
||||
}
|
||||
}
|
||||
AtomicInteger runCount1 = new AtomicInteger(0);
|
||||
sentenceConstituencyParse.taggedLabeledYield().forEach((LBW) -> {
|
||||
sentenceConstituencyParse1.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma())
|
||||
&& !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> {
|
||||
filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma());
|
||||
return _item;
|
||||
}).forEachOrdered((_item) -> {
|
||||
runCount1.getAndIncrement();
|
||||
});
|
||||
});
|
||||
score += runCount1.get() * 1500;
|
||||
}
|
||||
}
|
||||
sentenceConstituencyParseList.clear();
|
||||
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlistVector = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Integer> sentiment1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Integer> sentiment2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (CoreMap sentence : pipelineAnnotation1Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
sentiment1.put(sentiment1.size(), RNNCoreAnnotations.getPredictedClass(tree));
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
||||
simpleSMXlist.put(simpleSMXlist.size(), predictions);
|
||||
simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector);
|
||||
}
|
||||
for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree));
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
||||
score = simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> {
|
||||
subtracter *= 25;
|
||||
return subtracter;
|
||||
}).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator - _item);
|
||||
for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) {
|
||||
double dot = nodeVector.dot(simpleSMX);
|
||||
double elementSum = nodeVector.kron(simpleSMX).elementSum();
|
||||
elementSum = Math.round(elementSum * 100.0) / 100.0;
|
||||
if (dot < 0.1) {
|
||||
score += 256;
|
||||
});
|
||||
taggedwordlist1.clear();
|
||||
AtomicInteger runCount = new AtomicInteger(0);
|
||||
taggedwordlist2.forEach((TGWList) -> {
|
||||
TGWList.forEach((TaggedWord) -> {
|
||||
if (tgwlistIndex.values().contains(TaggedWord.tag())) {
|
||||
tgwlistIndex.values().remove(TaggedWord.tag());
|
||||
runCount.getAndIncrement();
|
||||
}
|
||||
});
|
||||
});
|
||||
tgwlistIndex.clear();
|
||||
taggedwordlist2.clear();
|
||||
score += runCount.get() * 64;
|
||||
ConcurrentMap<Integer, Tree> sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
try {
|
||||
for (CoreMap sentence : pipelineAnnotation1.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
||||
sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse);
|
||||
}
|
||||
if (elementSum < 0.1 && elementSum > 0.0) {
|
||||
score += 1300;
|
||||
} else if (elementSum > 0.1 && elementSum < 1.0) {
|
||||
score -= 1100;
|
||||
for (CoreMap sentence : pipelineAnnotation2.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
||||
GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse);
|
||||
Collection<TypedDependency> allTypedDependencies = gs.allTypedDependencies();
|
||||
ConcurrentMap<Integer, String> filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) {
|
||||
Set<Constituent> inT1notT2 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1);
|
||||
Set<Constituent> inT2notT1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse);
|
||||
ConcurrentMap<Integer, String> constiLabels = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (Constituent consti : inT1notT2) {
|
||||
for (Constituent consti1 : inT2notT1) {
|
||||
if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) {
|
||||
score += 64;
|
||||
constiLabels.put(constiLabels.size(), consti.value());
|
||||
}
|
||||
}
|
||||
}
|
||||
GrammaticalStructure gs1 = gsf.newGrammaticalStructure(sentenceConstituencyParse1);
|
||||
Collection<TypedDependency> allTypedDependencies1 = gs1.allTypedDependencies();
|
||||
for (TypedDependency TDY1 : allTypedDependencies1) {
|
||||
IndexedWord dep = TDY1.dep();
|
||||
IndexedWord gov = TDY1.gov();
|
||||
GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep);
|
||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 900;
|
||||
}
|
||||
GrammaticalRelation reln = TDY1.reln();
|
||||
if (reln.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 256;
|
||||
}
|
||||
}
|
||||
for (TypedDependency TDY : allTypedDependencies) {
|
||||
IndexedWord dep = TDY.dep();
|
||||
IndexedWord gov = TDY.gov();
|
||||
GrammaticalRelation grammaticalRelation = gs1.getGrammaticalRelation(gov, dep);
|
||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 900;
|
||||
}
|
||||
GrammaticalRelation reln = TDY.reln();
|
||||
if (reln.isApplicable(sentenceConstituencyParse1)) {
|
||||
score += 256;
|
||||
}
|
||||
}
|
||||
AtomicInteger runCount1 = new AtomicInteger(0);
|
||||
sentenceConstituencyParse.taggedLabeledYield().forEach((LBW) -> {
|
||||
sentenceConstituencyParse1.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma())
|
||||
&& !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> {
|
||||
filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma());
|
||||
return _item;
|
||||
}).forEachOrdered((_item) -> {
|
||||
runCount1.getAndIncrement();
|
||||
});
|
||||
});
|
||||
score += runCount1.get() * 1500;
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
System.out.println("pipelineAnnotation stacktrace: " + ex.getLocalizedMessage()+ "\n");
|
||||
}
|
||||
sentenceConstituencyParseList.clear();
|
||||
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlistVector = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Integer> sentiment1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Integer> sentiment2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (CoreMap sentence : pipelineAnnotation1Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
sentiment1.put(sentiment1.size(), RNNCoreAnnotations.getPredictedClass(tree));
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
||||
simpleSMXlist.put(simpleSMXlist.size(), predictions);
|
||||
simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector);
|
||||
}
|
||||
for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree));
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
||||
score = simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> {
|
||||
subtracter *= 25;
|
||||
return subtracter;
|
||||
}).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator - _item);
|
||||
for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) {
|
||||
double dot = nodeVector.dot(simpleSMX);
|
||||
double elementSum = nodeVector.kron(simpleSMX).elementSum();
|
||||
elementSum = Math.round(elementSum * 100.0) / 100.0;
|
||||
if (dot < 0.1) {
|
||||
score += 256;
|
||||
}
|
||||
if (elementSum < 0.1 && elementSum > 0.0) {
|
||||
score += 1300;
|
||||
} else if (elementSum > 0.1 && elementSum < 1.0) {
|
||||
score -= 1100;
|
||||
} else {
|
||||
score -= 1424;
|
||||
}
|
||||
}
|
||||
}
|
||||
score -= (sentiment1.size() > sentiment2.size() ? sentiment1.size() - sentiment2.size() : sentiment2.size() - sentiment1.size()) * 500;
|
||||
DocumentReaderAndWriter<CoreLabel> readerAndWriter = classifier.makePlainTextReaderAndWriter();
|
||||
List classifyRaw1 = classifier.classifyRaw(str, readerAndWriter);
|
||||
List classifyRaw2 = classifier.classifyRaw(str1, readerAndWriter);
|
||||
score -= (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200;
|
||||
int mainSentiment1 = 0;
|
||||
int longest1 = 0;
|
||||
int mainSentiment2 = 0;
|
||||
int longest2 = 0;
|
||||
for (CoreMap sentence : pipelineAnnotation1Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
|
||||
String partText = sentence.toString();
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
if (partText.length() > longest1) {
|
||||
mainSentiment1 = sentiment;
|
||||
longest1 = partText.length();
|
||||
}
|
||||
}
|
||||
for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
String partText = sentence.toString();
|
||||
if (partText.length() > longest2) {
|
||||
mainSentiment2 = sentiment;
|
||||
longest2 = partText.length();
|
||||
}
|
||||
}
|
||||
if (longest1 != longest2) {
|
||||
long deffLongest = longest1 > longest2 ? longest1 : longest2;
|
||||
long deffshorter = longest1 < longest2 ? longest1 : longest2;
|
||||
if (deffLongest >= (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) {
|
||||
score += (deffLongest - deffshorter) * 200;
|
||||
} else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) {
|
||||
score += (deffLongest - deffshorter) * 200;
|
||||
} else {
|
||||
score -= 1424;
|
||||
score -= (deffLongest - deffshorter) * 50;
|
||||
}
|
||||
}
|
||||
}
|
||||
score -= (sentiment1.size() > sentiment2.size() ? sentiment1.size() - sentiment2.size() : sentiment2.size() - sentiment1.size()) * 500;
|
||||
DocumentReaderAndWriter<CoreLabel> readerAndWriter = classifier.makePlainTextReaderAndWriter();
|
||||
List classifyRaw1 = classifier.classifyRaw(str, readerAndWriter);
|
||||
List classifyRaw2 = classifier.classifyRaw(str1, readerAndWriter);
|
||||
score -= (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200;
|
||||
int mainSentiment1 = 0;
|
||||
int longest1 = 0;
|
||||
int mainSentiment2 = 0;
|
||||
int longest2 = 0;
|
||||
for (CoreMap sentence : pipelineAnnotation1Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
|
||||
String partText = sentence.toString();
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
if (partText.length() > longest1) {
|
||||
mainSentiment1 = sentiment;
|
||||
longest1 = partText.length();
|
||||
int tokensCounter1 = 0;
|
||||
int tokensCounter2 = 0;
|
||||
int anotatorcounter1 = 0;
|
||||
int anotatorcounter2 = 0;
|
||||
int inflectedCounterPositive1 = 0;
|
||||
int inflectedCounterPositive2 = 0;
|
||||
int inflectedCounterNegative = 0;
|
||||
int MarkedContinuousCounter1 = 0;
|
||||
int MarkedContinuousCounter2 = 0;
|
||||
int UnmarkedPatternCounter = 0;
|
||||
ConcurrentMap<Integer, String> ITokenMapTag1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> ITokenMapTag2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenStems1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenStems2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenForm1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenForm2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetEntry1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetEntry2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetiPart1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetiPart2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenEntryPOS1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenEntryPOS2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
try {
|
||||
List<CoreMap> sentences = jmweStrAnnotation1.get(CoreAnnotations.SentencesAnnotation.class);
|
||||
for (CoreMap sentence : sentences) {
|
||||
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
|
||||
if (token.isInflected()) {
|
||||
inflectedCounterPositive1++;
|
||||
} else {
|
||||
inflectedCounterNegative++;
|
||||
}
|
||||
strTokenForm1.put(strTokenForm1.size() + 1, token.getForm());
|
||||
strTokenGetEntry1.put(strTokenGetEntry1.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
|
||||
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
||||
IMWEDesc entry = token.getEntry();
|
||||
MarkedContinuousCounter1 += entry.getMarkedContinuous();
|
||||
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
||||
for (IMWEDesc.IPart iPart : values) {
|
||||
strTokenGetiPart1.put(strTokenGetiPart1.size() + 1, iPart.getForm());
|
||||
}
|
||||
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
|
||||
strTokenEntryPOS1.put(strTokenEntryPOS1.size() + 1, strPostPrefix);
|
||||
}
|
||||
for (IToken tokens : token.getTokens()) {
|
||||
ITokenMapTag1.put(ITokenMapTag1.size() + 1, tokens.getTag());
|
||||
for (String strtoken : tokens.getStems()) {
|
||||
strTokenStems1.put(strTokenStems1.size() + 1, strtoken);
|
||||
}
|
||||
}
|
||||
tokensCounter1++;
|
||||
}
|
||||
anotatorcounter1++;
|
||||
}
|
||||
sentences = jmweStrAnnotation2.get(CoreAnnotations.SentencesAnnotation.class);
|
||||
for (CoreMap sentence : sentences) {
|
||||
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
|
||||
if (token.isInflected()) {
|
||||
inflectedCounterPositive2++;
|
||||
} else {
|
||||
inflectedCounterNegative--;
|
||||
}
|
||||
strTokenForm2.put(strTokenForm2.size() + 1, token.getForm());
|
||||
strTokenGetEntry2.put(strTokenGetEntry2.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
|
||||
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
||||
IMWEDesc entry = token.getEntry();
|
||||
MarkedContinuousCounter2 += entry.getMarkedContinuous();
|
||||
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
||||
for (IMWEDesc.IPart iPart : values) {
|
||||
strTokenGetiPart2.put(strTokenGetiPart2.size() + 1, iPart.getForm());
|
||||
}
|
||||
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
|
||||
strTokenEntryPOS2.put(strTokenEntryPOS2.size() + 1, strPostPrefix);
|
||||
}
|
||||
for (IToken tokens : token.getTokens()) {
|
||||
ITokenMapTag2.put(ITokenMapTag2.size() + 1, tokens.getTag());
|
||||
for (String strtoken : tokens.getStems()) {
|
||||
strTokenStems2.put(strTokenStems2.size() + 1, strtoken);
|
||||
}
|
||||
}
|
||||
tokensCounter2++;
|
||||
}
|
||||
anotatorcounter2++;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
System.out.println("SENTIMENT stacktrace: " + ex.getMessage() + "\n");
|
||||
}
|
||||
}
|
||||
for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
String partText = sentence.toString();
|
||||
if (partText.length() > longest2) {
|
||||
mainSentiment2 = sentiment;
|
||||
longest2 = partText.length();
|
||||
for (String strTokenPos1 : strTokenEntryPOS1.values()) {
|
||||
for (String strTokenPos2 : strTokenEntryPOS2.values()) {
|
||||
if (strTokenPos1.equals(strTokenPos2)) {
|
||||
score += 500;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (longest1 != longest2) {
|
||||
long deffLongest = longest1 > longest2 ? longest1 : longest2;
|
||||
long deffshorter = longest1 < longest2 ? longest1 : longest2;
|
||||
if (deffLongest >= (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) {
|
||||
score += (deffLongest - deffshorter) * 200;
|
||||
} else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) {
|
||||
score += (deffLongest - deffshorter) * 200;
|
||||
score += UnmarkedPatternCounter * 1600;
|
||||
if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) {
|
||||
score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500
|
||||
: (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500;
|
||||
}
|
||||
for (String strTokeniPart1 : strTokenGetiPart1.values()) {
|
||||
for (String strTokeniPart2 : strTokenGetiPart2.values()) {
|
||||
if (strTokeniPart1.equals(strTokeniPart2)) {
|
||||
score += 400;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strTokenEntry1 : strTokenGetEntry1.values()) {
|
||||
for (String strTokenEntry2 : strTokenGetEntry2.values()) {
|
||||
if (strTokenEntry1.equals(strTokenEntry2)) {
|
||||
score += 2500;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strmapTag : ITokenMapTag1.values()) {
|
||||
for (String strmapTag1 : ITokenMapTag2.values()) {
|
||||
if (strmapTag.equals(strmapTag1)) {
|
||||
score += 1450;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strTokenForm1itr1 : strTokenForm1.values()) {
|
||||
for (String strTokenForm1itr2 : strTokenForm2.values()) {
|
||||
if (strTokenForm1itr1.equals(strTokenForm1itr2)) {
|
||||
score += 2600;
|
||||
} else if (strTokenForm1itr1.contains(strTokenForm1itr2)) {
|
||||
score += 500;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strTokenStem : strTokenStems1.values()) {
|
||||
for (String strTokenStem1 : strTokenStems2.values()) {
|
||||
if (strTokenStem.equals(strTokenStem1)) {
|
||||
score += 1500;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (inflectedCounterPositive1 + inflectedCounterPositive2 > inflectedCounterNegative && inflectedCounterNegative > 0) {
|
||||
score += (inflectedCounterPositive1 - inflectedCounterNegative) * 650;
|
||||
}
|
||||
if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) {
|
||||
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550;
|
||||
}
|
||||
if (anotatorcounter1 > 1 && anotatorcounter2 > 1) {
|
||||
score += (anotatorcounter1 + anotatorcounter2) * 400;
|
||||
}
|
||||
if (tokensCounter1 > 0 && tokensCounter2 > 0) {
|
||||
score += (tokensCounter1 + tokensCounter2) * 400;
|
||||
} else {
|
||||
score -= (deffLongest - deffshorter) * 50;
|
||||
score -= tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
|
||||
}
|
||||
LevenshteinDistance leven = new LevenshteinDistance(str, str1);
|
||||
int SentenceScoreDiff = leven.computeLevenshteinDistance();
|
||||
SentenceScoreDiff *= 15;
|
||||
score -= SentenceScoreDiff;
|
||||
} catch (Exception ex) {
|
||||
System.out.println("SENTIMENT stacktrace Overall catch: " + ex.getMessage() + "\n");
|
||||
}
|
||||
List<CoreMap> sentences = jmweStrAnnotation1.get(CoreAnnotations.SentencesAnnotation.class);
|
||||
int tokensCounter1 = 0;
|
||||
int tokensCounter2 = 0;
|
||||
int anotatorcounter1 = 0;
|
||||
int anotatorcounter2 = 0;
|
||||
int inflectedCounterPositive1 = 0;
|
||||
int inflectedCounterPositive2 = 0;
|
||||
int inflectedCounterNegative = 0;
|
||||
int MarkedContinuousCounter1 = 0;
|
||||
int MarkedContinuousCounter2 = 0;
|
||||
int UnmarkedPatternCounter = 0;
|
||||
ConcurrentMap<Integer, String> ITokenMapTag1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> ITokenMapTag2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenStems1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenStems2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenForm1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenForm2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetEntry1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetEntry2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetiPart1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetiPart2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenEntryPOS1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenEntryPOS2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (CoreMap sentence : sentences) {
|
||||
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
|
||||
if (token.isInflected()) {
|
||||
inflectedCounterPositive1++;
|
||||
} else {
|
||||
inflectedCounterNegative++;
|
||||
}
|
||||
strTokenForm1.put(strTokenForm1.size() + 1, token.getForm());
|
||||
strTokenGetEntry1.put(strTokenGetEntry1.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
|
||||
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
||||
IMWEDesc entry = token.getEntry();
|
||||
MarkedContinuousCounter1 += entry.getMarkedContinuous();
|
||||
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
||||
for (IMWEDesc.IPart iPart : values) {
|
||||
strTokenGetiPart1.put(strTokenGetiPart1.size() + 1, iPart.getForm());
|
||||
}
|
||||
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
|
||||
strTokenEntryPOS1.put(strTokenEntryPOS1.size() + 1, strPostPrefix);
|
||||
}
|
||||
for (IToken tokens : token.getTokens()) {
|
||||
ITokenMapTag1.put(ITokenMapTag1.size() + 1, tokens.getTag());
|
||||
for (String strtoken : tokens.getStems()) {
|
||||
strTokenStems1.put(strTokenStems1.size() + 1, strtoken);
|
||||
}
|
||||
}
|
||||
tokensCounter1++;
|
||||
}
|
||||
anotatorcounter1++;
|
||||
}
|
||||
sentences = jmweStrAnnotation2.get(CoreAnnotations.SentencesAnnotation.class);
|
||||
for (CoreMap sentence : sentences) {
|
||||
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
|
||||
if (token.isInflected()) {
|
||||
inflectedCounterPositive2++;
|
||||
} else {
|
||||
inflectedCounterNegative--;
|
||||
}
|
||||
strTokenForm2.put(strTokenForm2.size() + 1, token.getForm());
|
||||
strTokenGetEntry2.put(strTokenGetEntry2.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
|
||||
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
||||
IMWEDesc entry = token.getEntry();
|
||||
MarkedContinuousCounter2 += entry.getMarkedContinuous();
|
||||
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
||||
for (IMWEDesc.IPart iPart : values) {
|
||||
strTokenGetiPart2.put(strTokenGetiPart2.size() + 1, iPart.getForm());
|
||||
}
|
||||
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
|
||||
strTokenEntryPOS2.put(strTokenEntryPOS2.size() + 1, strPostPrefix);
|
||||
}
|
||||
for (IToken tokens : token.getTokens()) {
|
||||
ITokenMapTag2.put(ITokenMapTag2.size() + 1, tokens.getTag());
|
||||
for (String strtoken : tokens.getStems()) {
|
||||
strTokenStems2.put(strTokenStems2.size() + 1, strtoken);
|
||||
}
|
||||
}
|
||||
tokensCounter2++;
|
||||
}
|
||||
anotatorcounter2++;
|
||||
}
|
||||
for (String strTokenPos1 : strTokenEntryPOS1.values()) {
|
||||
for (String strTokenPos2 : strTokenEntryPOS2.values()) {
|
||||
if (strTokenPos1.equals(strTokenPos2)) {
|
||||
score += 500;
|
||||
}
|
||||
}
|
||||
}
|
||||
score += UnmarkedPatternCounter * 1600;
|
||||
if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) {
|
||||
score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500
|
||||
: (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500;
|
||||
}
|
||||
for (String strTokeniPart1 : strTokenGetiPart1.values()) {
|
||||
for (String strTokeniPart2 : strTokenGetiPart2.values()) {
|
||||
if (strTokeniPart1.equals(strTokeniPart2)) {
|
||||
score += 400;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strTokenEntry1 : strTokenGetEntry1.values()) {
|
||||
for (String strTokenEntry2 : strTokenGetEntry2.values()) {
|
||||
if (strTokenEntry1.equals(strTokenEntry2)) {
|
||||
score += 2500;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strmapTag : ITokenMapTag1.values()) {
|
||||
for (String strmapTag1 : ITokenMapTag2.values()) {
|
||||
if (strmapTag.equals(strmapTag1)) {
|
||||
score += 1450;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strTokenForm1itr1 : strTokenForm1.values()) {
|
||||
for (String strTokenForm1itr2 : strTokenForm2.values()) {
|
||||
if (strTokenForm1itr1.equals(strTokenForm1itr2)) {
|
||||
score += 2600;
|
||||
} else if (strTokenForm1itr1.contains(strTokenForm1itr2)) {
|
||||
score += 500;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strTokenStem : strTokenStems1.values()) {
|
||||
for (String strTokenStem1 : strTokenStems2.values()) {
|
||||
if (strTokenStem.equals(strTokenStem1)) {
|
||||
score += 1500;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (inflectedCounterPositive1 + inflectedCounterPositive2 > inflectedCounterNegative && inflectedCounterNegative > 0) {
|
||||
score += (inflectedCounterPositive1 - inflectedCounterNegative) * 650;
|
||||
}
|
||||
if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) {
|
||||
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550;
|
||||
}
|
||||
if (anotatorcounter1 > 1 && anotatorcounter2 > 1) {
|
||||
score += (anotatorcounter1 + anotatorcounter2) * 400;
|
||||
}
|
||||
if (tokensCounter1 > 0 && tokensCounter2 > 0) {
|
||||
score += (tokensCounter1 + tokensCounter2) * 400;
|
||||
} else {
|
||||
score -= tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
|
||||
}
|
||||
LevenshteinDistance leven = new LevenshteinDistance(str, str1);
|
||||
int SentenceScoreDiff = leven.computeLevenshteinDistance();
|
||||
SentenceScoreDiff *= 15;
|
||||
score -= SentenceScoreDiff;
|
||||
System.out.println("Final current score: " + score + "\nSentence 1: " + str + "\nSentence 2: " + str1 + "\n");
|
||||
smxParam.setDistance(score);
|
||||
return smxParam;
|
||||
|
||||
Reference in New Issue
Block a user