2021-04-04 10:01:54 +02:00
|
|
|
package FunctionLayer.StanfordParser;
|
|
|
|
|
|
|
|
|
|
import FunctionLayer.LevenshteinDistance;
|
|
|
|
|
import FunctionLayer.Datahandler;
|
|
|
|
|
import FunctionLayer.SimilarityMatrix;
|
|
|
|
|
import FunctionLayer.StopwordAnnotator;
|
|
|
|
|
import com.google.common.collect.MapMaker;
|
|
|
|
|
import edu.mit.jmwe.data.IMWE;
|
|
|
|
|
import edu.mit.jmwe.data.IMWEDesc;
|
|
|
|
|
import edu.mit.jmwe.data.IToken;
|
|
|
|
|
import edu.stanford.nlp.ie.AbstractSequenceClassifier;
|
|
|
|
|
import edu.stanford.nlp.ling.CoreAnnotations;
|
|
|
|
|
import edu.stanford.nlp.ling.CoreLabel;
|
|
|
|
|
import edu.stanford.nlp.ling.HasWord;
|
|
|
|
|
import edu.stanford.nlp.ling.IndexedWord;
|
|
|
|
|
import edu.stanford.nlp.ling.JMWEAnnotation;
|
|
|
|
|
import edu.stanford.nlp.ling.TaggedWord;
|
|
|
|
|
import edu.stanford.nlp.neural.rnn.RNNCoreAnnotations;
|
|
|
|
|
import edu.stanford.nlp.pipeline.Annotation;
|
|
|
|
|
import edu.stanford.nlp.pipeline.CoreDocument;
|
|
|
|
|
import edu.stanford.nlp.pipeline.CoreEntityMention;
|
|
|
|
|
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
|
|
|
|
|
import edu.stanford.nlp.process.CoreLabelTokenFactory;
|
|
|
|
|
import edu.stanford.nlp.process.DocumentPreprocessor;
|
|
|
|
|
import edu.stanford.nlp.process.PTBTokenizer;
|
|
|
|
|
import edu.stanford.nlp.process.TokenizerFactory;
|
|
|
|
|
import edu.stanford.nlp.sentiment.SentimentCoreAnnotations;
|
|
|
|
|
import edu.stanford.nlp.sequences.DocumentReaderAndWriter;
|
|
|
|
|
import edu.stanford.nlp.tagger.maxent.MaxentTagger;
|
|
|
|
|
import edu.stanford.nlp.trees.Constituent;
|
|
|
|
|
import edu.stanford.nlp.trees.GrammaticalRelation;
|
|
|
|
|
import edu.stanford.nlp.trees.GrammaticalStructure;
|
|
|
|
|
import edu.stanford.nlp.trees.GrammaticalStructureFactory;
|
|
|
|
|
import edu.stanford.nlp.trees.Tree;
|
|
|
|
|
import edu.stanford.nlp.trees.TreeCoreAnnotations;
|
|
|
|
|
import edu.stanford.nlp.trees.TypedDependency;
|
|
|
|
|
import edu.stanford.nlp.trees.tregex.gui.Tdiff;
|
|
|
|
|
import edu.stanford.nlp.util.CoreMap;
|
|
|
|
|
import edu.stanford.nlp.util.Pair;
|
|
|
|
|
import java.io.StringReader;
|
|
|
|
|
import java.util.AbstractMap;
|
|
|
|
|
import java.util.ArrayList;
|
|
|
|
|
import java.util.Collection;
|
|
|
|
|
import java.util.List;
|
|
|
|
|
import java.util.Map;
|
|
|
|
|
import java.util.Objects;
|
|
|
|
|
import java.util.OptionalDouble;
|
|
|
|
|
import java.util.Set;
|
|
|
|
|
import java.util.concurrent.Callable;
|
|
|
|
|
import java.util.concurrent.ConcurrentMap;
|
|
|
|
|
import java.util.concurrent.atomic.AtomicInteger;
|
|
|
|
|
import org.apache.lucene.analysis.core.StopAnalyzer;
|
|
|
|
|
import org.ejml.simple.SimpleMatrix;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* To change this license header, choose License Headers in Project Properties.
|
|
|
|
|
* To change this template file, choose Tools | Templates
|
|
|
|
|
* and open the template in the editor.
|
|
|
|
|
*/
|
|
|
|
|
/**
|
|
|
|
|
*
|
|
|
|
|
* @author install1
|
|
|
|
|
*/
|
|
|
|
|
public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
|
|
|
|
|
|
|
|
|
private final SimilarityMatrix smxParam;
|
|
|
|
|
private final String str;
|
|
|
|
|
private final String str1;
|
|
|
|
|
private final MaxentTagger tagger;
|
|
|
|
|
private final GrammaticalStructureFactory gsf;
|
|
|
|
|
private final AbstractSequenceClassifier classifier;
|
|
|
|
|
private final Annotation jmweStrAnnotation1;
|
|
|
|
|
private final Annotation jmweStrAnnotation2;
|
|
|
|
|
private final Annotation pipelineAnnotation1;
|
|
|
|
|
private final Annotation pipelineAnnotation2;
|
|
|
|
|
private final Annotation pipelineAnnotation1Sentiment;
|
|
|
|
|
private final Annotation pipelineAnnotation2Sentiment;
|
|
|
|
|
private final CoreDocument pipelineCoreDcoument1;
|
|
|
|
|
private final CoreDocument pipelineCoreDcoument2;
|
|
|
|
|
private SentimentValueCache cacheSentiment1;
|
|
|
|
|
private SentimentValueCache cacheSentiment2;
|
|
|
|
|
|
|
|
|
|
public final SentimentValueCache getCacheSentiment1() {
|
|
|
|
|
return cacheSentiment1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public final SentimentValueCache getCacheSentiment2() {
|
|
|
|
|
return cacheSentiment2;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public SentimentAnalyzerTest(String str, String str1, SimilarityMatrix smxParam, Annotation str1Annotation, Annotation str2Annotation,
|
|
|
|
|
Annotation strPipeline1, Annotation strPipeline2, Annotation strPipeSentiment1, Annotation strPipeSentiment2,
|
|
|
|
|
CoreDocument pipelineCoreDcoument1, CoreDocument pipelineCoreDcoument2, SentimentValueCache cacheValue1, SentimentValueCache cacheValue2) {
|
|
|
|
|
this.str = str;
|
|
|
|
|
this.str1 = str1;
|
|
|
|
|
this.smxParam = smxParam;
|
|
|
|
|
this.tagger = Datahandler.getTagger();
|
|
|
|
|
this.gsf = Datahandler.getGsf();
|
|
|
|
|
this.classifier = Datahandler.getClassifier();
|
|
|
|
|
this.jmweStrAnnotation1 = str1Annotation;
|
|
|
|
|
this.jmweStrAnnotation2 = str2Annotation;
|
|
|
|
|
this.pipelineAnnotation1 = strPipeline1;
|
|
|
|
|
this.pipelineAnnotation2 = strPipeline2;
|
|
|
|
|
this.pipelineAnnotation1Sentiment = strPipeSentiment1;
|
|
|
|
|
this.pipelineAnnotation2Sentiment = strPipeSentiment2;
|
|
|
|
|
this.pipelineCoreDcoument1 = pipelineCoreDcoument1;
|
|
|
|
|
this.pipelineCoreDcoument2 = pipelineCoreDcoument2;
|
|
|
|
|
this.cacheSentiment1 = cacheValue1;
|
|
|
|
|
this.cacheSentiment2 = cacheValue2;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private List<List<TaggedWord>> getTaggedWordList(String message) {
|
|
|
|
|
List<List<TaggedWord>> taggedwordlist = new ArrayList();
|
|
|
|
|
DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(message));
|
|
|
|
|
TokenizerFactory<CoreLabel> ptbTokenizerFactory = PTBTokenizer.factory(new CoreLabelTokenFactory(), "untokenizable=firstDelete"); //noneDelete
|
|
|
|
|
tokenizer.setTokenizerFactory(ptbTokenizerFactory);
|
|
|
|
|
for (final List<HasWord> sentence : tokenizer) {
|
|
|
|
|
taggedwordlist.add(tagger.tagSentence(sentence));
|
|
|
|
|
}
|
|
|
|
|
return taggedwordlist;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private int tokenizeCounting(List<List<TaggedWord>> taggedwordlist) {
|
|
|
|
|
int counter = 0;
|
|
|
|
|
Collection<TaggedWord> taggedCollection = new ArrayList();
|
|
|
|
|
for (List<TaggedWord> taggedList : taggedwordlist) {
|
|
|
|
|
counter += taggedList.size();
|
|
|
|
|
}
|
|
|
|
|
return counter;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private ConcurrentMap<Integer, String> retrieveTGWListIndex(List<List<TaggedWord>> taggedwordlist) {
|
|
|
|
|
ConcurrentMap<Integer, String> tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
taggedwordlist.forEach((TGWList) -> {
|
|
|
|
|
TGWList.forEach((TaggedWord) -> {
|
|
|
|
|
if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) {
|
|
|
|
|
tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag());
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
return tgwlistIndex;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double iterateTrees(ConcurrentMap<Integer, Tree> sentenceConstituencyParseList2, ConcurrentMap<Integer, Tree> sentenceConstituencyParseList1,
|
|
|
|
|
Double score) {
|
|
|
|
|
double preConstituentsScore = score;
|
|
|
|
|
ConcurrentMap<Integer, Integer> constituentsMap = new MapMaker().concurrencyLevel(4).makeMap();
|
|
|
|
|
int constituencySize = sentenceConstituencyParseList1.size() + sentenceConstituencyParseList2.size();
|
|
|
|
|
for (final Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2.values()) {
|
|
|
|
|
int constiRelationsize = 0;
|
|
|
|
|
for (final Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1.values()) {
|
|
|
|
|
Set<Constituent> constinuent1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse2);
|
|
|
|
|
Set<Constituent> constinuent2 = Tdiff.markDiff(sentenceConstituencyParse2, sentenceConstituencyParse1);
|
|
|
|
|
ConcurrentMap<Integer, String> constiLabels = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
for (final Constituent consti : constinuent1) {
|
|
|
|
|
for (final Constituent consti1 : constinuent2) {
|
|
|
|
|
if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) {
|
|
|
|
|
constiLabels.put(constiLabels.size(), consti.value());
|
|
|
|
|
constiRelationsize++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
int constituents1 = constinuent1.size() - constiRelationsize;
|
|
|
|
|
int constituents2 = constinuent2.size() - constiRelationsize;
|
|
|
|
|
constituentsMap.put(constituentsMap.size(), constituents1);
|
|
|
|
|
constituentsMap.put(constituentsMap.size(), constituents2);
|
|
|
|
|
constituentsMap.put(constituentsMap.size(), constiRelationsize);
|
|
|
|
|
if (constituentsMap.size() < 4) {
|
|
|
|
|
if ((constituents1 * 5 < constituents2 || constituents2 * 5 < constituents1) && constituents1 > 0 && constituents2 > 0) {
|
|
|
|
|
score -= (constituents1 + constituents2) * 200;
|
|
|
|
|
} else if ((constituents1 == 0 || constituents2 == 0) && (constituents1 >= constituents2 + 4 || constituents2 >= constituents1 + 4)) {
|
|
|
|
|
score -= constituents1 > constituents2 ? constituents1 * 500 : constituents2 * 500;
|
|
|
|
|
} else if (constiRelationsize >= constituents1 + constituents2 && (constituents1 > 2 && constituents2 > 2)) {
|
|
|
|
|
score += (constiRelationsize + constituents1 + constituents2) * 350;
|
|
|
|
|
} else if (constituents1 >= 2 && constituents2 >= 2 && constituents1 * 1.5 > constituents2 && constituents2 * 1.5 > constituents1) {
|
|
|
|
|
if (constituents1 == constituents2 && constiRelationsize - constituents1 == 1) {
|
|
|
|
|
score += (constiRelationsize + constituents1 + constituents2) * 1550;
|
|
|
|
|
} else if (constiRelationsize >= constituents1 && constituents1 == constituents2) {
|
|
|
|
|
score -= (constiRelationsize + constituents1 + constituents2) * 550;
|
|
|
|
|
} else if (constiRelationsize < constituents1 && constiRelationsize < constituents2) {
|
|
|
|
|
score += 800;
|
|
|
|
|
} else if ((constiRelationsize == constituents1 || constiRelationsize == constituents2) && constituents1 * 1.5 > constituents2
|
|
|
|
|
&& constituents2 * 1.5 > constituents1) {
|
|
|
|
|
score += (constiRelationsize + constituents1 + constituents2) * 350;
|
|
|
|
|
}
|
|
|
|
|
} else if (constiRelationsize > constituents1 + constituents2) {
|
|
|
|
|
score += 2500;
|
|
|
|
|
} else if (constiRelationsize * 5 < constituents1 || constiRelationsize * 5 < constituents2) {
|
|
|
|
|
score -= (constituents1 + constituents2) * 400;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
score = preConstituentsScore;
|
|
|
|
|
int n1 = constituentsMap.get(0);
|
|
|
|
|
int n2 = constituentsMap.get(1);
|
|
|
|
|
int n3 = constituentsMap.get(2);
|
|
|
|
|
int cap = 0;
|
|
|
|
|
if (n1 > n2 && n1 > n3) {
|
|
|
|
|
cap = n1;
|
|
|
|
|
} else if (n2 > n3 && n2 > n1) {
|
|
|
|
|
cap = n2;
|
|
|
|
|
} else {
|
|
|
|
|
cap = n3;
|
|
|
|
|
}
|
|
|
|
|
int overheat = 0;
|
|
|
|
|
for (int iterator = 3; iterator < constituentsMap.size(); iterator++) {
|
|
|
|
|
Integer getConstituent = constituentsMap.get(iterator);
|
|
|
|
|
if (getConstituent > cap) {
|
|
|
|
|
overheat++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (overheat > 1) {
|
|
|
|
|
score -= overheat * 800;
|
|
|
|
|
} else {
|
|
|
|
|
score += 1300;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (constituencySize > 10) {
|
|
|
|
|
score -= constituencySize * 400;
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double typeDependenciesGrammaticalRelation(Collection<TypedDependency> allTypedDependencies1, Collection<TypedDependency> allTypedDependencies2,
|
|
|
|
|
Double score, ConcurrentMap<Integer, GrammaticalStructure> grammaticalMap1, ConcurrentMap<Integer, GrammaticalStructure> grammaticalMap2,
|
|
|
|
|
ConcurrentMap<Integer, Tree> sentenceConstituencyParseList1, ConcurrentMap<Integer, Tree> sentenceConstituencyParseList2) {
|
|
|
|
|
ConcurrentMap<Integer, Integer> alltypeDepsSizeMap = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
ConcurrentMap<Integer, Integer> summationMap = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
int relationApplicable1 = 0;
|
|
|
|
|
int relationApplicable2 = 0;
|
|
|
|
|
int grammaticalRelation1 = 0;
|
|
|
|
|
int grammaticalRelation2 = 0;
|
|
|
|
|
List<Tree> treeCollectionGramatical = new ArrayList();
|
|
|
|
|
List<Tree> treeCollectionReln = new ArrayList();
|
|
|
|
|
for (TypedDependency TDY1 : allTypedDependencies1) {
|
|
|
|
|
IndexedWord dep = TDY1.dep();
|
|
|
|
|
IndexedWord gov = TDY1.gov();
|
|
|
|
|
for (GrammaticalStructure gs : grammaticalMap1.values()) {
|
|
|
|
|
GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep);
|
|
|
|
|
for (Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2.values()) {
|
|
|
|
|
if (grammaticalRelation.isApplicable(sentenceConstituencyParse2) && !treeCollectionGramatical.contains(sentenceConstituencyParse2)) {
|
|
|
|
|
score += 700;
|
|
|
|
|
grammaticalRelation1++;
|
|
|
|
|
treeCollectionGramatical.add(sentenceConstituencyParse2);
|
|
|
|
|
}
|
|
|
|
|
GrammaticalRelation reln = TDY1.reln();
|
|
|
|
|
if (reln.isApplicable(sentenceConstituencyParse2) && !treeCollectionReln.contains(sentenceConstituencyParse2)) {
|
|
|
|
|
score += 525;
|
|
|
|
|
relationApplicable1++;
|
|
|
|
|
treeCollectionReln.add(sentenceConstituencyParse2);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
treeCollectionGramatical = new ArrayList();
|
|
|
|
|
treeCollectionReln = new ArrayList();
|
|
|
|
|
for (TypedDependency TDY : allTypedDependencies2) {
|
|
|
|
|
IndexedWord dep = TDY.dep();
|
|
|
|
|
IndexedWord gov = TDY.gov();
|
|
|
|
|
for (GrammaticalStructure gs : grammaticalMap2.values()) {
|
|
|
|
|
GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep);
|
|
|
|
|
for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1.values()) {
|
|
|
|
|
if (grammaticalRelation.isApplicable(sentenceConstituencyParse1) && !treeCollectionGramatical.contains(sentenceConstituencyParse1)) {
|
|
|
|
|
score += 700;
|
|
|
|
|
grammaticalRelation2++;
|
|
|
|
|
treeCollectionGramatical.add(sentenceConstituencyParse1);
|
|
|
|
|
}
|
|
|
|
|
GrammaticalRelation reln = TDY.reln();
|
|
|
|
|
//sentenceConstituencyParse1
|
|
|
|
|
if (reln.isApplicable(sentenceConstituencyParse1) && !treeCollectionReln.contains(sentenceConstituencyParse1)) {
|
|
|
|
|
score += 525;
|
|
|
|
|
relationApplicable2++;
|
|
|
|
|
treeCollectionReln.add(sentenceConstituencyParse1);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if ((grammaticalRelation1 == 0 && grammaticalRelation2 > 4) || (grammaticalRelation2 == 0 && grammaticalRelation1 > 4)) {
|
|
|
|
|
score -= 3450;
|
|
|
|
|
}
|
|
|
|
|
if (!allTypedDependencies1.isEmpty() || !allTypedDependencies2.isEmpty()) {
|
|
|
|
|
int allTypeDep1 = allTypedDependencies1.size();
|
|
|
|
|
int allTypeDep2 = allTypedDependencies2.size();
|
|
|
|
|
if (allTypeDep1 <= allTypeDep2 * 5 && allTypeDep2 <= allTypeDep1 * 5) {
|
|
|
|
|
if (allTypeDep1 > 0 && allTypeDep2 > 0 && (allTypeDep1 >= 5 || allTypeDep2 >= 5)) {
|
|
|
|
|
if ((allTypeDep1 + 1 == allTypeDep2 || allTypeDep2 + 1 == allTypeDep1)) {
|
|
|
|
|
score += allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 160 : (allTypeDep2 - allTypeDep1) * 160;
|
|
|
|
|
} else if (allTypeDep1 * 2.5 >= allTypeDep2 && allTypeDep2 * 2.5 >= allTypeDep1) {
|
|
|
|
|
score += allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 260 : (allTypeDep2 - allTypeDep1) * 260;
|
|
|
|
|
} else if (allTypeDep1 >= 5 && allTypeDep1 <= 10 && allTypeDep2 >= 5 && allTypeDep2 <= 10) {
|
|
|
|
|
score += allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 260 : (allTypeDep2 - allTypeDep1) * 260;
|
|
|
|
|
} else {
|
|
|
|
|
score -= allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 600 : (allTypeDep2 - allTypeDep1) * 600;
|
|
|
|
|
}
|
|
|
|
|
alltypeDepsSizeMap.put(alltypeDepsSizeMap.size() + 1, allTypeDep1);
|
|
|
|
|
alltypeDepsSizeMap.put(alltypeDepsSizeMap.size() + 1, allTypeDep2);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (allTypeDep1 >= 5 && allTypeDep2 >= 5) {
|
|
|
|
|
int largerTypeDep = allTypeDep1 > allTypeDep2 ? allTypeDep1 : allTypeDep2;
|
|
|
|
|
int smallerTypeDep = allTypeDep1 < allTypeDep2 ? allTypeDep1 : allTypeDep2;
|
|
|
|
|
int summation = (largerTypeDep * largerTypeDep) - (smallerTypeDep * smallerTypeDep);
|
|
|
|
|
if (summation / largerTypeDep < 15.0 && summation / largerTypeDep > 10.0 && smallerTypeDep * 2 > largerTypeDep
|
|
|
|
|
&& !summationMap.values().contains(summation)) {
|
|
|
|
|
score += summation * 80;
|
|
|
|
|
summationMap.put(summationMap.size() + 1, summation);
|
|
|
|
|
} else if (largerTypeDep == smallerTypeDep) {
|
|
|
|
|
score += 2500;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (relationApplicable1 > 0 && relationApplicable2 > 0 && relationApplicable1 != relationApplicable2) {
|
|
|
|
|
score += 600;
|
|
|
|
|
} else if (allTypeDep1 * 5 < allTypeDep2 || allTypeDep2 * 5 < allTypeDep1) {
|
|
|
|
|
score -= allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * (allTypeDep2 * 450)
|
|
|
|
|
: (allTypeDep2 - allTypeDep1) * (allTypeDep1 * 450);
|
|
|
|
|
}
|
|
|
|
|
if (relationApplicable1 > 1 && relationApplicable2 > 1 && relationApplicable1 * 3 > relationApplicable2
|
|
|
|
|
&& relationApplicable2 * 3 > relationApplicable1) {
|
|
|
|
|
score += relationApplicable1 > relationApplicable2 ? (relationApplicable1 - relationApplicable2) * 1500
|
|
|
|
|
: (relationApplicable2 - relationApplicable1) * 1500;
|
|
|
|
|
} else if (relationApplicable1 * 5 < relationApplicable2 || relationApplicable2 * 5 < relationApplicable1) {
|
|
|
|
|
score -= relationApplicable1 > relationApplicable2 ? (relationApplicable1 - relationApplicable2) * 500
|
|
|
|
|
: (relationApplicable2 - relationApplicable1) * 500;
|
|
|
|
|
}
|
|
|
|
|
if (grammaticalRelation1 > 0 && grammaticalRelation2 > 0 && grammaticalRelation1 * 3 > grammaticalRelation2
|
|
|
|
|
&& grammaticalRelation2 * 3 > grammaticalRelation1) {
|
|
|
|
|
score += grammaticalRelation1 > grammaticalRelation2 ? (grammaticalRelation1 - grammaticalRelation2) * 1500
|
|
|
|
|
: (grammaticalRelation2 - grammaticalRelation1) * 1500;
|
|
|
|
|
} else if (grammaticalRelation1 * 5 < grammaticalRelation2 || grammaticalRelation2 * 5 < grammaticalRelation1) {
|
|
|
|
|
score -= grammaticalRelation1 > grammaticalRelation2 ? (grammaticalRelation1 - grammaticalRelation2) * 500
|
|
|
|
|
: (grammaticalRelation2 - grammaticalRelation1) * 500;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
ConcurrentMap<Integer, String> filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
AtomicInteger runCount1 = new AtomicInteger(0);
|
|
|
|
|
for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1.values()) {
|
|
|
|
|
for (Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2.values()) {
|
|
|
|
|
sentenceConstituencyParse1.taggedLabeledYield().forEach((LBW) -> {
|
|
|
|
|
sentenceConstituencyParse2.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma())
|
|
|
|
|
&& !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> {
|
|
|
|
|
filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma());
|
|
|
|
|
return _item;
|
|
|
|
|
}).forEachOrdered((_item) -> {
|
|
|
|
|
runCount1.getAndIncrement();
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
score += runCount1.get() * 250;
|
|
|
|
|
int typeSizeSmallest = 100;
|
|
|
|
|
int typeSizeLargest = 0;
|
|
|
|
|
for (Integer i : alltypeDepsSizeMap.values()) {
|
|
|
|
|
if (i > typeSizeLargest) {
|
|
|
|
|
typeSizeLargest = i;
|
|
|
|
|
}
|
|
|
|
|
if (i < typeSizeSmallest) {
|
|
|
|
|
typeSizeSmallest = i;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (typeSizeLargest >= typeSizeSmallest * 3) {
|
|
|
|
|
score -= typeSizeLargest * 160;
|
|
|
|
|
}
|
|
|
|
|
typeSizeLargest = 0;
|
|
|
|
|
typeSizeSmallest = 100;
|
|
|
|
|
for (int i : summationMap.values()) {
|
|
|
|
|
if (i > typeSizeLargest) {
|
|
|
|
|
typeSizeLargest = i;
|
|
|
|
|
}
|
|
|
|
|
if (i < typeSizeSmallest) {
|
|
|
|
|
typeSizeSmallest = i;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (typeSizeLargest >= typeSizeSmallest * 3) {
|
|
|
|
|
score -= typeSizeLargest * 160;
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double simpleRNNMatrixCalculations(Double score, ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist1, ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist2) {
|
|
|
|
|
List<Double> iteratedDoubleList = new ArrayList();
|
|
|
|
|
List<Double> iterateddotPredictions = new ArrayList();
|
|
|
|
|
double dotpredictionTransfer = 0.0;
|
|
|
|
|
int iterationOverHeat = 0;
|
|
|
|
|
double scoreFallback = score;
|
|
|
|
|
for (SimpleMatrix simpleSMX2 : simpleSMXlist2.values()) {
|
|
|
|
|
ConcurrentMap<Integer, Double> AccumulateDotMap = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
ConcurrentMap<Integer, Double> subtractorMap = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
ConcurrentMap<Integer, Double> dotPredictions = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
ConcurrentMap<Integer, Double> DotOverTransfer = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
Double totalSubtraction = 0.0;
|
|
|
|
|
Double largest = 10.0;
|
|
|
|
|
Double shortest = 100.0;
|
|
|
|
|
for (SimpleMatrix simpleSMX1 : simpleSMXlist1.values()) {
|
|
|
|
|
double dotPrediction2 = simpleSMX2.dot(simpleSMX1) * 100;
|
|
|
|
|
double dotPrediction1 = simpleSMX1.dot(simpleSMX2) * 100;
|
|
|
|
|
AccumulateDotMap.put(AccumulateDotMap.size() + 1, dotPrediction1);
|
|
|
|
|
AccumulateDotMap.put(AccumulateDotMap.size() + 1, dotPrediction2);
|
|
|
|
|
double subtracter1 = dotPrediction1 > 50 ? dotPrediction1 - 100 : dotPrediction1 > 0 ? 100 - dotPrediction1 : 0;
|
|
|
|
|
double subtracter2 = dotPrediction2 > 50 ? dotPrediction2 - 100 : dotPrediction2 > 0 ? 100 - dotPrediction2 : 0;
|
|
|
|
|
subtractorMap.put(subtractorMap.size() + 1, subtracter1);
|
|
|
|
|
subtractorMap.put(subtractorMap.size() + 1, subtracter2);
|
|
|
|
|
dotpredictionTransfer = dotPrediction1;
|
|
|
|
|
if (!dotPredictions.values().contains(dotPrediction1)) {
|
|
|
|
|
for (Double transferDots : DotOverTransfer.values()) {
|
|
|
|
|
if (transferDots == dotPrediction1) {
|
|
|
|
|
totalSubtraction += transferDots;
|
|
|
|
|
} else {
|
|
|
|
|
score -= subtracter1 * 25;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
DotOverTransfer.put(DotOverTransfer.size(), dotPrediction1);
|
|
|
|
|
} else {
|
|
|
|
|
subtracter1 -= 100;
|
|
|
|
|
subtracter1 *= 25;
|
|
|
|
|
score += subtracter1 * dotPrediction1;
|
|
|
|
|
}
|
|
|
|
|
dotPredictions.put(dotPredictions.size() + 1, dotPrediction1);
|
|
|
|
|
if (!dotPredictions.values().contains(dotPrediction2)) {
|
|
|
|
|
for (Double transferDots : DotOverTransfer.values()) {
|
|
|
|
|
if (transferDots == dotPrediction2) {
|
|
|
|
|
totalSubtraction += transferDots;
|
|
|
|
|
} else {
|
|
|
|
|
score -= subtracter1 * 25;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
DotOverTransfer.put(DotOverTransfer.size(), dotPrediction2);
|
|
|
|
|
if (dotPrediction2 > largest) {
|
|
|
|
|
largest = dotPrediction2;
|
|
|
|
|
}
|
|
|
|
|
if (dotPrediction2 < shortest) {
|
|
|
|
|
shortest = dotPrediction2;
|
|
|
|
|
}
|
|
|
|
|
Double dotPredictionIntervalDifference = largest - shortest;
|
|
|
|
|
subtracter2 *= 25;
|
|
|
|
|
if (dotPredictionIntervalDifference < 5.0) {
|
|
|
|
|
if (dotPredictions.values().size() > 0) {
|
|
|
|
|
if (subtracter2 > 0) {
|
|
|
|
|
score -= subtracter2;
|
|
|
|
|
} else {
|
|
|
|
|
score += subtracter2;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
score -= subtracter2 / 10;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if (subtracter2 > 55.0 && subtracter2 < 82.0) {
|
|
|
|
|
if (dotPrediction2 > 30 && dotPrediction2 < 60) {
|
|
|
|
|
if (dotpredictionTransfer != 0.0 && (subtracter2 / dotPrediction2 < 1.3 || (subtracter2 / dotPrediction2 > 1.9
|
|
|
|
|
&& subtracter2 / dotPrediction2 < 1.99))) {
|
|
|
|
|
if (subtracter2 / dotPrediction2 < 1.248 && subtracter2 / dotPrediction2 > 1.238) {
|
|
|
|
|
score -= 2500;
|
|
|
|
|
} else {
|
|
|
|
|
score += 4500;
|
|
|
|
|
}
|
|
|
|
|
} else if (dotpredictionTransfer != 0.0 && subtracter2 / dotPrediction2 > 1.6 && subtracter2 / dotPrediction2 < 1.95) {
|
|
|
|
|
if (subtracter2 > 61.1 && subtracter2 < 61.9 && dotPrediction2 > 37.5 && dotPrediction2 < 38.2) {
|
|
|
|
|
score += 4500;
|
|
|
|
|
} else {
|
|
|
|
|
score -= 4500;
|
|
|
|
|
}
|
|
|
|
|
} else if (!iterateddotPredictions.contains(dotPrediction2)) {
|
|
|
|
|
score -= subtracter2 * dotPrediction2; // +=
|
|
|
|
|
iterateddotPredictions.add(dotPrediction2);
|
|
|
|
|
} else {
|
|
|
|
|
score -= 550;
|
|
|
|
|
}
|
|
|
|
|
} else if (dotPrediction2 < 25 && subtracter2 < 75) {
|
|
|
|
|
score -= dotPrediction2 * 250;
|
|
|
|
|
} else if (subtracter2 > 75.0) {
|
|
|
|
|
if (dotPrediction2 > 20.0 && dotPrediction2 < 23.0) {
|
|
|
|
|
score += 3500;
|
|
|
|
|
} else {
|
|
|
|
|
if (subtracter2 < 75.9 && dotPrediction2 > 24.0 && dotPrediction2 < 24.9) {
|
|
|
|
|
score += (subtracter2 * dotPrediction2) * 5;
|
|
|
|
|
} else {
|
|
|
|
|
score -= 4500;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
dotPredictions.put(dotPredictions.size() + 1, dotPrediction2);
|
|
|
|
|
iterationOverHeat++;
|
|
|
|
|
}
|
|
|
|
|
Double subTracPre = 0.0;
|
|
|
|
|
for (Double subtractors : subtractorMap.values()) {
|
|
|
|
|
if (Objects.equals(subTracPre, subtractors)) {
|
|
|
|
|
if (subTracPre > 43.5 && subTracPre < 50.0) {
|
|
|
|
|
score += (subTracPre * 15) / subtractorMap.values().size();
|
|
|
|
|
} else if (subTracPre > 60.0 && subTracPre < 66.5) {
|
|
|
|
|
score += (subTracPre * 15) / subtractorMap.values().size();
|
|
|
|
|
}
|
|
|
|
|
} else if (subTracPre > 75.0 && subTracPre < 90.0) {
|
|
|
|
|
score += (subTracPre * 50) / subtractorMap.values().size();
|
|
|
|
|
} else if (subTracPre >= 55.0) {
|
|
|
|
|
score -= 2800;
|
|
|
|
|
} else if (subTracPre < -25.0 && subTracPre > -45.0) {
|
|
|
|
|
score += subTracPre * 100;
|
|
|
|
|
}
|
|
|
|
|
subTracPre = subtractors;
|
|
|
|
|
}
|
|
|
|
|
if (totalSubtraction > 45.0) {
|
|
|
|
|
score -= totalSubtraction * 25;
|
|
|
|
|
} else {
|
|
|
|
|
score += totalSubtraction * 25;
|
|
|
|
|
}
|
|
|
|
|
Double preAccumulatorDot = 0.0;
|
|
|
|
|
Double postAccumulatorDot = 0.0;
|
|
|
|
|
for (Double accumulators : AccumulateDotMap.values()) {
|
|
|
|
|
if (Objects.equals(preAccumulatorDot, accumulators)) {
|
|
|
|
|
if (Objects.equals(postAccumulatorDot, accumulators)) {
|
|
|
|
|
score -= 1400;
|
|
|
|
|
}
|
|
|
|
|
postAccumulatorDot = accumulators;
|
|
|
|
|
}
|
|
|
|
|
preAccumulatorDot = accumulators;
|
|
|
|
|
}
|
|
|
|
|
subTracPre = 0.0;
|
|
|
|
|
for (Double subtractors : subtractorMap.values()) {
|
|
|
|
|
if (Objects.equals(subTracPre, subtractors) && subTracPre != 0.0) {
|
|
|
|
|
if (!iteratedDoubleList.contains(subTracPre)) {
|
|
|
|
|
score += 500;
|
|
|
|
|
iteratedDoubleList.add(subTracPre);
|
|
|
|
|
} else {
|
|
|
|
|
score -= 150;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
subTracPre = subtractors;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (iterationOverHeat > 12) {
|
|
|
|
|
score = scoreFallback;
|
|
|
|
|
score -= 2500;
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double simpleRNNMaxtrixVectors(Double score, ConcurrentMap<Integer, SimpleMatrix> simpleSMXlistVector1, ConcurrentMap<Integer, SimpleMatrix> simpleSMXlistVector2) {
|
|
|
|
|
ConcurrentMap<Integer, Double> elementSumCounter = new MapMaker().concurrencyLevel(3).makeMap();
|
|
|
|
|
ConcurrentMap<Integer, Double> dotMap = new MapMaker().concurrencyLevel(3).makeMap();
|
|
|
|
|
ConcurrentMap<Integer, Double> elementSumMap = new MapMaker().concurrencyLevel(3).makeMap();
|
|
|
|
|
ConcurrentMap<Integer, Double> dotSumMap = new MapMaker().concurrencyLevel(3).makeMap();
|
|
|
|
|
Double preDot = 0.0;
|
|
|
|
|
Double postDot = 0.0;
|
|
|
|
|
int iterateSize = simpleSMXlistVector1.values().size() + simpleSMXlistVector2.values().size();
|
|
|
|
|
for (SimpleMatrix simpleSMX2 : simpleSMXlistVector2.values()) {
|
|
|
|
|
for (SimpleMatrix simpleSMX1 : simpleSMXlistVector1.values()) {
|
|
|
|
|
double dot2 = simpleSMX2.dot(simpleSMX1);
|
|
|
|
|
double elementSum2 = simpleSMX2.kron(simpleSMX1).elementSum();
|
|
|
|
|
double dot1 = simpleSMX1.dot(simpleSMX2);
|
|
|
|
|
double elementSum1 = simpleSMX1.kron(simpleSMX2).elementSum();
|
|
|
|
|
if (preDot == dot2) {
|
|
|
|
|
if (postDot == dot2) {
|
|
|
|
|
score -= 500;
|
|
|
|
|
}
|
|
|
|
|
postDot = dot2;
|
|
|
|
|
}
|
|
|
|
|
if (preDot == dot1) {
|
|
|
|
|
if (postDot == dot1) {
|
|
|
|
|
score -= 500;
|
|
|
|
|
}
|
|
|
|
|
postDot = dot1;
|
|
|
|
|
}
|
|
|
|
|
preDot = dot1;
|
|
|
|
|
elementSum1 = Math.round(elementSum1 * 100.0) / 100.0;
|
|
|
|
|
elementSumCounter.put(elementSumCounter.size() + 1, elementSum1);
|
|
|
|
|
dotMap.put(dotMap.size() + 1, dot1);
|
|
|
|
|
preDot = dot2;
|
|
|
|
|
elementSum2 = Math.round(elementSum2 * 100.0) / 100.0;
|
|
|
|
|
elementSumCounter.put(elementSumCounter.size() + 1, elementSum2);
|
|
|
|
|
dotMap.put(dotMap.size() + 1, dot2);
|
|
|
|
|
if (!dotSumMap.values().contains(dot1)) {
|
|
|
|
|
if (dot1 < 0.1 && dot1 > 0.050) {
|
|
|
|
|
score += 256;
|
|
|
|
|
}
|
|
|
|
|
if (dot1 > 0.44 && dot1 < 0.60) {
|
|
|
|
|
score -= 2400;
|
|
|
|
|
} else if (dot1 > 0.40 && dot1 < 0.445) {
|
|
|
|
|
score += 3600;
|
|
|
|
|
}
|
|
|
|
|
dotSumMap.put(dotSumMap.size() + 1, dot1);
|
|
|
|
|
} else {
|
|
|
|
|
score -= 50;
|
|
|
|
|
}
|
|
|
|
|
if (!elementSumMap.values().contains(elementSum1)) {
|
|
|
|
|
if (elementSum1 < 0.01 && elementSum1 > 0.00) {
|
|
|
|
|
score += 1300;
|
|
|
|
|
} else if (elementSum1 > 0.1 && elementSum1 < 1.0) {
|
|
|
|
|
score += 1100;
|
|
|
|
|
} else {
|
|
|
|
|
score -= elementSum1 * 1024;
|
|
|
|
|
}
|
|
|
|
|
elementSumMap.put(elementSumMap.size() + 1, elementSum1);
|
|
|
|
|
} else {
|
|
|
|
|
score -= 50;
|
|
|
|
|
}
|
|
|
|
|
if (!dotSumMap.values().contains(dot2)) {
|
|
|
|
|
if (dot2 < 0.000) {
|
|
|
|
|
score += dot2 * 500;
|
|
|
|
|
} else if (dot2 < 0.1) {
|
|
|
|
|
score += 256;
|
|
|
|
|
}
|
|
|
|
|
if (dot2 > 0.50) {
|
|
|
|
|
score -= 1200;
|
|
|
|
|
}
|
|
|
|
|
dotSumMap.put(dotSumMap.size() + 1, dot2);
|
|
|
|
|
} else if (dot2 > 0.050 && dot2 < 0.10) {
|
|
|
|
|
score -= 350;
|
|
|
|
|
} else {
|
|
|
|
|
score = score > 0 ? score - dot2 * 1200 : score + dot2 * 1200;
|
|
|
|
|
}
|
|
|
|
|
if (!elementSumMap.values().contains(elementSum2)) {
|
|
|
|
|
if (elementSum2 < 0.01 && elementSum2 > 0.00) {
|
|
|
|
|
score += 3300;
|
|
|
|
|
} else if (elementSum2 > 0.1 && elementSum2 < 0.2) {
|
|
|
|
|
score += 1100;
|
|
|
|
|
} else {
|
|
|
|
|
score -= elementSum2 * 1024;
|
|
|
|
|
}
|
|
|
|
|
elementSumMap.put(elementSumMap.size() + 1, elementSum2);
|
|
|
|
|
} else if (elementSum2 > 0.050 && elementSum2 < 0.10) {
|
|
|
|
|
score += 750;
|
|
|
|
|
} else {
|
|
|
|
|
score = score > 0 ? score - elementSum2 * 1200 : score + elementSum2 * 1200;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (iterateSize > 5) {
|
|
|
|
|
score -= iterateSize * 400;
|
|
|
|
|
}
|
|
|
|
|
score = elementsAndDotsRelation(score, dotMap, elementSumCounter);
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double elementsAndDotsRelation(Double score, ConcurrentMap<Integer, Double> dotMap, ConcurrentMap<Integer, Double> elementSumCounter) {
|
|
|
|
|
OptionalDouble minvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).min();
|
|
|
|
|
OptionalDouble maxvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).max();
|
|
|
|
|
double total = minvalueDots.getAsDouble() + maxvalueDots.getAsDouble();
|
|
|
|
|
boolean permitted = false;
|
|
|
|
|
if (minvalueDots.getAsDouble() != maxvalueDots.getAsDouble()) {
|
|
|
|
|
permitted = true;
|
|
|
|
|
}
|
|
|
|
|
if (permitted) {
|
|
|
|
|
Double dotsVariance = maxvalueDots.getAsDouble() - minvalueDots.getAsDouble();
|
|
|
|
|
if (maxvalueDots.getAsDouble() > minvalueDots.getAsDouble() * 10 && (dotsVariance < 0.40
|
|
|
|
|
|| dotsVariance > 0.60)) {
|
|
|
|
|
score -= 5500;
|
|
|
|
|
} else if (minvalueDots.getAsDouble() < -0.10) {
|
|
|
|
|
score -= 3500;
|
|
|
|
|
} else if (dotsVariance > minvalueDots.getAsDouble() * 2) {
|
|
|
|
|
score += 3500;
|
|
|
|
|
} else if (minvalueDots.getAsDouble() * 2 > maxvalueDots.getAsDouble()) {
|
|
|
|
|
score -= 3500;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
OptionalDouble minvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).min();
|
|
|
|
|
OptionalDouble maxvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).max();
|
|
|
|
|
Double elementsVariance = maxvalueElements.getAsDouble() - minvalueElements.getAsDouble();
|
|
|
|
|
if (elementsVariance != 0.0) {
|
|
|
|
|
if (elementsVariance <= 0.01 && maxvalueElements.getAsDouble() <= 0.02) {
|
|
|
|
|
if (maxvalueElements.getAsDouble() < 0.01 && minvalueElements.getAsDouble() > -0.02 && minvalueElements.getAsDouble() < 0.0) {
|
|
|
|
|
score += 6500;
|
|
|
|
|
} else {
|
|
|
|
|
score -= 5500;
|
|
|
|
|
}
|
|
|
|
|
} else if (minvalueElements.getAsDouble() < 0.0 && minvalueElements.getAsDouble() - maxvalueElements.getAsDouble() < 0.50) {
|
|
|
|
|
score -= 4500;
|
|
|
|
|
} else if (elementsVariance * 2 >= maxvalueElements.getAsDouble() && elementsVariance < 0.1) {
|
|
|
|
|
score -= elementsVariance * 86000;
|
|
|
|
|
}
|
|
|
|
|
} else if (maxvalueElements.getAsDouble() == minvalueElements.getAsDouble() && maxvalueElements.getAsDouble() > 0.40
|
|
|
|
|
&& maxvalueElements.getAsDouble() < 0.60) {
|
|
|
|
|
score += 600;
|
|
|
|
|
} else if (maxvalueElements.getAsDouble() < 0.20 && minvalueElements.getAsDouble() >= 0.15) {
|
|
|
|
|
score += 2600;
|
|
|
|
|
} else if (maxvalueElements.getAsDouble() == minvalueElements.getAsDouble() && minvalueElements.getAsDouble() == 0.12) {
|
|
|
|
|
score += 2600;
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double sentimentMatrixVariances(Double score, int longest1, int longest2, int mainSentiment1, int mainSentiment2) {
|
|
|
|
|
if (longest1 != longest2) {
|
|
|
|
|
long deffLongest = longest1 > longest2 ? longest1 : longest2;
|
|
|
|
|
long deffshorter = longest1 < longest2 ? longest1 : longest2;
|
|
|
|
|
if (deffLongest > deffshorter * 5) {
|
|
|
|
|
score -= 5500;
|
|
|
|
|
} else if (deffLongest < (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) {
|
|
|
|
|
score += (deffLongest - deffshorter) * 20;
|
|
|
|
|
} else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) {
|
|
|
|
|
score += (deffLongest - deffshorter) * 20;
|
|
|
|
|
} else if (deffLongest - deffshorter < 2) {
|
|
|
|
|
score += (deffLongest - deffshorter) * 20;
|
|
|
|
|
} else if (deffshorter * 2 >= deffLongest && deffshorter * 2 < deffLongest + 5) {
|
|
|
|
|
score += (deffLongest - deffshorter) * 20;
|
|
|
|
|
} else {
|
|
|
|
|
score -= (deffLongest - deffshorter) * 50;
|
|
|
|
|
}
|
|
|
|
|
if (deffLongest - deffshorter <= 5) {
|
|
|
|
|
if (deffLongest < 24 && deffshorter > 20) {
|
|
|
|
|
score += 4500;
|
|
|
|
|
} else if (deffLongest - deffshorter == 5 && deffshorter >= 19 && deffLongest <= 24) {
|
|
|
|
|
score += 4500;
|
|
|
|
|
} else {
|
|
|
|
|
if (deffshorter > 26 && deffLongest < 35) {
|
|
|
|
|
score += 3900;
|
|
|
|
|
} else if (deffLongest - deffshorter == 1 && deffshorter > 15 && deffLongest < 25) {
|
|
|
|
|
score += 4900;
|
|
|
|
|
} else {
|
|
|
|
|
score -= 1900;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private final Map.Entry<Double, Map.Entry<SentimentValueCache, SentimentValueCache>> classifyRawEvaluation(Double score, SentimentValueCache cacheSentimentLocal1,
|
|
|
|
|
SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
if (cacheSentiment1 == null || cacheSentiment2 == null) {
|
|
|
|
|
DocumentReaderAndWriter<CoreLabel> readerAndWriter = classifier.makePlainTextReaderAndWriter();
|
|
|
|
|
if (cacheSentiment1 == null) {
|
|
|
|
|
cacheSentimentLocal1.setClassifyRaw(classifier.classifyRaw(str, readerAndWriter));
|
|
|
|
|
}
|
|
|
|
|
if (cacheSentiment2 == null) {
|
|
|
|
|
cacheSentimentLocal2.setClassifyRaw(classifier.classifyRaw(str1, readerAndWriter));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
final List classifyRaw1 = cacheSentiment1 == null ? cacheSentimentLocal1.getClassifyRaw() : cacheSentiment1.getClassifyRaw();
|
|
|
|
|
final List classifyRaw2 = cacheSentiment2 == null ? cacheSentimentLocal2.getClassifyRaw() : cacheSentiment2.getClassifyRaw();
|
|
|
|
|
score -= (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200;
|
|
|
|
|
Map.Entry< Double, Map.Entry<SentimentValueCache, SentimentValueCache>> entry
|
|
|
|
|
= new AbstractMap.SimpleEntry(score, new AbstractMap.SimpleEntry(cacheSentimentLocal1, cacheSentimentLocal2));
|
|
|
|
|
return entry;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double entryCountsRelation(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
int entry1 = cacheSentiment1 == null ? cacheSentimentLocal1.getEntryCounts().values().size() : cacheSentiment1.getEntryCounts().values().size();
|
|
|
|
|
int entry2 = cacheSentiment2 == null ? cacheSentimentLocal2.getEntryCounts().values().size() : cacheSentiment2.getEntryCounts().values().size();
|
|
|
|
|
if (entry1 > 0 && entry2 > 0) {
|
|
|
|
|
if ((entry1 >= entry2 * 5) || (entry2 >= entry1 * 5)) {
|
|
|
|
|
score -= entry1 > entry2 ? (entry1 - entry2) * 450 : (entry2 - entry1) * 450;
|
|
|
|
|
} else if ((entry1 >= entry2 * 50 || entry2 >= entry1 * 50)) {
|
|
|
|
|
score -= entry1 > entry2 ? entry1 * 180 : entry2 * 180;
|
|
|
|
|
} else if ((entry1 >= entry2 * 2 || entry2 >= entry1 * 2) && entry1 * 2 >= entry2 && entry2 * 2 >= entry1) {
|
|
|
|
|
score -= entry1 > entry2 ? (entry1 - entry2) * 450 : (entry2 - entry1) * 450;
|
|
|
|
|
} else if (entry1 * 3 >= entry2 && entry2 * 3 >= entry1) {
|
|
|
|
|
score += entry1 > entry2 ? (entry1 - entry2) * 550 : (entry2 - entry1) * 550;
|
|
|
|
|
} else if (entry1 > 10 && entry2 > 10 && entry1 * 2 > entry2 && entry2 * 2 > entry1) {
|
|
|
|
|
score += entry1 > entry2 ? entry2 * 600 : entry1 * 600;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private SentimentValueCache GrammaticStructureSetup(SentimentValueCache cacheSentimentLocal, Annotation pipelineAnnotation) {
|
|
|
|
|
for (CoreMap sentence : pipelineAnnotation.get(CoreAnnotations.SentencesAnnotation.class)) {
|
|
|
|
|
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
|
|
|
|
cacheSentimentLocal.addSentenceConstituencyParse(sentenceConstituencyParse);
|
|
|
|
|
GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse);
|
|
|
|
|
cacheSentimentLocal.addTypedDependencies(gs.allTypedDependencies());
|
|
|
|
|
cacheSentimentLocal.addGS(gs);
|
|
|
|
|
}
|
|
|
|
|
return cacheSentimentLocal;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private SentimentValueCache initializeCacheSetup(String str, SentimentValueCache cacheSentimentLocal) {
|
|
|
|
|
cacheSentimentLocal = new SentimentValueCache(str);
|
|
|
|
|
cacheSentimentLocal.setTaggedwords(getTaggedWordList(str));
|
|
|
|
|
cacheSentimentLocal.setCounter(tokenizeCounting(cacheSentimentLocal.getTaggedwordlist()));
|
|
|
|
|
return cacheSentimentLocal;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private SentimentValueCache sentimentCoreAnnotationSetup(Annotation pipelineAnnotationSentiment, SentimentValueCache cacheSentimentLocal) {
|
|
|
|
|
for (CoreMap sentence : pipelineAnnotationSentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
|
|
|
|
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
|
|
|
|
if (tree != null) {
|
|
|
|
|
int predictedClass = RNNCoreAnnotations.getPredictedClass(tree);
|
|
|
|
|
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
|
|
|
|
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
|
|
|
|
cacheSentimentLocal.addRNNPredictClass(predictedClass);
|
|
|
|
|
cacheSentimentLocal.addSimpleMatrix(predictions);
|
|
|
|
|
cacheSentimentLocal.addSimpleMatrixVector(nodeVector);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return cacheSentimentLocal;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private SentimentValueCache setupMainSentimentandLongestVal(Annotation pipelineAnnotationSentiment, SentimentValueCache cacheSentimentLocal) {
|
|
|
|
|
for (CoreMap sentence : pipelineAnnotationSentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
|
|
|
|
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
|
|
|
|
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
|
|
|
|
|
String partText = sentence.toString();
|
|
|
|
|
//SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
|
|
|
|
if (partText.length() > cacheSentimentLocal.getLongest()) {
|
|
|
|
|
cacheSentimentLocal.setMainSentiment(sentiment);
|
|
|
|
|
cacheSentimentLocal.setLongest(partText.length());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return cacheSentimentLocal;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private SentimentValueCache jmweAnnotationSetup(Annotation jmweStrAnnotation, SentimentValueCache cacheSentimentLocal) {
|
|
|
|
|
List<CoreMap> sentences = jmweStrAnnotation.get(CoreAnnotations.SentencesAnnotation.class);
|
|
|
|
|
Collection<IMWE<IToken>> tokeninflectionMap = new ArrayList();
|
|
|
|
|
int tokenadder = 0;
|
|
|
|
|
for (CoreMap sentence : sentences) {
|
|
|
|
|
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
|
|
|
|
|
if (token.isInflected()) {
|
|
|
|
|
cacheSentimentLocal.setInflectedCounterPositive(cacheSentimentLocal.getInflectedCounterPositive() + 1);
|
|
|
|
|
} else if (!tokeninflectionMap.contains(token)) {
|
|
|
|
|
cacheSentimentLocal.setInflectedCounterNegative(cacheSentimentLocal.getInflectedCounterNegative() + 1);
|
|
|
|
|
tokeninflectionMap.add(token);
|
|
|
|
|
}
|
|
|
|
|
cacheSentimentLocal.addstrTokenForm(token.getForm());
|
|
|
|
|
cacheSentimentLocal.addstrTokenGetEntry(token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
|
|
|
|
|
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
|
|
|
|
IMWEDesc entry = token.getEntry();
|
|
|
|
|
cacheSentimentLocal.setMarkedContinuousCounter(cacheSentimentLocal.getMarkedContinuousCounter() + entry.getMarkedContinuous());
|
|
|
|
|
cacheSentimentLocal.setUnmarkedPatternCounter(cacheSentimentLocal.getUnmarkedPatternCounter() + entry.getUnmarkedPattern());
|
|
|
|
|
for (IMWEDesc.IPart iPart : values) {
|
|
|
|
|
cacheSentimentLocal.addstrTokenGetiPart(iPart.getForm());
|
|
|
|
|
}
|
|
|
|
|
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
|
|
|
|
|
cacheSentimentLocal.addstrTokenEntryPOS(strPostPrefix);
|
|
|
|
|
}
|
|
|
|
|
for (int counts : entry.getCounts()) {
|
|
|
|
|
cacheSentimentLocal.addEntryCounts(counts);
|
|
|
|
|
}
|
|
|
|
|
for (IToken tokens : token.getTokens()) {
|
|
|
|
|
cacheSentimentLocal.addITokenMapTag(tokens.getTag());
|
|
|
|
|
for (String strtoken : tokens.getStems()) {
|
|
|
|
|
cacheSentimentLocal.addstrTokenStems(strtoken);
|
|
|
|
|
cacheSentimentLocal.setMarkedContiniousCounterEntries(cacheSentimentLocal.getMarkedContiniousCounterEntries() + 1);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
tokenadder += 1;
|
|
|
|
|
}
|
|
|
|
|
cacheSentimentLocal.setAnotatorcounter(cacheSentimentLocal.getAnotatorcounter() + 1);
|
|
|
|
|
}
|
|
|
|
|
cacheSentimentLocal.setTokensCounter(tokenadder);
|
|
|
|
|
return cacheSentimentLocal;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double entryCountsScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
ConcurrentMap<Integer, Integer> countsMap = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
int totalsize = cacheSentimentLocal1.getEntryCounts().values().size() + cacheSentimentLocal2.getEntryCounts().values().size();
|
|
|
|
|
for (int counts : cacheSentimentLocal1.getEntryCounts().values()) {
|
|
|
|
|
for (int counts1 : cacheSentimentLocal2.getEntryCounts().values()) {
|
|
|
|
|
if (counts > 0 && counts1 > 0) {
|
|
|
|
|
if (counts == counts1 && !countsMap.values().contains(counts)) {
|
|
|
|
|
score += (counts * 250) / totalsize;
|
|
|
|
|
countsMap.put(countsMap.size() + 1, counts);
|
|
|
|
|
} else if (counts * 3 < counts1 || counts1 * 3 < counts) {
|
|
|
|
|
score -= 600;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double tokenEntryPosScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
if (cacheSentimentLocal1.getstrTokenEntryPOS().values().size() > 1 && cacheSentimentLocal2.getstrTokenEntryPOS().values().size() > 1) {
|
|
|
|
|
for (String strTokenPos1 : cacheSentimentLocal1.getstrTokenEntryPOS().values()) {
|
|
|
|
|
for (String strTokenPos2 : cacheSentimentLocal2.getstrTokenEntryPOS().values()) {
|
|
|
|
|
if (strTokenPos1.equals(strTokenPos2)) {
|
|
|
|
|
score += 500;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
int posEntrySize1 = cacheSentimentLocal1.getstrTokenEntryPOS().values().size();
|
|
|
|
|
int posEntrySize2 = cacheSentimentLocal2.getstrTokenEntryPOS().values().size();
|
|
|
|
|
if (posEntrySize1 * 3 > posEntrySize2 && posEntrySize2 * 3 > posEntrySize1) {
|
|
|
|
|
score += posEntrySize1 > posEntrySize2 ? (posEntrySize1 - posEntrySize2) * 700 : (posEntrySize2 - posEntrySize1) * 700;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double unmarkedPatternCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
int UnmarkedPatternCounter1 = cacheSentimentLocal1.getUnmarkedPatternCounter();
|
|
|
|
|
int UnmarkedPatternCounter2 = cacheSentimentLocal2.getUnmarkedPatternCounter();
|
|
|
|
|
if (UnmarkedPatternCounter1 > 0 && UnmarkedPatternCounter2 > 0) {
|
|
|
|
|
if (UnmarkedPatternCounter1 < 100 && UnmarkedPatternCounter2 < 100) {
|
|
|
|
|
if (UnmarkedPatternCounter1 * 2 > UnmarkedPatternCounter2 && UnmarkedPatternCounter2 * 2 > UnmarkedPatternCounter1) {
|
|
|
|
|
score += 2500;
|
|
|
|
|
} else if (UnmarkedPatternCounter1 * 5 < UnmarkedPatternCounter2 || UnmarkedPatternCounter2 * 5 < UnmarkedPatternCounter1) {
|
|
|
|
|
score -= 4000;
|
|
|
|
|
}
|
|
|
|
|
} else if (UnmarkedPatternCounter1 > 500 && UnmarkedPatternCounter2 > 500) {
|
|
|
|
|
score -= UnmarkedPatternCounter1 >= UnmarkedPatternCounter2 ? UnmarkedPatternCounter1 * 3 : UnmarkedPatternCounter2 * 3;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double markedContiniousCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
int MarkedContinuousCounter1 = cacheSentimentLocal1.getMarkedContinuousCounter();
|
|
|
|
|
int MarkedContinuousCounter2 = cacheSentimentLocal2.getMarkedContinuousCounter();
|
|
|
|
|
int MarkedContiniousCounter1Entries = cacheSentimentLocal1.getMarkedContiniousCounterEntries();
|
|
|
|
|
int MarkedContiniousCounter2Entries = cacheSentimentLocal2.getMarkedContiniousCounterEntries();
|
|
|
|
|
if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) {
|
|
|
|
|
if (MarkedContinuousCounter1 > MarkedContinuousCounter2 * 50 || MarkedContinuousCounter2 > MarkedContinuousCounter1 * 50) {
|
|
|
|
|
score -= MarkedContinuousCounter1 > MarkedContinuousCounter2 ? MarkedContinuousCounter1 * 120 : MarkedContinuousCounter2 * 120;
|
|
|
|
|
} else if (!Objects.equals(MarkedContiniousCounter1Entries, MarkedContiniousCounter2Entries)
|
|
|
|
|
&& (MarkedContinuousCounter1 * 2 >= MarkedContinuousCounter2 * MarkedContinuousCounter1)
|
|
|
|
|
|| (MarkedContinuousCounter2 * 2 >= MarkedContinuousCounter1 * MarkedContinuousCounter2)) {
|
|
|
|
|
score += 4500;
|
|
|
|
|
} else if (MarkedContiniousCounter1Entries == 0 || MarkedContiniousCounter2Entries == 0) {
|
|
|
|
|
score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500
|
|
|
|
|
: (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500;
|
|
|
|
|
}
|
|
|
|
|
if (MarkedContiniousCounter1Entries > 0 && MarkedContiniousCounter2Entries > 0 && MarkedContinuousCounter1 > 0
|
|
|
|
|
&& MarkedContinuousCounter2 > 0 && MarkedContinuousCounter1 < MarkedContinuousCounter2 * 10
|
|
|
|
|
&& MarkedContinuousCounter2 < MarkedContinuousCounter1 * 10) {
|
|
|
|
|
if (MarkedContiniousCounter1Entries > MarkedContiniousCounter2Entries * 5
|
|
|
|
|
|| MarkedContiniousCounter2Entries > MarkedContiniousCounter1Entries * 5
|
|
|
|
|
|| MarkedContiniousCounter1Entries * 5 < MarkedContinuousCounter1
|
|
|
|
|
|| MarkedContiniousCounter1Entries * 5 < MarkedContinuousCounter2
|
|
|
|
|
|| MarkedContiniousCounter2Entries * 5 < MarkedContinuousCounter1
|
|
|
|
|
|| MarkedContiniousCounter2Entries * 5 < MarkedContinuousCounter2) {
|
|
|
|
|
score -= MarkedContinuousCounter1 > MarkedContinuousCounter2 ? MarkedContinuousCounter1 * 400 : MarkedContinuousCounter2 * 400;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double strTokensMapScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
ConcurrentMap<Integer, String> strtokensMap = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
for (String strTokeniPart1 : cacheSentimentLocal1.getstrTokenGetiPart().values()) {
|
|
|
|
|
for (String strTokeniPart2 : cacheSentimentLocal2.getstrTokenGetiPart().values()) {
|
|
|
|
|
if (strTokeniPart1.equals(strTokeniPart2) && !strtokensMap.values().contains(strTokeniPart2)) {
|
|
|
|
|
strtokensMap.put(strtokensMap.size() + 1, strTokeniPart2);
|
|
|
|
|
score += 800;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
int tokenIPartSize1 = cacheSentimentLocal1.getstrTokenGetiPart().values().size();
|
|
|
|
|
int tokenIPartSize2 = cacheSentimentLocal2.getstrTokenGetiPart().values().size();
|
|
|
|
|
int strTokenMapSize = strtokensMap.values().size();
|
|
|
|
|
if (tokenIPartSize1 * 2 > tokenIPartSize2 && tokenIPartSize2 * 2 > tokenIPartSize1) {
|
|
|
|
|
score += tokenIPartSize1 > tokenIPartSize2 ? (tokenIPartSize1 - tokenIPartSize2) * 700 : (tokenIPartSize2 - tokenIPartSize1) * 700;
|
|
|
|
|
score += strTokenMapSize * 600;
|
|
|
|
|
} else if (tokenIPartSize1 > 0 && tokenIPartSize2 > 0) {
|
|
|
|
|
score -= tokenIPartSize1 > tokenIPartSize2 ? (tokenIPartSize1 - tokenIPartSize2) * 700 : (tokenIPartSize2 - tokenIPartSize1) * 700;
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double strTokenEntryScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
int tokenEntry1 = cacheSentimentLocal1.getstrTokenGetEntry().values().size();
|
|
|
|
|
int tokenEntry2 = cacheSentimentLocal2.getstrTokenGetEntry().values().size();
|
|
|
|
|
boolean boundaryLeaks = false;
|
|
|
|
|
int remnantCounter = 0;
|
|
|
|
|
if (tokenEntry1 * 2 != tokenEntry2 && tokenEntry2 * 2 != tokenEntry1) {
|
|
|
|
|
boundaryLeaks = true;
|
|
|
|
|
}
|
|
|
|
|
ConcurrentMap<Integer, String> entryTokenMap = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
for (String strTokenEntry1 : cacheSentimentLocal1.getstrTokenGetEntry().values()) {
|
|
|
|
|
for (String strTokenEntry2 : cacheSentimentLocal2.getstrTokenGetEntry().values()) {
|
|
|
|
|
if (!entryTokenMap.values().contains(strTokenEntry2)) {
|
|
|
|
|
if (strTokenEntry1.equals(strTokenEntry2)) {
|
|
|
|
|
score += boundaryLeaks ? 2500 : 2500 / 2;
|
|
|
|
|
} else if (!boundaryLeaks) {
|
|
|
|
|
score -= 450;
|
|
|
|
|
} else {
|
|
|
|
|
remnantCounter++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
entryTokenMap.put(entryTokenMap.size() + 1, strTokenEntry2);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
score += remnantCounter * 250;
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double strTokenMapTagsScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
ConcurrentMap<Integer, String> iTokenMapTagsMap = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
for (String strmapTag : cacheSentimentLocal1.getITokenMapTag().values()) {
|
|
|
|
|
for (String strmapTag1 : cacheSentimentLocal2.getITokenMapTag().values()) {
|
|
|
|
|
if (strmapTag.equals(strmapTag1) && !iTokenMapTagsMap.values().contains(strmapTag1)) {
|
|
|
|
|
score -= 1450;
|
|
|
|
|
iTokenMapTagsMap.put(iTokenMapTagsMap.size() + 1, strmapTag);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
int mapTagsize1 = cacheSentimentLocal1.getITokenMapTag().values().size();
|
|
|
|
|
int mapTagsize2 = cacheSentimentLocal2.getITokenMapTag().values().size();
|
|
|
|
|
int tokenTagMapSize = iTokenMapTagsMap.values().size();
|
|
|
|
|
if (mapTagsize1 != 0 && mapTagsize2 != 0) {
|
|
|
|
|
if (mapTagsize1 * 2 > mapTagsize2 && mapTagsize2 * 2 > mapTagsize1) {
|
|
|
|
|
score += mapTagsize1 > mapTagsize2 ? (mapTagsize1 - mapTagsize2) * 700 : (mapTagsize2 - mapTagsize1) * 700;
|
|
|
|
|
score += tokenTagMapSize * 600;
|
|
|
|
|
} else {
|
|
|
|
|
score -= mapTagsize1 > mapTagsize2 ? (mapTagsize1 - mapTagsize2) * 700 : (mapTagsize2 - mapTagsize1) * 700;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double tokenformSizeScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
int tokenform1size = cacheSentimentLocal1.getstrTokenForm().values().size();
|
|
|
|
|
int tokenform2size = cacheSentimentLocal2.getstrTokenForm().values().size();
|
|
|
|
|
if (tokenform1size > 0 || tokenform2size > 0) {
|
|
|
|
|
if (tokenform1size < tokenform2size * 5 && tokenform2size < tokenform1size * 5) {
|
|
|
|
|
for (String strTokenForm1itr1 : cacheSentimentLocal1.getstrTokenForm().values()) {
|
|
|
|
|
for (String strTokenForm1itr2 : cacheSentimentLocal2.getstrTokenForm().values()) {
|
|
|
|
|
if (strTokenForm1itr1.equals(strTokenForm1itr2)) {
|
|
|
|
|
score -= 1600;
|
|
|
|
|
} else {
|
|
|
|
|
score += 500;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} else if (tokenform1size > 0 && tokenform2size > 0) {
|
|
|
|
|
if (tokenform1size * 2 >= tokenform2size && tokenform2size * 2 >= tokenform1size) {
|
|
|
|
|
score += tokenform1size > tokenform2size ? tokenform1size * 600 : tokenform2size * 600;
|
|
|
|
|
} else if (tokenform1size * 4 <= tokenform2size || tokenform2size * 4 <= tokenform1size) {
|
|
|
|
|
score -= tokenform1size > tokenform2size ? (tokenform1size - tokenform2size) * 600 : (tokenform2size - tokenform1size) * 600;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double tokenStemmingMapScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
ConcurrentMap<Integer, String> tokenStemmingMap = new MapMaker().concurrencyLevel(2).makeMap();
|
|
|
|
|
for (String strTokenStem : cacheSentimentLocal1.getstrTokenStems().values()) {
|
|
|
|
|
for (String strTokenStem1 : cacheSentimentLocal2.getstrTokenStems().values()) {
|
|
|
|
|
if (strTokenStem.equals(strTokenStem1) && !tokenStemmingMap.values().contains(strTokenStem)) {
|
|
|
|
|
score += 500;
|
|
|
|
|
tokenStemmingMap.put(tokenStemmingMap.size() + 1, strTokenStem);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double inflectedCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
int inflectedCounterPositive1 = cacheSentimentLocal1.getInflectedCounterPositive();
|
|
|
|
|
int inflectedCounterPositive2 = cacheSentimentLocal2.getInflectedCounterPositive();
|
|
|
|
|
int inflectedCounterNegative = cacheSentimentLocal1.getInflectedCounterNegative() > cacheSentimentLocal2.getInflectedCounterNegative()
|
|
|
|
|
? cacheSentimentLocal1.getInflectedCounterNegative() - cacheSentimentLocal2.getInflectedCounterNegative()
|
|
|
|
|
: cacheSentimentLocal2.getInflectedCounterNegative() - cacheSentimentLocal1.getInflectedCounterNegative();
|
|
|
|
|
if ((inflectedCounterPositive1 + inflectedCounterPositive2) > inflectedCounterNegative && inflectedCounterNegative > 0) {
|
|
|
|
|
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 650;
|
|
|
|
|
}
|
|
|
|
|
if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) {
|
|
|
|
|
if (inflectedCounterPositive1 * 2 > inflectedCounterPositive2 && inflectedCounterPositive2 * 2 > inflectedCounterPositive1
|
|
|
|
|
&& inflectedCounterNegative > 0) {
|
|
|
|
|
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) * 150) - (inflectedCounterNegative * 10);
|
|
|
|
|
} else if (inflectedCounterPositive1 * 5 < inflectedCounterPositive2 || inflectedCounterPositive2 * 5 < inflectedCounterPositive1) {
|
|
|
|
|
score -= inflectedCounterPositive1 > inflectedCounterPositive2 ? (inflectedCounterPositive1 - inflectedCounterPositive2) * 400
|
|
|
|
|
: (inflectedCounterPositive2 - inflectedCounterPositive1) * 400;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (inflectedCounterPositive1 >= 15 && inflectedCounterPositive2 >= 15) {
|
|
|
|
|
score -= 4000;
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double annotatorCountScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
int anotatorcounter1 = cacheSentimentLocal1.getAnotatorcounter();
|
|
|
|
|
int anotatorcounter2 = cacheSentimentLocal2.getAnotatorcounter();
|
|
|
|
|
if (anotatorcounter1 > 1 && anotatorcounter2 > 1) {
|
|
|
|
|
if (anotatorcounter1 * 2 > anotatorcounter2 && anotatorcounter2 * 2 > anotatorcounter1) {
|
|
|
|
|
score += anotatorcounter1 > anotatorcounter2 ? (anotatorcounter1 - anotatorcounter2) * 700
|
|
|
|
|
: (anotatorcounter2 - anotatorcounter1) * 700;
|
|
|
|
|
} else if (anotatorcounter1 * 5 < anotatorcounter2 || anotatorcounter2 * 5 < anotatorcounter1) {
|
|
|
|
|
score -= anotatorcounter1 > anotatorcounter2 ? (anotatorcounter1 - anotatorcounter2) * 400 : (anotatorcounter2 - anotatorcounter1) * 400;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double tokensCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
int tokensCounter1 = cacheSentimentLocal1.getTokensCounter();
|
|
|
|
|
int tokensCounter2 = cacheSentimentLocal2.getTokensCounter();
|
|
|
|
|
if ((tokensCounter1 > 1 && tokensCounter2 > 1) && tokensCounter1 < tokensCounter2 * 5 && tokensCounter2 < tokensCounter1 * 5) {
|
|
|
|
|
if (tokensCounter1 > tokensCounter2 / 2 && tokensCounter2 > tokensCounter1 / 2 && tokensCounter1 < 10 && tokensCounter2 < 10) {
|
|
|
|
|
score += (tokensCounter1 + tokensCounter2) * 500;
|
|
|
|
|
} else {
|
|
|
|
|
score -= 500;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
int elseint = tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
|
|
|
|
|
if ((tokensCounter1 > tokensCounter2 * 5 || tokensCounter2 > tokensCounter1 * 5)
|
|
|
|
|
&& tokensCounter1 > 0 && tokensCounter2 > 0) {
|
|
|
|
|
score -= tokensCounter1 > tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
|
|
|
|
|
} else if (elseint > 0 && tokensCounter1 > 0 && tokensCounter2 > 0) {
|
|
|
|
|
if (elseint > 500 && elseint < 2000) {
|
|
|
|
|
score += elseint * 10;
|
|
|
|
|
} else {
|
|
|
|
|
score -= elseint * 15;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if (elseint > 0 && elseint < 1500 && (tokensCounter1 > 0 || tokensCounter2 > 0) && tokensCounter1 <= 2 && tokensCounter2 <= 2) {
|
|
|
|
|
score -= elseint * 1.5;
|
|
|
|
|
} else if (tokensCounter1 < 5 && tokensCounter2 < 5) {
|
|
|
|
|
score -= elseint * (tokensCounter1 + tokensCounter2);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private SentimentValueCache setupNEREntitiesAndTokenTags(CoreDocument pipelineCoreDcoument, SentimentValueCache cacheSentimentLocal) {
|
|
|
|
|
for (CoreEntityMention em : pipelineCoreDcoument.entityMentions()) {
|
|
|
|
|
Set<Map.Entry<String, Double>> entrySet = em.entityTypeConfidences().entrySet();
|
|
|
|
|
String entityType = em.entityType();
|
|
|
|
|
Double EntityConfidences = 0.0;
|
|
|
|
|
for (Map.Entry<String, Double> entries : entrySet) {
|
|
|
|
|
EntityConfidences = entries.getValue();
|
|
|
|
|
}
|
|
|
|
|
List<CoreLabel> tokens = em.tokens();
|
|
|
|
|
for (CoreLabel token : tokens) {
|
|
|
|
|
if (token != null) {
|
|
|
|
|
if (!cacheSentimentLocal.getnerEntityTokenTags().values().contains(token.tag())) {
|
|
|
|
|
if (entityType.equals("PERSON") && EntityConfidences > 0.80) {
|
|
|
|
|
cacheSentimentLocal.addnerEntityTokenTags(token.tag());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (!cacheSentimentLocal.getnerEntities1().values().contains(em.text())) {
|
|
|
|
|
cacheSentimentLocal.addNEREntities1(em.text());
|
|
|
|
|
cacheSentimentLocal.addNEREntities2(em.entityType());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return cacheSentimentLocal;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double nerEntitiesAndTokenScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
List<String> entsCollection = new ArrayList();
|
|
|
|
|
for (String strEnts1 : cacheSentimentLocal1.getnerEntities1().values()) {
|
|
|
|
|
for (String strEnts2 : cacheSentimentLocal2.getnerEntities2().values()) {
|
|
|
|
|
if (strEnts1.equalsIgnoreCase(strEnts2) && !entsCollection.contains(strEnts1)) {
|
|
|
|
|
score += 2500;
|
|
|
|
|
entsCollection.add(strEnts1);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
entsCollection = new ArrayList();
|
|
|
|
|
for (String strToken : cacheSentimentLocal1.getnerEntityTokenTags().values()) {
|
|
|
|
|
for (String strToken1 : cacheSentimentLocal2.getnerEntityTokenTags().values()) {
|
|
|
|
|
if (strToken.equalsIgnoreCase(strToken1) && !entsCollection.contains(strToken)) {
|
|
|
|
|
score += 2000;
|
|
|
|
|
entsCollection.add(strToken);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private SentimentValueCache setupStoWordTokensLemma(Annotation pipelineAnnotationSentiment, SentimentValueCache cacheSentimentLocal) {
|
|
|
|
|
String customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with";
|
|
|
|
|
List<CoreLabel> tokensSentiment = pipelineAnnotationSentiment.get(CoreAnnotations.TokensAnnotation.class);
|
|
|
|
|
Set<?> stopWords = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
|
|
|
|
|
Set<?> stopWordsCustom = StopwordAnnotator.getStopWordList(customStopWordList, true);
|
|
|
|
|
for (CoreLabel token : tokensSentiment) {
|
|
|
|
|
Pair<Boolean, Boolean> stopword = token.get(StopwordAnnotator.class);
|
|
|
|
|
String word = token.word().toLowerCase();
|
|
|
|
|
if (stopWords.contains(word) || stopWordsCustom.contains(word)) {
|
|
|
|
|
cacheSentimentLocal.addstopwordTokens(word);
|
|
|
|
|
}
|
|
|
|
|
String lemma = token.lemma().toLowerCase();
|
|
|
|
|
if (stopWords.contains(lemma) || stopWordsCustom.contains(lemma)) {
|
|
|
|
|
cacheSentimentLocal.addStopWordLemma(lemma);
|
|
|
|
|
}
|
|
|
|
|
if (stopword.first() && stopword.second()) {
|
|
|
|
|
cacheSentimentLocal.setPairCounter(cacheSentimentLocal.getPairCounter() + 1);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return cacheSentimentLocal;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double stopWordTokenLemmaScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
Collection<String> stopWordCollection = new ArrayList();
|
|
|
|
|
for (String stopwords1 : cacheSentimentLocal1.getStopwordTokens().values()) {
|
|
|
|
|
for (String stopwords2 : cacheSentimentLocal2.getStopwordTokens().values()) {
|
|
|
|
|
if (stopwords1.equals(stopwords2) && !stopWordCollection.contains(stopwords1)) {
|
|
|
|
|
score -= 50;
|
|
|
|
|
stopWordCollection.add(stopwords1);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
stopWordCollection = new ArrayList();
|
|
|
|
|
for (String stopwords1 : cacheSentimentLocal1.getStopWordLemma().values()) {
|
|
|
|
|
for (String stopwords2 : cacheSentimentLocal2.getStopWordLemma().values()) {
|
|
|
|
|
if (stopwords1.equals(stopwords2) && !stopWordCollection.contains(stopwords1)) {
|
|
|
|
|
score -= 50;
|
|
|
|
|
stopWordCollection.add(stopwords1);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double stopwordTokenPairCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
if (!cacheSentimentLocal1.getStopwordTokens().values().isEmpty() && !cacheSentimentLocal2.getStopwordTokens().values().isEmpty()) {
|
|
|
|
|
int stopwordsize1 = cacheSentimentLocal1.getStopwordTokens().values().size();
|
|
|
|
|
int stopwordsize2 = cacheSentimentLocal2.getStopwordTokens().values().size();
|
|
|
|
|
if (stopwordsize1 * 5 < stopwordsize2 || stopwordsize2 * 5 < stopwordsize1) {
|
|
|
|
|
score -= stopwordsize1 > stopwordsize2 ? (stopwordsize1 - stopwordsize2) * 850 : (stopwordsize2 - stopwordsize1) * 850;
|
|
|
|
|
} else if (stopwordsize1 == stopwordsize2 && stopwordsize1 > 1) {
|
|
|
|
|
score -= stopwordsize1 * 450;
|
|
|
|
|
} else if ((stopwordsize1 / 2 == stopwordsize2 || stopwordsize2 / 2 == stopwordsize1) && stopwordsize1 + stopwordsize2 >= 4) {
|
|
|
|
|
score -= 2500;
|
|
|
|
|
} else {
|
|
|
|
|
score += stopwordsize1 > stopwordsize2 ? (stopwordsize1 - stopwordsize2) * 850 : (stopwordsize2 - stopwordsize1) * 850;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
int pairCounter1 = cacheSentimentLocal1.getPairCounter();
|
|
|
|
|
int pairCounter2 = cacheSentimentLocal2.getPairCounter();
|
|
|
|
|
if (pairCounter1 > 0 && pairCounter2 > 0) {
|
|
|
|
|
if (pairCounter1 * 5 <= pairCounter2 || pairCounter2 * 5 <= pairCounter1) {
|
|
|
|
|
score -= pairCounter1 > pairCounter2 ? (pairCounter1 - pairCounter2) * 1500 : (pairCounter2 - pairCounter1) * 1500;
|
|
|
|
|
} else if (pairCounter1 == pairCounter2 && pairCounter1 > 1) {
|
|
|
|
|
score -= pairCounter1 * 450;
|
|
|
|
|
} else if ((pairCounter1 / 2 == pairCounter2 || pairCounter2 / 2 == pairCounter1) && pairCounter1 + pairCounter2 >= 4) {
|
|
|
|
|
score -= 2500;
|
|
|
|
|
} else {
|
|
|
|
|
score += pairCounter1 > pairCounter2 ? (pairCounter1 - pairCounter2) * 700 : (pairCounter2 - pairCounter1) * 700;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Double tgwListScoreIncrementer(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
|
|
|
|
|
AtomicInteger runCount = new AtomicInteger(0);
|
|
|
|
|
cacheSentimentLocal1.getTgwlistIndex().values().forEach(TaggedWord -> {
|
|
|
|
|
if (!cacheSentimentLocal2.getTgwlistIndex().values().contains(TaggedWord)) {
|
|
|
|
|
cacheSentimentLocal2.addTgwlistIndex(TaggedWord);
|
|
|
|
|
runCount.getAndIncrement();
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
score += runCount.get() * 64;
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public final SimilarityMatrix call() {
|
|
|
|
|
Double score = -100.0;
|
|
|
|
|
SentimentValueCache cacheSentimentLocal1 = null;
|
|
|
|
|
SentimentValueCache cacheSentimentLocal2 = null;
|
|
|
|
|
int counter1;
|
|
|
|
|
int counter2;
|
|
|
|
|
try {
|
|
|
|
|
if (cacheSentiment1 == null) {
|
|
|
|
|
cacheSentimentLocal1 = initializeCacheSetup(str, cacheSentimentLocal1);
|
|
|
|
|
}
|
|
|
|
|
if (cacheSentiment2 == null) {
|
|
|
|
|
cacheSentimentLocal2 = initializeCacheSetup(str1, cacheSentimentLocal2);
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
counter1 = cacheSentiment1 == null ? cacheSentimentLocal1.getCounter() : cacheSentiment1.getCounter();
|
|
|
|
|
counter2 = cacheSentiment2 == null ? cacheSentimentLocal2.getCounter() : cacheSentiment2.getCounter();
|
|
|
|
|
|
|
|
|
|
final int overValue = (counter1 >= counter2 ? counter1 - counter2 : counter2 - counter1) * 32;
|
|
|
|
|
score -= overValue;
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
if (cacheSentiment1 == null) {
|
|
|
|
|
ConcurrentMap<Integer, String> retrieveTGWListIndex = retrieveTGWListIndex(cacheSentimentLocal1.getTaggedwordlist());
|
|
|
|
|
for (String str : retrieveTGWListIndex.values()) {
|
|
|
|
|
cacheSentimentLocal1.addTgwlistIndex(str);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (cacheSentiment2 == null) {
|
|
|
|
|
ConcurrentMap<Integer, String> retrieveTGWListIndex = retrieveTGWListIndex(cacheSentimentLocal2.getTaggedwordlist());
|
|
|
|
|
for (String str : retrieveTGWListIndex.values()) {
|
|
|
|
|
cacheSentimentLocal2.addTgwlistIndex(str);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = tgwListScoreIncrementer(score, cacheSentiment1 == null
|
|
|
|
|
? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null ? cacheSentimentLocal2 : cacheSentiment2);
|
|
|
|
|
if (cacheSentiment1 == null) {
|
|
|
|
|
cacheSentimentLocal1 = GrammaticStructureSetup(cacheSentimentLocal1, pipelineAnnotation1);
|
|
|
|
|
}
|
|
|
|
|
if (cacheSentiment2 == null) {
|
|
|
|
|
cacheSentimentLocal2 = GrammaticStructureSetup(cacheSentimentLocal2, pipelineAnnotation2);
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
ConcurrentMap<Integer, Tree> sentenceConstituencyParseList1 = null;
|
|
|
|
|
ConcurrentMap<Integer, Tree> sentenceConstituencyParseList2 = null;
|
|
|
|
|
try {
|
|
|
|
|
sentenceConstituencyParseList2 = cacheSentiment2 == null
|
|
|
|
|
? cacheSentimentLocal2.getSentenceConstituencyParseList() : cacheSentiment2.getSentenceConstituencyParseList();
|
|
|
|
|
sentenceConstituencyParseList1 = cacheSentiment1 == null
|
|
|
|
|
? cacheSentimentLocal1.getSentenceConstituencyParseList() : cacheSentiment1.getSentenceConstituencyParseList();
|
|
|
|
|
score = iterateTrees(sentenceConstituencyParseList2, sentenceConstituencyParseList1, score);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
Collection<TypedDependency> allTypedDependencies2 = cacheSentiment2 == null ? cacheSentimentLocal2.getAllTypedDependencies()
|
|
|
|
|
: cacheSentiment2.getAllTypedDependencies();
|
|
|
|
|
Collection<TypedDependency> allTypedDependencies1 = cacheSentiment1 == null ? cacheSentimentLocal1.getAllTypedDependencies()
|
|
|
|
|
: cacheSentiment1.getAllTypedDependencies();
|
|
|
|
|
|
|
|
|
|
ConcurrentMap<Integer, GrammaticalStructure> grammaticalMap1 = cacheSentiment1 == null ? cacheSentimentLocal1.getGs() : cacheSentiment1.getGs();
|
|
|
|
|
ConcurrentMap<Integer, GrammaticalStructure> grammaticalMap2 = cacheSentiment2 == null ? cacheSentimentLocal2.getGs() : cacheSentiment2.getGs();
|
|
|
|
|
score = typeDependenciesGrammaticalRelation(allTypedDependencies1, allTypedDependencies2, score, grammaticalMap1, grammaticalMap2,
|
|
|
|
|
sentenceConstituencyParseList1, sentenceConstituencyParseList2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
if (cacheSentiment1 == null) {
|
|
|
|
|
cacheSentimentLocal1 = sentimentCoreAnnotationSetup(pipelineAnnotation1Sentiment, cacheSentimentLocal1);
|
|
|
|
|
}
|
|
|
|
|
if (cacheSentiment2 == null) {
|
|
|
|
|
cacheSentimentLocal2 = sentimentCoreAnnotationSetup(pipelineAnnotation2Sentiment, cacheSentimentLocal2);
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
final ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist1 = cacheSentiment1 == null
|
|
|
|
|
? cacheSentimentLocal1.getSimpleSMXlist() : cacheSentiment1.getSimpleSMXlist();
|
|
|
|
|
final ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist2 = cacheSentiment2 == null
|
|
|
|
|
? cacheSentimentLocal2.getSimpleSMXlist() : cacheSentiment2.getSimpleSMXlist();
|
|
|
|
|
final ConcurrentMap<Integer, SimpleMatrix> simpleSMXlistVector1 = cacheSentiment1 == null
|
|
|
|
|
? cacheSentimentLocal1.getSimpleSMXlistVector() : cacheSentiment1.getSimpleSMXlistVector();
|
|
|
|
|
final ConcurrentMap<Integer, SimpleMatrix> simpleSMXlistVector2 = cacheSentiment2 == null
|
|
|
|
|
? cacheSentimentLocal2.getSimpleSMXlistVector() : cacheSentiment2.getSimpleSMXlistVector();
|
|
|
|
|
score = simpleRNNMatrixCalculations(score, simpleSMXlist1, simpleSMXlist2);
|
|
|
|
|
score = simpleRNNMaxtrixVectors(score, simpleSMXlistVector1, simpleSMXlistVector2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
int sentiment1 = cacheSentiment1 == null ? cacheSentimentLocal1.getRnnPrediectClassMap().size() : cacheSentiment1.getRnnPrediectClassMap().size();
|
|
|
|
|
int sentiment2 = cacheSentiment2 == null ? cacheSentimentLocal2.getRnnPrediectClassMap().size() : cacheSentiment2.getRnnPrediectClassMap().size();
|
|
|
|
|
score -= (sentiment1 > sentiment2 ? sentiment1 - sentiment2 : sentiment2 - sentiment1) * 500;
|
|
|
|
|
Map.Entry<Double, Map.Entry<SentimentValueCache, SentimentValueCache>> classifyRawEvaluationEntry = classifyRawEvaluation(score, cacheSentimentLocal1,
|
|
|
|
|
cacheSentimentLocal2);
|
|
|
|
|
score = classifyRawEvaluationEntry.getKey();
|
|
|
|
|
if (cacheSentiment1 == null) {
|
|
|
|
|
cacheSentimentLocal1 = classifyRawEvaluationEntry.getValue().getKey();
|
|
|
|
|
}
|
|
|
|
|
if (cacheSentiment2 == null) {
|
|
|
|
|
cacheSentimentLocal2 = classifyRawEvaluationEntry.getValue().getValue();
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
if (cacheSentiment1 == null) {
|
|
|
|
|
cacheSentimentLocal1 = setupMainSentimentandLongestVal(pipelineAnnotation1Sentiment, cacheSentimentLocal1);
|
|
|
|
|
}
|
|
|
|
|
if (cacheSentiment2 == null) {
|
|
|
|
|
cacheSentimentLocal2 = setupMainSentimentandLongestVal(pipelineAnnotation2Sentiment, cacheSentimentLocal2);
|
|
|
|
|
}
|
|
|
|
|
score = sentimentMatrixVariances(score, cacheSentiment1 == null ? cacheSentimentLocal1.getLongest() : cacheSentiment1.getLongest(),
|
|
|
|
|
cacheSentiment2 == null ? cacheSentimentLocal2.getLongest() : cacheSentiment2.getLongest(), cacheSentiment1 == null
|
|
|
|
|
? cacheSentimentLocal1.getMainSentiment() : cacheSentiment1.getMainSentiment(), cacheSentiment2 == null
|
|
|
|
|
? cacheSentimentLocal2.getMainSentiment() : cacheSentiment2.getMainSentiment());
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
if (cacheSentiment1 == null) {
|
|
|
|
|
cacheSentimentLocal1 = jmweAnnotationSetup(jmweStrAnnotation1, cacheSentimentLocal1);
|
|
|
|
|
}
|
|
|
|
|
if (cacheSentiment2 == null) {
|
|
|
|
|
cacheSentimentLocal2 = jmweAnnotationSetup(jmweStrAnnotation2, cacheSentimentLocal2);
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
SentimentValueCache scoringCache1 = cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1;
|
|
|
|
|
SentimentValueCache scoringCache2 = cacheSentiment2 == null ? cacheSentimentLocal2 : cacheSentiment2;
|
|
|
|
|
try {
|
|
|
|
|
score = entryCountsRelation(score, scoringCache1, scoringCache2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = entryCountsScoring(score, scoringCache1, scoringCache2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = tokenEntryPosScoring(score, scoringCache1, scoringCache2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = unmarkedPatternCounterScoring(score, scoringCache1, scoringCache2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = markedContiniousCounterScoring(score, scoringCache1, scoringCache2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = strTokensMapScoring(score, scoringCache1, scoringCache2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = strTokenEntryScoring(score, scoringCache1, scoringCache2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = strTokenMapTagsScoring(score, scoringCache1, scoringCache2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = tokenformSizeScoring(score, scoringCache1, scoringCache2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = tokenStemmingMapScoring(score, scoringCache1, scoringCache2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = inflectedCounterScoring(score, scoringCache1, scoringCache2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = annotatorCountScoring(score, scoringCache1, scoringCache2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = tokensCounterScoring(score, scoringCache1, scoringCache2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
LevenshteinDistance leven = new LevenshteinDistance(str, str1);
|
|
|
|
|
double SentenceScoreDiff = leven.computeLevenshteinDistance();
|
|
|
|
|
SentenceScoreDiff *= 15;
|
|
|
|
|
score -= SentenceScoreDiff;
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
if (cacheSentiment1 == null) {
|
|
|
|
|
cacheSentimentLocal1 = setupNEREntitiesAndTokenTags(pipelineCoreDcoument1, cacheSentimentLocal1);
|
|
|
|
|
}
|
|
|
|
|
if (cacheSentiment2 == null) {
|
|
|
|
|
cacheSentimentLocal2 = setupNEREntitiesAndTokenTags(pipelineCoreDcoument2, cacheSentimentLocal2);
|
|
|
|
|
}
|
|
|
|
|
score = nerEntitiesAndTokenScoring(score, cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null
|
|
|
|
|
? cacheSentimentLocal2 : cacheSentiment2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
if (cacheSentiment1 == null) {
|
|
|
|
|
cacheSentimentLocal1 = setupStoWordTokensLemma(pipelineAnnotation1Sentiment, cacheSentimentLocal1);
|
|
|
|
|
}
|
|
|
|
|
if (cacheSentiment2 == null) {
|
|
|
|
|
cacheSentimentLocal2 = setupStoWordTokensLemma(pipelineAnnotation2Sentiment, cacheSentimentLocal2);
|
|
|
|
|
}
|
|
|
|
|
score = stopWordTokenLemmaScoring(score, cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null
|
|
|
|
|
? cacheSentimentLocal2 : cacheSentiment2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
score = stopwordTokenPairCounterScoring(score, cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null
|
|
|
|
|
? cacheSentimentLocal2 : cacheSentiment2);
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
smxParam.setDistance(score);
|
|
|
|
|
try {
|
|
|
|
|
if (cacheSentiment1 == null) {
|
|
|
|
|
smxParam.setCacheValue1(cacheSentimentLocal1);
|
|
|
|
|
}
|
|
|
|
|
if (cacheSentiment2 == null) {
|
|
|
|
|
smxParam.setCacheValue2(cacheSentimentLocal2);
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
return smxParam;
|
|
|
|
|
}
|
|
|
|
|
}
|
