package FunctionLayer.StanfordParser; import FunctionLayer.LevenshteinDistance; import FunctionLayer.Datahandler; import FunctionLayer.SimilarityMatrix; import FunctionLayer.StopwordAnnotator; import com.google.common.collect.MapMaker; import edu.mit.jmwe.data.IMWE; import edu.mit.jmwe.data.IMWEDesc; import edu.mit.jmwe.data.IToken; import edu.stanford.nlp.ie.AbstractSequenceClassifier; import edu.stanford.nlp.ling.CoreAnnotations; import edu.stanford.nlp.ling.CoreLabel; import edu.stanford.nlp.ling.HasWord; import edu.stanford.nlp.ling.IndexedWord; import edu.stanford.nlp.ling.JMWEAnnotation; import edu.stanford.nlp.ling.TaggedWord; import edu.stanford.nlp.neural.rnn.RNNCoreAnnotations; import edu.stanford.nlp.pipeline.Annotation; import edu.stanford.nlp.pipeline.CoreDocument; import edu.stanford.nlp.pipeline.CoreEntityMention; import edu.stanford.nlp.pipeline.StanfordCoreNLP; import edu.stanford.nlp.process.CoreLabelTokenFactory; import edu.stanford.nlp.process.DocumentPreprocessor; import edu.stanford.nlp.process.PTBTokenizer; import edu.stanford.nlp.process.TokenizerFactory; import edu.stanford.nlp.sentiment.SentimentCoreAnnotations; import edu.stanford.nlp.sequences.DocumentReaderAndWriter; import edu.stanford.nlp.tagger.maxent.MaxentTagger; import edu.stanford.nlp.trees.Constituent; import edu.stanford.nlp.trees.GrammaticalRelation; import edu.stanford.nlp.trees.GrammaticalStructure; import edu.stanford.nlp.trees.GrammaticalStructureFactory; import edu.stanford.nlp.trees.Tree; import edu.stanford.nlp.trees.TreeCoreAnnotations; import edu.stanford.nlp.trees.TypedDependency; import edu.stanford.nlp.trees.tregex.gui.Tdiff; import edu.stanford.nlp.util.CoreMap; import edu.stanford.nlp.util.Pair; import java.io.StringReader; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.OptionalDouble; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.analysis.core.StopAnalyzer; import org.ejml.simple.SimpleMatrix; /* * To change this license header, choose License Headers in Project Properties. * To change this template file, choose Tools | Templates * and open the template in the editor. */ /** * * @author install1 */ public class SentimentAnalyzerTest implements Callable { private final SimilarityMatrix smxParam; private final String str; private final String str1; private final MaxentTagger tagger; private final GrammaticalStructureFactory gsf; private final AbstractSequenceClassifier classifier; private final Annotation jmweStrAnnotation1; private final Annotation jmweStrAnnotation2; private final Annotation pipelineAnnotation1; private final Annotation pipelineAnnotation2; private final Annotation pipelineAnnotation1Sentiment; private final Annotation pipelineAnnotation2Sentiment; private final CoreDocument pipelineCoreDcoument1; private final CoreDocument pipelineCoreDcoument2; private SentimentValueCache cacheSentiment1; private SentimentValueCache cacheSentiment2; public final SentimentValueCache getCacheSentiment1() { return cacheSentiment1; } public final SentimentValueCache getCacheSentiment2() { return cacheSentiment2; } public SentimentAnalyzerTest(String str, String str1, SimilarityMatrix smxParam, Annotation str1Annotation, Annotation str2Annotation, Annotation strPipeline1, Annotation strPipeline2, Annotation strPipeSentiment1, Annotation strPipeSentiment2, CoreDocument pipelineCoreDcoument1, CoreDocument pipelineCoreDcoument2, SentimentValueCache cacheValue1, SentimentValueCache cacheValue2) { this.str = str; this.str1 = str1; this.smxParam = smxParam; this.tagger = Datahandler.getTagger(); this.gsf = Datahandler.getGsf(); this.classifier = Datahandler.getClassifier(); this.jmweStrAnnotation1 = str1Annotation; this.jmweStrAnnotation2 = str2Annotation; this.pipelineAnnotation1 = strPipeline1; this.pipelineAnnotation2 = strPipeline2; this.pipelineAnnotation1Sentiment = strPipeSentiment1; this.pipelineAnnotation2Sentiment = strPipeSentiment2; this.pipelineCoreDcoument1 = pipelineCoreDcoument1; this.pipelineCoreDcoument2 = pipelineCoreDcoument2; this.cacheSentiment1 = cacheValue1; this.cacheSentiment2 = cacheValue2; } private List> getTaggedWordList(String message) { List> taggedwordlist = new ArrayList(); DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(message)); TokenizerFactory ptbTokenizerFactory = PTBTokenizer.factory(new CoreLabelTokenFactory(), "untokenizable=firstDelete"); //noneDelete tokenizer.setTokenizerFactory(ptbTokenizerFactory); for (final List sentence : tokenizer) { taggedwordlist.add(tagger.tagSentence(sentence)); } return taggedwordlist; } private int tokenizeCounting(List> taggedwordlist) { int counter = 0; Collection taggedCollection = new ArrayList(); for (List taggedList : taggedwordlist) { counter += taggedList.size(); } return counter; } private ConcurrentMap retrieveTGWListIndex(List> taggedwordlist) { ConcurrentMap tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap(); taggedwordlist.forEach((TGWList) -> { TGWList.forEach((TaggedWord) -> { if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) { tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag()); } }); }); return tgwlistIndex; } private Double iterateTrees(ConcurrentMap sentenceConstituencyParseList2, ConcurrentMap sentenceConstituencyParseList1, Double score) { double preConstituentsScore = score; ConcurrentMap constituentsMap = new MapMaker().concurrencyLevel(4).makeMap(); int constituencySize = sentenceConstituencyParseList1.size() + sentenceConstituencyParseList2.size(); for (final Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2.values()) { int constiRelationsize = 0; for (final Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1.values()) { Set constinuent1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse2); Set constinuent2 = Tdiff.markDiff(sentenceConstituencyParse2, sentenceConstituencyParse1); ConcurrentMap constiLabels = new MapMaker().concurrencyLevel(2).makeMap(); for (final Constituent consti : constinuent1) { for (final Constituent consti1 : constinuent2) { if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) { constiLabels.put(constiLabels.size(), consti.value()); constiRelationsize++; } } } int constituents1 = constinuent1.size() - constiRelationsize; int constituents2 = constinuent2.size() - constiRelationsize; constituentsMap.put(constituentsMap.size(), constituents1); constituentsMap.put(constituentsMap.size(), constituents2); constituentsMap.put(constituentsMap.size(), constiRelationsize); if (constituentsMap.size() < 4) { if ((constituents1 * 5 < constituents2 || constituents2 * 5 < constituents1) && constituents1 > 0 && constituents2 > 0) { score -= (constituents1 + constituents2) * 200; } else if ((constituents1 == 0 || constituents2 == 0) && (constituents1 >= constituents2 + 4 || constituents2 >= constituents1 + 4)) { score -= constituents1 > constituents2 ? constituents1 * 500 : constituents2 * 500; } else if (constiRelationsize >= constituents1 + constituents2 && (constituents1 > 2 && constituents2 > 2)) { score += (constiRelationsize + constituents1 + constituents2) * 350; } else if (constituents1 >= 2 && constituents2 >= 2 && constituents1 * 1.5 > constituents2 && constituents2 * 1.5 > constituents1) { if (constituents1 == constituents2 && constiRelationsize - constituents1 == 1) { score += (constiRelationsize + constituents1 + constituents2) * 1550; } else if (constiRelationsize >= constituents1 && constituents1 == constituents2) { score -= (constiRelationsize + constituents1 + constituents2) * 550; } else if (constiRelationsize < constituents1 && constiRelationsize < constituents2) { score += 800; } else if ((constiRelationsize == constituents1 || constiRelationsize == constituents2) && constituents1 * 1.5 > constituents2 && constituents2 * 1.5 > constituents1) { score += (constiRelationsize + constituents1 + constituents2) * 350; } } else if (constiRelationsize > constituents1 + constituents2) { score += 2500; } else if (constiRelationsize * 5 < constituents1 || constiRelationsize * 5 < constituents2) { score -= (constituents1 + constituents2) * 400; } } else { score = preConstituentsScore; int n1 = constituentsMap.get(0); int n2 = constituentsMap.get(1); int n3 = constituentsMap.get(2); int cap = 0; if (n1 > n2 && n1 > n3) { cap = n1; } else if (n2 > n3 && n2 > n1) { cap = n2; } else { cap = n3; } int overheat = 0; for (int iterator = 3; iterator < constituentsMap.size(); iterator++) { Integer getConstituent = constituentsMap.get(iterator); if (getConstituent > cap) { overheat++; } } if (overheat > 1) { score -= overheat * 800; } else { score += 1300; } } } } if (constituencySize > 10) { score -= constituencySize * 400; } return score; } private Double typeDependenciesGrammaticalRelation(Collection allTypedDependencies1, Collection allTypedDependencies2, Double score, ConcurrentMap grammaticalMap1, ConcurrentMap grammaticalMap2, ConcurrentMap sentenceConstituencyParseList1, ConcurrentMap sentenceConstituencyParseList2) { ConcurrentMap alltypeDepsSizeMap = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap summationMap = new MapMaker().concurrencyLevel(2).makeMap(); int relationApplicable1 = 0; int relationApplicable2 = 0; int grammaticalRelation1 = 0; int grammaticalRelation2 = 0; List treeCollectionGramatical = new ArrayList(); List treeCollectionReln = new ArrayList(); for (TypedDependency TDY1 : allTypedDependencies1) { IndexedWord dep = TDY1.dep(); IndexedWord gov = TDY1.gov(); for (GrammaticalStructure gs : grammaticalMap1.values()) { GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep); for (Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2.values()) { if (grammaticalRelation.isApplicable(sentenceConstituencyParse2) && !treeCollectionGramatical.contains(sentenceConstituencyParse2)) { score += 700; grammaticalRelation1++; treeCollectionGramatical.add(sentenceConstituencyParse2); } GrammaticalRelation reln = TDY1.reln(); if (reln.isApplicable(sentenceConstituencyParse2) && !treeCollectionReln.contains(sentenceConstituencyParse2)) { score += 525; relationApplicable1++; treeCollectionReln.add(sentenceConstituencyParse2); } } } } treeCollectionGramatical = new ArrayList(); treeCollectionReln = new ArrayList(); for (TypedDependency TDY : allTypedDependencies2) { IndexedWord dep = TDY.dep(); IndexedWord gov = TDY.gov(); for (GrammaticalStructure gs : grammaticalMap2.values()) { GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep); for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1.values()) { if (grammaticalRelation.isApplicable(sentenceConstituencyParse1) && !treeCollectionGramatical.contains(sentenceConstituencyParse1)) { score += 700; grammaticalRelation2++; treeCollectionGramatical.add(sentenceConstituencyParse1); } GrammaticalRelation reln = TDY.reln(); //sentenceConstituencyParse1 if (reln.isApplicable(sentenceConstituencyParse1) && !treeCollectionReln.contains(sentenceConstituencyParse1)) { score += 525; relationApplicable2++; treeCollectionReln.add(sentenceConstituencyParse1); } } } } if ((grammaticalRelation1 == 0 && grammaticalRelation2 > 4) || (grammaticalRelation2 == 0 && grammaticalRelation1 > 4)) { score -= 3450; } if (!allTypedDependencies1.isEmpty() || !allTypedDependencies2.isEmpty()) { int allTypeDep1 = allTypedDependencies1.size(); int allTypeDep2 = allTypedDependencies2.size(); if (allTypeDep1 <= allTypeDep2 * 5 && allTypeDep2 <= allTypeDep1 * 5) { if (allTypeDep1 > 0 && allTypeDep2 > 0 && (allTypeDep1 >= 5 || allTypeDep2 >= 5)) { if ((allTypeDep1 + 1 == allTypeDep2 || allTypeDep2 + 1 == allTypeDep1)) { score += allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 160 : (allTypeDep2 - allTypeDep1) * 160; } else if (allTypeDep1 * 2.5 >= allTypeDep2 && allTypeDep2 * 2.5 >= allTypeDep1) { score += allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 260 : (allTypeDep2 - allTypeDep1) * 260; } else if (allTypeDep1 >= 5 && allTypeDep1 <= 10 && allTypeDep2 >= 5 && allTypeDep2 <= 10) { score += allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 260 : (allTypeDep2 - allTypeDep1) * 260; } else { score -= allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 600 : (allTypeDep2 - allTypeDep1) * 600; } alltypeDepsSizeMap.put(alltypeDepsSizeMap.size() + 1, allTypeDep1); alltypeDepsSizeMap.put(alltypeDepsSizeMap.size() + 1, allTypeDep2); } } if (allTypeDep1 >= 5 && allTypeDep2 >= 5) { int largerTypeDep = allTypeDep1 > allTypeDep2 ? allTypeDep1 : allTypeDep2; int smallerTypeDep = allTypeDep1 < allTypeDep2 ? allTypeDep1 : allTypeDep2; int summation = (largerTypeDep * largerTypeDep) - (smallerTypeDep * smallerTypeDep); if (summation / largerTypeDep < 15.0 && summation / largerTypeDep > 10.0 && smallerTypeDep * 2 > largerTypeDep && !summationMap.values().contains(summation)) { score += summation * 80; summationMap.put(summationMap.size() + 1, summation); } else if (largerTypeDep == smallerTypeDep) { score += 2500; } } if (relationApplicable1 > 0 && relationApplicable2 > 0 && relationApplicable1 != relationApplicable2) { score += 600; } else if (allTypeDep1 * 5 < allTypeDep2 || allTypeDep2 * 5 < allTypeDep1) { score -= allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * (allTypeDep2 * 450) : (allTypeDep2 - allTypeDep1) * (allTypeDep1 * 450); } if (relationApplicable1 > 1 && relationApplicable2 > 1 && relationApplicable1 * 3 > relationApplicable2 && relationApplicable2 * 3 > relationApplicable1) { score += relationApplicable1 > relationApplicable2 ? (relationApplicable1 - relationApplicable2) * 1500 : (relationApplicable2 - relationApplicable1) * 1500; } else if (relationApplicable1 * 5 < relationApplicable2 || relationApplicable2 * 5 < relationApplicable1) { score -= relationApplicable1 > relationApplicable2 ? (relationApplicable1 - relationApplicable2) * 500 : (relationApplicable2 - relationApplicable1) * 500; } if (grammaticalRelation1 > 0 && grammaticalRelation2 > 0 && grammaticalRelation1 * 3 > grammaticalRelation2 && grammaticalRelation2 * 3 > grammaticalRelation1) { score += grammaticalRelation1 > grammaticalRelation2 ? (grammaticalRelation1 - grammaticalRelation2) * 1500 : (grammaticalRelation2 - grammaticalRelation1) * 1500; } else if (grammaticalRelation1 * 5 < grammaticalRelation2 || grammaticalRelation2 * 5 < grammaticalRelation1) { score -= grammaticalRelation1 > grammaticalRelation2 ? (grammaticalRelation1 - grammaticalRelation2) * 500 : (grammaticalRelation2 - grammaticalRelation1) * 500; } } ConcurrentMap filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap(); AtomicInteger runCount1 = new AtomicInteger(0); for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1.values()) { for (Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2.values()) { sentenceConstituencyParse1.taggedLabeledYield().forEach((LBW) -> { sentenceConstituencyParse2.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma()) && !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> { filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma()); return _item; }).forEachOrdered((_item) -> { runCount1.getAndIncrement(); }); }); } } score += runCount1.get() * 250; int typeSizeSmallest = 100; int typeSizeLargest = 0; for (Integer i : alltypeDepsSizeMap.values()) { if (i > typeSizeLargest) { typeSizeLargest = i; } if (i < typeSizeSmallest) { typeSizeSmallest = i; } } if (typeSizeLargest >= typeSizeSmallest * 3) { score -= typeSizeLargest * 160; } typeSizeLargest = 0; typeSizeSmallest = 100; for (int i : summationMap.values()) { if (i > typeSizeLargest) { typeSizeLargest = i; } if (i < typeSizeSmallest) { typeSizeSmallest = i; } } if (typeSizeLargest >= typeSizeSmallest * 3) { score -= typeSizeLargest * 160; } return score; } private Double simpleRNNMatrixCalculations(Double score, ConcurrentMap simpleSMXlist1, ConcurrentMap simpleSMXlist2) { List iteratedDoubleList = new ArrayList(); List iterateddotPredictions = new ArrayList(); double dotpredictionTransfer = 0.0; int iterationOverHeat = 0; double scoreFallback = score; for (SimpleMatrix simpleSMX2 : simpleSMXlist2.values()) { ConcurrentMap AccumulateDotMap = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap subtractorMap = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap dotPredictions = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap DotOverTransfer = new MapMaker().concurrencyLevel(2).makeMap(); Double totalSubtraction = 0.0; Double largest = 10.0; Double shortest = 100.0; for (SimpleMatrix simpleSMX1 : simpleSMXlist1.values()) { double dotPrediction2 = simpleSMX2.dot(simpleSMX1) * 100; double dotPrediction1 = simpleSMX1.dot(simpleSMX2) * 100; AccumulateDotMap.put(AccumulateDotMap.size() + 1, dotPrediction1); AccumulateDotMap.put(AccumulateDotMap.size() + 1, dotPrediction2); double subtracter1 = dotPrediction1 > 50 ? dotPrediction1 - 100 : dotPrediction1 > 0 ? 100 - dotPrediction1 : 0; double subtracter2 = dotPrediction2 > 50 ? dotPrediction2 - 100 : dotPrediction2 > 0 ? 100 - dotPrediction2 : 0; subtractorMap.put(subtractorMap.size() + 1, subtracter1); subtractorMap.put(subtractorMap.size() + 1, subtracter2); dotpredictionTransfer = dotPrediction1; if (!dotPredictions.values().contains(dotPrediction1)) { for (Double transferDots : DotOverTransfer.values()) { if (transferDots == dotPrediction1) { totalSubtraction += transferDots; } else { score -= subtracter1 * 25; } } DotOverTransfer.put(DotOverTransfer.size(), dotPrediction1); } else { subtracter1 -= 100; subtracter1 *= 25; score += subtracter1 * dotPrediction1; } dotPredictions.put(dotPredictions.size() + 1, dotPrediction1); if (!dotPredictions.values().contains(dotPrediction2)) { for (Double transferDots : DotOverTransfer.values()) { if (transferDots == dotPrediction2) { totalSubtraction += transferDots; } else { score -= subtracter1 * 25; } } DotOverTransfer.put(DotOverTransfer.size(), dotPrediction2); if (dotPrediction2 > largest) { largest = dotPrediction2; } if (dotPrediction2 < shortest) { shortest = dotPrediction2; } Double dotPredictionIntervalDifference = largest - shortest; subtracter2 *= 25; if (dotPredictionIntervalDifference < 5.0) { if (dotPredictions.values().size() > 0) { if (subtracter2 > 0) { score -= subtracter2; } else { score += subtracter2; } } } else { score -= subtracter2 / 10; } } else { if (subtracter2 > 55.0 && subtracter2 < 82.0) { if (dotPrediction2 > 30 && dotPrediction2 < 60) { if (dotpredictionTransfer != 0.0 && (subtracter2 / dotPrediction2 < 1.3 || (subtracter2 / dotPrediction2 > 1.9 && subtracter2 / dotPrediction2 < 1.99))) { if (subtracter2 / dotPrediction2 < 1.248 && subtracter2 / dotPrediction2 > 1.238) { score -= 2500; } else { score += 4500; } } else if (dotpredictionTransfer != 0.0 && subtracter2 / dotPrediction2 > 1.6 && subtracter2 / dotPrediction2 < 1.95) { if (subtracter2 > 61.1 && subtracter2 < 61.9 && dotPrediction2 > 37.5 && dotPrediction2 < 38.2) { score += 4500; } else { score -= 4500; } } else if (!iterateddotPredictions.contains(dotPrediction2)) { score -= subtracter2 * dotPrediction2; // += iterateddotPredictions.add(dotPrediction2); } else { score -= 550; } } else if (dotPrediction2 < 25 && subtracter2 < 75) { score -= dotPrediction2 * 250; } else if (subtracter2 > 75.0) { if (dotPrediction2 > 20.0 && dotPrediction2 < 23.0) { score += 3500; } else { if (subtracter2 < 75.9 && dotPrediction2 > 24.0 && dotPrediction2 < 24.9) { score += (subtracter2 * dotPrediction2) * 5; } else { score -= 4500; } } } } } dotPredictions.put(dotPredictions.size() + 1, dotPrediction2); iterationOverHeat++; } Double subTracPre = 0.0; for (Double subtractors : subtractorMap.values()) { if (Objects.equals(subTracPre, subtractors)) { if (subTracPre > 43.5 && subTracPre < 50.0) { score += (subTracPre * 15) / subtractorMap.values().size(); } else if (subTracPre > 60.0 && subTracPre < 66.5) { score += (subTracPre * 15) / subtractorMap.values().size(); } } else if (subTracPre > 75.0 && subTracPre < 90.0) { score += (subTracPre * 50) / subtractorMap.values().size(); } else if (subTracPre >= 55.0) { score -= 2800; } else if (subTracPre < -25.0 && subTracPre > -45.0) { score += subTracPre * 100; } subTracPre = subtractors; } if (totalSubtraction > 45.0) { score -= totalSubtraction * 25; } else { score += totalSubtraction * 25; } Double preAccumulatorDot = 0.0; Double postAccumulatorDot = 0.0; for (Double accumulators : AccumulateDotMap.values()) { if (Objects.equals(preAccumulatorDot, accumulators)) { if (Objects.equals(postAccumulatorDot, accumulators)) { score -= 1400; } postAccumulatorDot = accumulators; } preAccumulatorDot = accumulators; } subTracPre = 0.0; for (Double subtractors : subtractorMap.values()) { if (Objects.equals(subTracPre, subtractors) && subTracPre != 0.0) { if (!iteratedDoubleList.contains(subTracPre)) { score += 500; iteratedDoubleList.add(subTracPre); } else { score -= 150; } } subTracPre = subtractors; } } if (iterationOverHeat > 12) { score = scoreFallback; score -= 2500; } return score; } private Double simpleRNNMaxtrixVectors(Double score, ConcurrentMap simpleSMXlistVector1, ConcurrentMap simpleSMXlistVector2) { ConcurrentMap elementSumCounter = new MapMaker().concurrencyLevel(3).makeMap(); ConcurrentMap dotMap = new MapMaker().concurrencyLevel(3).makeMap(); ConcurrentMap elementSumMap = new MapMaker().concurrencyLevel(3).makeMap(); ConcurrentMap dotSumMap = new MapMaker().concurrencyLevel(3).makeMap(); Double preDot = 0.0; Double postDot = 0.0; int iterateSize = simpleSMXlistVector1.values().size() + simpleSMXlistVector2.values().size(); for (SimpleMatrix simpleSMX2 : simpleSMXlistVector2.values()) { for (SimpleMatrix simpleSMX1 : simpleSMXlistVector1.values()) { double dot2 = simpleSMX2.dot(simpleSMX1); double elementSum2 = simpleSMX2.kron(simpleSMX1).elementSum(); double dot1 = simpleSMX1.dot(simpleSMX2); double elementSum1 = simpleSMX1.kron(simpleSMX2).elementSum(); if (preDot == dot2) { if (postDot == dot2) { score -= 500; } postDot = dot2; } if (preDot == dot1) { if (postDot == dot1) { score -= 500; } postDot = dot1; } preDot = dot1; elementSum1 = Math.round(elementSum1 * 100.0) / 100.0; elementSumCounter.put(elementSumCounter.size() + 1, elementSum1); dotMap.put(dotMap.size() + 1, dot1); preDot = dot2; elementSum2 = Math.round(elementSum2 * 100.0) / 100.0; elementSumCounter.put(elementSumCounter.size() + 1, elementSum2); dotMap.put(dotMap.size() + 1, dot2); if (!dotSumMap.values().contains(dot1)) { if (dot1 < 0.1 && dot1 > 0.050) { score += 256; } if (dot1 > 0.44 && dot1 < 0.60) { score -= 2400; } else if (dot1 > 0.40 && dot1 < 0.445) { score += 3600; } dotSumMap.put(dotSumMap.size() + 1, dot1); } else { score -= 50; } if (!elementSumMap.values().contains(elementSum1)) { if (elementSum1 < 0.01 && elementSum1 > 0.00) { score += 1300; } else if (elementSum1 > 0.1 && elementSum1 < 1.0) { score += 1100; } else { score -= elementSum1 * 1024; } elementSumMap.put(elementSumMap.size() + 1, elementSum1); } else { score -= 50; } if (!dotSumMap.values().contains(dot2)) { if (dot2 < 0.000) { score += dot2 * 500; } else if (dot2 < 0.1) { score += 256; } if (dot2 > 0.50) { score -= 1200; } dotSumMap.put(dotSumMap.size() + 1, dot2); } else if (dot2 > 0.050 && dot2 < 0.10) { score -= 350; } else { score = score > 0 ? score - dot2 * 1200 : score + dot2 * 1200; } if (!elementSumMap.values().contains(elementSum2)) { if (elementSum2 < 0.01 && elementSum2 > 0.00) { score += 3300; } else if (elementSum2 > 0.1 && elementSum2 < 0.2) { score += 1100; } else { score -= elementSum2 * 1024; } elementSumMap.put(elementSumMap.size() + 1, elementSum2); } else if (elementSum2 > 0.050 && elementSum2 < 0.10) { score += 750; } else { score = score > 0 ? score - elementSum2 * 1200 : score + elementSum2 * 1200; } } } if (iterateSize > 5) { score -= iterateSize * 400; } score = elementsAndDotsRelation(score, dotMap, elementSumCounter); return score; } private Double elementsAndDotsRelation(Double score, ConcurrentMap dotMap, ConcurrentMap elementSumCounter) { OptionalDouble minvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).min(); OptionalDouble maxvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).max(); double total = minvalueDots.getAsDouble() + maxvalueDots.getAsDouble(); boolean permitted = false; if (minvalueDots.getAsDouble() != maxvalueDots.getAsDouble()) { permitted = true; } if (permitted) { Double dotsVariance = maxvalueDots.getAsDouble() - minvalueDots.getAsDouble(); if (maxvalueDots.getAsDouble() > minvalueDots.getAsDouble() * 10 && (dotsVariance < 0.40 || dotsVariance > 0.60)) { score -= 5500; } else if (minvalueDots.getAsDouble() < -0.10) { score -= 3500; } else if (dotsVariance > minvalueDots.getAsDouble() * 2) { score += 3500; } else if (minvalueDots.getAsDouble() * 2 > maxvalueDots.getAsDouble()) { score -= 3500; } } OptionalDouble minvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).min(); OptionalDouble maxvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).max(); Double elementsVariance = maxvalueElements.getAsDouble() - minvalueElements.getAsDouble(); if (elementsVariance != 0.0) { if (elementsVariance <= 0.01 && maxvalueElements.getAsDouble() <= 0.02) { if (maxvalueElements.getAsDouble() < 0.01 && minvalueElements.getAsDouble() > -0.02 && minvalueElements.getAsDouble() < 0.0) { score += 6500; } else { score -= 5500; } } else if (minvalueElements.getAsDouble() < 0.0 && minvalueElements.getAsDouble() - maxvalueElements.getAsDouble() < 0.50) { score -= 4500; } else if (elementsVariance * 2 >= maxvalueElements.getAsDouble() && elementsVariance < 0.1) { score -= elementsVariance * 86000; } } else if (maxvalueElements.getAsDouble() == minvalueElements.getAsDouble() && maxvalueElements.getAsDouble() > 0.40 && maxvalueElements.getAsDouble() < 0.60) { score += 600; } else if (maxvalueElements.getAsDouble() < 0.20 && minvalueElements.getAsDouble() >= 0.15) { score += 2600; } else if (maxvalueElements.getAsDouble() == minvalueElements.getAsDouble() && minvalueElements.getAsDouble() == 0.12) { score += 2600; } return score; } private Double sentimentMatrixVariances(Double score, int longest1, int longest2, int mainSentiment1, int mainSentiment2) { if (longest1 != longest2) { long deffLongest = longest1 > longest2 ? longest1 : longest2; long deffshorter = longest1 < longest2 ? longest1 : longest2; if (deffLongest > deffshorter * 5) { score -= 5500; } else if (deffLongest < (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) { score += (deffLongest - deffshorter) * 20; } else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) { score += (deffLongest - deffshorter) * 20; } else if (deffLongest - deffshorter < 2) { score += (deffLongest - deffshorter) * 20; } else if (deffshorter * 2 >= deffLongest && deffshorter * 2 < deffLongest + 5) { score += (deffLongest - deffshorter) * 20; } else { score -= (deffLongest - deffshorter) * 50; } if (deffLongest - deffshorter <= 5) { if (deffLongest < 24 && deffshorter > 20) { score += 4500; } else if (deffLongest - deffshorter == 5 && deffshorter >= 19 && deffLongest <= 24) { score += 4500; } else { if (deffshorter > 26 && deffLongest < 35) { score += 3900; } else if (deffLongest - deffshorter == 1 && deffshorter > 15 && deffLongest < 25) { score += 4900; } else { score -= 1900; } } } } return score; } private final Map.Entry> classifyRawEvaluation(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { if (cacheSentiment1 == null || cacheSentiment2 == null) { DocumentReaderAndWriter readerAndWriter = classifier.makePlainTextReaderAndWriter(); if (cacheSentiment1 == null) { cacheSentimentLocal1.setClassifyRaw(classifier.classifyRaw(str, readerAndWriter)); } if (cacheSentiment2 == null) { cacheSentimentLocal2.setClassifyRaw(classifier.classifyRaw(str1, readerAndWriter)); } } final List classifyRaw1 = cacheSentiment1 == null ? cacheSentimentLocal1.getClassifyRaw() : cacheSentiment1.getClassifyRaw(); final List classifyRaw2 = cacheSentiment2 == null ? cacheSentimentLocal2.getClassifyRaw() : cacheSentiment2.getClassifyRaw(); score -= (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200; Map.Entry< Double, Map.Entry> entry = new AbstractMap.SimpleEntry(score, new AbstractMap.SimpleEntry(cacheSentimentLocal1, cacheSentimentLocal2)); return entry; } private Double entryCountsRelation(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { int entry1 = cacheSentiment1 == null ? cacheSentimentLocal1.getEntryCounts().values().size() : cacheSentiment1.getEntryCounts().values().size(); int entry2 = cacheSentiment2 == null ? cacheSentimentLocal2.getEntryCounts().values().size() : cacheSentiment2.getEntryCounts().values().size(); if (entry1 > 0 && entry2 > 0) { if ((entry1 >= entry2 * 5) || (entry2 >= entry1 * 5)) { score -= entry1 > entry2 ? (entry1 - entry2) * 450 : (entry2 - entry1) * 450; } else if ((entry1 >= entry2 * 50 || entry2 >= entry1 * 50)) { score -= entry1 > entry2 ? entry1 * 180 : entry2 * 180; } else if ((entry1 >= entry2 * 2 || entry2 >= entry1 * 2) && entry1 * 2 >= entry2 && entry2 * 2 >= entry1) { score -= entry1 > entry2 ? (entry1 - entry2) * 450 : (entry2 - entry1) * 450; } else if (entry1 * 3 >= entry2 && entry2 * 3 >= entry1) { score += entry1 > entry2 ? (entry1 - entry2) * 550 : (entry2 - entry1) * 550; } else if (entry1 > 10 && entry2 > 10 && entry1 * 2 > entry2 && entry2 * 2 > entry1) { score += entry1 > entry2 ? entry2 * 600 : entry1 * 600; } } return score; } private SentimentValueCache GrammaticStructureSetup(SentimentValueCache cacheSentimentLocal, Annotation pipelineAnnotation) { for (CoreMap sentence : pipelineAnnotation.get(CoreAnnotations.SentencesAnnotation.class)) { Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); cacheSentimentLocal.addSentenceConstituencyParse(sentenceConstituencyParse); GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse); cacheSentimentLocal.addTypedDependencies(gs.allTypedDependencies()); cacheSentimentLocal.addGS(gs); } return cacheSentimentLocal; } private SentimentValueCache initializeCacheSetup(String str, SentimentValueCache cacheSentimentLocal) { cacheSentimentLocal = new SentimentValueCache(str); cacheSentimentLocal.setTaggedwords(getTaggedWordList(str)); cacheSentimentLocal.setCounter(tokenizeCounting(cacheSentimentLocal.getTaggedwordlist())); return cacheSentimentLocal; } private SentimentValueCache sentimentCoreAnnotationSetup(Annotation pipelineAnnotationSentiment, SentimentValueCache cacheSentimentLocal) { for (CoreMap sentence : pipelineAnnotationSentiment.get(CoreAnnotations.SentencesAnnotation.class)) { Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); if (tree != null) { int predictedClass = RNNCoreAnnotations.getPredictedClass(tree); SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); cacheSentimentLocal.addRNNPredictClass(predictedClass); cacheSentimentLocal.addSimpleMatrix(predictions); cacheSentimentLocal.addSimpleMatrixVector(nodeVector); } } return cacheSentimentLocal; } private SentimentValueCache setupMainSentimentandLongestVal(Annotation pipelineAnnotationSentiment, SentimentValueCache cacheSentimentLocal) { for (CoreMap sentence : pipelineAnnotationSentiment.get(CoreAnnotations.SentencesAnnotation.class)) { Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); int sentiment = RNNCoreAnnotations.getPredictedClass(tree); String partText = sentence.toString(); //SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); if (partText.length() > cacheSentimentLocal.getLongest()) { cacheSentimentLocal.setMainSentiment(sentiment); cacheSentimentLocal.setLongest(partText.length()); } } return cacheSentimentLocal; } private SentimentValueCache jmweAnnotationSetup(Annotation jmweStrAnnotation, SentimentValueCache cacheSentimentLocal) { List sentences = jmweStrAnnotation.get(CoreAnnotations.SentencesAnnotation.class); Collection> tokeninflectionMap = new ArrayList(); int tokenadder = 0; for (CoreMap sentence : sentences) { for (IMWE token : sentence.get(JMWEAnnotation.class)) { if (token.isInflected()) { cacheSentimentLocal.setInflectedCounterPositive(cacheSentimentLocal.getInflectedCounterPositive() + 1); } else if (!tokeninflectionMap.contains(token)) { cacheSentimentLocal.setInflectedCounterNegative(cacheSentimentLocal.getInflectedCounterNegative() + 1); tokeninflectionMap.add(token); } cacheSentimentLocal.addstrTokenForm(token.getForm()); cacheSentimentLocal.addstrTokenGetEntry(token.getEntry().toString().substring(token.getEntry().toString().length() - 1)); Collection values = token.getPartMap().values(); IMWEDesc entry = token.getEntry(); cacheSentimentLocal.setMarkedContinuousCounter(cacheSentimentLocal.getMarkedContinuousCounter() + entry.getMarkedContinuous()); cacheSentimentLocal.setUnmarkedPatternCounter(cacheSentimentLocal.getUnmarkedPatternCounter() + entry.getUnmarkedPattern()); for (IMWEDesc.IPart iPart : values) { cacheSentimentLocal.addstrTokenGetiPart(iPart.getForm()); } for (String strPostPrefix : entry.getPOS().getPrefixes()) { cacheSentimentLocal.addstrTokenEntryPOS(strPostPrefix); } for (int counts : entry.getCounts()) { cacheSentimentLocal.addEntryCounts(counts); } for (IToken tokens : token.getTokens()) { cacheSentimentLocal.addITokenMapTag(tokens.getTag()); for (String strtoken : tokens.getStems()) { cacheSentimentLocal.addstrTokenStems(strtoken); cacheSentimentLocal.setMarkedContiniousCounterEntries(cacheSentimentLocal.getMarkedContiniousCounterEntries() + 1); } } tokenadder += 1; } cacheSentimentLocal.setAnotatorcounter(cacheSentimentLocal.getAnotatorcounter() + 1); } cacheSentimentLocal.setTokensCounter(tokenadder); return cacheSentimentLocal; } private Double entryCountsScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { ConcurrentMap countsMap = new MapMaker().concurrencyLevel(2).makeMap(); int totalsize = cacheSentimentLocal1.getEntryCounts().values().size() + cacheSentimentLocal2.getEntryCounts().values().size(); for (int counts : cacheSentimentLocal1.getEntryCounts().values()) { for (int counts1 : cacheSentimentLocal2.getEntryCounts().values()) { if (counts > 0 && counts1 > 0) { if (counts == counts1 && !countsMap.values().contains(counts)) { score += (counts * 250) / totalsize; countsMap.put(countsMap.size() + 1, counts); } else if (counts * 3 < counts1 || counts1 * 3 < counts) { score -= 600; } } } } return score; } private Double tokenEntryPosScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { if (cacheSentimentLocal1.getstrTokenEntryPOS().values().size() > 1 && cacheSentimentLocal2.getstrTokenEntryPOS().values().size() > 1) { for (String strTokenPos1 : cacheSentimentLocal1.getstrTokenEntryPOS().values()) { for (String strTokenPos2 : cacheSentimentLocal2.getstrTokenEntryPOS().values()) { if (strTokenPos1.equals(strTokenPos2)) { score += 500; } } } int posEntrySize1 = cacheSentimentLocal1.getstrTokenEntryPOS().values().size(); int posEntrySize2 = cacheSentimentLocal2.getstrTokenEntryPOS().values().size(); if (posEntrySize1 * 3 > posEntrySize2 && posEntrySize2 * 3 > posEntrySize1) { score += posEntrySize1 > posEntrySize2 ? (posEntrySize1 - posEntrySize2) * 700 : (posEntrySize2 - posEntrySize1) * 700; } } return score; } private Double unmarkedPatternCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { int UnmarkedPatternCounter1 = cacheSentimentLocal1.getUnmarkedPatternCounter(); int UnmarkedPatternCounter2 = cacheSentimentLocal2.getUnmarkedPatternCounter(); if (UnmarkedPatternCounter1 > 0 && UnmarkedPatternCounter2 > 0) { if (UnmarkedPatternCounter1 < 100 && UnmarkedPatternCounter2 < 100) { if (UnmarkedPatternCounter1 * 2 > UnmarkedPatternCounter2 && UnmarkedPatternCounter2 * 2 > UnmarkedPatternCounter1) { score += 2500; } else if (UnmarkedPatternCounter1 * 5 < UnmarkedPatternCounter2 || UnmarkedPatternCounter2 * 5 < UnmarkedPatternCounter1) { score -= 4000; } } else if (UnmarkedPatternCounter1 > 500 && UnmarkedPatternCounter2 > 500) { score -= UnmarkedPatternCounter1 >= UnmarkedPatternCounter2 ? UnmarkedPatternCounter1 * 3 : UnmarkedPatternCounter2 * 3; } } return score; } private Double markedContiniousCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { int MarkedContinuousCounter1 = cacheSentimentLocal1.getMarkedContinuousCounter(); int MarkedContinuousCounter2 = cacheSentimentLocal2.getMarkedContinuousCounter(); int MarkedContiniousCounter1Entries = cacheSentimentLocal1.getMarkedContiniousCounterEntries(); int MarkedContiniousCounter2Entries = cacheSentimentLocal2.getMarkedContiniousCounterEntries(); if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) { if (MarkedContinuousCounter1 > MarkedContinuousCounter2 * 50 || MarkedContinuousCounter2 > MarkedContinuousCounter1 * 50) { score -= MarkedContinuousCounter1 > MarkedContinuousCounter2 ? MarkedContinuousCounter1 * 120 : MarkedContinuousCounter2 * 120; } else if (!Objects.equals(MarkedContiniousCounter1Entries, MarkedContiniousCounter2Entries) && (MarkedContinuousCounter1 * 2 >= MarkedContinuousCounter2 * MarkedContinuousCounter1) || (MarkedContinuousCounter2 * 2 >= MarkedContinuousCounter1 * MarkedContinuousCounter2)) { score += 4500; } else if (MarkedContiniousCounter1Entries == 0 || MarkedContiniousCounter2Entries == 0) { score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500 : (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500; } if (MarkedContiniousCounter1Entries > 0 && MarkedContiniousCounter2Entries > 0 && MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0 && MarkedContinuousCounter1 < MarkedContinuousCounter2 * 10 && MarkedContinuousCounter2 < MarkedContinuousCounter1 * 10) { if (MarkedContiniousCounter1Entries > MarkedContiniousCounter2Entries * 5 || MarkedContiniousCounter2Entries > MarkedContiniousCounter1Entries * 5 || MarkedContiniousCounter1Entries * 5 < MarkedContinuousCounter1 || MarkedContiniousCounter1Entries * 5 < MarkedContinuousCounter2 || MarkedContiniousCounter2Entries * 5 < MarkedContinuousCounter1 || MarkedContiniousCounter2Entries * 5 < MarkedContinuousCounter2) { score -= MarkedContinuousCounter1 > MarkedContinuousCounter2 ? MarkedContinuousCounter1 * 400 : MarkedContinuousCounter2 * 400; } } } return score; } private Double strTokensMapScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { ConcurrentMap strtokensMap = new MapMaker().concurrencyLevel(2).makeMap(); for (String strTokeniPart1 : cacheSentimentLocal1.getstrTokenGetiPart().values()) { for (String strTokeniPart2 : cacheSentimentLocal2.getstrTokenGetiPart().values()) { if (strTokeniPart1.equals(strTokeniPart2) && !strtokensMap.values().contains(strTokeniPart2)) { strtokensMap.put(strtokensMap.size() + 1, strTokeniPart2); score += 800; } } } int tokenIPartSize1 = cacheSentimentLocal1.getstrTokenGetiPart().values().size(); int tokenIPartSize2 = cacheSentimentLocal2.getstrTokenGetiPart().values().size(); int strTokenMapSize = strtokensMap.values().size(); if (tokenIPartSize1 * 2 > tokenIPartSize2 && tokenIPartSize2 * 2 > tokenIPartSize1) { score += tokenIPartSize1 > tokenIPartSize2 ? (tokenIPartSize1 - tokenIPartSize2) * 700 : (tokenIPartSize2 - tokenIPartSize1) * 700; score += strTokenMapSize * 600; } else if (tokenIPartSize1 > 0 && tokenIPartSize2 > 0) { score -= tokenIPartSize1 > tokenIPartSize2 ? (tokenIPartSize1 - tokenIPartSize2) * 700 : (tokenIPartSize2 - tokenIPartSize1) * 700; } return score; } private Double strTokenEntryScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { int tokenEntry1 = cacheSentimentLocal1.getstrTokenGetEntry().values().size(); int tokenEntry2 = cacheSentimentLocal2.getstrTokenGetEntry().values().size(); boolean boundaryLeaks = false; int remnantCounter = 0; if (tokenEntry1 * 2 != tokenEntry2 && tokenEntry2 * 2 != tokenEntry1) { boundaryLeaks = true; } ConcurrentMap entryTokenMap = new MapMaker().concurrencyLevel(2).makeMap(); for (String strTokenEntry1 : cacheSentimentLocal1.getstrTokenGetEntry().values()) { for (String strTokenEntry2 : cacheSentimentLocal2.getstrTokenGetEntry().values()) { if (!entryTokenMap.values().contains(strTokenEntry2)) { if (strTokenEntry1.equals(strTokenEntry2)) { score += boundaryLeaks ? 2500 : 2500 / 2; } else if (!boundaryLeaks) { score -= 450; } else { remnantCounter++; } } entryTokenMap.put(entryTokenMap.size() + 1, strTokenEntry2); } } score += remnantCounter * 250; return score; } private Double strTokenMapTagsScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { ConcurrentMap iTokenMapTagsMap = new MapMaker().concurrencyLevel(2).makeMap(); for (String strmapTag : cacheSentimentLocal1.getITokenMapTag().values()) { for (String strmapTag1 : cacheSentimentLocal2.getITokenMapTag().values()) { if (strmapTag.equals(strmapTag1) && !iTokenMapTagsMap.values().contains(strmapTag1)) { score -= 1450; iTokenMapTagsMap.put(iTokenMapTagsMap.size() + 1, strmapTag); } } } int mapTagsize1 = cacheSentimentLocal1.getITokenMapTag().values().size(); int mapTagsize2 = cacheSentimentLocal2.getITokenMapTag().values().size(); int tokenTagMapSize = iTokenMapTagsMap.values().size(); if (mapTagsize1 != 0 && mapTagsize2 != 0) { if (mapTagsize1 * 2 > mapTagsize2 && mapTagsize2 * 2 > mapTagsize1) { score += mapTagsize1 > mapTagsize2 ? (mapTagsize1 - mapTagsize2) * 700 : (mapTagsize2 - mapTagsize1) * 700; score += tokenTagMapSize * 600; } else { score -= mapTagsize1 > mapTagsize2 ? (mapTagsize1 - mapTagsize2) * 700 : (mapTagsize2 - mapTagsize1) * 700; } } return score; } private Double tokenformSizeScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { int tokenform1size = cacheSentimentLocal1.getstrTokenForm().values().size(); int tokenform2size = cacheSentimentLocal2.getstrTokenForm().values().size(); if (tokenform1size > 0 || tokenform2size > 0) { if (tokenform1size < tokenform2size * 5 && tokenform2size < tokenform1size * 5) { for (String strTokenForm1itr1 : cacheSentimentLocal1.getstrTokenForm().values()) { for (String strTokenForm1itr2 : cacheSentimentLocal2.getstrTokenForm().values()) { if (strTokenForm1itr1.equals(strTokenForm1itr2)) { score -= 1600; } else { score += 500; } } } } else if (tokenform1size > 0 && tokenform2size > 0) { if (tokenform1size * 2 >= tokenform2size && tokenform2size * 2 >= tokenform1size) { score += tokenform1size > tokenform2size ? tokenform1size * 600 : tokenform2size * 600; } else if (tokenform1size * 4 <= tokenform2size || tokenform2size * 4 <= tokenform1size) { score -= tokenform1size > tokenform2size ? (tokenform1size - tokenform2size) * 600 : (tokenform2size - tokenform1size) * 600; } } } return score; } private Double tokenStemmingMapScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { ConcurrentMap tokenStemmingMap = new MapMaker().concurrencyLevel(2).makeMap(); for (String strTokenStem : cacheSentimentLocal1.getstrTokenStems().values()) { for (String strTokenStem1 : cacheSentimentLocal2.getstrTokenStems().values()) { if (strTokenStem.equals(strTokenStem1) && !tokenStemmingMap.values().contains(strTokenStem)) { score += 500; tokenStemmingMap.put(tokenStemmingMap.size() + 1, strTokenStem); } } } return score; } private Double inflectedCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { int inflectedCounterPositive1 = cacheSentimentLocal1.getInflectedCounterPositive(); int inflectedCounterPositive2 = cacheSentimentLocal2.getInflectedCounterPositive(); int inflectedCounterNegative = cacheSentimentLocal1.getInflectedCounterNegative() > cacheSentimentLocal2.getInflectedCounterNegative() ? cacheSentimentLocal1.getInflectedCounterNegative() - cacheSentimentLocal2.getInflectedCounterNegative() : cacheSentimentLocal2.getInflectedCounterNegative() - cacheSentimentLocal1.getInflectedCounterNegative(); if ((inflectedCounterPositive1 + inflectedCounterPositive2) > inflectedCounterNegative && inflectedCounterNegative > 0) { score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 650; } if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) { if (inflectedCounterPositive1 * 2 > inflectedCounterPositive2 && inflectedCounterPositive2 * 2 > inflectedCounterPositive1 && inflectedCounterNegative > 0) { score += ((inflectedCounterPositive1 + inflectedCounterPositive2) * 150) - (inflectedCounterNegative * 10); } else if (inflectedCounterPositive1 * 5 < inflectedCounterPositive2 || inflectedCounterPositive2 * 5 < inflectedCounterPositive1) { score -= inflectedCounterPositive1 > inflectedCounterPositive2 ? (inflectedCounterPositive1 - inflectedCounterPositive2) * 400 : (inflectedCounterPositive2 - inflectedCounterPositive1) * 400; } } if (inflectedCounterPositive1 >= 15 && inflectedCounterPositive2 >= 15) { score -= 4000; } return score; } private Double annotatorCountScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { int anotatorcounter1 = cacheSentimentLocal1.getAnotatorcounter(); int anotatorcounter2 = cacheSentimentLocal2.getAnotatorcounter(); if (anotatorcounter1 > 1 && anotatorcounter2 > 1) { if (anotatorcounter1 * 2 > anotatorcounter2 && anotatorcounter2 * 2 > anotatorcounter1) { score += anotatorcounter1 > anotatorcounter2 ? (anotatorcounter1 - anotatorcounter2) * 700 : (anotatorcounter2 - anotatorcounter1) * 700; } else if (anotatorcounter1 * 5 < anotatorcounter2 || anotatorcounter2 * 5 < anotatorcounter1) { score -= anotatorcounter1 > anotatorcounter2 ? (anotatorcounter1 - anotatorcounter2) * 400 : (anotatorcounter2 - anotatorcounter1) * 400; } } return score; } private Double tokensCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { int tokensCounter1 = cacheSentimentLocal1.getTokensCounter(); int tokensCounter2 = cacheSentimentLocal2.getTokensCounter(); if ((tokensCounter1 > 1 && tokensCounter2 > 1) && tokensCounter1 < tokensCounter2 * 5 && tokensCounter2 < tokensCounter1 * 5) { if (tokensCounter1 > tokensCounter2 / 2 && tokensCounter2 > tokensCounter1 / 2 && tokensCounter1 < 10 && tokensCounter2 < 10) { score += (tokensCounter1 + tokensCounter2) * 500; } else { score -= 500; } } else { int elseint = tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500; if ((tokensCounter1 > tokensCounter2 * 5 || tokensCounter2 > tokensCounter1 * 5) && tokensCounter1 > 0 && tokensCounter2 > 0) { score -= tokensCounter1 > tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500; } else if (elseint > 0 && tokensCounter1 > 0 && tokensCounter2 > 0) { if (elseint > 500 && elseint < 2000) { score += elseint * 10; } else { score -= elseint * 15; } } else { if (elseint > 0 && elseint < 1500 && (tokensCounter1 > 0 || tokensCounter2 > 0) && tokensCounter1 <= 2 && tokensCounter2 <= 2) { score -= elseint * 1.5; } else if (tokensCounter1 < 5 && tokensCounter2 < 5) { score -= elseint * (tokensCounter1 + tokensCounter2); } } } return score; } private SentimentValueCache setupNEREntitiesAndTokenTags(CoreDocument pipelineCoreDcoument, SentimentValueCache cacheSentimentLocal) { for (CoreEntityMention em : pipelineCoreDcoument.entityMentions()) { Set> entrySet = em.entityTypeConfidences().entrySet(); String entityType = em.entityType(); Double EntityConfidences = 0.0; for (Map.Entry entries : entrySet) { EntityConfidences = entries.getValue(); } List tokens = em.tokens(); for (CoreLabel token : tokens) { if (token != null) { if (!cacheSentimentLocal.getnerEntityTokenTags().values().contains(token.tag())) { if (entityType.equals("PERSON") && EntityConfidences > 0.80) { cacheSentimentLocal.addnerEntityTokenTags(token.tag()); } } } } if (!cacheSentimentLocal.getnerEntities1().values().contains(em.text())) { cacheSentimentLocal.addNEREntities1(em.text()); cacheSentimentLocal.addNEREntities2(em.entityType()); } } return cacheSentimentLocal; } private Double nerEntitiesAndTokenScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { List entsCollection = new ArrayList(); for (String strEnts1 : cacheSentimentLocal1.getnerEntities1().values()) { for (String strEnts2 : cacheSentimentLocal2.getnerEntities2().values()) { if (strEnts1.equalsIgnoreCase(strEnts2) && !entsCollection.contains(strEnts1)) { score += 2500; entsCollection.add(strEnts1); } } } entsCollection = new ArrayList(); for (String strToken : cacheSentimentLocal1.getnerEntityTokenTags().values()) { for (String strToken1 : cacheSentimentLocal2.getnerEntityTokenTags().values()) { if (strToken.equalsIgnoreCase(strToken1) && !entsCollection.contains(strToken)) { score += 2000; entsCollection.add(strToken); } } } return score; } private SentimentValueCache setupStoWordTokensLemma(Annotation pipelineAnnotationSentiment, SentimentValueCache cacheSentimentLocal) { String customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with"; List tokensSentiment = pipelineAnnotationSentiment.get(CoreAnnotations.TokensAnnotation.class); Set stopWords = StopAnalyzer.ENGLISH_STOP_WORDS_SET; Set stopWordsCustom = StopwordAnnotator.getStopWordList(customStopWordList, true); for (CoreLabel token : tokensSentiment) { Pair stopword = token.get(StopwordAnnotator.class); String word = token.word().toLowerCase(); if (stopWords.contains(word) || stopWordsCustom.contains(word)) { cacheSentimentLocal.addstopwordTokens(word); } String lemma = token.lemma().toLowerCase(); if (stopWords.contains(lemma) || stopWordsCustom.contains(lemma)) { cacheSentimentLocal.addStopWordLemma(lemma); } if (stopword.first() && stopword.second()) { cacheSentimentLocal.setPairCounter(cacheSentimentLocal.getPairCounter() + 1); } } return cacheSentimentLocal; } private Double stopWordTokenLemmaScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { Collection stopWordCollection = new ArrayList(); for (String stopwords1 : cacheSentimentLocal1.getStopwordTokens().values()) { for (String stopwords2 : cacheSentimentLocal2.getStopwordTokens().values()) { if (stopwords1.equals(stopwords2) && !stopWordCollection.contains(stopwords1)) { score -= 50; stopWordCollection.add(stopwords1); } } } stopWordCollection = new ArrayList(); for (String stopwords1 : cacheSentimentLocal1.getStopWordLemma().values()) { for (String stopwords2 : cacheSentimentLocal2.getStopWordLemma().values()) { if (stopwords1.equals(stopwords2) && !stopWordCollection.contains(stopwords1)) { score -= 50; stopWordCollection.add(stopwords1); } } } return score; } private Double stopwordTokenPairCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { if (!cacheSentimentLocal1.getStopwordTokens().values().isEmpty() && !cacheSentimentLocal2.getStopwordTokens().values().isEmpty()) { int stopwordsize1 = cacheSentimentLocal1.getStopwordTokens().values().size(); int stopwordsize2 = cacheSentimentLocal2.getStopwordTokens().values().size(); if (stopwordsize1 * 5 < stopwordsize2 || stopwordsize2 * 5 < stopwordsize1) { score -= stopwordsize1 > stopwordsize2 ? (stopwordsize1 - stopwordsize2) * 850 : (stopwordsize2 - stopwordsize1) * 850; } else if (stopwordsize1 == stopwordsize2 && stopwordsize1 > 1) { score -= stopwordsize1 * 450; } else if ((stopwordsize1 / 2 == stopwordsize2 || stopwordsize2 / 2 == stopwordsize1) && stopwordsize1 + stopwordsize2 >= 4) { score -= 2500; } else { score += stopwordsize1 > stopwordsize2 ? (stopwordsize1 - stopwordsize2) * 850 : (stopwordsize2 - stopwordsize1) * 850; } } int pairCounter1 = cacheSentimentLocal1.getPairCounter(); int pairCounter2 = cacheSentimentLocal2.getPairCounter(); if (pairCounter1 > 0 && pairCounter2 > 0) { if (pairCounter1 * 5 <= pairCounter2 || pairCounter2 * 5 <= pairCounter1) { score -= pairCounter1 > pairCounter2 ? (pairCounter1 - pairCounter2) * 1500 : (pairCounter2 - pairCounter1) * 1500; } else if (pairCounter1 == pairCounter2 && pairCounter1 > 1) { score -= pairCounter1 * 450; } else if ((pairCounter1 / 2 == pairCounter2 || pairCounter2 / 2 == pairCounter1) && pairCounter1 + pairCounter2 >= 4) { score -= 2500; } else { score += pairCounter1 > pairCounter2 ? (pairCounter1 - pairCounter2) * 700 : (pairCounter2 - pairCounter1) * 700; } } return score; } private Double tgwListScoreIncrementer(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { AtomicInteger runCount = new AtomicInteger(0); cacheSentimentLocal1.getTgwlistIndex().values().forEach(TaggedWord -> { if (!cacheSentimentLocal2.getTgwlistIndex().values().contains(TaggedWord)) { cacheSentimentLocal2.addTgwlistIndex(TaggedWord); runCount.getAndIncrement(); } }); score += runCount.get() * 64; return score; } @Override public final SimilarityMatrix call() { Double score = -100.0; SentimentValueCache cacheSentimentLocal1 = null; SentimentValueCache cacheSentimentLocal2 = null; int counter1; int counter2; try { if (cacheSentiment1 == null) { cacheSentimentLocal1 = initializeCacheSetup(str, cacheSentimentLocal1); } if (cacheSentiment2 == null) { cacheSentimentLocal2 = initializeCacheSetup(str1, cacheSentimentLocal2); } } catch (Exception ex) { } try { counter1 = cacheSentiment1 == null ? cacheSentimentLocal1.getCounter() : cacheSentiment1.getCounter(); counter2 = cacheSentiment2 == null ? cacheSentimentLocal2.getCounter() : cacheSentiment2.getCounter(); final int overValue = (counter1 >= counter2 ? counter1 - counter2 : counter2 - counter1) * 32; score -= overValue; } catch (Exception ex) { } try { if (cacheSentiment1 == null) { ConcurrentMap retrieveTGWListIndex = retrieveTGWListIndex(cacheSentimentLocal1.getTaggedwordlist()); for (String str : retrieveTGWListIndex.values()) { cacheSentimentLocal1.addTgwlistIndex(str); } } if (cacheSentiment2 == null) { ConcurrentMap retrieveTGWListIndex = retrieveTGWListIndex(cacheSentimentLocal2.getTaggedwordlist()); for (String str : retrieveTGWListIndex.values()) { cacheSentimentLocal2.addTgwlistIndex(str); } } } catch (Exception ex) { } try { score = tgwListScoreIncrementer(score, cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null ? cacheSentimentLocal2 : cacheSentiment2); if (cacheSentiment1 == null) { cacheSentimentLocal1 = GrammaticStructureSetup(cacheSentimentLocal1, pipelineAnnotation1); } if (cacheSentiment2 == null) { cacheSentimentLocal2 = GrammaticStructureSetup(cacheSentimentLocal2, pipelineAnnotation2); } } catch (Exception ex) { } ConcurrentMap sentenceConstituencyParseList1 = null; ConcurrentMap sentenceConstituencyParseList2 = null; try { sentenceConstituencyParseList2 = cacheSentiment2 == null ? cacheSentimentLocal2.getSentenceConstituencyParseList() : cacheSentiment2.getSentenceConstituencyParseList(); sentenceConstituencyParseList1 = cacheSentiment1 == null ? cacheSentimentLocal1.getSentenceConstituencyParseList() : cacheSentiment1.getSentenceConstituencyParseList(); score = iterateTrees(sentenceConstituencyParseList2, sentenceConstituencyParseList1, score); } catch (Exception ex) { } try { Collection allTypedDependencies2 = cacheSentiment2 == null ? cacheSentimentLocal2.getAllTypedDependencies() : cacheSentiment2.getAllTypedDependencies(); Collection allTypedDependencies1 = cacheSentiment1 == null ? cacheSentimentLocal1.getAllTypedDependencies() : cacheSentiment1.getAllTypedDependencies(); ConcurrentMap grammaticalMap1 = cacheSentiment1 == null ? cacheSentimentLocal1.getGs() : cacheSentiment1.getGs(); ConcurrentMap grammaticalMap2 = cacheSentiment2 == null ? cacheSentimentLocal2.getGs() : cacheSentiment2.getGs(); score = typeDependenciesGrammaticalRelation(allTypedDependencies1, allTypedDependencies2, score, grammaticalMap1, grammaticalMap2, sentenceConstituencyParseList1, sentenceConstituencyParseList2); } catch (Exception ex) { } try { if (cacheSentiment1 == null) { cacheSentimentLocal1 = sentimentCoreAnnotationSetup(pipelineAnnotation1Sentiment, cacheSentimentLocal1); } if (cacheSentiment2 == null) { cacheSentimentLocal2 = sentimentCoreAnnotationSetup(pipelineAnnotation2Sentiment, cacheSentimentLocal2); } } catch (Exception ex) { } try { final ConcurrentMap simpleSMXlist1 = cacheSentiment1 == null ? cacheSentimentLocal1.getSimpleSMXlist() : cacheSentiment1.getSimpleSMXlist(); final ConcurrentMap simpleSMXlist2 = cacheSentiment2 == null ? cacheSentimentLocal2.getSimpleSMXlist() : cacheSentiment2.getSimpleSMXlist(); final ConcurrentMap simpleSMXlistVector1 = cacheSentiment1 == null ? cacheSentimentLocal1.getSimpleSMXlistVector() : cacheSentiment1.getSimpleSMXlistVector(); final ConcurrentMap simpleSMXlistVector2 = cacheSentiment2 == null ? cacheSentimentLocal2.getSimpleSMXlistVector() : cacheSentiment2.getSimpleSMXlistVector(); score = simpleRNNMatrixCalculations(score, simpleSMXlist1, simpleSMXlist2); score = simpleRNNMaxtrixVectors(score, simpleSMXlistVector1, simpleSMXlistVector2); } catch (Exception ex) { } try { int sentiment1 = cacheSentiment1 == null ? cacheSentimentLocal1.getRnnPrediectClassMap().size() : cacheSentiment1.getRnnPrediectClassMap().size(); int sentiment2 = cacheSentiment2 == null ? cacheSentimentLocal2.getRnnPrediectClassMap().size() : cacheSentiment2.getRnnPrediectClassMap().size(); score -= (sentiment1 > sentiment2 ? sentiment1 - sentiment2 : sentiment2 - sentiment1) * 500; Map.Entry> classifyRawEvaluationEntry = classifyRawEvaluation(score, cacheSentimentLocal1, cacheSentimentLocal2); score = classifyRawEvaluationEntry.getKey(); if (cacheSentiment1 == null) { cacheSentimentLocal1 = classifyRawEvaluationEntry.getValue().getKey(); } if (cacheSentiment2 == null) { cacheSentimentLocal2 = classifyRawEvaluationEntry.getValue().getValue(); } } catch (Exception ex) { } try { if (cacheSentiment1 == null) { cacheSentimentLocal1 = setupMainSentimentandLongestVal(pipelineAnnotation1Sentiment, cacheSentimentLocal1); } if (cacheSentiment2 == null) { cacheSentimentLocal2 = setupMainSentimentandLongestVal(pipelineAnnotation2Sentiment, cacheSentimentLocal2); } score = sentimentMatrixVariances(score, cacheSentiment1 == null ? cacheSentimentLocal1.getLongest() : cacheSentiment1.getLongest(), cacheSentiment2 == null ? cacheSentimentLocal2.getLongest() : cacheSentiment2.getLongest(), cacheSentiment1 == null ? cacheSentimentLocal1.getMainSentiment() : cacheSentiment1.getMainSentiment(), cacheSentiment2 == null ? cacheSentimentLocal2.getMainSentiment() : cacheSentiment2.getMainSentiment()); } catch (Exception ex) { } try { if (cacheSentiment1 == null) { cacheSentimentLocal1 = jmweAnnotationSetup(jmweStrAnnotation1, cacheSentimentLocal1); } if (cacheSentiment2 == null) { cacheSentimentLocal2 = jmweAnnotationSetup(jmweStrAnnotation2, cacheSentimentLocal2); } } catch (Exception ex) { } SentimentValueCache scoringCache1 = cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1; SentimentValueCache scoringCache2 = cacheSentiment2 == null ? cacheSentimentLocal2 : cacheSentiment2; try { score = entryCountsRelation(score, scoringCache1, scoringCache2); } catch (Exception ex) { } try { score = entryCountsScoring(score, scoringCache1, scoringCache2); } catch (Exception ex) { } try { score = tokenEntryPosScoring(score, scoringCache1, scoringCache2); } catch (Exception ex) { } try { score = unmarkedPatternCounterScoring(score, scoringCache1, scoringCache2); } catch (Exception ex) { } try { score = markedContiniousCounterScoring(score, scoringCache1, scoringCache2); } catch (Exception ex) { } try { score = strTokensMapScoring(score, scoringCache1, scoringCache2); } catch (Exception ex) { } try { score = strTokenEntryScoring(score, scoringCache1, scoringCache2); } catch (Exception ex) { } try { score = strTokenMapTagsScoring(score, scoringCache1, scoringCache2); } catch (Exception ex) { } try { score = tokenformSizeScoring(score, scoringCache1, scoringCache2); } catch (Exception ex) { } try { score = tokenStemmingMapScoring(score, scoringCache1, scoringCache2); } catch (Exception ex) { } try { score = inflectedCounterScoring(score, scoringCache1, scoringCache2); } catch (Exception ex) { } try { score = annotatorCountScoring(score, scoringCache1, scoringCache2); } catch (Exception ex) { } try { score = tokensCounterScoring(score, scoringCache1, scoringCache2); } catch (Exception ex) { } try { LevenshteinDistance leven = new LevenshteinDistance(str, str1); double SentenceScoreDiff = leven.computeLevenshteinDistance(); SentenceScoreDiff *= 15; score -= SentenceScoreDiff; } catch (Exception ex) { } try { if (cacheSentiment1 == null) { cacheSentimentLocal1 = setupNEREntitiesAndTokenTags(pipelineCoreDcoument1, cacheSentimentLocal1); } if (cacheSentiment2 == null) { cacheSentimentLocal2 = setupNEREntitiesAndTokenTags(pipelineCoreDcoument2, cacheSentimentLocal2); } score = nerEntitiesAndTokenScoring(score, cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null ? cacheSentimentLocal2 : cacheSentiment2); } catch (Exception ex) { } try { if (cacheSentiment1 == null) { cacheSentimentLocal1 = setupStoWordTokensLemma(pipelineAnnotation1Sentiment, cacheSentimentLocal1); } if (cacheSentiment2 == null) { cacheSentimentLocal2 = setupStoWordTokensLemma(pipelineAnnotation2Sentiment, cacheSentimentLocal2); } score = stopWordTokenLemmaScoring(score, cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null ? cacheSentimentLocal2 : cacheSentiment2); } catch (Exception ex) { } try { score = stopwordTokenPairCounterScoring(score, cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null ? cacheSentimentLocal2 : cacheSentiment2); } catch (Exception ex) { } smxParam.setDistance(score); try { if (cacheSentiment1 == null) { smxParam.setCacheValue1(cacheSentimentLocal1); } if (cacheSentiment2 == null) { smxParam.setCacheValue2(cacheSentimentLocal2); } } catch (Exception ex) { } return smxParam; } }