you know that feeling when you cant tell if you forgot to add something
This commit is contained in:
jenzur
+334
-114
@@ -3,6 +3,7 @@ package FunctionLayer.StanfordParser;
|
||||
import FunctionLayer.LevenshteinDistance;
|
||||
import FunctionLayer.Datahandler;
|
||||
import FunctionLayer.SimilarityMatrix;
|
||||
import FunctionLayer.StopwordAnnotator;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import edu.mit.jmwe.data.IMWE;
|
||||
import edu.mit.jmwe.data.IMWEDesc;
|
||||
@@ -36,6 +37,7 @@ import edu.stanford.nlp.trees.TreeCoreAnnotations;
|
||||
import edu.stanford.nlp.trees.TypedDependency;
|
||||
import edu.stanford.nlp.trees.tregex.gui.Tdiff;
|
||||
import edu.stanford.nlp.util.CoreMap;
|
||||
import edu.stanford.nlp.util.Pair;
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
@@ -49,6 +51,7 @@ import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.BinaryOperator;
|
||||
import java.util.function.Function;
|
||||
import org.apache.lucene.analysis.core.StopAnalyzer;
|
||||
import org.ejml.simple.SimpleMatrix;
|
||||
|
||||
/*
|
||||
@@ -146,6 +149,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
});
|
||||
});
|
||||
score += runCount.get() * 64;
|
||||
////System.out.println("score post runCountGet: " + score + "\n");
|
||||
ConcurrentMap<Integer, Tree> sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
try {
|
||||
for (CoreMap sentence : pipelineAnnotation1.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
@@ -174,10 +178,14 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
}
|
||||
int constituents1 = constinuent1.size() - constiRelationsize;
|
||||
int constituents2 = constinuent2.size() - constiRelationsize;
|
||||
if (constituents1 > 0 && constituents2 > 0) {
|
||||
if (constituents1 * 5 < constituents2 || constituents2 * 5 < constituents1) {
|
||||
score -= (constituents1 + constituents2) * 200;
|
||||
} else if (constituents1 == 0 || constituents2 == 0) {
|
||||
score -= constiRelationsize * 200;
|
||||
} else {
|
||||
score += constiRelationsize * 200;
|
||||
score += constiRelationsize * 160;
|
||||
//System.out.println("score post constiRelationsize: " + score + "\nconstituents1: " + constituents1
|
||||
// + "\nconstituents2: " + constituents2 + "\nconstiRelationsize: " + constiRelationsize + "\n");
|
||||
}
|
||||
GrammaticalStructure gs1 = gsf.newGrammaticalStructure(sentenceConstituencyParse1);
|
||||
Collection<TypedDependency> allTypedDependencies1 = gs1.allTypedDependencies();
|
||||
@@ -190,7 +198,8 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
IndexedWord gov = TDY1.gov();
|
||||
GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep);
|
||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 1900;
|
||||
score += 700;
|
||||
//System.out.println("grammaticalRelation applicable score: " + score + "\n");
|
||||
grammaticalRelation1++;
|
||||
}
|
||||
GrammaticalRelation reln = TDY1.reln();
|
||||
@@ -205,49 +214,85 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
GrammaticalRelation grammaticalRelation = gs1.getGrammaticalRelation(gov, dep);
|
||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 900;
|
||||
//System.out.println("grammaticalRelation appliceable score: " + score + "\n");
|
||||
grammaticalRelation2++;
|
||||
}
|
||||
GrammaticalRelation reln = TDY.reln();
|
||||
if (reln.isApplicable(sentenceConstituencyParse1)) {
|
||||
score += 525;
|
||||
//System.out.println("reln appliceable score: " + score + "\n");
|
||||
relationApplicable2++;
|
||||
}
|
||||
}
|
||||
if ((grammaticalRelation1 == 0 && grammaticalRelation2 > 0) || (grammaticalRelation2 == 0 && grammaticalRelation1 > 0)) {
|
||||
if ((grammaticalRelation1 == 0 && grammaticalRelation2 > 4) || (grammaticalRelation2 == 0 && grammaticalRelation1 > 4)) {
|
||||
score -= 3450;
|
||||
//System.out.println("grammaticalRelation1 score trim: " + score + "\ngrammaticalRelation1: " + grammaticalRelation1
|
||||
// + "\ngrammaticalRelation2: " + grammaticalRelation2 + "\n");
|
||||
}
|
||||
if (!allTypedDependencies.isEmpty() || !allTypedDependencies1.isEmpty()) {
|
||||
int allTypeDep1 = allTypedDependencies.size();
|
||||
int allTypeDep2 = allTypedDependencies1.size();
|
||||
if (allTypeDep1 <= allTypeDep2 * 5 && allTypeDep2 <= allTypeDep1 * 5) {
|
||||
if (!alltypeDepsSizeMap.values().contains(allTypeDep1)) {
|
||||
score += allTypeDep1 * 600;
|
||||
if (allTypeDep1 > 0 && allTypeDep2 > 0) {
|
||||
if (allTypeDep1 * 2 <= allTypeDep2 || allTypeDep2 * 2 <= allTypeDep1) {
|
||||
score -= allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 160 : (allTypeDep2 - allTypeDep1) * 160;
|
||||
//System.out.println(" allTypeDep score: " + score + "\nallTypeDep1: " + allTypeDep1 + "\nallTypeDep2: "
|
||||
// + allTypeDep2 + "\n");
|
||||
} else {
|
||||
score += allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 600 : (allTypeDep2 - allTypeDep1) * 600;
|
||||
//System.out.println(" allTypeDep score: " + score + "\nallTypeDep1: " + allTypeDep1 + "\nallTypeDep2: "
|
||||
// + allTypeDep2 + "\n");
|
||||
}
|
||||
alltypeDepsSizeMap.put(alltypeDepsSizeMap.size() + 1, allTypeDep1);
|
||||
}
|
||||
if (!alltypeDepsSizeMap.values().contains(allTypeDep1)) {
|
||||
score += allTypeDep2 * 600;
|
||||
alltypeDepsSizeMap.put(alltypeDepsSizeMap.size() + 1, allTypeDep2);
|
||||
}
|
||||
}
|
||||
if (allTypeDep1 >= 5 && allTypeDep2 >= 5) {
|
||||
int largerTypeDep = allTypeDep1 > allTypeDep2 ? allTypeDep1 : allTypeDep2;
|
||||
int smallerTypeDep = allTypeDep1 < allTypeDep2 ? allTypeDep1 : allTypeDep2;
|
||||
int summation = largerTypeDep * largerTypeDep - smallerTypeDep * smallerTypeDep;
|
||||
if (summation > 50 && summation < 75) {
|
||||
int summation = (largerTypeDep * largerTypeDep) - (smallerTypeDep * smallerTypeDep);
|
||||
if (summation / largerTypeDep < 15.0 && summation / largerTypeDep > 10.0 && smallerTypeDep * 2 > largerTypeDep
|
||||
&& !summationMap.values().contains(summation)) {
|
||||
score += summation * 80;
|
||||
} else if (!summationMap.values().contains(summation)) {
|
||||
score -= largerTypeDep * 500;
|
||||
summationMap.put(summationMap.size() + 1, summation);
|
||||
//System.out.println("score post summation: " + score + "\nsummation: " + summation + "\n");
|
||||
} else if (largerTypeDep == smallerTypeDep) {
|
||||
score += 2500;
|
||||
//System.out.println("score largerTypeDep equals smallerTypeDep: " + score + "\nlargerTypeDep: " + largerTypeDep + "\n");
|
||||
}
|
||||
}
|
||||
if (relationApplicable1 > 0 && relationApplicable2 > 0 && relationApplicable1 == relationApplicable2
|
||||
&& grammaticalRelation1 > 0 && grammaticalRelation2 > 0 && grammaticalRelation1 == grammaticalRelation2) {
|
||||
score += 3500;
|
||||
} else {
|
||||
score += allTypeDep1 > allTypeDep2
|
||||
? (allTypeDep2 - allTypeDep1) * (allTypeDep2 * 50)
|
||||
: (allTypeDep1 - allTypeDep2) * (allTypeDep1 * 50);
|
||||
//System.out.println("score relationApplicable equal: " + score + "\n");
|
||||
} else if (allTypeDep1 * 5 < allTypeDep2 || allTypeDep2 * 5 < allTypeDep1) {
|
||||
score -= allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * (allTypeDep2 * 450)
|
||||
: (allTypeDep2 - allTypeDep1) * (allTypeDep1 * 450);
|
||||
//System.out.println("score minus grammaticalRelation equal: " + score + "\n");
|
||||
}
|
||||
if (relationApplicable1 > 1 && relationApplicable2 > 1 && relationApplicable1 * 3 > relationApplicable2
|
||||
&& relationApplicable2 * 3 > relationApplicable1) {
|
||||
score += relationApplicable1 > relationApplicable2 ? (relationApplicable1 - relationApplicable2) * 1500
|
||||
: (relationApplicable2 - relationApplicable1) * 1500;
|
||||
//System.out.println("score relationApplicable plus: " + score + "\n");
|
||||
} else if (relationApplicable1 * 5 < relationApplicable2 || relationApplicable2 * 5 < relationApplicable1) {
|
||||
score -= relationApplicable1 > relationApplicable2 ? (relationApplicable1 - relationApplicable2) * 500
|
||||
: (relationApplicable2 - relationApplicable1) * 500;
|
||||
//System.out.println("score relationApplicable minus: " + score + "\n");
|
||||
}
|
||||
if (grammaticalRelation1 > 0 && grammaticalRelation2 > 0 && grammaticalRelation1 * 3 > grammaticalRelation2
|
||||
&& grammaticalRelation2 * 3 > grammaticalRelation1) {
|
||||
score += grammaticalRelation1 > grammaticalRelation2 ? (grammaticalRelation1 - grammaticalRelation2) * 1500
|
||||
: (grammaticalRelation2 - grammaticalRelation1) * 1500;
|
||||
//System.out.println("score grammaticalRelation plus: " + score + "\n");
|
||||
} else if (grammaticalRelation1 * 5 < grammaticalRelation2 || grammaticalRelation2 * 5 < grammaticalRelation1) {
|
||||
score -= grammaticalRelation1 > grammaticalRelation2 ? (grammaticalRelation1 - grammaticalRelation2) * 500
|
||||
: (grammaticalRelation2 - grammaticalRelation1) * 500;
|
||||
//System.out.println("score grammaticalRelation minus: " + score + "\n");
|
||||
}
|
||||
//System.out.println("score post relationApplicable1 veri: " + score + "\nrelationApplicable1: " + relationApplicable1
|
||||
// + "\nrelationApplicable2: " + relationApplicable2 + "\ngrammaticalRelation1: " + grammaticalRelation1 + "\n"
|
||||
// + "grammaticalRelation2: " + grammaticalRelation2 + "\n");
|
||||
}
|
||||
AtomicInteger runCount1 = new AtomicInteger(0);
|
||||
sentenceConstituencyParse.taggedLabeledYield().forEach((LBW) -> {
|
||||
@@ -259,9 +304,10 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
runCount1.getAndIncrement();
|
||||
});
|
||||
});
|
||||
score += runCount1.get() * 1500;
|
||||
score += runCount1.get() * 250;
|
||||
}
|
||||
}
|
||||
//System.out.println("score pre typeSizeSmallest: " + score + "\n");
|
||||
int typeSizeSmallest = 100;
|
||||
int typeSizeLargest = 0;
|
||||
for (Integer i : alltypeDepsSizeMap.values()) {
|
||||
@@ -273,7 +319,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
}
|
||||
}
|
||||
if (typeSizeLargest >= typeSizeSmallest * 3) {
|
||||
score -= typeSizeLargest * 1600;
|
||||
score -= typeSizeLargest * 160;
|
||||
}
|
||||
typeSizeLargest = 0;
|
||||
typeSizeSmallest = 100;
|
||||
@@ -286,10 +332,10 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
}
|
||||
}
|
||||
if (typeSizeLargest >= typeSizeSmallest * 3) {
|
||||
score -= typeSizeLargest * 1600;
|
||||
score -= typeSizeLargest * 160;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
System.out.println("pipelineAnnotation stacktrace: " + ex.getLocalizedMessage() + "\n");
|
||||
//System.out.println("pipelineAnnotation stacktrace: " + ex.getLocalizedMessage() + "\n");
|
||||
}
|
||||
sentenceConstituencyParseList.clear();
|
||||
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
@@ -308,6 +354,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
ConcurrentMap<Integer, Double> dotMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Double> elementSumMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Double> dotSumMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
//System.out.println("score pre pipelineAnnotation2Sentiment: " + score + "\n");
|
||||
for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree));
|
||||
@@ -332,59 +379,77 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
}
|
||||
Double dotPredictionIntervalDifference = largest - shortest;
|
||||
subtracter *= 25;
|
||||
//System.out.println("subtracter: " + subtracter + "\n");
|
||||
if (dotPredictionIntervalDifference < 5.0) {
|
||||
if (dotPredictions.values().size() > 0) {
|
||||
score += subtracter;
|
||||
if (subtracter > 0) {
|
||||
score -= subtracter;
|
||||
} else {
|
||||
score += subtracter;
|
||||
//System.out.println("score + subtracter: " + score + "\nsubtracter: " + subtracter + "\n");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
score -= subtracter;
|
||||
score -= subtracter / 10;
|
||||
}
|
||||
} else {
|
||||
subtracter -= 100;
|
||||
subtracter *= 25;
|
||||
score += subtracter * dotPrediction;
|
||||
//System.out.println("score + subtracter * dotPrediction: " + score + "\nsubtracter: " + subtracter + "\ndotPrediction: "
|
||||
//+ dotPrediction + "\n");
|
||||
}
|
||||
dotPredictions.put(dotPredictions.size() + 1, dotPrediction);
|
||||
}
|
||||
//System.out.println("score post subtracter1: " + score + "\n");
|
||||
Double subTracPre = 0.0;
|
||||
for (Double subtractors : subtractorMap.values()) {
|
||||
if (Objects.equals(subTracPre, subtractors)) {
|
||||
score -= 2000;
|
||||
score -= 1500;
|
||||
//System.out.println("score minus subTracPre equals: " + score + "\nsubTracPre: " + subTracPre + "\n");
|
||||
}
|
||||
subTracPre = subtractors;
|
||||
}
|
||||
ConcurrentMap<Integer, Double> DotOverTransfer = dotPredictions;
|
||||
dotPredictions = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
Double totalSubtraction = 0.0;
|
||||
for (SimpleMatrix simpleSMX : simpleSMXlist.values()) {
|
||||
double dotPrediction = simpleSMX.dot(predictions) * 100;
|
||||
AccumulateDotMap.put(AccumulateDotMap.size() + 1, dotPrediction);
|
||||
double subtracter = dotPrediction > 50 ? dotPrediction - 100 : dotPrediction > 0 ? 100 - dotPrediction : 0;
|
||||
//System.out.println("dotPrediction: " + dotPrediction + "\nsubtracter: " + subtracter + "\n");
|
||||
subtractorMap.put(subtractorMap.size() + 1, subtracter);
|
||||
if (!dotPredictions.values().contains(dotPrediction)) {
|
||||
subtracter *= 25;
|
||||
int match = 0;
|
||||
for (Double transferDots : DotOverTransfer.values()) {
|
||||
if (transferDots == dotPrediction) {
|
||||
score += subtracter;
|
||||
match++;
|
||||
totalSubtraction += transferDots;
|
||||
} else {
|
||||
score -= subtracter * 25;
|
||||
//System.out.println("score minus subtracter: " + score + "\nsubtracter: " + subtracter + "\n");
|
||||
}
|
||||
}
|
||||
if (match == 0) {
|
||||
score -= subtracter;
|
||||
//System.out.println("transferDots: " + transferDots + "\n");
|
||||
}
|
||||
} else {
|
||||
subtracter -= 100;
|
||||
subtracter *= 25;
|
||||
score += subtracter * dotPrediction;
|
||||
score -= subtracter * dotPrediction;
|
||||
//System.out.println("score minus subtracter * dotPrediction 2: " + score + "\ndotPrediction: "
|
||||
// + dotPrediction + "\n");
|
||||
}
|
||||
dotPredictions.put(dotPredictions.size() + 1, dotPrediction);
|
||||
}
|
||||
if (totalSubtraction > 45.0) {
|
||||
score -= totalSubtraction * 25;
|
||||
} else {
|
||||
score += totalSubtraction * 25;
|
||||
}
|
||||
//System.out.println("score post totalSubtraction: " + score + "\ntotalSubtraction: " + totalSubtraction + "\n");
|
||||
Double preAccumulatorDot = 0.0;
|
||||
Double postAccumulatorDot = 0.0;
|
||||
for (Double accumulators : AccumulateDotMap.values()) {
|
||||
if (Objects.equals(preAccumulatorDot, accumulators)) {
|
||||
if (Objects.equals(postAccumulatorDot, accumulators)) {
|
||||
score -= 4000;
|
||||
score -= 1400;
|
||||
}
|
||||
postAccumulatorDot = accumulators;
|
||||
}
|
||||
@@ -393,7 +458,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
subTracPre = 0.0;
|
||||
for (Double subtractors : subtractorMap.values()) {
|
||||
if (Objects.equals(subTracPre, subtractors)) {
|
||||
score -= 2000;
|
||||
score -= 500;
|
||||
}
|
||||
subTracPre = subtractors;
|
||||
}
|
||||
@@ -404,7 +469,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
double elementSum = nodeVector.kron(simpleSMX).elementSum();
|
||||
if (preDot == dot) {
|
||||
if (postDot == dot) {
|
||||
score -= 4000;
|
||||
score -= 500;
|
||||
}
|
||||
postDot = dot;
|
||||
}
|
||||
@@ -414,28 +479,34 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
dotMap.put(dotMap.size() + 1, dot);
|
||||
if (!dotSumMap.values().contains(dot)) {
|
||||
if (dot < 0.000) {
|
||||
score += dot * 1500;
|
||||
score += dot * 500;
|
||||
//System.out.println("score + dot * 500: " + score + "\ndot: " + dot + "\n");
|
||||
} else if (dot < 0.1) {
|
||||
score += 256;
|
||||
//System.out.println("score + 256: " + score + "\ndot: " + dot + "<n");
|
||||
}
|
||||
if (dot > 0.50) {
|
||||
score -= 2400;
|
||||
score -= 1200;
|
||||
}
|
||||
dotSumMap.put(dotSumMap.size() + 1, dot);
|
||||
} else {
|
||||
score -= 750;
|
||||
score -= 250;
|
||||
}
|
||||
if (!elementSumMap.values().contains(elementSum)) {
|
||||
if (elementSum < 0.01 && elementSum > 0.00) {
|
||||
score += 3300;
|
||||
//System.out.println("score elementSum < 0.01 && elementSum > 0.00: " + score + "\nelementSum: "
|
||||
// + elementSum + "\n");
|
||||
} else if (elementSum > 0.1 && elementSum < 0.2) {
|
||||
score += 1100;
|
||||
//System.out.println("score elementSum < 0.01 && elementSum > 0.00: " + score + "\nelementSum: "
|
||||
// + elementSum + "\n");
|
||||
} else {
|
||||
score -= elementSum * 1424;
|
||||
score -= elementSum * 1024;
|
||||
}
|
||||
elementSumMap.put(elementSumMap.size() + 1, elementSum);
|
||||
} else {
|
||||
score -= 750;
|
||||
score -= 250;
|
||||
}
|
||||
}
|
||||
for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) {
|
||||
@@ -443,7 +514,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
double elementSum = simpleSMX.kron(nodeVector).elementSum();
|
||||
if (preDot == dot) {
|
||||
if (postDot == dot) {
|
||||
score -= 4000;
|
||||
score -= 500;
|
||||
}
|
||||
postDot = dot;
|
||||
}
|
||||
@@ -454,28 +525,35 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
if (!dotSumMap.values().contains(dot)) {
|
||||
if (dot < 0.1) {
|
||||
score += 256;
|
||||
//System.out.println("score dot < 0.1: " + score + "\ndot: "
|
||||
// + dot + "\n");
|
||||
}
|
||||
if (dot > 0.50) {
|
||||
score -= 2400;
|
||||
score -= 1400;
|
||||
}
|
||||
dotSumMap.put(dotSumMap.size() + 1, dot);
|
||||
} else {
|
||||
score -= 750;
|
||||
score -= 250;
|
||||
}
|
||||
if (!elementSumMap.values().contains(elementSum)) {
|
||||
if (elementSum < 0.01 && elementSum > 0.00) {
|
||||
score += 1300;
|
||||
//System.out.println("score elementSum < 0.01 && elementSum > 0.00: " + score + "\nelementSum: "
|
||||
// + elementSum + "\n");
|
||||
} else if (elementSum > 0.1 && elementSum < 1.0) {
|
||||
score += 1100;
|
||||
//System.out.println("score elementSum < 0.01 && elementSum > 0.00: " + score + "\nelementSum: "
|
||||
// + elementSum + "\n");
|
||||
} else {
|
||||
score -= elementSum * 1424;
|
||||
score -= elementSum * 1024;
|
||||
}
|
||||
elementSumMap.put(elementSumMap.size() + 1, elementSum);
|
||||
} else {
|
||||
score -= 750;
|
||||
score -= 250;
|
||||
}
|
||||
}
|
||||
}
|
||||
//System.out.println("score post sentiment analyzer2: " + score + "\n");
|
||||
OptionalDouble minvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).min();
|
||||
OptionalDouble maxvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).max();
|
||||
double total = minvalueDots.getAsDouble() + maxvalueDots.getAsDouble();
|
||||
@@ -485,36 +563,43 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
}
|
||||
if (permitted) {
|
||||
Double dotsVariance = maxvalueDots.getAsDouble() - minvalueDots.getAsDouble();
|
||||
//System.out.println("maxvalueDots.getAsDouble():" + maxvalueDots.getAsDouble() + "\nminvalueDots.getAsDouble():"
|
||||
// + minvalueDots.getAsDouble() + "\ndotsVariance: " + dotsVariance + "\n");
|
||||
if (maxvalueDots.getAsDouble() > minvalueDots.getAsDouble() * 10) {
|
||||
score -= 5500;
|
||||
} else if (minvalueDots.getAsDouble() < -0.10) {
|
||||
score -= 3500;
|
||||
} else if (dotsVariance < 0.5) {
|
||||
score += 3500;
|
||||
} else if (dotsVariance < 0.5 && dotsVariance > 0.1) {
|
||||
score -= 3500;
|
||||
} else if (dotsVariance > minvalueDots.getAsDouble() * 2) {
|
||||
score += 3500;
|
||||
//System.out.println("varians 4 score. " + score + "\n");
|
||||
} else if (minvalueDots.getAsDouble() * 3 > maxvalueDots.getAsDouble() && maxvalueDots.getAsDouble() < 0.1001) {
|
||||
score += dotsVariance * 200000;
|
||||
}
|
||||
}
|
||||
//System.out.println("score post dotsVariance: " + score + "\n");
|
||||
OptionalDouble minvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).min();
|
||||
OptionalDouble maxvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).max();
|
||||
Double elementsVariance = maxvalueElements.getAsDouble() - minvalueElements.getAsDouble();
|
||||
//System.out.println("elementsVariance: " + elementsVariance + "\nmaxvalueElements.getAsDouble(): "
|
||||
// + maxvalueElements.getAsDouble() + "\nminvalueElements.getAsDouble(): " + minvalueElements.getAsDouble() + "\n");
|
||||
if (elementsVariance == 0.0) {
|
||||
score -= 550;
|
||||
} else if (elementsVariance < 0.02 && elementsVariance > -0.01) {
|
||||
score += 3500;
|
||||
} else if (elementsVariance < 0.5 && maxvalueElements.getAsDouble() > 0.0 && minvalueElements.getAsDouble() > 0.0 && elementsVariance > 0.000) {
|
||||
score += 3500;
|
||||
} else if (minvalueElements.getAsDouble() < 0.0 && minvalueElements.getAsDouble() - maxvalueElements.getAsDouble() < 0.50) {
|
||||
score -= 2500;
|
||||
} else if (elementsVariance * 2 >= maxvalueElements.getAsDouble() && elementsVariance < 0.1) {
|
||||
score -= elementsVariance * 86000;
|
||||
}
|
||||
|
||||
//System.out.println("score post elementsVariance: " + score + "\n");
|
||||
score -= (sentiment1.size() > sentiment2.size() ? sentiment1.size() - sentiment2.size() : sentiment2.size() - sentiment1.size()) * 500;
|
||||
|
||||
DocumentReaderAndWriter<CoreLabel> readerAndWriter = classifier.makePlainTextReaderAndWriter();
|
||||
List classifyRaw1 = classifier.classifyRaw(str, readerAndWriter);
|
||||
List classifyRaw2 = classifier.classifyRaw(str1, readerAndWriter);
|
||||
score -= (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200;
|
||||
|
||||
//System.out.println("score post classifyRaw: " + score + "\n");
|
||||
int mainSentiment1 = 0;
|
||||
int longest1 = 0;
|
||||
int mainSentiment2 = 0;
|
||||
@@ -539,24 +624,25 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
longest2 = partText.length();
|
||||
}
|
||||
}
|
||||
//System.out.println("score post pipelineAnnotation2Sentiment: " + score + "\n");
|
||||
if (longest1 != longest2) {
|
||||
long deffLongest = longest1 > longest2 ? longest1 : longest2;
|
||||
long deffshorter = longest1 < longest2 ? longest1 : longest2;
|
||||
if (deffLongest > deffshorter * 5) {
|
||||
score -= 5500;
|
||||
} else if (deffLongest < (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) {
|
||||
score += (deffLongest - deffshorter) * 120;
|
||||
score += (deffLongest - deffshorter) * 20;
|
||||
} else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) {
|
||||
score += (deffLongest - deffshorter) * 20;
|
||||
} else if (deffLongest - deffshorter < 2) {
|
||||
score += (deffLongest + deffshorter) * 40;
|
||||
score += (deffLongest - deffshorter) * 20;
|
||||
} else if (deffshorter * 2 >= deffLongest && deffshorter * 2 < deffLongest + 5) {
|
||||
score += deffLongest * 20;
|
||||
score += (deffLongest - deffshorter) * 20;
|
||||
} else {
|
||||
score -= (deffLongest - deffshorter) * 50;
|
||||
}
|
||||
if (deffLongest - deffshorter <= 5) {
|
||||
score += 2500;
|
||||
score += 250;
|
||||
}
|
||||
}
|
||||
int tokensCounter1 = 0;
|
||||
@@ -570,7 +656,8 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
int MarkedContinuousCounter2 = 0;
|
||||
Integer MarkedContiniousCounter1Entries = 0;
|
||||
Integer MarkedContiniousCounter2Entries = 0;
|
||||
int UnmarkedPatternCounter = 0;
|
||||
int UnmarkedPatternCounter1 = 0;
|
||||
int UnmarkedPatternCounter2 = 0;
|
||||
ConcurrentMap<Integer, String> ITokenMapTag1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> ITokenMapTag2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenStems1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
@@ -599,7 +686,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
||||
IMWEDesc entry = token.getEntry();
|
||||
MarkedContinuousCounter1 += entry.getMarkedContinuous();
|
||||
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
||||
UnmarkedPatternCounter1 += entry.getUnmarkedPattern();
|
||||
for (IMWEDesc.IPart iPart : values) {
|
||||
strTokenGetiPart1.put(strTokenGetiPart1.size() + 1, iPart.getForm());
|
||||
}
|
||||
@@ -633,7 +720,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
||||
IMWEDesc entry = token.getEntry();
|
||||
MarkedContinuousCounter2 += entry.getMarkedContinuous();
|
||||
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
||||
UnmarkedPatternCounter2 += entry.getUnmarkedPattern();
|
||||
for (IMWEDesc.IPart iPart : values) {
|
||||
strTokenGetiPart2.put(strTokenGetiPart2.size() + 1, iPart.getForm());
|
||||
}
|
||||
@@ -655,26 +742,33 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
anotatorcounter2++;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
System.out.println("SENTIMENT stacktrace: " + ex.getMessage() + "\n");
|
||||
//System.out.println("SENTIMENT stacktrace: " + ex.getMessage() + "\n");
|
||||
}
|
||||
|
||||
int entry1 = entryCounts1.values().size();
|
||||
int entry2 = entryCounts2.values().size();
|
||||
if ((entry1 >= entry2 * 5 && entry2 > 0) || (entry2 >= entry1 * 5 && entry1 > 0)) {
|
||||
score -= entry1 > entry2 ? (entry1 - entry2) * 450 : (entry2 - entry1) * 450;
|
||||
} else if (entry1 >= entry2 * 50 || entry2 >= entry1 * 50) {
|
||||
score -= entry1 > entry2 ? entry1 * 180 : entry2 * 180;
|
||||
} else if (entry1 >= entry2 * 2 || entry2 >= entry1 * 2) {
|
||||
score += entry1 > entry2 ? (entry1 - entry2) * 450 : (entry2 - entry1) * 450;
|
||||
} else if (entry1 == 0 && entry2 == 0) {
|
||||
score -= 4500;
|
||||
} else if (entry1 == entry2) {
|
||||
score += 5500;
|
||||
//System.out.println("score post JMWEAnnotation: " + score + "\nentry1: " + entry1 + "\nentry2: " + entry2 + "\n");
|
||||
if (entry1 > 0 && entry2 > 0) {
|
||||
if ((entry1 >= entry2 * 5) || (entry2 >= entry1 * 5)) {
|
||||
score -= entry1 > entry2 ? (entry1 - entry2) * 450 : (entry2 - entry1) * 450;
|
||||
//System.out.println("1");
|
||||
} else if ((entry1 >= entry2 * 50 || entry2 >= entry1 * 50)) {
|
||||
score -= entry1 > entry2 ? entry1 * 180 : entry2 * 180;
|
||||
//System.out.println("2");
|
||||
} else if (entry1 >= entry2 * 2 || entry2 >= entry1 * 2) {
|
||||
score += entry1 > entry2 ? (entry1 - entry2) * 450 : (entry2 - entry1) * 450;
|
||||
//System.out.println("3");
|
||||
} else if (entry1 > 10 && entry2 > 10 && entry1 * 2 > entry2 && entry2 * 2 > entry1) {
|
||||
score += entry1 > entry2 ? entry2 * 600 : entry1 * 600;
|
||||
//System.out.println("6");
|
||||
}
|
||||
}
|
||||
ConcurrentMap<Integer, Integer> countsMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (int counts : entryCounts1.values()) {
|
||||
for (int counts1 : entryCounts2.values()) {
|
||||
if (counts == counts1 && counts > 0 && !countsMap.values().contains(counts)) {
|
||||
score += counts * 250;
|
||||
//System.out.println("score post counts: " + score + "\nCounts: " + counts + "\n");
|
||||
countsMap.put(countsMap.size() + 1, counts);
|
||||
}
|
||||
}
|
||||
@@ -684,23 +778,27 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
for (String strTokenPos2 : strTokenEntryPOS2.values()) {
|
||||
if (strTokenPos1.equals(strTokenPos2)) {
|
||||
score += 500;
|
||||
|
||||
} else {
|
||||
score -= 650;
|
||||
|
||||
//System.out.println("strTokenEntryPOS score: " + score + "\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (UnmarkedPatternCounter > 0 && UnmarkedPatternCounter < 5) {
|
||||
score -= UnmarkedPatternCounter * 1600;
|
||||
} else {
|
||||
score -= UnmarkedPatternCounter * 10;
|
||||
//System.out.println("score pre UnmarkedPatternCounter: " + score + "\nUnmarkedPatternCounter1: " + UnmarkedPatternCounter1
|
||||
// + "\nUnmarkedPatternCounter2: " + UnmarkedPatternCounter2 + "\n");
|
||||
if (UnmarkedPatternCounter1 > 0 && UnmarkedPatternCounter2 > 0) {
|
||||
if (UnmarkedPatternCounter1 * 2 > UnmarkedPatternCounter2 && UnmarkedPatternCounter2 * 2 > UnmarkedPatternCounter1) {
|
||||
score += 2500;
|
||||
} else if (UnmarkedPatternCounter1 * 5 < UnmarkedPatternCounter2 || UnmarkedPatternCounter2 * 5 < UnmarkedPatternCounter1) {
|
||||
score -= 4000;
|
||||
}
|
||||
}
|
||||
|
||||
//System.out.println("score post UnmarkedPatternCounter: " + score + "\n");
|
||||
if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) {
|
||||
if (MarkedContinuousCounter1 > MarkedContinuousCounter2 * 50 || MarkedContinuousCounter2 > MarkedContinuousCounter1 * 50) {
|
||||
score -= MarkedContinuousCounter1 > MarkedContinuousCounter2 ? MarkedContinuousCounter1 * 120 : MarkedContinuousCounter2 * 120;
|
||||
//System.out.println("score post MarkedContinuousCounter too big: " + score + "\n");
|
||||
} else if (!Objects.equals(MarkedContiniousCounter1Entries, MarkedContiniousCounter2Entries)
|
||||
&& (MarkedContinuousCounter1 * 2 >= MarkedContinuousCounter2 * MarkedContinuousCounter1)
|
||||
|| (MarkedContinuousCounter2 * 2 >= MarkedContinuousCounter1 * MarkedContinuousCounter2)) {
|
||||
@@ -719,6 +817,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
|| MarkedContiniousCounter2Entries * 5 < MarkedContinuousCounter1
|
||||
|| MarkedContiniousCounter2Entries * 5 < MarkedContinuousCounter2) {
|
||||
score -= MarkedContinuousCounter1 > MarkedContinuousCounter2 ? MarkedContinuousCounter1 * 400 : MarkedContinuousCounter2 * 400;
|
||||
//System.out.println("score post MarkedContinuousCounter: " + score + "\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -730,6 +829,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
score += 400;
|
||||
} else {
|
||||
score -= 200;
|
||||
//System.out.println("score minus strTokenGetiPart: " + score + "\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -747,7 +847,8 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
if (strTokenEntry1.equals(strTokenEntry2)) {
|
||||
score += boundariyLeacks ? 2500 : 2500 / 2;
|
||||
} else if (!boundariyLeacks) {
|
||||
score -= 1250;
|
||||
score -= 450;
|
||||
//System.out.println("boundariyLeacks score: " + score + "\n");
|
||||
} else {
|
||||
remnantCounter++;
|
||||
}
|
||||
@@ -755,7 +856,9 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
entryTokenMap.put(entryTokenMap.size() + 1, strTokenEntry2);
|
||||
}
|
||||
}
|
||||
score -= remnantCounter * 250;
|
||||
//System.out.println("score pre remnantCounter: " + score + "\n");
|
||||
score += remnantCounter * 250;
|
||||
//System.out.println("score post remnantCounter: " + score + "\n");
|
||||
ConcurrentMap<Integer, String> iTokenMapTagsMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (String strmapTag : ITokenMapTag1.values()) {
|
||||
for (String strmapTag1 : ITokenMapTag2.values()) {
|
||||
@@ -767,6 +870,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
}
|
||||
}
|
||||
}
|
||||
//System.out.println("score post strmapTag: " + score + "\n");
|
||||
int tokenform1size = strTokenForm1.values().size();
|
||||
int tokenform2size = strTokenForm2.values().size();
|
||||
if (tokenform1size > 0 || tokenform2size > 0) {
|
||||
@@ -777,50 +881,81 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
score -= 1600;
|
||||
} else {
|
||||
score += 500;
|
||||
//System.out.println("tokenform1size score500: " + score + "\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (tokenform1size > 0 && tokenform2size > 0) {
|
||||
score += tokenform1size > tokenform2size ? tokenform1size * 1600 : tokenform2size * 1600;
|
||||
if (tokenform1size * 2 >= tokenform2size && tokenform2size * 2 >= tokenform1size) {
|
||||
score += tokenform1size > tokenform2size ? tokenform1size * 600 : tokenform2size * 600;
|
||||
} else if (tokenform1size * 4 <= tokenform2size || tokenform2size * 4 <= tokenform1size) {
|
||||
score -= tokenform1size > tokenform2size ? (tokenform1size - tokenform2size) * 600 : (tokenform2size - tokenform1size) * 600;
|
||||
}
|
||||
//System.out.println("tokenform1size score: " + score + "\ntokenform1size: " + tokenform1size + "\ntokenform2size: "
|
||||
// + tokenform2size + "\n");
|
||||
}
|
||||
} else {
|
||||
tokenform1size = tokenform1size > 0 ? tokenform1size : 1;
|
||||
tokenform2size = tokenform2size > 0 ? tokenform2size : 1;
|
||||
score -= (tokenform1size + tokenform2size) * 1200;
|
||||
}
|
||||
//System.out.println("Score pre tokenStemmingMap: " + score + "\n");
|
||||
ConcurrentMap<Integer, String> tokenStemmingMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (String strTokenStem : strTokenStems1.values()) {
|
||||
for (String strTokenStem1 : strTokenStems2.values()) {
|
||||
if (strTokenStem.equals(strTokenStem1)) {
|
||||
if (strTokenStem.equals(strTokenStem1) && !tokenStemmingMap.values().contains(strTokenStem)) {
|
||||
score += 1500;
|
||||
} else if (!tokenStemmingMap.values().contains(strTokenStem)) {
|
||||
score -= 150;
|
||||
tokenStemmingMap.put(tokenStemmingMap.size() + 1, strTokenStem);
|
||||
}
|
||||
//System.out.println("score strTokenStem: " + score + "\n");
|
||||
}
|
||||
}
|
||||
//System.out.println("Score pre inflected: " + score + "\n");
|
||||
//System.out.println("inflectedCounterPositive1: " + inflectedCounterPositive1 + "\ninflectedCounterPositive2: "
|
||||
// + inflectedCounterPositive2 + "\ninflectedCounterNegative: " + inflectedCounterNegative + "\n");
|
||||
if (inflectedCounterPositive1 + inflectedCounterPositive2 > inflectedCounterNegative && inflectedCounterNegative > 0) {
|
||||
score += (inflectedCounterPositive1 - inflectedCounterNegative) * 650;
|
||||
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 650;
|
||||
//System.out.println("score inflectedCounterPositive plus: " + score + "\n");
|
||||
}
|
||||
if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) {
|
||||
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550;
|
||||
}
|
||||
if (anotatorcounter1 > 1 && anotatorcounter2 > 1) {
|
||||
score += (anotatorcounter1 - anotatorcounter2) * 400;
|
||||
}
|
||||
if ((tokensCounter1 > 0 && tokensCounter2 > 0) && tokensCounter1 < tokensCounter2 * 5 && tokensCounter2 < tokensCounter1 * 5) {
|
||||
score += (tokensCounter1 + tokensCounter2) * 1400;
|
||||
} else {
|
||||
int elseint = tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
|
||||
if ((tokensCounter1 > tokensCounter2 * 5 || tokensCounter2 > tokensCounter1 * 5)
|
||||
&& tokensCounter1 > 0 && tokensCounter2 > 0) {
|
||||
score -= (tokensCounter1 + tokensCounter2) * 1500;
|
||||
} else if (elseint > 0 && tokensCounter1 > 0 && tokensCounter2 > 0) {
|
||||
score += elseint * 2;
|
||||
} else if (elseint == 0) {
|
||||
score += 1500;
|
||||
if (inflectedCounterPositive1 * 2 > inflectedCounterPositive2 && inflectedCounterPositive2 * 2 > inflectedCounterPositive1) {
|
||||
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550;
|
||||
//System.out.println("score plus inflectedCounterPositive * 2: " + score + "\n");
|
||||
} else if (inflectedCounterPositive1 * 5 < inflectedCounterPositive2 || inflectedCounterPositive2 * 5 < inflectedCounterPositive1) {
|
||||
score -= inflectedCounterPositive1 > inflectedCounterPositive2 ? (inflectedCounterPositive1 - inflectedCounterPositive2) * 400
|
||||
: (inflectedCounterPositive2 - inflectedCounterPositive1) * 400;
|
||||
//System.out.println("score minus inflectedCounterPositive * 2: " + score + "\n");
|
||||
}
|
||||
}
|
||||
//System.out.println("anotatorcounter1: " + anotatorcounter1 + "\nanotatorcounter2: " + anotatorcounter2 + "\n");
|
||||
if (anotatorcounter1 > 1 && anotatorcounter2 > 1) {
|
||||
if (anotatorcounter1 * 2 > anotatorcounter2 && anotatorcounter2 * 2 > anotatorcounter1) {
|
||||
score += anotatorcounter1 > anotatorcounter2 ? (anotatorcounter1 - anotatorcounter2) * 700
|
||||
: (anotatorcounter2 - anotatorcounter1) * 700;
|
||||
//System.out.println("score plus anotatorcounter: " + score + "\n");
|
||||
} else if (anotatorcounter1 * 5 < anotatorcounter2 || anotatorcounter2 * 5 < anotatorcounter1) {
|
||||
score -= anotatorcounter1 > anotatorcounter2 ? (anotatorcounter1 - anotatorcounter2) * 400 : (anotatorcounter2 - anotatorcounter1) * 400;
|
||||
//System.out.println("score minus anotatorcounter: " + score + "\n");
|
||||
}
|
||||
}
|
||||
//System.out.println("tokensCounter1: " + tokensCounter1 + "\ntokensCounter2: " + tokensCounter2 + "\n");
|
||||
if ((tokensCounter1 > 1 && tokensCounter2 > 1) && tokensCounter1 < tokensCounter2 * 5 && tokensCounter2 < tokensCounter1 * 5) {
|
||||
if (tokensCounter1 > tokensCounter2 / 2 && tokensCounter2 > tokensCounter1 / 2) {
|
||||
score += (tokensCounter1 + tokensCounter2) * 1400;
|
||||
//System.out.println("score plus tokensCounter: " + score + "\n");
|
||||
} else {
|
||||
score -= 3500;
|
||||
//System.out.println("score minus tokensCounter: " + score + "\n");
|
||||
}
|
||||
} else {
|
||||
int elseint = tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
|
||||
//System.out.println("elseint: " + elseint + "<n");
|
||||
if ((tokensCounter1 > tokensCounter2 * 5 || tokensCounter2 > tokensCounter1 * 5)
|
||||
&& tokensCounter1 > 0 && tokensCounter2 > 0) {
|
||||
score -= tokensCounter1 > tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
|
||||
//System.out.println("score post tokensCounter: " + score + "\n");
|
||||
} else if (elseint > 0 && tokensCounter1 > 0 && tokensCounter2 > 0) {
|
||||
score -= elseint * 2;
|
||||
//System.out.println("score post elseint: " + elseint + "\n");
|
||||
}
|
||||
}
|
||||
//System.out.println("Score Pre levenhstein: " + score + "\n");
|
||||
LevenshteinDistance leven = new LevenshteinDistance(str, str1);
|
||||
double SentenceScoreDiff = leven.computeLevenshteinDistance();
|
||||
SentenceScoreDiff *= 15;
|
||||
@@ -841,9 +976,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
List<CoreLabel> tokens = em.tokens();
|
||||
for (CoreLabel token : tokens) {
|
||||
if (!nerEntityTokenTags1.values().contains(token.tag())) {
|
||||
if (entityType.equals("PERSON") && EntityConfidences < 0.80) {
|
||||
score -= 6000;
|
||||
} else {
|
||||
if (entityType.equals("PERSON") && EntityConfidences > 0.80) {
|
||||
nerEntityTokenTags1.put(nerEntityTokenTags1.size() + 1, token.tag());
|
||||
}
|
||||
}
|
||||
@@ -863,9 +996,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
List<CoreLabel> tokens = em.tokens();
|
||||
for (CoreLabel token : tokens) {
|
||||
if (!nerEntityTokenTags2.values().contains(token.tag())) {
|
||||
if (entityType.equals("PERSON") && EntityConfidences < 0.80) {
|
||||
score -= 6000;
|
||||
} else {
|
||||
if (entityType.equals("PERSON") && EntityConfidences > 0.80) {
|
||||
nerEntityTokenTags2.put(nerEntityTokenTags2.size() + 1, token.tag());
|
||||
}
|
||||
}
|
||||
@@ -875,26 +1006,115 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
nerEntities4.put(nerEntities4.size() + 1, em.entityType());
|
||||
}
|
||||
}
|
||||
//System.out.println("score post PERSON trim: " + score + "\n");
|
||||
for (String strEnts1 : nerEntities1.values()) {
|
||||
Collection<String> values = nerEntities2.values();
|
||||
for (String strEnts2 : values) {
|
||||
if (strEnts1.equalsIgnoreCase(strEnts2)) {
|
||||
score += 7500;
|
||||
score += 2500;
|
||||
//System.out.println("score strEnts1 plus: " + score + "\n");
|
||||
} else {
|
||||
score -= 150;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strEnts1 : nerEntities3.values()) {
|
||||
if (nerEntities4.values().contains(strEnts1)) {
|
||||
score += 3500;
|
||||
score -= 1500;
|
||||
//System.out.println("score nerEntities4 minus: " + score + "\n");
|
||||
} else {
|
||||
score -= 150;
|
||||
}
|
||||
}
|
||||
for (String strToken : nerEntityTokenTags1.values()) {
|
||||
if (nerEntityTokenTags2.values().contains(strToken)) {
|
||||
score += 2500;
|
||||
score += 2000;
|
||||
//System.out.println("score nerEntities4 plus: " + score + "\n");
|
||||
} else {
|
||||
score -= 150;
|
||||
}
|
||||
}
|
||||
//System.out.println("score pre stopwordTokens: " + score + "\n");
|
||||
ConcurrentMap<Integer, String> stopwordTokens = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> stopwordTokens1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> stopWordLemma = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> stopWordLemma1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
Integer pairCounter1 = 0;
|
||||
Integer pairCounter2 = 0;
|
||||
String customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with";
|
||||
List<CoreLabel> tokensSentiment1 = pipelineAnnotation1Sentiment.get(CoreAnnotations.TokensAnnotation.class);
|
||||
List<CoreLabel> tokensSentiment2 = pipelineAnnotation2Sentiment.get(CoreAnnotations.TokensAnnotation.class);
|
||||
Set<?> stopWords = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
|
||||
Set<?> stopWordsCustom = StopwordAnnotator.getStopWordList(customStopWordList, true);
|
||||
for (CoreLabel token : tokensSentiment1) {
|
||||
Pair<Boolean, Boolean> stopword = token.get(StopwordAnnotator.class);
|
||||
String word = token.word().toLowerCase();
|
||||
if (stopWords.contains(word) || stopWordsCustom.contains(word)) {
|
||||
stopwordTokens.put(stopwordTokens.size(), word);
|
||||
}
|
||||
String lemma = token.lemma().toLowerCase();
|
||||
if (stopWords.contains(lemma) || stopWordsCustom.contains(lemma)) {
|
||||
stopWordLemma.put(stopWordLemma.size(), lemma);
|
||||
}
|
||||
if (stopword.first() && stopword.second()) {
|
||||
pairCounter1++;
|
||||
}
|
||||
//System.out.println("stopword Pair: " + stopword.first() + " " + stopword.second() + "\nword: "
|
||||
// + word + "\nlemma: " + lemma + "\n");
|
||||
}
|
||||
for (CoreLabel token : tokensSentiment2) {
|
||||
Pair<Boolean, Boolean> stopword = token.get(StopwordAnnotator.class);
|
||||
String word = token.word().toLowerCase();
|
||||
if (stopWords.contains(word) || stopWordsCustom.contains(word)) {
|
||||
stopwordTokens1.put(stopwordTokens1.size(), word);
|
||||
}
|
||||
String lemma = token.lemma().toLowerCase();
|
||||
if (stopWords.contains(lemma) || stopWordsCustom.contains(lemma)) {
|
||||
stopWordLemma1.put(stopWordLemma1.size(), lemma);
|
||||
}
|
||||
if (stopword.first() && stopword.second()) {
|
||||
pairCounter2++;
|
||||
}
|
||||
//System.out.println("stopword Pair: " + stopword.first() + " " + stopword.second() + "\nword: "
|
||||
// + word + "\nlemma: " + lemma + "\n");
|
||||
}
|
||||
for (String stopwords1 : stopwordTokens.values()) {
|
||||
for (String stopwords2 : stopwordTokens1.values()) {
|
||||
if (stopwords1.equals(stopwords2)) {
|
||||
score -= 500;
|
||||
//System.out.println("score stopwordsToken: " + score + "\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String stopwords1 : stopWordLemma.values()) {
|
||||
for (String stopwords2 : stopWordLemma1.values()) {
|
||||
if (stopwords1.equals(stopwords2)) {
|
||||
score -= 500;
|
||||
//System.out.println("score stopwords Lemma: " + score + "\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!stopwordTokens.values().isEmpty() && !stopwordTokens1.values().isEmpty()) {
|
||||
int stopwordsize1 = stopwordTokens.values().size();
|
||||
int stopwordsize2 = stopwordTokens1.values().size();
|
||||
if (stopwordsize1 * 5 < stopwordsize2 || stopwordsize2 * 5 < stopwordsize1) {
|
||||
score -= stopwordsize1 > stopwordsize2 ? (stopwordsize1 - stopwordsize2) * 850 : (stopwordsize2 - stopwordsize1) * 850;
|
||||
} else {
|
||||
score += stopwordsize1 > stopwordsize2 ? (stopwordsize1 - stopwordsize2) * 850 : (stopwordsize2 - stopwordsize1) * 850;;
|
||||
}
|
||||
//System.out.println("score post stopwordsize: " + score + "\nstopwordsize1: " + stopwordsize1 + "\nstopwordsize2: "
|
||||
// + stopwordsize2 + "\n");
|
||||
}
|
||||
if (pairCounter1 > 0 && pairCounter2 > 0) {
|
||||
if (pairCounter1 * 3 <= pairCounter2 || pairCounter2 * 3 <= pairCounter1) {
|
||||
score -= pairCounter1 > pairCounter2 ? (pairCounter1 - pairCounter2) * 1500 : (pairCounter2 - pairCounter1) * 1500;
|
||||
} else {
|
||||
score += pairCounter1 > pairCounter2 ? (pairCounter1 - pairCounter2) * 700 : (pairCounter2 - pairCounter1) * 700;
|
||||
}
|
||||
//System.out.println("score post pairCounter: " + score + "\npairCounter1: " + pairCounter1 + "\npairCounter2: " + pairCounter2 + "\n");
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
System.out.println("SENTIMENT stacktrace Overall catch: " + ex.getMessage() + "\n");
|
||||
//System.out.println("SENTIMENT stacktrace Overall catch: " + ex.getMessage() + "\n");
|
||||
}
|
||||
System.out.println("Final current score: " + score + "\nSentence 1: " + str + "\nSentence 2: " + str1 + "\n");
|
||||
smxParam.setDistance(score);
|
||||
|
||||
Reference in New Issue
Block a user