UNLOZE/projects-jenz
10
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

WHY THE FUCK CANT YOU JUST TRANSFER A SimilarityMatrix OBJECT LIST LIKE ANY OTHER NORMAL COLLECTION

Browse Source
This commit is contained in:
jenzur
2019-05-24 16:08:25 +02:00
parent a4139e4ae4
commit f54c447eb6
3 changed files with 318 additions and 408 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java
+177 -102
View File
@@ -68,8 +68,6 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
private final String str1;
private final MaxentTagger tagger;
private final GrammaticalStructureFactory gsf;
private final StanfordCoreNLP pipeline;
private final StanfordCoreNLP pipelineSentiment;
private final AbstractSequenceClassifier classifier;
private final Annotation jmweStrAnnotation1;
private final Annotation jmweStrAnnotation2;
@@ -97,8 +95,6 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
this.str1 = str1;
this.smxParam = smxParam;
this.tagger = Datahandler.getTagger();
this.pipeline = Datahandler.getPipeline();
this.pipelineSentiment = Datahandler.getPipelineSentiment();
this.gsf = Datahandler.getGsf();
this.classifier = Datahandler.getClassifier();
this.jmweStrAnnotation1 = str1Annotation;
@@ -190,13 +186,14 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
for (Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2.values()) {
if (grammaticalRelation.isApplicable(sentenceConstituencyParse2)) {
score += 700;
//System.out.println("grammaticalRelation applicable score: " + score + "\n");
// System.out.println("grammaticalRelation applicable score: " + score + "\n");
grammaticalRelation1++;
}
GrammaticalRelation reln = TDY1.reln();
//sentenceConstituencyParse1
if (reln.isApplicable(sentenceConstituencyParse2)) {
score += 525;
// System.out.println("reln1 applicable score: " + score + "\n");
relationApplicable1++;
}
}
@@ -210,22 +207,23 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1.values()) {
if (grammaticalRelation.isApplicable(sentenceConstituencyParse1)) {
score += 700;
//System.out.println("grammaticalRelation applicable score: " + score + "\n");
// System.out.println("grammaticalRelation applicable score: " + score + "\n");
grammaticalRelation2++;
}
GrammaticalRelation reln = TDY.reln();
//sentenceConstituencyParse1
if (reln.isApplicable(sentenceConstituencyParse1)) {
score += 525;
// System.out.println("reln2 applicable score: " + score + "\n");
relationApplicable2++;
}
}
}
}
// System.out.println("\ngrammaticalRelation1: " + grammaticalRelation1 + "\ngrammaticalRelation2: " + grammaticalRelation2 + "\n");
if ((grammaticalRelation1 == 0 && grammaticalRelation2 > 4) || (grammaticalRelation2 == 0 && grammaticalRelation1 > 4)) {
score -= 3450;
//System.out.println("grammaticalRelation1 score trim: " + score + "\ngrammaticalRelation1: " + grammaticalRelation1
// + "\ngrammaticalRelation2: " + grammaticalRelation2 + "\n");
// System.out.println("grammaticalRelation score trim: " + score + "\n");
}
if (!allTypedDependencies1.isEmpty() || !allTypedDependencies2.isEmpty()) {
int allTypeDep1 = allTypedDependencies1.size();
@@ -233,13 +231,13 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
if (allTypeDep1 <= allTypeDep2 * 5 && allTypeDep2 <= allTypeDep1 * 5) {
if (allTypeDep1 > 0 && allTypeDep2 > 0) {
if (allTypeDep1 * 2 <= allTypeDep2 || allTypeDep2 * 2 <= allTypeDep1) {
score -= allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 160 : (allTypeDep2 - allTypeDep1) * 160;
//System.out.println(" allTypeDep score: " + score + "\nallTypeDep1: " + allTypeDep1 + "\nallTypeDep2: "
// + allTypeDep2 + "\n");
score += allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 160 : (allTypeDep2 - allTypeDep1) * 160;
// System.out.println(" allTypeDep plus score: " + score + "\nallTypeDep1: " + allTypeDep1 + "\nallTypeDep2: "
// + allTypeDep2 + "\n");
} else {
score += allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 600 : (allTypeDep2 - allTypeDep1) * 600;
//System.out.println(" allTypeDep score: " + score + "\nallTypeDep1: " + allTypeDep1 + "\nallTypeDep2: "
// + allTypeDep2 + "\n");
score -= allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 600 : (allTypeDep2 - allTypeDep1) * 600;
// System.out.println(" allTypeDep minus score: " + score + "\nallTypeDep1: " + allTypeDep1 + "\nallTypeDep2: "
// + allTypeDep2 + "\n");
}
alltypeDepsSizeMap.put(alltypeDepsSizeMap.size() + 1, allTypeDep1);
alltypeDepsSizeMap.put(alltypeDepsSizeMap.size() + 1, allTypeDep2);
@@ -253,44 +251,43 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
&& !summationMap.values().contains(summation)) {
score += summation * 80;
summationMap.put(summationMap.size() + 1, summation);
//System.out.println("score post summation: " + score + "\nsummation: " + summation + "\n");
// System.out.println("score post summation: " + score + "\nsummation: " + summation + "\n");
} else if (largerTypeDep == smallerTypeDep) {
score += 2500;
//System.out.println("score largerTypeDep equals smallerTypeDep: " + score + "\nlargerTypeDep: " + largerTypeDep + "\n");
// System.out.println("score largerTypeDep equals smallerTypeDep: " + score + "\nlargerTypeDep: " + largerTypeDep + "\n");
}
}
if (relationApplicable1 > 0 && relationApplicable2 > 0 && relationApplicable1 == relationApplicable2
&& grammaticalRelation1 > 0 && grammaticalRelation2 > 0 && grammaticalRelation1 == grammaticalRelation2) {
score += 3500;
//System.out.println("score relationApplicable equal: " + score + "\n");
if (relationApplicable1 > 0 && relationApplicable2 > 0 && relationApplicable1 != relationApplicable2) {
score += 1500;
// System.out.println("score relationApplicable equal: " + score + "\n");
} else if (allTypeDep1 * 5 < allTypeDep2 || allTypeDep2 * 5 < allTypeDep1) {
score -= allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * (allTypeDep2 * 450)
: (allTypeDep2 - allTypeDep1) * (allTypeDep1 * 450);
//System.out.println("score minus grammaticalRelation equal: " + score + "\n");
// System.out.println("score minus grammaticalRelation equal: " + score + "\n");
}
if (relationApplicable1 > 1 && relationApplicable2 > 1 && relationApplicable1 * 3 > relationApplicable2
&& relationApplicable2 * 3 > relationApplicable1) {
score += relationApplicable1 > relationApplicable2 ? (relationApplicable1 - relationApplicable2) * 1500
: (relationApplicable2 - relationApplicable1) * 1500;
//System.out.println("score relationApplicable plus: " + score + "\n");
// System.out.println("score relationApplicable plus: " + score + "\n");
} else if (relationApplicable1 * 5 < relationApplicable2 || relationApplicable2 * 5 < relationApplicable1) {
score -= relationApplicable1 > relationApplicable2 ? (relationApplicable1 - relationApplicable2) * 500
: (relationApplicable2 - relationApplicable1) * 500;
//System.out.println("score relationApplicable minus: " + score + "\n");
// System.out.println("score relationApplicable minus: " + score + "\n");
}
if (grammaticalRelation1 > 0 && grammaticalRelation2 > 0 && grammaticalRelation1 * 3 > grammaticalRelation2
&& grammaticalRelation2 * 3 > grammaticalRelation1) {
score += grammaticalRelation1 > grammaticalRelation2 ? (grammaticalRelation1 - grammaticalRelation2) * 1500
: (grammaticalRelation2 - grammaticalRelation1) * 1500;
//System.out.println("score grammaticalRelation plus: " + score + "\n");
// System.out.println("score grammaticalRelation plus: " + score + "\n");
} else if (grammaticalRelation1 * 5 < grammaticalRelation2 || grammaticalRelation2 * 5 < grammaticalRelation1) {
score -= grammaticalRelation1 > grammaticalRelation2 ? (grammaticalRelation1 - grammaticalRelation2) * 500
: (grammaticalRelation2 - grammaticalRelation1) * 500;
//System.out.println("score grammaticalRelation minus: " + score + "\n");
// System.out.println("score grammaticalRelation minus: " + score + "\n");
}
//System.out.println("score post relationApplicable1 veri: " + score + "\nrelationApplicable1: " + relationApplicable1
// + "\nrelationApplicable2: " + relationApplicable2 + "\ngrammaticalRelation1: " + grammaticalRelation1 + "\n"
// + "grammaticalRelation2: " + grammaticalRelation2 + "\n");
// System.out.println("score post relationApplicable1 veri: " + score + "\nrelationApplicable1: " + relationApplicable1
// + "\nrelationApplicable2: " + relationApplicable2 + "\ngrammaticalRelation1: " + grammaticalRelation1 + "\n"
// + "grammaticalRelation2: " + grammaticalRelation2 + "\n");
}
ConcurrentMap<Integer, String> filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap();
AtomicInteger runCount1 = new AtomicInteger(0);
@@ -338,13 +335,12 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
return score;
}
private final Double simpleRNNMatrixCalculations(Double score, ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist1, ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist2) {
private Double simpleRNNMatrixCalculations(Double score, ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist1, ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist2) {
for (SimpleMatrix simpleSMX2 : simpleSMXlist2.values()) {
ConcurrentMap<Integer, Double> AccumulateDotMap = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Double> subtractorMap = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Double> dotPredictions = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Double> DotOverTransfer = dotPredictions;
dotPredictions = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Double> DotOverTransfer = new MapMaker().concurrencyLevel(2).makeMap();
Double totalSubtraction = 0.0;
Double largest = 10.0;
Double shortest = 100.0;
@@ -357,26 +353,38 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
double subtracter2 = dotPrediction2 > 50 ? dotPrediction2 - 100 : dotPrediction2 > 0 ? 100 - dotPrediction2 : 0;
subtractorMap.put(subtractorMap.size() + 1, subtracter1);
subtractorMap.put(subtractorMap.size() + 1, subtracter2);
//System.out.println("dotPrediction: " + dotPrediction + "\nsubtracter: " + subtracter + "\n");
// System.out.println("dotPrediction: " + dotPrediction1 + "\n");
if (!dotPredictions.values().contains(dotPrediction1)) {
for (Double transferDots : DotOverTransfer.values()) {
if (transferDots == dotPrediction1) {
totalSubtraction += transferDots;
} else {
score -= subtracter1 * 25;
//System.out.println("score minus subtracter: " + score + "\nsubtracter: " + subtracter + "\n");
// System.out.println("score minus subtracter: " + score + "\n");
}
//System.out.println("transferDots: " + transferDots + "\n");
// System.out.println("transferDots: " + transferDots + "\n");
}
DotOverTransfer.put(DotOverTransfer.size(), dotPrediction1);
} else {
// System.out.println("subtracter1 pre: " + subtracter1 + "\n");
subtracter1 -= 100;
subtracter1 *= 25;
score -= subtracter1 * dotPrediction1;
//System.out.println("score minus subtracter * dotPrediction 2: " + score + "\ndotPrediction: "
// + dotPrediction + "\n");
score += subtracter1 * dotPrediction1;
// System.out.println("subtracter1 post: " + subtracter1 + "\n");
// System.out.println("score minus subtracter * dotPrediction 2: " + score + "\ndotPrediction1: " + dotPrediction1 + "\n");
}
dotPredictions.put(dotPredictions.size() + 1, dotPrediction1);
if (!dotPredictions.values().contains(dotPrediction2)) {
for (Double transferDots : DotOverTransfer.values()) {
if (transferDots == dotPrediction2) {
totalSubtraction += transferDots;
} else {
score -= subtracter1 * 25;
// System.out.println("score minus subtracter: " + score + "\n");
}
// System.out.println("transferDots: " + transferDots + "\n");
}
DotOverTransfer.put(DotOverTransfer.size(), dotPrediction2);
if (dotPrediction2 > largest) {
largest = dotPrediction2;
}
@@ -392,42 +400,51 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
score -= subtracter2;
} else {
score += subtracter2;
//System.out.println("score + subtracter: " + score + "\nsubtracter: " + subtracter + "\n");
// System.out.println("score + subtracter: " + score + "\nsubtracter2: " + subtracter2 + "\n");
}
}
} else {
score -= subtracter2 / 10;
}
} else {
subtracter2 -= 100;
subtracter2 *= 25;
score += subtracter2 * dotPrediction2;
//System.out.println("score + subtracter * dotPrediction: " + score + "\nsubtracter: " + subtracter + "\ndotPrediction: "
//+ dotPrediction + "\n");
} else if (dotPrediction2 < 22.0 || dotPrediction2 > 40.0) {
//System.out.println("subtracter2: " + subtracter2 + "\n");
if (subtracter2 > 55.0 && subtracter2 < 82.0) {
score += subtracter2 * dotPrediction2;
// System.out.println("score: " + score + "\ndotPrediction2: " + dotPrediction2 + "\n");
}
}
dotPredictions.put(dotPredictions.size() + 1, dotPrediction2);
}
//System.out.println("score post subtracter1: " + score + "\n");
// System.out.println("score post subtracter1: " + score + "\n");
Double subTracPre = 0.0;
for (Double subtractors : subtractorMap.values()) {
if (Objects.equals(subTracPre, subtractors)) {
score -= 1500;
//System.out.println("score minus subTracPre equals: " + score + "\nsubTracPre: " + subTracPre + "\n");
if (Objects.equals(subTracPre, subtractors) && subTracPre < 70.0 && subTracPre > 20.0) {
score += (subTracPre * 10) / subtractorMap.values().size();
// System.out.println("score subTracPre plus equals: " + score + "\nsubTracPre: " + subTracPre + "\n");
} else {
if (subTracPre > 0.0) {
score -= (subTracPre * 50) / subtractorMap.values().size();
} else {
score += (subTracPre * 50) / subtractorMap.values().size();
}
// System.out.println("score subTracPre minus equals: " + score + "\nsubTracPre: " + subTracPre + "\n");
}
subTracPre = subtractors;
}
if (totalSubtraction > 45.0) {
score -= totalSubtraction * 25;
} else {
score += totalSubtraction * 25;
}
//System.out.println("score post totalSubtraction: " + score + "\ntotalSubtraction: " + totalSubtraction + "\n");
// System.out.println("score post totalSubtraction: " + score + "\ntotalSubtraction: " + totalSubtraction + "\n");
Double preAccumulatorDot = 0.0;
Double postAccumulatorDot = 0.0;
for (Double accumulators : AccumulateDotMap.values()) {
if (Objects.equals(preAccumulatorDot, accumulators)) {
if (Objects.equals(postAccumulatorDot, accumulators)) {
score -= 1400;
// System.out.println("score minus postAccumulatorDot: " + score + "\n");
}
postAccumulatorDot = accumulators;
}
@@ -436,7 +453,8 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
subTracPre = 0.0;
for (Double subtractors : subtractorMap.values()) {
if (Objects.equals(subTracPre, subtractors)) {
score -= 500;
score += 500;
// System.out.println("score minus subTracPre: " + score + "\n");
}
subTracPre = subtractors;
}
@@ -674,15 +692,13 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
private SentimentValueCache sentimentCoreAnnotationSetup(Annotation pipelineAnnotationSentiment, SentimentValueCache cacheSentimentLocal) {
for (CoreMap sentence : pipelineAnnotationSentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
int predictedClass = RNNCoreAnnotations.getPredictedClass(tree);
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
try {
if (tree != null) {
int predictedClass = RNNCoreAnnotations.getPredictedClass(tree);
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
cacheSentimentLocal.addRNNPredictClass(predictedClass);
cacheSentimentLocal.addSimpleMatrix(predictions);
cacheSentimentLocal.addSimpleMatrixVector(nodeVector);
} catch (Exception ex) {
System.out.println("ex: " + ex.getLocalizedMessage() + "\n");
}
}
return cacheSentimentLocal;
@@ -704,12 +720,15 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
private SentimentValueCache jmweAnnotationSetup(Annotation jmweStrAnnotation, SentimentValueCache cacheSentimentLocal) {
List<CoreMap> sentences = jmweStrAnnotation.get(CoreAnnotations.SentencesAnnotation.class);
Collection<IMWE<IToken>> tokeninflectionMap = new ArrayList();
int tokenadder = 0;
for (CoreMap sentence : sentences) {
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
if (token.isInflected()) {
cacheSentimentLocal.setInflectedCounterPositive(cacheSentimentLocal.getInflectedCounterPositive() + 1);
} else {
} else if (!tokeninflectionMap.contains(token)) {
cacheSentimentLocal.setInflectedCounterNegative(cacheSentimentLocal.getInflectedCounterNegative() + 1);
tokeninflectionMap.add(token);
}
cacheSentimentLocal.addstrTokenForm(token.getForm());
cacheSentimentLocal.addstrTokenGetEntry(token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
@@ -733,21 +752,29 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
cacheSentimentLocal.setMarkedContiniousCounterEntries(cacheSentimentLocal.getMarkedContiniousCounterEntries() + 1);
}
}
cacheSentimentLocal.setTokensCounter(cacheSentimentLocal.getTokensCounter() + 1);
tokenadder += 1;
}
cacheSentimentLocal.setAnotatorcounter(cacheSentimentLocal.getAnotatorcounter() + 1);
}
cacheSentimentLocal.setTokensCounter(tokenadder);
return cacheSentimentLocal;
}
private Double entryCountsScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
ConcurrentMap<Integer, Integer> countsMap = new MapMaker().concurrencyLevel(2).makeMap();
int totalsize = cacheSentimentLocal1.getEntryCounts().values().size() + cacheSentimentLocal2.getEntryCounts().values().size();
for (int counts : cacheSentimentLocal1.getEntryCounts().values()) {
for (int counts1 : cacheSentimentLocal2.getEntryCounts().values()) {
if (counts == counts1 && counts > 0 && !countsMap.values().contains(counts)) {
score += counts * 250;
//System.out.println("score post counts: " + score + "\nCounts: " + counts + "\n");
countsMap.put(countsMap.size() + 1, counts);
if (counts > 0 && counts1 > 0) {
//System.out.println("counts1: " + counts + "\ncounts2: " + counts1 + "\n");
if (counts == counts1 && !countsMap.values().contains(counts)) {
score += (counts * 250) / totalsize;
// System.out.println("score post counts plus: " + score + "\ntotalsize: " + totalsize + "\n");
countsMap.put(countsMap.size() + 1, counts);
} else if (counts * 3 < counts1 || counts1 * 3 < counts) {
score -= 600;
// System.out.println("score post counts minus: " + score + "\n");
}
}
}
}
@@ -760,12 +787,17 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
for (String strTokenPos2 : cacheSentimentLocal2.getstrTokenEntryPOS().values()) {
if (strTokenPos1.equals(strTokenPos2)) {
score += 500;
} else {
score -= 650;
//System.out.println("strTokenEntryPOS score: " + score + "\n");
// System.out.println("strTokenEntryPOS score: " + score + "\n");
}
}
}
int posEntrySize1 = cacheSentimentLocal1.getstrTokenEntryPOS().values().size();
int posEntrySize2 = cacheSentimentLocal2.getstrTokenEntryPOS().values().size();
if (posEntrySize1 * 3 > posEntrySize2 && posEntrySize2 * 3 > posEntrySize1) {
score += posEntrySize1 > posEntrySize2 ? (posEntrySize1 - posEntrySize2) * 700 : (posEntrySize2 - posEntrySize1) * 700;
//System.out.println("posEntrySize plus score: " + score + "\n");
}
// System.out.println("posEntrySize1: " + posEntrySize1 + "\nposEntrySize2: " + posEntrySize2 + "\n");
}
return score;
}
@@ -773,11 +805,19 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
private Double unmarkedPatternCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
int UnmarkedPatternCounter1 = cacheSentimentLocal1.getUnmarkedPatternCounter();
int UnmarkedPatternCounter2 = cacheSentimentLocal2.getUnmarkedPatternCounter();
// System.out.println("UnmarkedPatternCounter1: " + UnmarkedPatternCounter1 + "\nUnmarkedPatternCounter2: " + UnmarkedPatternCounter2 + "\n");
if (UnmarkedPatternCounter1 > 0 && UnmarkedPatternCounter2 > 0) {
if (UnmarkedPatternCounter1 * 2 > UnmarkedPatternCounter2 && UnmarkedPatternCounter2 * 2 > UnmarkedPatternCounter1) {
score += 2500;
} else if (UnmarkedPatternCounter1 * 5 < UnmarkedPatternCounter2 || UnmarkedPatternCounter2 * 5 < UnmarkedPatternCounter1) {
score -= 4000;
if (UnmarkedPatternCounter1 < 100 && UnmarkedPatternCounter2 < 100) {
if (UnmarkedPatternCounter1 * 2 > UnmarkedPatternCounter2 && UnmarkedPatternCounter2 * 2 > UnmarkedPatternCounter1) {
score += 2500;
// System.out.println("score plus UnmarkedPattern: " + score + "\n");
} else if (UnmarkedPatternCounter1 * 5 < UnmarkedPatternCounter2 || UnmarkedPatternCounter2 * 5 < UnmarkedPatternCounter1) {
score -= 4000;
// System.out.println("score minus UnmarkedPattern: " + score + "\n");
}
} else {
score -= 2500;
// System.out.println("score minus UnmarkedPattern10>: " + score + "\n");
}
}
return score;
@@ -823,13 +863,24 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
for (String strTokeniPart2 : cacheSentimentLocal2.getstrTokenGetiPart().values()) {
if (strTokeniPart1.equals(strTokeniPart2) && !strtokensMap.values().contains(strTokeniPart2)) {
strtokensMap.put(strtokensMap.size() + 1, strTokeniPart2);
score += 400;
} else {
score -= 200;
//System.out.println("score minus strTokenGetiPart: " + score + "\n");
score += 800;
// System.out.println("score minus strTokenGetiPart: " + score + "\n");
}
}
}
int tokenIPartSize1 = cacheSentimentLocal1.getstrTokenGetiPart().values().size();
int tokenIPartSize2 = cacheSentimentLocal2.getstrTokenGetiPart().values().size();
int strTokenMapSize = strtokensMap.values().size();
if (tokenIPartSize1 * 2 > tokenIPartSize2 && tokenIPartSize2 * 2 > tokenIPartSize1) {
score += tokenIPartSize1 > tokenIPartSize2 ? (tokenIPartSize1 - tokenIPartSize2) * 700 : (tokenIPartSize2 - tokenIPartSize1) * 700;
score += strTokenMapSize * 600;
//System.out.println("tokenIPartSize plus score: " + score + "\ntokenIPartSize1: " + tokenIPartSize1 + "\ntokenIPartSize2: "
// + tokenIPartSize2 + "\nstrTokenMapSize: " + strTokenMapSize + "\n");
} else if (tokenIPartSize1 > 0 && tokenIPartSize2 > 0) {
score -= tokenIPartSize1 > tokenIPartSize2 ? (tokenIPartSize1 - tokenIPartSize2) * 700 : (tokenIPartSize2 - tokenIPartSize1) * 700;
// System.out.println("tokenIPartSize minus score: " + score + "\ntokenIPartSize1: " + tokenIPartSize1 + "\ntokenIPartSize2: "
// + tokenIPartSize2 + "\n");
}
return score;
}
@@ -866,14 +917,24 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
ConcurrentMap<Integer, String> iTokenMapTagsMap = new MapMaker().concurrencyLevel(2).makeMap();
for (String strmapTag : cacheSentimentLocal1.getITokenMapTag().values()) {
for (String strmapTag1 : cacheSentimentLocal2.getITokenMapTag().values()) {
if (strmapTag.equals(strmapTag1)) {
score -= 1450;
} else if (!iTokenMapTagsMap.values().contains(strmapTag)) {
score += 725;
if (strmapTag.equals(strmapTag1) && !iTokenMapTagsMap.values().contains(strmapTag1)) {
score += 1450;
iTokenMapTagsMap.put(iTokenMapTagsMap.size() + 1, strmapTag);
}
}
}
int mapTagsize1 = cacheSentimentLocal1.getITokenMapTag().values().size();
int mapTagsize2 = cacheSentimentLocal2.getITokenMapTag().values().size();
int tokenTagMapSize = iTokenMapTagsMap.values().size();
if (mapTagsize1 * 2 > mapTagsize2 && mapTagsize2 * 2 > mapTagsize1) {
score += mapTagsize1 > mapTagsize2 ? (mapTagsize1 - mapTagsize2) * 700 : (mapTagsize2 - mapTagsize1) * 700;
score += tokenTagMapSize * 600;
// System.out.println("tokenIPartSize 2 plus score: " + score + "\n");
//System.out.println("mapTagsize1: " + mapTagsize1 + "\nmapTagsize2: " + mapTagsize2 + "\ntokenTagMapSize: "
// + tokenTagMapSize + "\n");
} else {
score -= mapTagsize1 > mapTagsize2 ? (mapTagsize1 - mapTagsize2) * 700 : (mapTagsize2 - mapTagsize1) * 700;;
}
return score;
}
@@ -910,10 +971,10 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
for (String strTokenStem : cacheSentimentLocal1.getstrTokenStems().values()) {
for (String strTokenStem1 : cacheSentimentLocal2.getstrTokenStems().values()) {
if (strTokenStem.equals(strTokenStem1) && !tokenStemmingMap.values().contains(strTokenStem)) {
score += 1500;
score -= 4500;
tokenStemmingMap.put(tokenStemmingMap.size() + 1, strTokenStem);
// System.out.println("score minus strTokenStem: " + score + "\n");
}
//System.out.println("score strTokenStem: " + score + "\n");
}
}
return score;
@@ -922,21 +983,24 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
private Double inflectedCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
int inflectedCounterPositive1 = cacheSentimentLocal1.getInflectedCounterPositive();
int inflectedCounterPositive2 = cacheSentimentLocal2.getInflectedCounterPositive();
int inflectedCounterNegative = cacheSentimentLocal1.getInflectedCounterNegative() + cacheSentimentLocal2.getInflectedCounterNegative();
int inflectedCounterNegative = cacheSentimentLocal1.getInflectedCounterNegative() > cacheSentimentLocal2.getInflectedCounterNegative()
? cacheSentimentLocal1.getInflectedCounterNegative() - cacheSentimentLocal2.getInflectedCounterNegative()
: cacheSentimentLocal2.getInflectedCounterNegative() - cacheSentimentLocal1.getInflectedCounterNegative();
//System.out.println("inflectedCounterPositive1: " + inflectedCounterPositive1 + "\ninflectedCounterPositive2: "
// + inflectedCounterPositive2 + "\ninflectedCounterNegative: " + inflectedCounterNegative + "\n");
if (inflectedCounterPositive1 + inflectedCounterPositive2 > inflectedCounterNegative && inflectedCounterNegative > 0) {
//+inflectedCounterPositive2 + "\ninflectedCounterNegative: " + inflectedCounterNegative + "\n" );
if ((inflectedCounterPositive1 + inflectedCounterPositive2) > inflectedCounterNegative && inflectedCounterNegative > 0) {
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 650;
//System.out.println("score inflectedCounterPositive plus: " + score + "\n");
}
if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) {
if (inflectedCounterPositive1 * 2 > inflectedCounterPositive2 && inflectedCounterPositive2 * 2 > inflectedCounterPositive1) {
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550;
//System.out.println("score plus inflectedCounterPositive * 2: " + score + "\n");
if (inflectedCounterPositive1 * 2 > inflectedCounterPositive2 && inflectedCounterPositive2 * 2 > inflectedCounterPositive1
&& inflectedCounterNegative > 0) {
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) * 150) - (inflectedCounterNegative * 10);
// System.out.println("score plus inflectedCounterPositive * 2: " + score + "\n");
} else if (inflectedCounterPositive1 * 5 < inflectedCounterPositive2 || inflectedCounterPositive2 * 5 < inflectedCounterPositive1) {
score -= inflectedCounterPositive1 > inflectedCounterPositive2 ? (inflectedCounterPositive1 - inflectedCounterPositive2) * 400
: (inflectedCounterPositive2 - inflectedCounterPositive1) * 400;
//System.out.println("score minus inflectedCounterPositive * 2: " + score + "\n");
// System.out.println("score minus inflectedCounterPositive * 2: " + score + "\n");
}
}
return score;
@@ -962,22 +1026,23 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
private Double tokensCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) {
int tokensCounter1 = cacheSentimentLocal1.getTokensCounter();
int tokensCounter2 = cacheSentimentLocal2.getTokensCounter();
//System.out.println("tokensCounter1: " + tokensCounter1 + "\ntokensCounter2: " + tokensCounter2 + "\n");
// System.out.println("tokensCounter1: " + tokensCounter1 + "\ntokensCounter2: " + tokensCounter2 + "\n");
if ((tokensCounter1 > 1 && tokensCounter2 > 1) && tokensCounter1 < tokensCounter2 * 5 && tokensCounter2 < tokensCounter1 * 5) {
if (tokensCounter1 > tokensCounter2 / 2 && tokensCounter2 > tokensCounter1 / 2) {
score += (tokensCounter1 + tokensCounter2) * 1400;
//System.out.println("score plus tokensCounter: " + score + "\n");
if (tokensCounter1 > tokensCounter2 / 2 && tokensCounter2 > tokensCounter1 / 2 && tokensCounter1 < 4 && tokensCounter2 < 4) {
//8000 score hardcap
score += (tokensCounter1 + tokensCounter2) * 400;
// System.out.println("score plus tokensCounter: " + score + "\n");
} else {
score -= 3500;
//System.out.println("score minus tokensCounter: " + score + "\n");
}
} else {
int elseint = tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
//System.out.println("elseint: " + elseint + "<n");
// System.out.println("elseint: " + elseint + "<n");
if ((tokensCounter1 > tokensCounter2 * 5 || tokensCounter2 > tokensCounter1 * 5)
&& tokensCounter1 > 0 && tokensCounter2 > 0) {
score -= tokensCounter1 > tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
//System.out.println("score post tokensCounter: " + score + "\n");
// System.out.println("score post tokensCounter: " + score + "\n");
} else if (elseint > 0 && tokensCounter1 > 0 && tokensCounter2 > 0) {
score -= elseint * 2;
//System.out.println("score post elseint: " + elseint + "\n");
@@ -996,14 +1061,12 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
}
List<CoreLabel> tokens = em.tokens();
for (CoreLabel token : tokens) {
try {
if (token != null) {
if (!cacheSentimentLocal.getnerEntityTokenTags().values().contains(token.tag())) {
if (entityType.equals("PERSON") && EntityConfidences > 0.80) {
cacheSentimentLocal.addnerEntityTokenTags(token.tag());
}
}
} catch (Exception ex) {
//System.out.println("failed corelabel ex: " + ex.getLocalizedMessage() + "\n" + ex.getCause() + "\n");
}
}
if (!cacheSentimentLocal.getnerEntities1().values().contains(em.text())) {
@@ -1157,7 +1220,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
}
score = tgwListScoreIncrementer(score, cacheSentiment1 == null
? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null ? cacheSentimentLocal2 : cacheSentiment2);
// System.out.println("score post runCountGet: " + score + "\n");
//System.out.println("score post tgwListScoreIncrementer: " + score + "\n");
if (cacheSentiment1 == null) {
cacheSentimentLocal1 = GrammaticStructureSetup(cacheSentimentLocal1, pipelineAnnotation1);
}
@@ -1169,6 +1232,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
ConcurrentMap<Integer, Tree> sentenceConstituencyParseList1 = cacheSentiment1 == null
? cacheSentimentLocal1.getSentenceConstituencyParseList() : cacheSentiment1.getSentenceConstituencyParseList();
score = iterateTrees(sentenceConstituencyParseList2, sentenceConstituencyParseList1, score);
//System.out.println("score post iterateTrees: " + score + "\n");
Collection<TypedDependency> allTypedDependencies2 = cacheSentiment2 == null ? cacheSentimentLocal2.getAllTypedDependencies()
: cacheSentiment2.getAllTypedDependencies();
Collection<TypedDependency> allTypedDependencies1 = cacheSentiment1 == null ? cacheSentimentLocal1.getAllTypedDependencies()
@@ -1177,6 +1241,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
ConcurrentMap<Integer, GrammaticalStructure> grammaticalMap2 = cacheSentiment2 == null ? cacheSentimentLocal2.getGs() : cacheSentiment2.getGs();
score = typeDependenciesGrammaticalRelation(allTypedDependencies1, allTypedDependencies2, score, grammaticalMap1, grammaticalMap2,
sentenceConstituencyParseList1, sentenceConstituencyParseList2);
// System.out.println("score post typeDependenciesGrammaticalRelation: " + score + "\n");
if (cacheSentiment1 == null) {
cacheSentimentLocal1 = sentimentCoreAnnotationSetup(pipelineAnnotation1Sentiment, cacheSentimentLocal1);
}
@@ -1191,12 +1256,13 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
? cacheSentimentLocal1.getSimpleSMXlistVector() : cacheSentiment1.getSimpleSMXlistVector();
final ConcurrentMap<Integer, SimpleMatrix> simpleSMXlistVector2 = cacheSentiment2 == null
? cacheSentimentLocal2.getSimpleSMXlistVector() : cacheSentiment2.getSimpleSMXlistVector();
//System.out.println("score pre pipelineAnnotation2Sentiment: " + score + "\n");
//System.out.println("score pre simpleRNNMatrixCalculations: " + score + "\n");
score = simpleRNNMatrixCalculations(score, simpleSMXlist1, simpleSMXlist2);
//System.out.println("score pre simpleRNNMaxtrixVectors: " + score + "\n");
score = simpleRNNMaxtrixVectors(score, simpleSMXlistVector1, simpleSMXlistVector2);
int sentiment1 = cacheSentiment1 == null ? cacheSentimentLocal1.getRnnPrediectClassMap().size() : cacheSentiment1.getRnnPrediectClassMap().size();
int sentiment2 = cacheSentiment2 == null ? cacheSentimentLocal2.getRnnPrediectClassMap().size() : cacheSentiment2.getRnnPrediectClassMap().size();
//System.out.println("score post elementsVariance: " + score + "\n");
//System.out.println("score pre sentiment trim: " + score + "\n");
score -= (sentiment1 > sentiment2 ? sentiment1 - sentiment2 : sentiment2 - sentiment1) * 500;
Map.Entry<Double, Map.Entry<SentimentValueCache, SentimentValueCache>> classifyRawEvaluationEntry = classifyRawEvaluation(score, cacheSentimentLocal1,
cacheSentimentLocal2);
@@ -1227,13 +1293,19 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
SentimentValueCache scoringCache1 = cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1;
SentimentValueCache scoringCache2 = cacheSentiment2 == null ? cacheSentimentLocal2 : cacheSentiment2;
score = entryCountsRelation(score, scoringCache1, scoringCache2);
//System.out.println("score post entryCountsRelation: " + score + "\n");
score = entryCountsScoring(score, scoringCache1, scoringCache2);
// System.out.println("score post entryCountsScoring: " + score + "\n");
score = tokenEntryPosScoring(score, scoringCache1, scoringCache2);
//System.out.println("score post tokenEntryPosScoring: " + score + "\n");
score = unmarkedPatternCounterScoring(score, scoringCache1, scoringCache2);
//System.out.println("score post UnmarkedPatternCounter: " + score + "\n");
// System.out.println("score post UnmarkedPatternCounter: " + score + "\n");
score = markedContiniousCounterScoring(score, scoringCache1, scoringCache2);
// System.out.println("score post markedContiniousCounterScoring: " + score + "\n");
score = strTokensMapScoring(score, scoringCache1, scoringCache2);
// System.out.println("score post strTokensMapScoring: " + score + "\n");
score = strTokenEntryScoring(score, scoringCache1, scoringCache2);
//System.out.println("score post strTokenEntryScoring: " + score + "\n");
score = strTokenMapTagsScoring(score, scoringCache1, scoringCache2);
//System.out.println("score post strmapTag: " + score + "\n");
score = tokenformSizeScoring(score, scoringCache1, scoringCache2);
@@ -1241,7 +1313,9 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
score = tokenStemmingMapScoring(score, scoringCache1, scoringCache2);
//System.out.println("Score pre inflected: " + score + "\n");
score = inflectedCounterScoring(score, scoringCache1, scoringCache2);
//System.out.println("Score pre annotatorCountScoring: " + score + "\n");
score = annotatorCountScoring(score, scoringCache1, scoringCache2);
//System.out.println("Score pre tokensCounterScoring: " + score + "\n");
score = tokensCounterScoring(score, scoringCache1, scoringCache2);
//System.out.println("Score Pre levenhstein: " + score + "\n");
LevenshteinDistance leven = new LevenshteinDistance(str, str1);
@@ -1254,10 +1328,10 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
if (cacheSentiment2 == null) {
cacheSentimentLocal2 = setupNEREntitiesAndTokenTags(pipelineCoreDcoument2, cacheSentimentLocal2);
}
//System.out.println("score post PERSON trim: " + score + "\n");
//System.out.println("score post SentenceScoreDiff trim: " + score + "\n");
score = nerEntitiesAndTokenScoring(score, cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null
? cacheSentimentLocal2 : cacheSentiment2);
//System.out.println("score pre stopwordTokens: " + score + "\n");
//System.out.println("score post nerEntitiesAndTokenScoring: " + score + "\n");
if (cacheSentiment1 == null) {
cacheSentimentLocal1 = setupStoWordTokensLemma(pipelineAnnotation1Sentiment, cacheSentimentLocal1);
}
@@ -1266,9 +1340,10 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
}
score = stopWordTokenLemmaScoring(score, cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null
? cacheSentimentLocal2 : cacheSentiment2);
//System.out.println("score post stopWordTokenLemmaScoring: " + score + "\n");
score = stopwordTokenPairCounterScoring(score, cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null
? cacheSentimentLocal2 : cacheSentiment2);
// System.out.println("Final current score: " + score + "\nSentence 1: " + str + "\nSentence 2: " + str1 + "\n");
// System.out.println("Final current score: " + score + "\nSentence 1: " + str + "\nSentence 2: " + str1 + "\n");
smxParam.setDistance(score);
if (cacheSentiment1 == null) {
smxParam.setCacheValue1(cacheSentimentLocal1);