UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

removed levenhstein callable implementation, simplified sentence fetching from DB, moved from unloze db to a small vps db, instead of iterating 1 in updatematrix is 25 now, updated sentiment analyzing, also reduced get time of results to 5 seconds, a bunch of trivial things also got changed

Browse Source
This commit is contained in:
jenzur 2019-03-29 12:34:40 +01:00
parent 296be21753
commit 9e68cbb283
7 changed files with 457 additions and 450 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ArtificialAutism/src/main/java/DataLayer/DBCPDataSource.java
View File

@ -20,9 +20,9 @@ public class DBCPDataSource {
static { static {
try { try {
ds.setDriver(new com.mysql.cj.jdbc.Driver()); ds.setDriver(new com.mysql.cj.jdbc.Driver());
ds.setUrl("jdbc:mysql://151.80.230.149:3306/ArtificialAutism?useLegacyDatetimeCode=false&serverTimezone=UTC"); ds.setUrl("jdbc:mysql://104.248.40.216:3306/ArtificialAutism?useLegacyDatetimeCode=false&serverTimezone=UTC");
ds.setUsername("ArtificialAutism"); ds.setUsername("root");
ds.setPassword("b423b54bwbfb1340438fn"); ds.setPassword("fb345972349fnsDW234/¤)#2");
ds.setMaxTotal(-1); ds.setMaxTotal(-1);
ds.setMinIdle(5); ds.setMinIdle(5);
ds.setMaxIdle(-1); ds.setMaxIdle(-1);

14
ArtificialAutism/src/main/java/DataLayer/DataMapper.java
View File

@ -140,42 +140,36 @@ public class DataMapper {
public static LinkedHashMap<String, LinkedHashMap<String, Double>> getAllRelationScores() { public static LinkedHashMap<String, LinkedHashMap<String, Double>> getAllRelationScores() {
int count = getSementicsDBRows(); int count = getSementicsDBRows();
int counter2 = 0;
int hardCapRetrieveCount = 500000;
LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMX = new LinkedHashMap(); LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMX = new LinkedHashMap();
while (count > counter2) {
try (Connection l_cCon = DBCPDataSource.getConnection()) { try (Connection l_cCon = DBCPDataSource.getConnection()) {
l_cCon.setAutoCommit(false); l_cCon.setAutoCommit(false);
String l_sSQL = "SELECT * FROM `WordMatrix` WHERE ID > " + counter2 + " AND ID < " + (counter2 + hardCapRetrieveCount); String l_sSQL = "SELECT * FROM `WordMatrix`";
try (PreparedStatement l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY, try (PreparedStatement l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY)) { java.sql.ResultSet.CONCUR_READ_ONLY)) {
l_pStatement.setFetchSize(Integer.MIN_VALUE); l_pStatement.setFetchSize(Integer.MIN_VALUE);
try (ResultSet l_rsSearch = l_pStatement.executeQuery()) { try (ResultSet l_rsSearch = l_pStatement.executeQuery()) {
int i = 0; int i = 0;
LinkedHashMap<String, Double> LHMLocal = new LinkedHashMap(); LinkedHashMap<String, Double> LHMLocal = new LinkedHashMap();
while (l_rsSearch.next() && i < hardCapRetrieveCount) { while (l_rsSearch.next()) {
String str1 = l_rsSearch.getString(1); String str1 = l_rsSearch.getString(1);
String str2 = l_rsSearch.getString(2); String str2 = l_rsSearch.getString(2);
Double score = l_rsSearch.getDouble(3); Double score = l_rsSearch.getDouble(3);
LHMLocal.put(str2, score); LHMLocal.put(str2, score);
while (l_rsSearch.next() && i < hardCapRetrieveCount && str1.equals(l_rsSearch.getString(1))) { while (l_rsSearch.next() && str1.equals(l_rsSearch.getString(1))) {
str2 = l_rsSearch.getString(2); str2 = l_rsSearch.getString(2);
score = l_rsSearch.getDouble(3); score = l_rsSearch.getDouble(3);
LHMLocal.put(str2, score); LHMLocal.put(str2, score);
i++; i++;
counter2++;
} }
LHMSMX.put(str1, LHMLocal); LHMSMX.put(str1, LHMLocal);
System.out.println("i: " + i + "\n" + "free memory: " + Runtime.getRuntime().freeMemory() + "\ncounter2: " + counter2 + "\n"); System.out.println("i: " + i + "\n" + "free memory: " + Runtime.getRuntime().freeMemory() + "\n");
i++; i++;
counter2++;
} }
} }
} }
} catch (SQLException ex) { } catch (SQLException ex) {
Logger.getLogger(DataMapper.class.getName()).log(Level.SEVERE, null, ex); Logger.getLogger(DataMapper.class.getName()).log(Level.SEVERE, null, ex);
} }
}
return LHMSMX; return LHMSMX;
} }

88
ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java
View File

@ -29,13 +29,11 @@ import java.io.StringReader;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Properties; import java.util.Properties;
import java.util.Random;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
@ -56,6 +54,7 @@ public class Datahandler {
public static final long EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(6, TimeUnit.MINUTES); public static final long EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(6, TimeUnit.MINUTES);
public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS); public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS);
public static Datahandler instance = new Datahandler(); public static Datahandler instance = new Datahandler();
private static volatile Double minDistance;
private volatile boolean refreshMatrixFromDB; private volatile boolean refreshMatrixFromDB;
private static volatile int secondaryIterator = 0; private static volatile int secondaryIterator = 0;
private final ConcurrentMap<Integer, String> stringCache; private final ConcurrentMap<Integer, String> stringCache;
@ -66,6 +65,7 @@ public class Datahandler {
private final Stopwatch stopwatch; private final Stopwatch stopwatch;
private final Stopwatch stopwatch1; private final Stopwatch stopwatch1;
private ForkJoinPool executor; private ForkJoinPool executor;
private static String similar = "";
private static String shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz"; private static String shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz";
private static String sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz"; private static String sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz";
private static String lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz"; private static String lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz";
@ -278,6 +278,7 @@ public class Datahandler {
if (stringCache.values().size() > 10 && !refreshMatrixFromDB) { if (stringCache.values().size() > 10 && !refreshMatrixFromDB) {
ConcurrentMap<Integer, String> stringCachelocal = stringCache; ConcurrentMap<Integer, String> stringCachelocal = stringCache;
int selectUpdate = -1; int selectUpdate = -1;
int iteratorCap = 25;
LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX; LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
int ij2 = 0; int ij2 = 0;
for (String str : stringCachelocal.values()) { for (String str : stringCachelocal.values()) {
@ -290,30 +291,42 @@ public class Datahandler {
} }
if (selectUpdate == -1 || selectUpdate + 1 == stringCachelocal.size()) { if (selectUpdate == -1 || selectUpdate + 1 == stringCachelocal.size()) {
int valueSize = stringCachelocal.size(); int valueSize = stringCachelocal.size();
if (secondaryIterator + 1 >= valueSize) { if (secondaryIterator + iteratorCap >= valueSize) {
secondaryIterator = 0; secondaryIterator = 0;
} }
selectUpdate = secondaryIterator; selectUpdate = secondaryIterator;
secondaryIterator++; secondaryIterator += iteratorCap;
} }
final String getStringCacheStr = stringCachelocal.getOrDefault(selectUpdate, null); final ConcurrentMap<Integer, String> getStringCacheMap = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); for (int i = 0; i < iteratorCap; i++) {
getStringCacheMap.put(i, stringCachelocal.get(selectUpdate));
selectUpdate++;
}
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateMap = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
getStringCacheMap.values().forEach((getStringCacheStr) -> {
stringCachelocal.values().forEach((str1) -> { stringCachelocal.values().forEach((str1) -> {
if (!getStringCacheStr.equals(str1)) {
boolean present = false; boolean present = false;
LinkedHashMap<String, Double> orDefault = lHMSMX.getOrDefault(getStringCacheStr, null); LinkedHashMap<String, Double> orDefault = lHMSMX.getOrDefault(getStringCacheStr, null);
if (orDefault != null) { if (orDefault != null) {
Double orDefault1 = orDefault.getOrDefault(str1, null); Collection<String> strkeys = orDefault.keySet();
if (orDefault1 != null) { for (String strkey : strkeys) {
if (strkey.equals(str1)) {
present = true; present = true;
break;
}
} }
} }
if (!present) { if (!present) {
orDefault = lHMSMX.getOrDefault(str1, null); orDefault = lHMSMX.getOrDefault(str1, null);
if (orDefault != null) { if (orDefault != null) {
Double orDefault1 = orDefault.getOrDefault(getStringCacheStr, null); Collection<String> strkeys = orDefault.keySet();
if (orDefault1 != null) { for (String strkey : strkeys) {
if (strkey.equals(getStringCacheStr)) {
present = true; present = true;
break;
}
} }
} }
} }
@ -330,6 +343,7 @@ public class Datahandler {
pipelineSentimentAnnotationCache.get(getStringCacheStr), pipelineSentimentAnnotationCache.get(str1)); pipelineSentimentAnnotationCache.get(getStringCacheStr), pipelineSentimentAnnotationCache.get(str1));
futures.put(futures.size() + 1, executor.submit(worker)); futures.put(futures.size() + 1, executor.submit(worker));
} }
}
}); });
System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); System.out.println("finished worker assignment, futures size: " + futures.size() + "\n");
futures.values().parallelStream().forEach((future) -> { futures.values().parallelStream().forEach((future) -> {
@ -344,13 +358,14 @@ public class Datahandler {
LinkedHashMap<String, Double> getFuture = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); LinkedHashMap<String, Double> getFuture = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
getFuture.put(SMX.getSecondaryString(), SMX.getDistance()); getFuture.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), getFuture); lHMSMX.put(SMX.getPrimaryString(), getFuture);
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); matrixUpdateMap.put(matrixUpdateMap.size() + 1, SMX);
} }
}); });
});
new Thread(() -> { new Thread(() -> {
try { try {
if (!matrixUpdateList.isEmpty()) { if (!matrixUpdateMap.isEmpty()) {
DataMapper.insertSementicMatrixes(matrixUpdateList); DataMapper.insertSementicMatrixes(matrixUpdateMap);
System.out.println("finished datamapper semetic insert"); System.out.println("finished datamapper semetic insert");
} }
} catch (CustomError ex) { } catch (CustomError ex) {
@ -359,7 +374,6 @@ public class Datahandler {
} }
}).start(); }).start();
} }
} }
public synchronized void checkIfUpdateStrings(boolean hlStatsMsg) throws CustomError { public synchronized void checkIfUpdateStrings(boolean hlStatsMsg) throws CustomError {
@ -398,7 +412,6 @@ public class Datahandler {
System.out.println("pre mostSimilarSTR \n"); System.out.println("pre mostSimilarSTR \n");
String mostSimilarSTR = mostSimilar(str, strArrs); String mostSimilarSTR = mostSimilar(str, strArrs);
if (mostSimilarSTR != null) { if (mostSimilarSTR != null) {
System.out.println("mostSimilarSTR; " + mostSimilarSTR + "\n");
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.getOrDefault(mostSimilarSTR, null); LinkedHashMap<String, Double> orDefault = LHMSMXLocal.getOrDefault(mostSimilarSTR, null);
if (orDefault != null) { if (orDefault != null) {
for (Entry<String, Double> entrySet : orDefault.entrySet()) { for (Entry<String, Double> entrySet : orDefault.entrySet()) {
@ -452,7 +465,6 @@ public class Datahandler {
futureslocal.put(futureslocal.size() + 1, executor.submit(worker)); futureslocal.put(futureslocal.size() + 1, executor.submit(worker));
} }
}); });
int index = 0;
futureslocal.values().parallelStream().forEach((future) -> { futureslocal.values().parallelStream().forEach((future) -> {
SimilarityMatrix SMX = new SimilarityMatrix("", ""); SimilarityMatrix SMX = new SimilarityMatrix("", "");
try { try {
@ -464,15 +476,10 @@ public class Datahandler {
}); });
for (SimilarityMatrix SMX : futurereturn.values()) { for (SimilarityMatrix SMX : futurereturn.values()) {
double distance = SMX.getDistance(); double distance = SMX.getDistance();
/*
System.out.println("index: " + index + "\nfutures size: " + futureslocal.values().size() + "\nScore: " + SMX.getDistance() + "\nSecondary: "
+ SMX.getSecondaryString() + "\nPrimary: " + SMX.getPrimaryString() + "\n");
*/
if (distance > Score) { if (distance > Score) {
Score = distance; Score = distance;
SMXreturn = SMX; SMXreturn = SMX;
} }
index++;
} }
System.out.println("Reached end: secondary: " + SMXreturn.getSecondaryString() + "\nPrimarY: " + SMXreturn.getPrimaryString() System.out.println("Reached end: secondary: " + SMXreturn.getSecondaryString() + "\nPrimarY: " + SMXreturn.getPrimaryString()
+ "\nScore: " + SMXreturn.getDistance()); + "\nScore: " + SMXreturn.getDistance());
@ -480,30 +487,27 @@ public class Datahandler {
} }
public String mostSimilar(String toBeCompared, ConcurrentMap<Integer, String> concurrentStrings) { public String mostSimilar(String toBeCompared, ConcurrentMap<Integer, String> concurrentStrings) {
int minDistance = 7; similar = "";
String similar = ""; minDistance = 7.5;
List<Future<ConcurrentMap<String, Integer>>> futures = new ArrayList();
ConcurrentMap<String, Integer> futuresreturnvalues = new MapMaker().concurrencyLevel(2).makeMap();
concurrentStrings.values().parallelStream().forEach((str) -> { concurrentStrings.values().parallelStream().forEach((str) -> {
Callable<ConcurrentMap<String, Integer>> worker = new LevenshteinDistance(toBeCompared, str); LevenshteinDistance leven = new LevenshteinDistance(toBeCompared, str);
futures.add(executor.submit(worker)); double distance = leven.computeLevenshteinDistance();
}); if (distance < minDistance) {
futures.parallelStream().forEach((future) -> { minDistance = distance;
try { System.out.println("distance: " + distance + "\n");
ConcurrentMap<String, Integer> get = future.get(); similar = str;
get.entrySet().forEach((str) -> {
futuresreturnvalues.put(str.getKey(), str.getValue());
});
} catch (NullPointerException | InterruptedException | ExecutionException ex) {
System.out.println("failed future\nex: " + ex.getMessage() + "\n");
} }
}); });
for (Entry<String, Integer> entritr : futuresreturnvalues.entrySet()) { LinkedHashMap<String, Double> orDefault = lHMSMX.getOrDefault(similar, null);
int distance = entritr.getValue(); if (orDefault == null) {
if (distance < minDistance) { return null;
System.out.println("distance: " + distance + "\n"); }
minDistance = distance; Double maxDistance = 0.0;
similar = entritr.getKey(); for (Entry<String, Double> defaultEntry : orDefault.entrySet()) {
Double value = defaultEntry.getValue();
if (value > maxDistance) {
maxDistance = value;
similar = defaultEntry.getKey();
} }
} }
return similar; return similar;

25
ArtificialAutism/src/main/java/FunctionLayer/LevenshteinDistance.java
View File

@ -15,11 +15,9 @@ import java.util.concurrent.ConcurrentMap;
* *
* @author install1 * @author install1
*/ */
public class LevenshteinDistance implements Callable<ConcurrentMap<String, Integer>> { public class LevenshteinDistance {
private CharSequence lhs; private CharSequence lhs;
private CharSequence rhs; private CharSequence rhs;
private ConcurrentMap<String, Integer> distanceEntry = new MapMaker().concurrencyLevel(2).makeMap();
private static int minimum(int a, int b, int c) { private static int minimum(int a, int b, int c) {
return Math.min(Math.min(a, b), c); return Math.min(Math.min(a, b), c);
@ -48,25 +46,4 @@ public class LevenshteinDistance implements Callable<ConcurrentMap<String, Integ
} }
return distance[lhs.length()][rhs.length()]; return distance[lhs.length()][rhs.length()];
} }
@Override
public ConcurrentMap<String, Integer> call() {
int[][] distance = new int[lhs.length() + 1][rhs.length() + 1];
for (int i = 0; i <= lhs.length(); i++) {
distance[i][0] = i;
}
for (int j = 1; j <= rhs.length(); j++) {
distance[0][j] = j;
}
for (int i = 1; i <= lhs.length(); i++) {
for (int j = 1; j <= rhs.length(); j++) {
distance[i][j] = minimum(
distance[i - 1][j] + 1,
distance[i][j - 1] + 1,
distance[i - 1][j - 1] + ((lhs.charAt(i - 1) == rhs.charAt(j - 1)) ? 0 : 1));
}
}
distanceEntry.put(lhs.toString(), distance[lhs.length()][rhs.length()]);
return distanceEntry;
}
} }

4
ArtificialAutism/src/main/java/FunctionLayer/PipelineJMWESingleton.java
View File

@ -28,7 +28,6 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
@ -67,8 +66,7 @@ public class PipelineJMWESingleton {
} }
IMWEDetector detector = getDetector(index, detectorName); IMWEDetector detector = getDetector(index, detectorName);
ConcurrentMap<String, Annotation> returnAnnotations = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<String, Annotation> returnAnnotations = new MapMaker().concurrencyLevel(2).makeMap();
Date startDate = new Date(); strvalues.forEach(str -> {
strvalues.parallelStream().forEach(str -> {
Annotation annoStr = new Annotation(str); Annotation annoStr = new Annotation(str);
returnAnnotations.put(str, annoStr); returnAnnotations.put(str, annoStr);
}); });

79
ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java
View File

@ -38,6 +38,7 @@ import java.io.StringReader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.OptionalDouble;
import java.util.Set; import java.util.Set;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
@ -114,7 +115,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
counter = taggedwordlist2.stream().map((taggedlist2) -> taggedlist2.size()).reduce(counter, Integer::sum); counter = taggedwordlist2.stream().map((taggedlist2) -> taggedlist2.size()).reduce(counter, Integer::sum);
counter1 = taggedwordlist1.stream().map((taggedlist1) -> taggedlist1.size()).reduce(counter1, Integer::sum); counter1 = taggedwordlist1.stream().map((taggedlist1) -> taggedlist1.size()).reduce(counter1, Integer::sum);
int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter; int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter;
overValue *= 16; overValue *= 32;
score -= overValue; score -= overValue;
ConcurrentMap<Integer, String> tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, String> tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap();
taggedwordlist1.forEach((TGWList) -> { taggedwordlist1.forEach((TGWList) -> {
@ -124,7 +125,6 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
} }
}); });
}); });
taggedwordlist1.clear();
AtomicInteger runCount = new AtomicInteger(0); AtomicInteger runCount = new AtomicInteger(0);
taggedwordlist2.forEach((TGWList) -> { taggedwordlist2.forEach((TGWList) -> {
TGWList.forEach((TaggedWord) -> { TGWList.forEach((TaggedWord) -> {
@ -134,8 +134,6 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
} }
}); });
}); });
tgwlistIndex.clear();
taggedwordlist2.clear();
score += runCount.get() * 64; score += runCount.get() * 64;
ConcurrentMap<Integer, Tree> sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, Tree> sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap();
try { try {
@ -144,22 +142,30 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse); sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse);
} }
for (CoreMap sentence : pipelineAnnotation2.get(CoreAnnotations.SentencesAnnotation.class)) { for (CoreMap sentence : pipelineAnnotation2.get(CoreAnnotations.SentencesAnnotation.class)) {
int constiRelationsize = 0;
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse); GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse);
Collection<TypedDependency> allTypedDependencies = gs.allTypedDependencies(); Collection<TypedDependency> allTypedDependencies = gs.allTypedDependencies();
ConcurrentMap<Integer, String> filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, String> filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap();
for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) { for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) {
Set<Constituent> inT1notT2 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1); Set<Constituent> constinuent1 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1);
Set<Constituent> inT2notT1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse); Set<Constituent> constinuent2 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse);
ConcurrentMap<Integer, String> constiLabels = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, String> constiLabels = new MapMaker().concurrencyLevel(2).makeMap();
for (Constituent consti : inT1notT2) { for (Constituent consti : constinuent1) {
for (Constituent consti1 : inT2notT1) { for (Constituent consti1 : constinuent2) {
if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) { if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) {
score += 64;
constiLabels.put(constiLabels.size(), consti.value()); constiLabels.put(constiLabels.size(), consti.value());
constiRelationsize++;
} }
} }
} }
int constituents1 = constinuent1.size() - constiRelationsize;
int constituents2 = constinuent2.size() - constiRelationsize;
if (constituents1 > 0 && constituents2 > 0) {
score -= (constituents1 + constituents2) * 200;
} else {
score += constiRelationsize * 200;
}
GrammaticalStructure gs1 = gsf.newGrammaticalStructure(sentenceConstituencyParse1); GrammaticalStructure gs1 = gsf.newGrammaticalStructure(sentenceConstituencyParse1);
Collection<TypedDependency> allTypedDependencies1 = gs1.allTypedDependencies(); Collection<TypedDependency> allTypedDependencies1 = gs1.allTypedDependencies();
for (TypedDependency TDY1 : allTypedDependencies1) { for (TypedDependency TDY1 : allTypedDependencies1) {
@ -167,11 +173,11 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
IndexedWord gov = TDY1.gov(); IndexedWord gov = TDY1.gov();
GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep); GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep);
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) { if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
score += 900; score += 1900;
} }
GrammaticalRelation reln = TDY1.reln(); GrammaticalRelation reln = TDY1.reln();
if (reln.isApplicable(sentenceConstituencyParse)) { if (reln.isApplicable(sentenceConstituencyParse)) {
score += 256; score += 525;
} }
} }
for (TypedDependency TDY : allTypedDependencies) { for (TypedDependency TDY : allTypedDependencies) {
@ -183,7 +189,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
} }
GrammaticalRelation reln = TDY.reln(); GrammaticalRelation reln = TDY.reln();
if (reln.isApplicable(sentenceConstituencyParse1)) { if (reln.isApplicable(sentenceConstituencyParse1)) {
score += 256; score += 525;
} }
} }
AtomicInteger runCount1 = new AtomicInteger(0); AtomicInteger runCount1 = new AtomicInteger(0);
@ -215,31 +221,52 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
simpleSMXlist.put(simpleSMXlist.size(), predictions); simpleSMXlist.put(simpleSMXlist.size(), predictions);
simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector); simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector);
} }
ConcurrentMap<Integer, Double> elementSumCounter = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Double> dotMap = new MapMaker().concurrencyLevel(2).makeMap();
for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) { for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree)); sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree));
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
score = simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> { score += simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : dot > 0 ? 50 - dot : 50).map((subtracter) -> {
subtracter *= 25; subtracter *= 25; //25
return subtracter; return subtracter;
}).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator - _item); }).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator + _item);
for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) { for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) {
double dot = nodeVector.dot(simpleSMX); double dot = nodeVector.dot(simpleSMX);
double elementSum = nodeVector.kron(simpleSMX).elementSum(); double elementSum = nodeVector.kron(simpleSMX).elementSum();
elementSum = Math.round(elementSum * 100.0) / 100.0; elementSum = Math.round(elementSum * 100.0) / 100.0;
elementSumCounter.put(elementSumCounter.size() + 1, elementSum);
dotMap.put(dotMap.size() + 1, dot);
if (dot < 0.1) { if (dot < 0.1) {
score += 256; score += 256;
} }
if (elementSum < 0.1 && elementSum > 0.0) { if (dot > 0.50) {
score -= 2400;
}
if (elementSum < 0.01 && elementSum > 0.00) {
score += 1300; score += 1300;
} else if (elementSum > 0.1 && elementSum < 1.0) { } else if (elementSum > 0.1 && elementSum < 1.0) {
score -= 1100; score += 1100;
} else { } else {
score -= 1424; score -= elementSum * 1424;
} }
} }
} }
if (dotMap.values().size() > 1) {
OptionalDouble minvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).min();
OptionalDouble maxvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).max();
if (maxvalueDots.getAsDouble() - minvalueDots.getAsDouble() < 0.05) {
score += 3500;
}
}
if (elementSumCounter.values().size() > 1){
OptionalDouble minvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).min();
OptionalDouble maxvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).max();
if (maxvalueElements.getAsDouble() - minvalueElements.getAsDouble() < 0.05) {
score += 3500;
}
}
score -= (sentiment1.size() > sentiment2.size() ? sentiment1.size() - sentiment2.size() : sentiment2.size() - sentiment1.size()) * 500; score -= (sentiment1.size() > sentiment2.size() ? sentiment1.size() - sentiment2.size() : sentiment2.size() - sentiment1.size()) * 500;
DocumentReaderAndWriter<CoreLabel> readerAndWriter = classifier.makePlainTextReaderAndWriter(); DocumentReaderAndWriter<CoreLabel> readerAndWriter = classifier.makePlainTextReaderAndWriter();
List classifyRaw1 = classifier.classifyRaw(str, readerAndWriter); List classifyRaw1 = classifier.classifyRaw(str, readerAndWriter);
@ -272,10 +299,15 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
if (longest1 != longest2) { if (longest1 != longest2) {
long deffLongest = longest1 > longest2 ? longest1 : longest2; long deffLongest = longest1 > longest2 ? longest1 : longest2;
long deffshorter = longest1 < longest2 ? longest1 : longest2; long deffshorter = longest1 < longest2 ? longest1 : longest2;
if (deffLongest >= (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) { //deffLongest >= (deffshorter * 2)
score += (deffLongest - deffshorter) * 200; if (deffLongest < (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) {
score += (deffLongest - deffshorter) * 120;
} else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) { } else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) {
score += (deffLongest - deffshorter) * 200; score += (deffLongest - deffshorter) * 120;
} else if (deffLongest - deffshorter < 2) {
score += (deffLongest + deffshorter) * 40;
} else if (deffLongest - deffshorter <= 5){
score += 2500;
} else{ } else{
score -= (deffLongest - deffshorter) * 50; score -= (deffLongest - deffshorter) * 50;
} }
@ -373,7 +405,9 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
} }
} }
} }
if (UnmarkedPatternCounter > 0 && UnmarkedPatternCounter < 5) {
score += UnmarkedPatternCounter * 1600; score += UnmarkedPatternCounter * 1600;
}
if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) { if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) {
score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500 score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500
: (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500; : (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500;
@ -427,7 +461,8 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
if (tokensCounter1 > 0 && tokensCounter2 > 0) { if (tokensCounter1 > 0 && tokensCounter2 > 0) {
score += (tokensCounter1 + tokensCounter2) * 400; score += (tokensCounter1 + tokensCounter2) * 400;
} else { } else {
score -= tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500; int elseint = tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
score -= elseint;
} }
LevenshteinDistance leven = new LevenshteinDistance(str, str1); LevenshteinDistance leven = new LevenshteinDistance(str, str1);
double SentenceScoreDiff = leven.computeLevenshteinDistance(); double SentenceScoreDiff = leven.computeLevenshteinDistance();

1
ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java
View File

@ -21,7 +21,6 @@ import FunctionLayer.PipelineJMWESingleton;
import java.io.IOException; import java.io.IOException;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.List; import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
import org.javacord.api.DiscordApi; import org.javacord.api.DiscordApi;