Merge branch 'master' of ssh://git.unloze.com:2222/UNLOZE/projects-jenz

This commit is contained in:
Christian 2020-12-15 23:46:49 +01:00
commit 99389c2892
65 changed files with 241 additions and 181 deletions

View File

@ -12,3 +12,5 @@ now requires about 4 GB ram atleast to run.
also now requires nlp.stanford.edu/software/stanford-corenlp-full-2018-10-05.zip
because it needs lexicalized parsers which shift reduce parser does not posses.
also requires the regular stanford parser except of just shift reduce parser
jars too large for /lib folder: CorenlpModels-1.0, srParser-1, ws4j-1.0.1, StanfordParserModel-1.0

View File

@ -1,99 +1,127 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.mycompany</groupId>
<artifactId>ArtificialAutism</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
<version>2.5.0</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.13</version>
</dependency>
<dependency>
<groupId>org.javacord</groupId>
<artifactId>javacord-api</artifactId>
<version>3.0.1</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.javacord</groupId>
<artifactId>javacord</artifactId>
<version>3.0.1</version>
<type>pom</type>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>26.0-jre</version>
</dependency>
<dependency>
<groupId>com.github.mpkorstanje</groupId>
<artifactId>simmetrics-core</artifactId>
<version>4.1.1</version>
</dependency>
<dependency>
<groupId>ws4j</groupId>
<artifactId>ws4j</artifactId>
<version>1.0.1</version>
</dependency>
<dependency>
<groupId>StanfordParser</groupId>
<artifactId>StanfordParser</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>StanfordParserModel</groupId>
<artifactId>StanfordParserModel</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>srParser</groupId>
<artifactId>srParser</artifactId>
<version>1</version>
</dependency>
</dependencies>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<mainClass>PresentationLayer.DiscordHandler</mainClass>
</properties>
<build>
<plugins>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<phase>install</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<mainClass>${mainClass}</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
</plugins>
</build>
</project>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.mycompany</groupId>
<artifactId>ArtificialAutism</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
<version>2.5.0</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.13</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>26.0-jre</version>
</dependency>
<dependency>
<groupId>com.github.mpkorstanje</groupId>
<artifactId>simmetrics-core</artifactId>
<version>4.1.1</version>
</dependency>
<dependency>
<groupId>ws4j</groupId>
<artifactId>ws4j</artifactId>
<version>1.0.1</version>
</dependency>
<dependency>
<groupId>StanfordParser</groupId>
<artifactId>StanfordParser</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>StanfordParserModel</groupId>
<artifactId>StanfordParserModel</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>srParser</groupId>
<artifactId>srParser</artifactId>
<version>1</version>
</dependency>
<dependency>
<groupId>ejml</groupId>
<artifactId>ejml</artifactId>
<version>0.2.3</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>jmweAnno</groupId>
<artifactId>jmweAnno</artifactId>
<version>1.0</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>edu.mit.jmwe</groupId>
<artifactId>edu.mit.jmwe</artifactId>
<version>1.0.2</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>CoreNLP</groupId>
<artifactId>CoreNLP</artifactId>
<version>1.0</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>7.2.0</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>CorenlpModels</groupId>
<artifactId>CorenlpModels</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>com.discord4j</groupId>
<artifactId>discord4j-core</artifactId>
<version>3.1.1</version>
</dependency>
</dependencies>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<mainClass>PresentationLayer.DiscordHandler</mainClass>
</properties>
<build>
<plugins>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<phase>install</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<mainClass>${mainClass}</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
</plugins>
</build>
</project>

View File

@ -179,7 +179,7 @@ public class Datahandler {
public void updateStringCache() {
try {
checkIfUpdateStrings(true);
checkIfUpdateStrings();
} catch (CustomError ex) {
Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
}
@ -225,7 +225,7 @@ public class Datahandler {
hlStatsMessages.put(str, hlStatsMessages.size());
}
}
int capacity = 50;
int capacity = 150;
hlStatsMessages.keySet().forEach(str -> {
if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) {
String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null);
@ -318,8 +318,16 @@ public class Datahandler {
SentimentValueCache sentimentCacheStr = sentimentCachingMap.getOrDefault(strCollection, null);
List<SimilarityMatrix> smxReturnList = new ArrayList();
List<String> randomIterationComparision = new ArrayList();
int iteratecap = strCacheLocal.size() > prefix_size ? strCacheLocal.size() - prefix_size : strCacheLocal.size();
if (strCacheLocal.size() < prefix_size)
{
for (String colStr : strCollection)
{
strCacheLocal.put(strCacheLocal.size(), colStr);
}
}
int iteratecap = strCacheLocal.size() > prefix_size ? strCacheLocal.size() - prefix_size : strCacheLocal.size() - 1;
int iterator = ThreadLocalRandom.current().nextInt(0, iteratecap);
int iterated = 0;
for (String str1 : strCacheLocal.values()) {
if (iterated >= iterator && iterated < iterator + prefix_size) {
@ -390,7 +398,6 @@ public class Datahandler {
//System.out.println("strmap siuze: " + strmap.size());
List<SimilarityMatrix> StrComparringNoSentenceRelationMap = StrComparringNoSentenceRelationMap(strCacheLocal, strmap.values(),
localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap);
System.out.println("StrComparringNoSentenceRelationMap size: " + StrComparringNoSentenceRelationMap.size());
Collections.sort(StrComparringNoSentenceRelationMap, (e1, e2) -> e1.getPrimaryString().compareTo(e2.getPrimaryString()));
ConcurrentMap<Integer, String> strmapreturn = futuresReturnOverallEvaluation(StrComparringNoSentenceRelationMap);
//System.out.println("strmapreturn size: " + strmapreturn.size());
@ -398,7 +405,7 @@ public class Datahandler {
}
private ConcurrentMap<Integer, String> removeNonSensicalStrings(ConcurrentMap<Integer, String> strmap) {
final ConcurrentMap<Integer, String> strCacheLocal = stringCache.size() < 150 ? strmap : stringCache;
final ConcurrentMap<Integer, String> strCacheLocal = stringCache;
final ConcurrentMap<String, Annotation> localJMWEMap = getMultipleJMWEAnnotation(strmap.values());
final ConcurrentMap<String, Annotation> localPipelineAnnotation = getMultiplePipelineAnnotation(strmap.values());
final ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation = getMultiplePipelineSentimentAnnotation(strmap.values());
@ -406,16 +413,13 @@ public class Datahandler {
return stringIteratorComparator(strmap, strCacheLocal, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap);
}
public synchronized void checkIfUpdateStrings(boolean hlStatsMsg) throws CustomError {
public synchronized void checkIfUpdateStrings() throws CustomError {
if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
ConcurrentMap<Integer, String> str = MessageResponseHandler.getStr();
System.out.println("str size: " + str.size());
str = cutContent(str, hlStatsMsg);
str = filterContent(str);
str = removeSlacks(str);
//System.out.println("finished removeSlacks \n" + str.size() + "\n");
str = removeNonSensicalStrings(str);
System.out.println("removeNonSensicalStrings str size POST: " + str.size() + "\n");
//System.out.println("removeNonSensicalStrings str size POST: " + str.size() + "\n");
str = annotationCacheUpdate(str);
System.out.println("annotationCacheUpdate str size POST: " + str.size() + "\n");
ConcurrentMap<Integer, String> strf = str;
@ -616,20 +620,6 @@ public class Datahandler {
return 0.0;
}
public static ConcurrentMap<Integer, String> cutContent(ConcurrentMap<Integer, String> str, boolean hlStatsMsg) {
ConcurrentMap<Integer, String> returnlist = new MapMaker().concurrencyLevel(2).makeMap();
str.values().forEach(str1 -> {
int iend = str1.indexOf("content: ");
if (iend != -1) {
String trs = str1.substring(iend + 9);
returnlist.put(returnlist.size() + 1, trs.substring(0, trs.length() - 1));
} else if (hlStatsMsg) {
returnlist.put(returnlist.size() + 1, str1);
}
});
return returnlist;
}
public static ConcurrentMap<Integer, String> filterContent(ConcurrentMap<Integer, String> str) {
ConcurrentMap<Integer, String> strlistreturn = new MapMaker().concurrencyLevel(2).makeMap();
str.values().forEach(str1 -> {
@ -641,12 +631,15 @@ public class Datahandler {
if (str1.contains("AM*")) {
str1 = str1.substring(str1.indexOf("AM*") + 3);
}
for (Character c : str1.toCharArray()) {
if (c == '?' || c == '°') {
if (str1.contains("?") || str1.contains("°"))
{
if (!str1.contains("http"))
{
str1 = str1.replace("?", " <:wlenny:514861023002624001> ");
str1 = str1.replace("°", " <:wlenny:514861023002624001> ");
}
}
if (str1.contains("(Counter-Terrorist)")) {
str1 = str1.replace("(Counter-Terrorist)", " ");
}
@ -730,38 +723,13 @@ public class Datahandler {
}
str1 = str1.trim();
if (str1.length() > 2 && (!str1.startsWith("!"))) {
strlistreturn.put(strlistreturn.size() + 1, str1);
strlistreturn.put(strlistreturn.size(), str1);
}
}
});
return strlistreturn;
}
private ConcurrentMap<Integer, String> removeSlacks(ConcurrentMap<Integer, String> str) {
ConcurrentMap<Integer, String> strreturn = new MapMaker().concurrencyLevel(2).makeMap();
if (stringCache.isEmpty()) {
return str;
}
Collection<String> values = stringCache.values();
str.values().forEach(str1 -> {
boolean tooclosematch = false;
for (String strVals : values) {
LevenshteinDistance leven = new LevenshteinDistance(strVals, str1);
double Distance = leven.computeLevenshteinDistance();
Double maxpermittedDistance = 2.5;
if (Distance < maxpermittedDistance) {
tooclosematch = true;
break;
}
}
if (!tooclosematch) {
strreturn.put(strreturn.size() + 1, str1);
//System.out.println("adding strreturn str1: " + str1 + "\n");
}
});
return strreturn;
}
private ConcurrentMap<Integer, String> annotationCacheUpdate(ConcurrentMap<Integer, String> strmap) {
ConcurrentMap<String, Annotation> jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strmap.values());
for (Entry<String, Annotation> jmweitr : jmweAnnotation.entrySet()) {

View File

@ -6,13 +6,17 @@
package FunctionLayer;
import PresentationLayer.DiscordHandler;
import discord4j.core.event.domain.message.MessageCreateEvent;
import discord4j.core.object.entity.User;
import discord4j.core.object.entity.channel.TextChannel;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.javacord.api.DiscordApi;
import org.javacord.api.entity.user.User;
import org.javacord.api.event.message.MessageCreateEvent;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
/**
*
@ -26,17 +30,25 @@ public class DoStuff {
return occupied;
}
public static void doStuff(MessageCreateEvent event, DiscordApi api) {
if (!event.getMessage().getAuthor().isYourself()) {
public static void doStuff(MessageCreateEvent event, String usernameBot) {
String username = null;
try {
username = event.getMessage().getAuthor().get().getUsername();
} catch (java.util.NoSuchElementException e) {
username = null;
}
if (username != null && !username.equals(usernameBot)) {
occupied = true;
String strtest = event.getServerTextChannel().get().getCategory().toString();
strtest = strtest.substring(9, strtest.length() - 1);
TextChannel block = event.getMessage().getChannel().cast(TextChannel.class).block();
String name = block.getCategory().block().getName();
name = name.toLowerCase();
String channelName = block.getName().toLowerCase();
boolean channelpermissionsDenied = false;
switch (strtest) {
case "Public Area": {
switch (name) {
case "public area": {
break;
}
case "Information Area": {
case "information area": {
break;
}
default: {
@ -44,25 +56,35 @@ public class DoStuff {
break;
}
}
List<User> blockLast = event.getMessage().getUserMentions().buffer().blockLast();
String content = event.getMessage().getContent();
if (!channelpermissionsDenied) {
List<User> userlist = event.getMessage().getMentionedUsers();
String strresult = event.getMessage().toString();
if (userlist != null) {
for (User user : userlist) {
strresult = strresult.replace(user.getIdAsString(), "");
if (blockLast != null)
{
for (User user : blockLast) {
content = content.replace(user.getId().asString(), "");
}
}
MessageResponseHandler.getMessage(strresult);
MessageResponseHandler.getMessage(content);
}
if (event.getMessage().getMentionedUsers().contains(api.getYourself())
|| event.getServerTextChannel().get().toString().contains("general-autism")) {
boolean mentionedBot = false;
if (blockLast != null){
for (User user : blockLast)
{
if (user.getUsername().equals(usernameBot))
{
mentionedBot = true;
break;
}
}
}
if (mentionedBot || channelName.contains("general-autism")) {
try {
String ResponseStr;
String person = event.getMessageAuthor().getName();
ResponseStr = MessageResponseHandler.selectReponseMessage(event.getMessage().toString(), person);
ResponseStr = MessageResponseHandler.selectReponseMessage(content, username);
if (!ResponseStr.isEmpty()) {
System.out.print("\nResponseStr3: " + ResponseStr + "\n");
event.getChannel().sendMessage(ResponseStr);
event.getMessage().getChannel().block().createMessage(ResponseStr).block();
}
} catch (CustomError ex) {
Logger.getLogger(DoStuff.class.getName()).log(Level.SEVERE, null, ex);
@ -71,7 +93,7 @@ public class DoStuff {
}
new Thread(() -> {
try {
Datahandler.instance.checkIfUpdateStrings(false);
Datahandler.instance.checkIfUpdateStrings();
} catch (CustomError ex) {
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
}

View File

@ -46,7 +46,6 @@ public class MessageResponseHandler {
public static String selectReponseMessage(String toString, String personName) throws CustomError {
ConcurrentMap<Integer, String> str1 = new MapMaker().concurrencyLevel(6).makeMap();
str1.put(str1.size() + 1, toString);
str1 = Datahandler.cutContent(str1, false);
String strreturn = "";
for (String str : str1.values()) {
if (!str.isEmpty()) {

View File

@ -17,14 +17,19 @@ package PresentationLayer;
import FunctionLayer.Datahandler;
import FunctionLayer.DoStuff;
import FunctionLayer.PipelineJMWESingleton;
import discord4j.core.DiscordClient;
import discord4j.core.GatewayDiscordClient;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Timer;
import java.util.TimerTask;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.javacord.api.DiscordApi;
import org.javacord.api.DiscordApiBuilder;
import DataLayer.settings;
import discord4j.common.util.Snowflake;
import discord4j.core.event.domain.message.MessageCreateEvent;
import java.math.BigInteger;
/**
*
@ -48,15 +53,19 @@ public class DiscordHandler {
System.out.println("FINISHED ALL ANNOTATIONS");
Datahandler.instance.addHLstatsMessages();
Datahandler.instance.updateStringCache();
String token = "NTI5NzAxNTk5NjAyMjc4NDAx.Dw0vDg.7-aMjVWdQMYPl8qVNyvTCPS5F_A";
DiscordApi api = new DiscordApiBuilder().setToken(token).login().join();
//String token = "NTI5NzAxNTk5NjAyMjc4NDAx.Dw0vDg.7-aMjVWdQMYPl8qVNyvTCPS5F_A";
String token = new settings().getDiscordToken();
final DiscordClient client = DiscordClient.create(token);
final GatewayDiscordClient gateway = client.login().block();
String usernameBot = gateway.getSelf().block().getUsername();
new Thread(() -> {
Datahandler.instance.update_autismo_socket_msg();
}).start();
api.addMessageCreateListener(event -> {
gateway.on(MessageCreateEvent.class).subscribe(event -> {
if (!FunctionLayer.DoStuff.isOccupied()) {
FunctionLayer.DoStuff.doStuff(event, api);
FunctionLayer.DoStuff.doStuff(event, usernameBot);
}
});
gateway.onDisconnect().block();
}
}

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -1,5 +1,5 @@
#Generated by Maven
#Sat Feb 02 20:31:02 CET 2019
#Mon Nov 02 11:08:19 CET 2020
version=1.0
groupId=com.mycompany
artifactId=ArtificialAutism

View File

@ -0,0 +1,17 @@
FunctionLayer\Datahandler.class
FunctionLayer\Datahandler$AnnotationCollector.class
FunctionLayer\SimilarityMatrix.class
FunctionLayer\StanfordParser\SentimentValueCache.class
DataLayer\DBCPDataSource.class
DataLayer\settings.class
FunctionLayer\LevenshteinDistance.class
FunctionLayer\CustomError.class
FunctionLayer\StanfordParser\SentimentAnalyzerTest.class
DataLayer\DataMapper.class
FunctionLayer\StopwordAnnotator.class
DataLayer\DBCPDataSourceHLstats.class
FunctionLayer\DoStuff.class
FunctionLayer\PipelineJMWESingleton.class
PresentationLayer\DiscordHandler.class
FunctionLayer\Datahandler$1.class
FunctionLayer\MessageResponseHandler.class

View File

@ -0,0 +1,15 @@
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\FunctionLayer\DoStuff.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\PresentationLayer\DiscordHandler.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\DataLayer\DBCPDataSourceHLstats.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\DataLayer\settings.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\FunctionLayer\LevenshteinDistance.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\FunctionLayer\StopwordAnnotator.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\FunctionLayer\StanfordParser\SentimentAnalyzerTest.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\FunctionLayer\PipelineJMWESingleton.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\DataLayer\DataMapper.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\FunctionLayer\CustomError.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\FunctionLayer\SimilarityMatrix.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\DataLayer\DBCPDataSource.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\FunctionLayer\MessageResponseHandler.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\FunctionLayer\Datahandler.java
E:\stationær backup filer\Projects\ArtificialAutism\src\main\java\FunctionLayer\StanfordParser\SentimentValueCache.java