autism
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package DataLayer;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import org.apache.commons.dbcp2.BasicDataSource;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class DBCPDataSource {
|
||||
private static BasicDataSource ds = new BasicDataSource();
|
||||
static {
|
||||
try {
|
||||
ds.setDriver(new com.mysql.cj.jdbc.Driver());
|
||||
ds.setUrl("jdbc:mysql://151.80.230.149:3306/ArtificialAutism?useLegacyDatetimeCode=false&serverTimezone=UTC");
|
||||
//ds.setUrl("jdbc:mysql://localhost:3306/ArtificialAutism?useLegacyDatetimeCode=false&serverTimezone=UTC");
|
||||
ds.setUsername("ArtificialAutism");
|
||||
ds.setPassword("b423b54bwbfb1340438fn");
|
||||
ds.setMaxTotal(-1);
|
||||
ds.setMinIdle(5);
|
||||
ds.setMaxIdle(-1);
|
||||
ds.setMaxOpenPreparedStatements(100);
|
||||
} catch (SQLException ex) {
|
||||
Logger.getLogger(DBCPDataSource.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
public static Connection getConnection() throws SQLException {
|
||||
return ds.getConnection();
|
||||
}
|
||||
|
||||
private DBCPDataSource() {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package DataLayer;
|
||||
|
||||
import FunctionLayer.SimilarityMatrix;
|
||||
import FunctionLayer.CustomError;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class DataMapper {
|
||||
|
||||
public static void createTables() throws CustomError {
|
||||
Connection l_cCon = null;
|
||||
PreparedStatement l_pStatement = null;
|
||||
ResultSet l_rsSearch = null;
|
||||
try {
|
||||
l_cCon = DBCPDataSource.getConnection();
|
||||
String l_sSQL = "CREATE TABLE IF NOT EXISTS `ArtificialAutism`.`Sentences` (`Strings` VARCHAR(256) NOT NULL, PRIMARY KEY (`Strings`))\n"
|
||||
+ "ENGINE = InnoDB;";
|
||||
l_pStatement = l_cCon.prepareStatement(l_sSQL);
|
||||
l_pStatement.execute();
|
||||
l_sSQL = "CREATE TABLE IF NOT EXISTS `ArtificialAutism`.`WordMatrix` (`Str1` VARCHAR(254) NOT NULL, `Str2` VARCHAR(254) NOT NULL,\n"
|
||||
+ " `Distance` DOUBLE NOT NULL, `ID` INT NOT NULL AUTO_INCREMENT,\n"
|
||||
+ " PRIMARY KEY (`ID`))\n"
|
||||
+ "ENGINE = InnoDB;";
|
||||
l_pStatement = l_cCon.prepareStatement(l_sSQL);
|
||||
l_pStatement.execute();
|
||||
} catch (SQLException ex) {
|
||||
throw new CustomError("failed in DataMapper " + ex.getMessage());
|
||||
} finally {
|
||||
CloseConnections(l_pStatement, l_rsSearch, l_cCon);
|
||||
}
|
||||
}
|
||||
|
||||
public static List<String> getAllStrings() throws CustomError {
|
||||
List<String> str = new ArrayList();
|
||||
Connection l_cCon = null;
|
||||
PreparedStatement l_pStatement = null;
|
||||
ResultSet l_rsSearch = null;
|
||||
try {
|
||||
l_cCon = DBCPDataSource.getConnection();
|
||||
String l_sSQL = "SELECT * FROM `Sentences`";
|
||||
l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
|
||||
java.sql.ResultSet.CONCUR_READ_ONLY);
|
||||
l_pStatement.setFetchSize(Integer.MIN_VALUE);
|
||||
l_rsSearch = l_pStatement.executeQuery();
|
||||
while (l_rsSearch.next()) {
|
||||
str.add(l_rsSearch.getString(1));
|
||||
}
|
||||
} catch (SQLException ex) {
|
||||
throw new CustomError("failed in DataMapper " + ex.getMessage());
|
||||
} finally {
|
||||
CloseConnections(l_pStatement, l_rsSearch, l_cCon);
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
public static void InsertMYSQLStrings(List<String> str) throws CustomError {
|
||||
Connection l_cCon = null;
|
||||
PreparedStatement l_pStatement = null;
|
||||
ResultSet l_rsSearch = null;
|
||||
String l_sSQL = "INSERT IGNORE `Sentences` (`Strings`) VALUES (?)";
|
||||
try {
|
||||
if (str != null && str.size() > 0) {
|
||||
l_cCon = DBCPDataSource.getConnection();
|
||||
l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
|
||||
java.sql.ResultSet.CONCUR_READ_ONLY);
|
||||
l_pStatement.setFetchSize(Integer.MIN_VALUE);
|
||||
for (String str1 : str) {
|
||||
System.out.println("adding str1: " + str1 + "\n");
|
||||
l_pStatement.setString(1, str1);
|
||||
l_pStatement.addBatch();
|
||||
}
|
||||
l_pStatement.executeBatch();
|
||||
}
|
||||
} catch (SQLException ex) {
|
||||
throw new CustomError("failed in DataMapper " + ex.getMessage());
|
||||
} finally {
|
||||
CloseConnections(l_pStatement, l_rsSearch, l_cCon);
|
||||
}
|
||||
}
|
||||
|
||||
public static int getSementicsDBRows() {
|
||||
int count = 0;
|
||||
try (Connection l_cCon = DBCPDataSource.getConnection()) {
|
||||
try (Statement s = l_cCon.createStatement();
|
||||
ResultSet r = s.executeQuery("SELECT COUNT(*) AS rowcount FROM WordMatrix")) {
|
||||
r.next();
|
||||
count = r.getInt("rowcount");
|
||||
}
|
||||
} catch (SQLException ex) {
|
||||
Logger.getLogger(DataMapper.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
public static List<SimilarityMatrix> getAllSementicMatrixes() throws CustomError {
|
||||
//https://stackoverflow.com/questions/5157476/resultset-behavior-with-mysql-database-does-it-store-all-rows-in-memory/5159999#5159999
|
||||
//https://stackoverflow.com/questions/3682614/how-to-read-all-rows-from-huge-table
|
||||
int count = getSementicsDBRows();
|
||||
int counter2 = 0;
|
||||
int hardCapRetrieveCount = 500000;
|
||||
List<SimilarityMatrix> WS4JList = new ArrayList(count + 1);
|
||||
while (count > counter2) {
|
||||
try (Connection l_cCon = DBCPDataSource.getConnection()) {
|
||||
l_cCon.setAutoCommit(false);
|
||||
String l_sSQL = "SELECT * FROM `WordMatrix` WHERE ID > " + counter2 + " AND ID < " + (counter2 + hardCapRetrieveCount);
|
||||
try (PreparedStatement l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
|
||||
java.sql.ResultSet.CONCUR_READ_ONLY)) {
|
||||
l_pStatement.setFetchSize(Integer.MIN_VALUE);
|
||||
try (ResultSet l_rsSearch = l_pStatement.executeQuery()) {
|
||||
int i = 0;
|
||||
while (l_rsSearch.next() && i < hardCapRetrieveCount) {
|
||||
SimilarityMatrix ws4j = new SimilarityMatrix(l_rsSearch.getString(1), l_rsSearch.getString(2), l_rsSearch.getDouble(3));
|
||||
//find something cheaper than arraylist probably
|
||||
WS4JList.add(ws4j);
|
||||
System.out.println("i: " + i + "\n" + "free memory: " + Runtime.getRuntime().freeMemory() + "\ncounter2: " + counter2 + "\n");
|
||||
i++;
|
||||
counter2++;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (SQLException ex) {
|
||||
Logger.getLogger(DataMapper.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
return WS4JList;
|
||||
}
|
||||
|
||||
public static void insertSementicMatrixes(List<SimilarityMatrix> WS4JListUpdate) throws CustomError {
|
||||
Connection l_cCon = null;
|
||||
PreparedStatement l_pStatement = null;
|
||||
ResultSet l_rsSearch = null;
|
||||
String l_sSQL = "INSERT IGNORE `WordMatrix` (`Str1`,`Str2`,`Distance`) VALUES (?, ?, ?)";
|
||||
try {
|
||||
l_cCon = DBCPDataSource.getConnection();
|
||||
l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
|
||||
java.sql.ResultSet.CONCUR_READ_ONLY);
|
||||
l_pStatement.setFetchSize(Integer.MIN_VALUE);
|
||||
System.out.println("Matrix update size: " + WS4JListUpdate.size());
|
||||
for (SimilarityMatrix ws4j : WS4JListUpdate) {
|
||||
l_pStatement.setString(1, ws4j.getPrimaryString());
|
||||
l_pStatement.setString(2, ws4j.getSecondaryString());
|
||||
l_pStatement.setDouble(3, ws4j.getDistance());
|
||||
l_pStatement.addBatch();
|
||||
}
|
||||
l_pStatement.executeBatch();
|
||||
} catch (SQLException ex) {
|
||||
throw new CustomError("failed in DataMapper " + ex.getMessage());
|
||||
} finally {
|
||||
CloseConnections(l_pStatement, l_rsSearch, l_cCon);
|
||||
}
|
||||
}
|
||||
|
||||
public static void CloseConnections(PreparedStatement ps, ResultSet rs, Connection con) {
|
||||
if (rs != null) {
|
||||
try {
|
||||
rs.close();
|
||||
} catch (SQLException ex) {
|
||||
Logger.getLogger(DataMapper.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
if (ps != null) {
|
||||
try {
|
||||
ps.close();
|
||||
} catch (SQLException ex) {
|
||||
Logger.getLogger(DataMapper.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
if (con != null) {
|
||||
try {
|
||||
con.close();
|
||||
} catch (SQLException ex) {
|
||||
Logger.getLogger(DataMapper.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class CustomError extends Exception {
|
||||
|
||||
public CustomError(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class LevenshteinDistance {
|
||||
|
||||
private static int minimum(int a, int b, int c) {
|
||||
return Math.min(Math.min(a, b), c);
|
||||
}
|
||||
|
||||
public static int computeLevenshteinDistance(CharSequence lhs, CharSequence rhs) {
|
||||
int[][] distance = new int[lhs.length() + 1][rhs.length() + 1];
|
||||
|
||||
for (int i = 0; i <= lhs.length(); i++) {
|
||||
distance[i][0] = i;
|
||||
}
|
||||
for (int j = 1; j <= rhs.length(); j++) {
|
||||
distance[0][j] = j;
|
||||
}
|
||||
for (int i = 1; i <= lhs.length(); i++) {
|
||||
for (int j = 1; j <= rhs.length(); j++) {
|
||||
distance[i][j] = minimum(
|
||||
distance[i - 1][j] + 1,
|
||||
distance[i][j - 1] + 1,
|
||||
distance[i - 1][j - 1] + ((lhs.charAt(i - 1) == rhs.charAt(j - 1)) ? 0 : 1));
|
||||
}
|
||||
}
|
||||
return distance[lhs.length()][rhs.length()];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,656 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
//https://stackoverflow.com/questions/43935229/hashmap-with-8-million-entries-becomes-slow
|
||||
//http://www2.imm.dtu.dk/pubdb/views/edoc_download.php/6364/pdf/imm6364.pdf
|
||||
*/
|
||||
package FunctionLayer;
|
||||
|
||||
import DataLayer.DataMapper;
|
||||
import FunctionLayer.StanfordParser.SentimentAnalyzerTest;
|
||||
import com.google.common.base.Stopwatch;
|
||||
import com.google.common.collect.ArrayListMultimap;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import com.google.common.collect.Multimap;
|
||||
import edu.stanford.nlp.ling.HasWord;
|
||||
import edu.stanford.nlp.ling.Label;
|
||||
import edu.stanford.nlp.ling.TaggedWord;
|
||||
import edu.stanford.nlp.ling.Word;
|
||||
import edu.stanford.nlp.parser.lexparser.LexicalizedParser;
|
||||
import edu.stanford.nlp.parser.shiftreduce.ShiftReduceParser;
|
||||
import edu.stanford.nlp.process.DocumentPreprocessor;
|
||||
import edu.stanford.nlp.tagger.maxent.MaxentTagger;
|
||||
import edu.stanford.nlp.trees.Tree;
|
||||
import edu.stanford.nlp.trees.TreebankLanguagePack;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class MYSQLDatahandler {
|
||||
|
||||
public static final long EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(10, TimeUnit.MINUTES);
|
||||
public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS);
|
||||
public static MYSQLDatahandler instance = new MYSQLDatahandler();
|
||||
public static int SemeticsUpdateCount;
|
||||
public static int threadCounter = 0;
|
||||
private volatile boolean RefreshMatrixFromDB;
|
||||
private final ConcurrentMap<Integer, String> StringCache;
|
||||
private List<SimilarityMatrix> SimilaritySMXList = new ArrayList();
|
||||
private List<String> multiprocessCalculations = new ArrayList();
|
||||
private final Stopwatch stopwatch;
|
||||
private final Stopwatch stopwatch1;
|
||||
|
||||
public MYSQLDatahandler() {
|
||||
this.stopwatch = Stopwatch.createUnstarted();
|
||||
this.stopwatch1 = Stopwatch.createStarted();
|
||||
this.StringCache = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
}
|
||||
|
||||
private Map<Integer, String> getCache() throws SQLException, IOException, CustomError {
|
||||
List<String> strlist;
|
||||
strlist = DataMapper.getAllStrings();
|
||||
LinkedHashMap<Integer, String> LHM = new LinkedHashMap();
|
||||
int i = 0;
|
||||
for (String str : strlist) {
|
||||
LHM.put(i, str);
|
||||
i++;
|
||||
}
|
||||
return LHM;
|
||||
}
|
||||
|
||||
public void initiateMYSQL() throws SQLException, IOException {
|
||||
try {
|
||||
DataMapper.createTables();
|
||||
StringCache.putAll(getCache());
|
||||
SimilaritySMXList = DataMapper.getAllSementicMatrixes();
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(MYSQLDatahandler.class
|
||||
.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void checkIfUpdateMatrixes() {
|
||||
RefreshMatrixFromDB = false;
|
||||
int calculationBoundaries = 10;
|
||||
int updateBadgesInteger = 500;
|
||||
if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) {
|
||||
RefreshMatrixFromDB = true;
|
||||
if (threadCounter == 0) {
|
||||
try {
|
||||
SimilaritySMXList = DataMapper.getAllSementicMatrixes();
|
||||
stopwatch1.reset();
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (StringCache.values().size() > 10) {
|
||||
if (!RefreshMatrixFromDB && multiprocessCalculations.size() <= (calculationBoundaries * calculationBoundaries)) {
|
||||
threadCounter++;
|
||||
List<String> strList = new ArrayList(StringCache.values());
|
||||
SemeticsUpdateCount = new Random().nextInt(strList.size() - 6);
|
||||
int beginindex = SemeticsUpdateCount;
|
||||
SemeticsUpdateCount += calculationBoundaries / 2;
|
||||
int temp = SemeticsUpdateCount;
|
||||
List<String> strIndexNavigator = new ArrayList();
|
||||
strList.subList(beginindex, temp).forEach((str) -> {
|
||||
strIndexNavigator.add(str);
|
||||
multiprocessCalculations.add(str);
|
||||
});
|
||||
new Thread(() -> {
|
||||
List<String> strIndexNavigatorL = new ArrayList(strIndexNavigator);
|
||||
List<String> strIndexAll = new ArrayList(strList);
|
||||
List<String> randomIndexesToUpdate = new ArrayList();
|
||||
int indexes = updateBadgesInteger;
|
||||
if (indexes >= strIndexAll.size()) {
|
||||
indexes = strIndexAll.size() - 1;
|
||||
}
|
||||
int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes);
|
||||
strIndexAll.subList(beginindexes, beginindexes + indexes).forEach((str) -> {
|
||||
randomIndexesToUpdate.add(str);
|
||||
});
|
||||
List<SimilarityMatrix> matrixUpdateList = new ArrayList();
|
||||
strIndexNavigatorL.forEach((str) -> {
|
||||
randomIndexesToUpdate.stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> {
|
||||
boolean present = false;
|
||||
if (multiprocessCalculations.contains(str1)) {
|
||||
present = true;
|
||||
} else {
|
||||
for (SimilarityMatrix SMX : SimilaritySMXList) {
|
||||
if (SMX.getPrimaryString().equals(str) && SMX.getSecondaryString().equals(str1)) {
|
||||
present = true;
|
||||
break;
|
||||
}
|
||||
if (SMX.getPrimaryString().equals(str1) && SMX.getSecondaryString().equals(str)) {
|
||||
present = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!present) {
|
||||
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
|
||||
double score = SentimentAnalyzerTest.sentimentanalyzing(str, str1);
|
||||
SMX.setDistance(score);
|
||||
matrixUpdateList.add(SMX);
|
||||
SimilaritySMXList.add(SMX);
|
||||
}
|
||||
});
|
||||
});
|
||||
new Thread(() -> {
|
||||
try {
|
||||
if (!matrixUpdateList.isEmpty()) {
|
||||
DataMapper.insertSementicMatrixes(matrixUpdateList);
|
||||
System.out.println("finished datamapper semetic insert");
|
||||
}
|
||||
threadCounter--;
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(MYSQLDatahandler.class
|
||||
.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}).start();
|
||||
}).
|
||||
start();
|
||||
try {
|
||||
wait(800);
|
||||
} catch (InterruptedException ex) {
|
||||
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
} else {
|
||||
if (threadCounter == 0) {
|
||||
List<String> strList = new ArrayList(StringCache.values());
|
||||
List<SimilarityMatrix> matrixUpdateList = new ArrayList();
|
||||
List<String> randomStrList = new ArrayList();
|
||||
int indexes = updateBadgesInteger;
|
||||
if (indexes >= strList.size()) {
|
||||
indexes = strList.size() - 1;
|
||||
}
|
||||
int beginindexes = new Random().nextInt((strList.size()) - indexes);
|
||||
strList.subList(beginindexes, beginindexes + indexes).forEach((str) -> {
|
||||
randomStrList.add(str);
|
||||
});
|
||||
multiprocessCalculations.forEach((str) -> {
|
||||
randomStrList.forEach((str1) -> {
|
||||
boolean present = false;
|
||||
for (SimilarityMatrix SMX : SimilaritySMXList) {
|
||||
if (SMX.getPrimaryString().equals(str) && SMX.getSecondaryString().equals(str1)) {
|
||||
present = true;
|
||||
break;
|
||||
}
|
||||
if (SMX.getPrimaryString().equals(str1) && SMX.getSecondaryString().equals(str)) {
|
||||
present = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!present) {
|
||||
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
|
||||
double score = SentimentAnalyzerTest.sentimentanalyzing(str, str1);
|
||||
SMX.setDistance(score);
|
||||
matrixUpdateList.add(SMX);
|
||||
SimilaritySMXList.add(SMX);
|
||||
}
|
||||
});
|
||||
});
|
||||
try {
|
||||
if (!matrixUpdateList.isEmpty()) {
|
||||
DataMapper.insertSementicMatrixes(matrixUpdateList);
|
||||
System.out.println("finished datamapper semetic insert");
|
||||
}
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(MYSQLDatahandler.class
|
||||
.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
multiprocessCalculations = new ArrayList();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void checkIfUpdateStrings() throws CustomError {
|
||||
if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
|
||||
new Thread(() -> {
|
||||
List<String> str = MessageResponseHandler.getStr();
|
||||
str = cutContent(str);
|
||||
str = filterContent(str);
|
||||
str = removeSlacks(str);
|
||||
List<String> strUpdate = new ArrayList();
|
||||
strUpdate.addAll(str);
|
||||
try {
|
||||
DataMapper.InsertMYSQLStrings(strUpdate);
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(MYSQLDatahandler.class
|
||||
.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
MessageResponseHandler.setStr(new ArrayList());
|
||||
int j = StringCache.size() + 1;
|
||||
for (String str1 : strUpdate) {
|
||||
StringCache.put(j, str1);
|
||||
j++;
|
||||
}
|
||||
}).start();
|
||||
if (!stopwatch.isRunning()) {
|
||||
stopwatch.start();
|
||||
} else {
|
||||
stopwatch.reset();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public String getResponseMsg(String str) throws CustomError {
|
||||
double Score = -10000;
|
||||
SimilarityMatrix SMXreturn = null;
|
||||
List<String> strLocal = new ArrayList(StringCache.values());
|
||||
for (String str1 : strLocal) {
|
||||
if (str.equals(str1)) {
|
||||
Iterator<SimilarityMatrix> SMXITR = SimilaritySMXList.iterator();
|
||||
while (SMXITR.hasNext()) {
|
||||
SimilarityMatrix SMX = SMXITR.next();
|
||||
if (SMX.getPrimaryString().equals(str) || SMX.getSecondaryString().equals(str)) {
|
||||
double smxDistance = SMX.getDistance();
|
||||
if (smxDistance > Score) {
|
||||
Score = smxDistance;
|
||||
SMXreturn = SMX;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (SMXreturn != null) {
|
||||
if (SMXreturn.getPrimaryString().equals(str)) {
|
||||
return SMXreturn.getSecondaryString();
|
||||
} else {
|
||||
return SMXreturn.getPrimaryString();
|
||||
}
|
||||
}
|
||||
String[] toArray = strLocal.toArray(new String[strLocal.size()]);
|
||||
String mostSimilarSTR = mostSimilar(str, toArray);
|
||||
Iterator<SimilarityMatrix> SMXITR = SimilaritySMXList.iterator();
|
||||
while (SMXITR.hasNext()) {
|
||||
System.out.println("mostSimilarSTR; " + mostSimilarSTR + "\n");
|
||||
mostSimilarSTR = mostSimilarSTR.trim();
|
||||
SimilarityMatrix SMX = SMXITR.next();
|
||||
if (SMX.getPrimaryString().trim().equals(mostSimilarSTR) || SMX.getSecondaryString().trim().equals(mostSimilarSTR)) {
|
||||
double smxDistance = SMX.getDistance();
|
||||
if (smxDistance > Score) {
|
||||
Score = smxDistance;
|
||||
SMXreturn = SMX;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (SMXreturn != null) {
|
||||
if (SMXreturn.getPrimaryString().equals(str)) {
|
||||
return SMXreturn.getSecondaryString();
|
||||
} else {
|
||||
return SMXreturn.getPrimaryString();
|
||||
}
|
||||
}
|
||||
SMXITR = SimilaritySMXList.iterator();
|
||||
while (SMXITR.hasNext()) {
|
||||
SimilarityMatrix SMX = SMXITR.next();
|
||||
if (SMX.getPrimaryString().contains(mostSimilarSTR) || SMX.getSecondaryString().contains(mostSimilarSTR)) {
|
||||
double smxDistance = SMX.getDistance();
|
||||
if (smxDistance > Score) {
|
||||
Score = smxDistance;
|
||||
SMXreturn = SMX;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (SMXreturn != null) {
|
||||
if (SMXreturn.getPrimaryString().equals(str)) {
|
||||
return SMXreturn.getSecondaryString();
|
||||
} else {
|
||||
return SMXreturn.getPrimaryString();
|
||||
}
|
||||
}
|
||||
return "how did you manage to reach this, AAAAAAAAAAAA end my suffering";
|
||||
}
|
||||
|
||||
public String mostSimilar(String toBeCompared, String[] strings) {
|
||||
int minDistance = Integer.MAX_VALUE;
|
||||
String similar = "";
|
||||
for (String str : strings) {
|
||||
int d = LevenshteinDistance.computeLevenshteinDistance(str, toBeCompared);
|
||||
if (d < minDistance) {
|
||||
minDistance = d;
|
||||
similar = str;
|
||||
}
|
||||
}
|
||||
return similar;
|
||||
}
|
||||
|
||||
public static List<String> cutContent(List<String> str) {
|
||||
List<String> returnlist = new ArrayList();
|
||||
for (String str1 : str) {
|
||||
int iend = str1.indexOf("content: ");
|
||||
if (iend != -1) {
|
||||
String trs = str1.substring(iend + 9);
|
||||
returnlist.add(trs.substring(0, trs.length() - 1));
|
||||
}
|
||||
}
|
||||
return returnlist;
|
||||
}
|
||||
|
||||
public static List<String> filterContent(List<String> str) {
|
||||
List<String> strlistreturn = new ArrayList();
|
||||
for (String str1 : str) {
|
||||
if (str1.isEmpty() || str1.length() < 3) {
|
||||
continue;
|
||||
}
|
||||
str1 = str1.trim();
|
||||
if (str1.contains("PM*")) {
|
||||
str1 = str1.substring(str1.indexOf("PM*") + 5);
|
||||
}
|
||||
if (str1.contains("AM*")) {
|
||||
str1 = str1.substring(str1.indexOf("AM*") + 5);
|
||||
}
|
||||
for (Character c : str1.toCharArray()) {
|
||||
if (c == '?' || c == '°') {
|
||||
str1 = str1.replace("?", " <:wlenny:514861023002624001> ");
|
||||
str1 = str1.replace("°", " <:wlenny:514861023002624001> ");
|
||||
}
|
||||
}
|
||||
if (str1.contains("(Counter-Terrorist)")) {
|
||||
str1 = str1.replace("(Counter-Terrorist)", " ");
|
||||
}
|
||||
if (str1.contains("(Terrorist)")) {
|
||||
str1 = str1.replace("(Terrorist)", " ");
|
||||
}
|
||||
if (str1.contains("(Spectator)")) {
|
||||
str1 = str1.replace("(Spectator)", " ");
|
||||
}
|
||||
if (str1.contains("*DEAD*")) {
|
||||
str1 = str1.replace("*DEAD*", " ");
|
||||
}
|
||||
if (str1.contains("{red}")) {
|
||||
str1 = str1.replace("{red}", " ");
|
||||
}
|
||||
if (str1.contains("{orange}")) {
|
||||
str1 = str1.replace("{orange}", " ");
|
||||
}
|
||||
if (str1.contains("{yellow}")) {
|
||||
str1 = str1.replace("{yellow}", " ");
|
||||
}
|
||||
if (str1.contains("{green}")) {
|
||||
str1 = str1.replace("{green}", " ");
|
||||
}
|
||||
if (str1.contains("{lightblue}")) {
|
||||
str1 = str1.replace("{lightblue}", " ");
|
||||
}
|
||||
if (str1.contains("{blue}")) {
|
||||
str1 = str1.replace("{blue}", " ");
|
||||
}
|
||||
if (str1.contains("{purple}")) {
|
||||
str1 = str1.replace("{purple}", " ");
|
||||
}
|
||||
if (str1.contains("{white}")) {
|
||||
str1 = str1.replace("{white}", " ");
|
||||
}
|
||||
str1 = str1.trim();
|
||||
if (str1.length() > 2 && (!str1.startsWith("!"))) {
|
||||
strlistreturn.add(str1);
|
||||
}
|
||||
}
|
||||
return strlistreturn;
|
||||
}
|
||||
|
||||
/*
|
||||
public static List<String> cutLongsFromEmotes(List<String> str) {
|
||||
List<String> strreturn = new ArrayList();
|
||||
int emotesTraceLong = 17;
|
||||
for (String str1 : str) {
|
||||
StringBuilder SB = new StringBuilder();
|
||||
int counter = 0;
|
||||
int i = 0;
|
||||
for (Character c : str1.toCharArray()) {
|
||||
i++;
|
||||
if (Character.isDigit(c)) {
|
||||
counter++;
|
||||
if (counter > emotesTraceLong && str1.length() > i + 2) {
|
||||
SB.append(str1.substring(0, i - emotesTraceLong));
|
||||
SB.append(str1.substring(i + 1, str1.length()));
|
||||
}
|
||||
} else {
|
||||
counter = 0;
|
||||
}
|
||||
}
|
||||
if (SB.toString().isEmpty()) {
|
||||
strreturn.add(str1);
|
||||
} else {
|
||||
strreturn.add(SB.toString());
|
||||
}
|
||||
}
|
||||
return strreturn;
|
||||
}
|
||||
*/
|
||||
private List<String> removeSlacks(List<String> str) {
|
||||
ShiftReduceParser model = SentimentAnalyzerTest.getModel();
|
||||
MaxentTagger tagger = SentimentAnalyzerTest.getTagger();
|
||||
List<TaggedWord> taggedWords;
|
||||
List<String> strreturn = new ArrayList();
|
||||
for (String str1 : str) {
|
||||
int counter = 0;
|
||||
List<String> TGWList = new ArrayList();
|
||||
DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1));
|
||||
for (List<HasWord> sentence : tokenizer) {
|
||||
List<TaggedWord> tagged1 = tagger.tagSentence(sentence);
|
||||
Tree tree = model.apply(tagged1);
|
||||
taggedWords = tree.taggedYield();
|
||||
for (TaggedWord TGW : taggedWords) {
|
||||
if (!TGWList.contains(TGW.tag()) && !TGW.tag().equals(":") && !TGW.word().equals(TGW.tag())) {
|
||||
TGWList.add(TGW.tag());
|
||||
counter++;
|
||||
}
|
||||
if (counter > 3) {
|
||||
int addCounter = 0;
|
||||
List<Word> wordList = new ArrayList();
|
||||
for (Word lab : tree.yieldWords()) {
|
||||
if (lab != null && lab.word() != null) {
|
||||
//System.out.println("lab: " + lab + " \n");
|
||||
if (!wordList.contains(lab) && !lab.equals(":")) {
|
||||
wordList.add(lab);
|
||||
addCounter++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (addCounter > 3) {
|
||||
addCounter = 0;
|
||||
List<HasWord> HWlist = new ArrayList();
|
||||
for (HasWord HW : tree.yieldHasWord()) {
|
||||
if (HW != null && HW.word() != null && !HWlist.contains(HW)) {
|
||||
//System.out.println("HasWord: " + HW + "\n");
|
||||
addCounter++;
|
||||
HWlist.add(HW);
|
||||
}
|
||||
}
|
||||
if (addCounter > 3) {
|
||||
boolean tooclosematch = false;
|
||||
for (String strVals : StringCache.values()) {
|
||||
double Distance = LevenshteinDistance.computeLevenshteinDistance(strVals, str1);
|
||||
int maxpermittedDistance = 5;
|
||||
if (Distance < maxpermittedDistance) {
|
||||
tooclosematch = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!tooclosematch) {
|
||||
strreturn.add(str1);
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (counter > 3) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return strreturn;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @throws CustomError
|
||||
* @deprecated
|
||||
*/
|
||||
public synchronized void calculateStrings() throws CustomError {
|
||||
//linkedhashmap?
|
||||
int initiallimit = 5;
|
||||
int listindex = 0;
|
||||
List<SimilarityMatrix> WS4JList = DataMapper.getAllSementicMatrixes();
|
||||
List<SimilarityMatrix> WS4JListUpdate = new ArrayList();
|
||||
List<String> sentencesList = DataMapper.getAllStrings();
|
||||
for (int i = 1; i < initiallimit; i++) {
|
||||
if (!sentencesList.get(i).isEmpty()) {
|
||||
//System.out.print("i: " + i + "\n sentencesList i: " + sentencesList.get(i) + "\n");
|
||||
String[] words1 = sentencesList.get(i).split(" ");
|
||||
for (String words11 : words1) {
|
||||
if (!words11.isEmpty() && words11.length() > 3) {
|
||||
String str = words11;
|
||||
if (!str.isEmpty() && str.length() > 3) {
|
||||
//SecondaryPredicate, no secondary key present with word
|
||||
Predicate<SimilarityMatrix> SecondaryPredicate = e -> str.equals(e.getSecondaryString());
|
||||
//no primary key present with the word
|
||||
if (!WS4JList.stream().filter(e -> str.equals(e.getPrimaryString())).findAny().isPresent()) {
|
||||
WS4JListUpdate = createWS4JWordMatrix(str, sentencesList, WS4JListUpdate, WS4JList, SecondaryPredicate);
|
||||
for (; listindex < WS4JListUpdate.size(); listindex++) {
|
||||
WS4JList.add(WS4JListUpdate.get(listindex));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
//System.out.println("\nWS4JListUpdate size: " + WS4JListUpdate.size());
|
||||
DataMapper.insertSementicMatrixes(WS4JListUpdate);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param str
|
||||
* @param strlist
|
||||
* @param ws4jlist
|
||||
* @param oldlist
|
||||
* @param SecondaryPredicate
|
||||
* @return
|
||||
* @deprecated
|
||||
*/
|
||||
public List<SimilarityMatrix> createWS4JWordMatrix(String str, List<String> strlist, List<SimilarityMatrix> ws4jlist,
|
||||
List<SimilarityMatrix> oldlist, Predicate<SimilarityMatrix> SecondaryPredicate) {
|
||||
for (String str1 : strlist) {
|
||||
if (!str1.isEmpty()) {
|
||||
String[] words1 = str1.split(" ");
|
||||
for (int j = 0; j < words1.length; j++) {
|
||||
if (!words1[j].isEmpty() && words1[j].length() > 3) {
|
||||
String strword = words1[j];
|
||||
if (!strword.isEmpty() && strword.length() > 3 && !strword.equals(str)) {
|
||||
Predicate<SimilarityMatrix> PrimaryPredicate = e -> strword.equals(e.getPrimaryString());
|
||||
if (!oldlist.stream().filter(PrimaryPredicate.and(SecondaryPredicate)).findAny().isPresent()) {
|
||||
//System.out.println("creating SimilarityMatrix with STR: " + str + "\n strword: " + strword + "\n");
|
||||
SimilarityMatrix ws4j = new SimilarityMatrix(str, strword);
|
||||
/*
|
||||
double addingDistance = ws4j.getDistanceCalculations();
|
||||
if (addingDistance > 0.00) {
|
||||
//System.out.println("added! \n");
|
||||
ws4j.setDistance(addingDistance);
|
||||
ws4jlist.add(ws4j);
|
||||
}
|
||||
*/
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ws4jlist;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stanford Parser method to update calculations to the DB
|
||||
*
|
||||
* @deprecated
|
||||
* @throws FunctionLayer.CustomError
|
||||
*/
|
||||
public void sentimentAnalyzingStringsToDB() throws CustomError {
|
||||
|
||||
List<String> sentencesList = DataMapper.getAllStrings();
|
||||
List<SimilarityMatrix> WS4JList = DataMapper.getAllSementicMatrixes();
|
||||
List<SimilarityMatrix> WS4JListUpdate = new ArrayList();
|
||||
int listindex = 0;
|
||||
for (int i = 0; i < sentencesList.size() - 3000; i++) {
|
||||
String str = sentencesList.get(i);
|
||||
if (!str.isEmpty() && str.length() > 3) {
|
||||
//System.out.println("i: " + i + "\n");
|
||||
Predicate<SimilarityMatrix> SecondaryPredicate = e -> str.equals(e.getSecondaryString());
|
||||
if (!WS4JList.stream().filter(e -> str.equals(e.getPrimaryString())).findAny().isPresent()) {
|
||||
//WS4JListUpdate = addStringMatrixes(str, sentencesList, SecondaryPredicate, WS4JListUpdate, WS4JList);
|
||||
for (; listindex < WS4JListUpdate.size(); listindex++) {
|
||||
WS4JList.add(WS4JListUpdate.get(listindex));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// System.out.println("\n WS4JListUpdate size: " + WS4JListUpdate.size());
|
||||
DataMapper.insertSementicMatrixes(WS4JListUpdate);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated @param str Primary string which is checked, the filter
|
||||
* ensures primary string has not been calculated yet
|
||||
* @param sentencesList the full string list where each unique primary has
|
||||
* to calculate its value to the rest of the DB
|
||||
* @param SecondaryPredicate ensures primary string is not already
|
||||
* calculated somewhere with another string
|
||||
* @param WS4JListUpdate Matrix list to update in DB with new Sentences
|
||||
* @param OldWS4JList Check if str1 already has primary entry with
|
||||
* co-responding SecondaryPredicate
|
||||
* @return Updated List which has to be inserted to the DB
|
||||
*/
|
||||
private List<SimilarityMatrix> addStringMatrixes(String str, List<String> sentencesList,
|
||||
Predicate<SimilarityMatrix> SecondaryPredicate, List<SimilarityMatrix> WS4JListUpdate,
|
||||
List<SimilarityMatrix> OldWS4JList, LexicalizedParser lp, TreebankLanguagePack tlp) {
|
||||
double valuerange = -100.0;
|
||||
for (int i = 0; i < sentencesList.size(); i++) {
|
||||
String str1 = sentencesList.get(i);
|
||||
if (!str1.isEmpty() && str1.length() > 3) {
|
||||
Predicate<SimilarityMatrix> PrimaryPredicate = e -> str1.equals(e.getPrimaryString());
|
||||
if (!OldWS4JList.stream().filter(PrimaryPredicate.and(SecondaryPredicate)).findAny().isPresent()) {
|
||||
double s = -100.0;
|
||||
if (s > valuerange) {
|
||||
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
|
||||
SMX.setDistance(s);
|
||||
/*
|
||||
System.out.println("SMX added: \n Primary: " + SMX.getPrimaryString() + "\n Secondary: " + SMX.getSecondaryString()
|
||||
+ "\n Score: " + SMX.getDistance() + "\n");
|
||||
*/
|
||||
WS4JListUpdate.add(SMX);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
str parameter is primary and not used as primary if reaching here
|
||||
secondary predicate ensures primary does not already exist as secondary with co-responding strlist primary
|
||||
*/
|
||||
return WS4JListUpdate;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
//https://www.programcreek.com/java-api-examples/index.php?source_dir=simmetrics-master/simmetrics-core/src/main/java/org/simmetrics/metrics/JaroWinkler.java#
|
||||
//https://stackoverflow.com/questions/36032958/semantic-matching-in-ws4j-at-sentence-level
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class MessageResponseHandler {
|
||||
|
||||
private static List<String> str = new ArrayList();
|
||||
|
||||
public static List<String> getStr() {
|
||||
return str;
|
||||
}
|
||||
|
||||
public static void setStr(List<String> str) {
|
||||
MessageResponseHandler.str = str;
|
||||
}
|
||||
|
||||
public static void getMessage(String message) {
|
||||
if (message != null && !message.isEmpty()) {
|
||||
message = message.replace("@", "");
|
||||
if (message.startsWith("[ *")) {
|
||||
message = message.substring(message.indexOf("]"));
|
||||
}
|
||||
str.add(message);
|
||||
}
|
||||
}
|
||||
|
||||
public static String selectReponseMessage(String toString) throws CustomError {
|
||||
List<String> str1 = new ArrayList();
|
||||
str1.add(toString);
|
||||
str1 = MYSQLDatahandler.cutContent(str1);
|
||||
String strreturn = str1.get(0);
|
||||
String getResponseMsg = MYSQLDatahandler.instance.getResponseMsg(strreturn);
|
||||
return getResponseMsg;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer;
|
||||
|
||||
import com.google.common.collect.Multimap;
|
||||
import com.google.common.collect.Multiset;
|
||||
import java.util.Collection;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class SimilarityMatrix{
|
||||
|
||||
private String PrimaryString;
|
||||
private String SecondaryString;
|
||||
private double distance;
|
||||
|
||||
public double getDistance() {
|
||||
return distance;
|
||||
}
|
||||
|
||||
public void setDistance(double distance) {
|
||||
this.distance = distance;
|
||||
}
|
||||
|
||||
public SimilarityMatrix(String str1, String str2) {
|
||||
this.PrimaryString = str1;
|
||||
this.SecondaryString = str2;
|
||||
}
|
||||
|
||||
public SimilarityMatrix(String str1, String str2, double result) {
|
||||
this.PrimaryString = str1;
|
||||
this.SecondaryString = str2;
|
||||
this.distance = result;
|
||||
}
|
||||
|
||||
/*
|
||||
public double getDistanceCalculations() {
|
||||
ILexicalDatabase db = new NictWordNet();
|
||||
WS4JConfiguration.getInstance().setMFS(true);
|
||||
RelatednessCalculator rc1 = new WuPalmer(db);
|
||||
RelatednessCalculator rc2 = new Resnik(db);
|
||||
RelatednessCalculator rc3 = new JiangConrath(db);
|
||||
RelatednessCalculator rc4 = new Lin(db);
|
||||
RelatednessCalculator rc5 = new LeacockChodorow(db);
|
||||
RelatednessCalculator rc6 = new Path(db);
|
||||
RelatednessCalculator rc7 = new Lesk(db);
|
||||
RelatednessCalculator rc8 = new HirstStOnge(db);
|
||||
double maxScore = -1D;
|
||||
List<RelatednessCalculator> RCList = new ArrayList();
|
||||
RCList.add(rc1);
|
||||
RCList.add(rc2);
|
||||
RCList.add(rc3);
|
||||
RCList.add(rc4);
|
||||
RCList.add(rc5);
|
||||
RCList.add(rc6);
|
||||
RCList.add(rc7);
|
||||
RCList.add(rc8);
|
||||
for (RelatednessCalculator rc : RCList) {
|
||||
double s = rc.calcRelatednessOfWords(PrimaryString, SecondaryString);
|
||||
s /= 1000;
|
||||
if (s > 0.000000) {
|
||||
System.out.println("s: " + s + "\n" + " PrimaryString: " + PrimaryString + "\n" + " SecondaryString: " + SecondaryString + "\n"
|
||||
+ " rc: " + rc.toString() + "\n");
|
||||
}
|
||||
String str = String.format("%.12f", s);
|
||||
if (str.contains(",")) {
|
||||
str = str.substring(0, str.indexOf(","));
|
||||
}
|
||||
int strend = str.length() > 6 ? 6 : str.length();
|
||||
str = str.substring(0, strend);
|
||||
double score = Double.valueOf(str);
|
||||
if (score > maxScore) {
|
||||
maxScore = score;
|
||||
}
|
||||
}
|
||||
return maxScore == -1D ? 0.00 : maxScore;
|
||||
}
|
||||
*/
|
||||
public String getPrimaryString() {
|
||||
return PrimaryString;
|
||||
}
|
||||
|
||||
public void setPrimaryString(String PrimaryString) {
|
||||
this.PrimaryString = PrimaryString;
|
||||
}
|
||||
|
||||
public String getSecondaryString() {
|
||||
return SecondaryString;
|
||||
}
|
||||
|
||||
public void setSecondaryString(String SecondaryString) {
|
||||
this.SecondaryString = SecondaryString;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+259
@@ -0,0 +1,259 @@
|
||||
package FunctionLayer.StanfordParser;
|
||||
|
||||
import FunctionLayer.LevenshteinDistance;
|
||||
import edu.stanford.nlp.ling.CoreAnnotations;
|
||||
import edu.stanford.nlp.ling.CoreLabel;
|
||||
import edu.stanford.nlp.ling.HasWord;
|
||||
import edu.stanford.nlp.ling.IndexedWord;
|
||||
import edu.stanford.nlp.ling.Label;
|
||||
import edu.stanford.nlp.ling.TaggedWord;
|
||||
import edu.stanford.nlp.neural.rnn.RNNCoreAnnotations;
|
||||
import edu.stanford.nlp.parser.lexparser.LexicalizedParser;
|
||||
import edu.stanford.nlp.parser.shiftreduce.ShiftReduceParser;
|
||||
import edu.stanford.nlp.pipeline.Annotation;
|
||||
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
|
||||
import edu.stanford.nlp.process.DocumentPreprocessor;
|
||||
import edu.stanford.nlp.process.Tokenizer;
|
||||
import edu.stanford.nlp.sentiment.SentimentCoreAnnotations;
|
||||
import edu.stanford.nlp.tagger.maxent.MaxentTagger;
|
||||
import edu.stanford.nlp.trees.Constituent;
|
||||
import edu.stanford.nlp.trees.GrammaticalRelation;
|
||||
import edu.stanford.nlp.trees.GrammaticalStructure;
|
||||
import edu.stanford.nlp.trees.GrammaticalStructureFactory;
|
||||
import edu.stanford.nlp.trees.Tree;
|
||||
import edu.stanford.nlp.trees.TreeCoreAnnotations;
|
||||
import edu.stanford.nlp.trees.TreebankLanguagePack;
|
||||
import edu.stanford.nlp.trees.TypedDependency;
|
||||
import edu.stanford.nlp.trees.tregex.gui.Tdiff;
|
||||
import edu.stanford.nlp.util.CoreMap;
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import org.ejml.simple.SimpleMatrix;
|
||||
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class SentimentAnalyzerTest {
|
||||
|
||||
private static String modelPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz";
|
||||
private static String sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz";
|
||||
private static String parserModelPathUD = "edu/stanford/nlp/models/parser/nndep/english_UD.gz";
|
||||
private static String lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz";
|
||||
private static String taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger";
|
||||
private static MaxentTagger tagger;
|
||||
private static ShiftReduceParser model;
|
||||
private static String[] options = {"-maxLength", "100"};
|
||||
private static LexicalizedParser lp;
|
||||
private static TreebankLanguagePack tlp;
|
||||
private static Properties props = new Properties();
|
||||
private static Properties propsSentiment = new Properties();
|
||||
private static GrammaticalStructureFactory gsf;
|
||||
private static StanfordCoreNLP pipeline;
|
||||
private static StanfordCoreNLP pipelineSentiment;
|
||||
|
||||
public static void shiftReduceParserInitiate() {
|
||||
model = ShiftReduceParser.loadModel(modelPath, options);
|
||||
tagger = new MaxentTagger(taggerPath);
|
||||
lp = LexicalizedParser.loadModel(lexParserEnglishRNN, options);
|
||||
tlp = lp.getOp().langpack();
|
||||
gsf = tlp.grammaticalStructureFactory();
|
||||
props.setProperty("annotators", "tokenize,ssplit,pos,lemma,parse");
|
||||
// set up pipeline properties
|
||||
props.setProperty("parse.model", modelPath);
|
||||
// use faster shift reduce parser
|
||||
props.setProperty("parse.maxlen", "100");
|
||||
props.setProperty("parse.binaryTrees", "true");
|
||||
propsSentiment.setProperty("annotators", "tokenize, ssplit, parse, sentiment");
|
||||
propsSentiment.setProperty("parse.model", lexParserEnglishRNN);
|
||||
propsSentiment.setProperty("parse.maxlen", "100");
|
||||
pipeline = new StanfordCoreNLP(props);
|
||||
pipelineSentiment = new StanfordCoreNLP(propsSentiment);
|
||||
}
|
||||
|
||||
public static ShiftReduceParser getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public static MaxentTagger getTagger() {
|
||||
return tagger;
|
||||
}
|
||||
|
||||
public static double sentimentanalyzing(String str, String str1) {
|
||||
double score = -100.0;
|
||||
List<List<TaggedWord>> taggedwordlist1 = new ArrayList();
|
||||
List<List<TaggedWord>> taggedwordlist2 = new ArrayList();
|
||||
DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1));
|
||||
for (List<HasWord> sentence : tokenizer) {
|
||||
List<TaggedWord> tagged1 = tagger.tagSentence(sentence);
|
||||
Tree tree = model.apply(tagged1);
|
||||
ArrayList<TaggedWord> taggedYield = tree.taggedYield();
|
||||
taggedwordlist1.add(taggedYield);
|
||||
}
|
||||
tokenizer = new DocumentPreprocessor(new StringReader(str));
|
||||
for (List<HasWord> sentence : tokenizer) {
|
||||
List<TaggedWord> tagged1 = tagger.tagSentence(sentence);
|
||||
Tree tree = model.apply(tagged1);
|
||||
ArrayList<TaggedWord> taggedYield = tree.taggedYield();
|
||||
taggedwordlist2.add(taggedYield);
|
||||
}
|
||||
int counter = 0;
|
||||
int counter1 = 0;
|
||||
for (List<TaggedWord> taggedlist2 : taggedwordlist2) {
|
||||
counter += taggedlist2.size();
|
||||
}
|
||||
for (List<TaggedWord> taggedlist1 : taggedwordlist1) {
|
||||
counter1 += taggedlist1.size();
|
||||
}
|
||||
int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter;
|
||||
overValue *= 16;
|
||||
while (overValue > 0) {
|
||||
overValue--;
|
||||
score--;
|
||||
}
|
||||
System.out.println("Score Post overValue: " + score + "\n");
|
||||
for (List<TaggedWord> TGWList : taggedwordlist1) {
|
||||
for (TaggedWord TGW : TGWList) {
|
||||
List<String> tgwlist1 = new ArrayList();
|
||||
for (List<TaggedWord> taggedlist2 : taggedwordlist2) {
|
||||
for (TaggedWord TGW1 : taggedlist2) {
|
||||
if (TGW.tag().equals(TGW1.tag()) && !TGW.tag().equals(":") && !tgwlist1.contains(TGW1.tag())) {
|
||||
score += 64;
|
||||
tgwlist1.add(TGW.tag());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// System.out.println("Score: " + score);
|
||||
Annotation annotation = new Annotation(str1);
|
||||
pipeline.annotate(annotation);
|
||||
List<Tree> sentenceConstituencyParseList = new ArrayList();
|
||||
for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
||||
sentenceConstituencyParseList.add(sentenceConstituencyParse);
|
||||
}
|
||||
Annotation annotation1 = new Annotation(str);
|
||||
pipeline.annotate(annotation1);
|
||||
for (CoreMap sentence : annotation1.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
||||
GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse);
|
||||
Collection<TypedDependency> allTypedDependencies = gs.allTypedDependencies();
|
||||
List<String> filerTreeContent = new ArrayList();
|
||||
for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList) {
|
||||
Set<Constituent> inT1notT2 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1);
|
||||
Set<Constituent> inT2notT1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse);
|
||||
List<String> constiLabels = new ArrayList();
|
||||
for (Constituent consti : inT1notT2) {
|
||||
for (Constituent consti1 : inT2notT1) {
|
||||
if (consti.value().equals(consti1.value()) && !constiLabels.contains(consti.value())) {
|
||||
//System.out.println("consti value: " + consti.value() + "\n");
|
||||
score += 64; //256
|
||||
constiLabels.add(consti.value());
|
||||
}
|
||||
}
|
||||
}
|
||||
GrammaticalStructure gs1 = gsf.newGrammaticalStructure(sentenceConstituencyParse1);
|
||||
Collection<TypedDependency> allTypedDependencies1 = gs1.allTypedDependencies();
|
||||
for (TypedDependency TDY1 : allTypedDependencies1) {
|
||||
IndexedWord dep = TDY1.dep();
|
||||
IndexedWord gov = TDY1.gov();
|
||||
GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep);
|
||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 900;
|
||||
//System.out.println("grammaticalRelation applicable: " + score + "\n");
|
||||
}
|
||||
GrammaticalRelation reln = TDY1.reln();
|
||||
if (reln.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 256;
|
||||
}
|
||||
}
|
||||
for (TypedDependency TDY : allTypedDependencies) {
|
||||
IndexedWord dep = TDY.dep();
|
||||
IndexedWord gov = TDY.gov();
|
||||
GrammaticalRelation grammaticalRelation = gs1.getGrammaticalRelation(gov, dep);
|
||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 900;
|
||||
//System.out.println("grammaticalRelation applicable: " + score + "\n");
|
||||
}
|
||||
GrammaticalRelation reln = TDY.reln();
|
||||
if (reln.isApplicable(sentenceConstituencyParse1)) {
|
||||
score += 256;
|
||||
}
|
||||
}
|
||||
for (CoreLabel LBW : sentenceConstituencyParse.taggedLabeledYield()) {
|
||||
for (CoreLabel LBW1 : sentenceConstituencyParse1.taggedLabeledYield()) {
|
||||
if (LBW.lemma().equals(LBW1.lemma()) && !filerTreeContent.contains(LBW.lemma())) {
|
||||
filerTreeContent.add(LBW.lemma());
|
||||
score += 1500;
|
||||
//System.out.println("lemma: " + LBW.lemma() + "\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Annotation annotationSentiment1 = pipelineSentiment.process(str);
|
||||
List<SimpleMatrix> simpleSMXlist = new ArrayList();
|
||||
List<SimpleMatrix> simpleSMXlistVector = new ArrayList();
|
||||
for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
||||
simpleSMXlist.add(predictions);
|
||||
simpleSMXlistVector.add(nodeVector);
|
||||
}
|
||||
annotationSentiment1 = pipelineSentiment.process(str1);
|
||||
for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
||||
for (SimpleMatrix simpleSMX : simpleSMXlist) {
|
||||
double dot = predictions.dot(simpleSMX) * 100;
|
||||
//System.out.println("dot value: " + dot + "\n");
|
||||
double subtracter = dot > 50 ? dot - 50 : 50 - dot;
|
||||
System.out.println("score pre dot: " + score + "\nsubtracter: " + subtracter + "\n");
|
||||
subtracter *= 25;
|
||||
while (subtracter > 0) {
|
||||
subtracter--;
|
||||
score--;
|
||||
}
|
||||
System.out.println("score post dot: " + score + "\n");
|
||||
}
|
||||
for (SimpleMatrix simpleSMX : simpleSMXlistVector) {
|
||||
double dot = nodeVector.dot(simpleSMX);
|
||||
double elementSum = nodeVector.kron(simpleSMX).elementSum();
|
||||
elementSum = Math.round(elementSum * 100.0) / 100.0;
|
||||
System.out.println("kron SMX elementSum: " + elementSum + "\n");
|
||||
if (dot < 0.1) {
|
||||
// System.out.println("\ndot VECTOR: " + dot + "\n");
|
||||
score += 256;
|
||||
}
|
||||
if (elementSum < 0.1 && elementSum > 0.0) {
|
||||
score += 1300;
|
||||
} else if (elementSum > 0.1 && elementSum < 1.0) {
|
||||
score -= 1100;
|
||||
} else {
|
||||
score -= 1424;
|
||||
}
|
||||
}
|
||||
}
|
||||
int SentenceScoreDiff = LevenshteinDistance.computeLevenshteinDistance(str, str1);
|
||||
SentenceScoreDiff *= 15;
|
||||
// System.out.println("score pre levenhstein substract: " + score + "\nSentenceScoreDiff: " + SentenceScoreDiff + "\n");
|
||||
while (SentenceScoreDiff > 0) {
|
||||
SentenceScoreDiff--;
|
||||
score--;
|
||||
}
|
||||
System.out.println("Final current score: " + score + "\nSentences: " + str + "\n" + str1 + "\n\n\n");
|
||||
return score;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer.StanfordParser;
|
||||
|
||||
import edu.cmu.lti.lexical_db.ILexicalDatabase;
|
||||
import edu.cmu.lti.lexical_db.NictWordNet;
|
||||
import edu.cmu.lti.ws4j.RelatednessCalculator;
|
||||
import edu.cmu.lti.ws4j.impl.HirstStOnge;
|
||||
import edu.cmu.lti.ws4j.impl.JiangConrath;
|
||||
import edu.cmu.lti.ws4j.impl.LeacockChodorow;
|
||||
import edu.cmu.lti.ws4j.impl.Lesk;
|
||||
import edu.cmu.lti.ws4j.impl.Lin;
|
||||
import edu.cmu.lti.ws4j.impl.Path;
|
||||
import edu.cmu.lti.ws4j.impl.Resnik;
|
||||
import edu.cmu.lti.ws4j.impl.WuPalmer;
|
||||
import edu.cmu.lti.ws4j.util.WS4JConfiguration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class SimilarityMatrix {
|
||||
|
||||
private String PrimaryString;
|
||||
private String SecondaryString;
|
||||
private double distance;
|
||||
|
||||
public double getDistance() {
|
||||
return distance;
|
||||
}
|
||||
|
||||
public void setDistance(double distance) {
|
||||
this.distance = distance;
|
||||
}
|
||||
|
||||
public SimilarityMatrix(String str1, String str2) {
|
||||
this.PrimaryString = str1;
|
||||
this.SecondaryString = str2;
|
||||
}
|
||||
|
||||
public SimilarityMatrix(String str1, String str2, double result) {
|
||||
this.PrimaryString = str1;
|
||||
this.SecondaryString = str2;
|
||||
this.distance = result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated
|
||||
* @return ws4j distance caluclation add infinitum absurdum
|
||||
*/
|
||||
public double getDistanceCalculations() {
|
||||
ILexicalDatabase db = new NictWordNet();
|
||||
WS4JConfiguration.getInstance().setMFS(true);
|
||||
RelatednessCalculator rc1 = new WuPalmer(db);
|
||||
RelatednessCalculator rc2 = new Resnik(db);
|
||||
RelatednessCalculator rc3 = new JiangConrath(db);
|
||||
RelatednessCalculator rc4 = new Lin(db);
|
||||
RelatednessCalculator rc5 = new LeacockChodorow(db);
|
||||
RelatednessCalculator rc6 = new Path(db);
|
||||
RelatednessCalculator rc7 = new Lesk(db);
|
||||
RelatednessCalculator rc8 = new HirstStOnge(db);
|
||||
double maxScore = -1D;
|
||||
List<RelatednessCalculator> RCList = new ArrayList();
|
||||
RCList.add(rc1);
|
||||
RCList.add(rc2);
|
||||
RCList.add(rc3);
|
||||
RCList.add(rc4);
|
||||
RCList.add(rc5);
|
||||
RCList.add(rc6);
|
||||
RCList.add(rc7);
|
||||
RCList.add(rc8);
|
||||
for (RelatednessCalculator rc : RCList) {
|
||||
double s = rc.calcRelatednessOfWords(PrimaryString, SecondaryString);
|
||||
s /= 1000;
|
||||
if (s > 0.000000) {
|
||||
System.out.println("s: " + s + "\n" + " PrimaryString: " + PrimaryString + "\n" + " SecondaryString: " + SecondaryString + "\n"
|
||||
+ " rc: " + rc.toString() + "\n");
|
||||
}
|
||||
String str = String.format("%.12f", s);
|
||||
if (str.contains(",")) {
|
||||
str = str.substring(0, str.indexOf(","));
|
||||
}
|
||||
int strend = str.length() > 6 ? 6 : str.length();
|
||||
str = str.substring(0, strend);
|
||||
double score = Double.valueOf(str);
|
||||
if (score > maxScore) {
|
||||
maxScore = score;
|
||||
}
|
||||
}
|
||||
return maxScore == -1D ? 0.00 : maxScore;
|
||||
}
|
||||
|
||||
public String getPrimaryString() {
|
||||
return PrimaryString;
|
||||
}
|
||||
|
||||
public void setPrimaryString(String PrimaryString) {
|
||||
this.PrimaryString = PrimaryString;
|
||||
}
|
||||
|
||||
public String getSecondaryString() {
|
||||
return SecondaryString;
|
||||
}
|
||||
|
||||
public void setSecondaryString(String SecondaryString) {
|
||||
this.SecondaryString = SecondaryString;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package FunctionLayer.misc;
|
||||
|
||||
import edu.stanford.nlp.ling.HasWord;
|
||||
import edu.stanford.nlp.ling.IndexedWord;
|
||||
import edu.stanford.nlp.ling.TaggedWord;
|
||||
import edu.stanford.nlp.ling.Word;
|
||||
import edu.stanford.nlp.parser.lexparser.LexicalizedParser;
|
||||
import edu.stanford.nlp.process.DocumentPreprocessor;
|
||||
import edu.stanford.nlp.process.Tokenizer;
|
||||
import edu.stanford.nlp.trees.GrammaticalStructure;
|
||||
import edu.stanford.nlp.trees.GrammaticalStructureFactory;
|
||||
import edu.stanford.nlp.trees.Tree;
|
||||
import edu.stanford.nlp.trees.TreebankLanguagePack;
|
||||
import edu.stanford.nlp.trees.TypedDependency;
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class SentimentAnalyzerTest {
|
||||
|
||||
public static SentimentAnalyzerTest instance = new SentimentAnalyzerTest();
|
||||
public static String grammar = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz";
|
||||
public static String[] options = {"-maxLength", "80", "-retainTmpSubcategories"};
|
||||
|
||||
public static LexicalizedParser initiateLexicalizedParser() {
|
||||
LexicalizedParser lp = LexicalizedParser.loadModel(grammar, options);
|
||||
return lp;
|
||||
}
|
||||
|
||||
public static TreebankLanguagePack initiateTreebankLanguagePack(LexicalizedParser lp) {
|
||||
TreebankLanguagePack tlp = lp.getOp().langpack();
|
||||
return tlp;
|
||||
}
|
||||
|
||||
public double sentimentanalyzing(String str, String str1, double sreturn, LexicalizedParser lp, TreebankLanguagePack tlp) {
|
||||
Iterable<List<? extends HasWord>> sentences;
|
||||
Tokenizer<? extends HasWord> toke
|
||||
= tlp.getTokenizerFactory().getTokenizer(new StringReader(str));
|
||||
List<? extends HasWord> sentence = toke.tokenize();
|
||||
String[] sent3 = {str1};
|
||||
String[] tag3 = {"PRP", "MD", "VB", "PRP", "."}; // Parser gets second "can" wrong without help
|
||||
List<TaggedWord> sentence2 = new ArrayList<>();
|
||||
for (int i = 0; i < sent3.length; i++) {
|
||||
sentence2.add(new TaggedWord(sent3[i], tag3[i]));
|
||||
}
|
||||
//parse.pennPrint();
|
||||
List<List<? extends HasWord>> tmp
|
||||
= new ArrayList<>();
|
||||
tmp.add(sentence2);
|
||||
tmp.add(sentence);
|
||||
sentences = tmp;
|
||||
for (List<? extends HasWord> sentence1 : sentences) {
|
||||
Tree parse1 = lp.parse(sentence1);
|
||||
GrammaticalStructureFactory gsf = tlp.grammaticalStructureFactory();
|
||||
GrammaticalStructure gs = gsf.newGrammaticalStructure(parse1);
|
||||
double score = parse1.score();
|
||||
if (score > sreturn) {
|
||||
//System.out.println("\n score : " + score + "\n");
|
||||
sreturn = score;
|
||||
}
|
||||
}
|
||||
return sreturn;
|
||||
}
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer.misc;
|
||||
|
||||
import DataLayer.DataMapper;
|
||||
import FunctionLayer.CustomError;
|
||||
import FunctionLayer.SimilarityMatrix;
|
||||
import static FunctionLayer.MYSQLDatahandler.EXPIRE_TIME_IN_SECONDS;
|
||||
import com.google.common.base.Stopwatch;
|
||||
import com.google.common.collect.ArrayListMultimap;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import com.google.common.collect.Multimap;
|
||||
import java.io.IOException;
|
||||
import java.sql.SQLException;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class SentimentSimilarityCacheObsolete {
|
||||
|
||||
public static final long EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(5, TimeUnit.HOURS);
|
||||
private static ConcurrentMap<String, List<SimilarityMatrix>> SimilarityMatrixCache;
|
||||
private static Stopwatch stopwatch;
|
||||
|
||||
public SentimentSimilarityCacheObsolete(ConcurrentMap<Integer, String> StringCache, Stopwatch stopwatch) {
|
||||
this.stopwatch = Stopwatch.createUnstarted();
|
||||
this.SimilarityMatrixCache = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
}
|
||||
|
||||
public void clearConCurrentMaps() {
|
||||
SimilarityMatrixCache.clear();
|
||||
}
|
||||
|
||||
private Multimap<String, SimilarityMatrix> getCache() throws SQLException, IOException, CustomError {
|
||||
List<SimilarityMatrix> matrixlist;
|
||||
matrixlist = DataMapper.getAllSementicMatrixes();
|
||||
Multimap<String, SimilarityMatrix> LHM = ArrayListMultimap.create();
|
||||
for (int i = 0; i < matrixlist.size(); i++) {
|
||||
LHM.put(matrixlist.get(i).getPrimaryString(), matrixlist.get(i));
|
||||
}
|
||||
return LHM;
|
||||
}
|
||||
|
||||
public static String getResponseStr(String strreturn) {
|
||||
String str = "";
|
||||
if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
|
||||
|
||||
if (!stopwatch.isRunning()) {
|
||||
stopwatch.start();
|
||||
} else {
|
||||
stopwatch.reset();
|
||||
}
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
https://github.com/DonatoMeoli/WS4J
|
||||
*/
|
||||
package FunctionLayer.misc;
|
||||
|
||||
import edu.cmu.lti.jawjaw.pobj.POS;
|
||||
import edu.cmu.lti.lexical_db.ILexicalDatabase;
|
||||
import edu.cmu.lti.lexical_db.NictWordNet;
|
||||
import edu.cmu.lti.lexical_db.data.Concept;
|
||||
import edu.cmu.lti.ws4j.Relatedness;
|
||||
import edu.cmu.lti.ws4j.RelatednessCalculator;
|
||||
import edu.cmu.lti.ws4j.impl.HirstStOnge;
|
||||
import edu.cmu.lti.ws4j.impl.JiangConrath;
|
||||
import edu.cmu.lti.ws4j.impl.LeacockChodorow;
|
||||
import edu.cmu.lti.ws4j.impl.Lesk;
|
||||
import edu.cmu.lti.ws4j.impl.Lin;
|
||||
import edu.cmu.lti.ws4j.impl.Path;
|
||||
import edu.cmu.lti.ws4j.impl.Resnik;
|
||||
import edu.cmu.lti.ws4j.impl.WuPalmer;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
* https://www.programcreek.com/java-api-examples/?api=edu.cmu.lti.ws4j.RelatednessCalculator
|
||||
* https://stackoverflow.com/questions/36300485/how-to-resolve-the-difference-between-the-values-attained-in-the-web-api-and-the
|
||||
*/
|
||||
public class WordNetSimalarityObsolete {
|
||||
|
||||
private static ILexicalDatabase db = new NictWordNet();
|
||||
private static RelatednessCalculator rc1 = new WuPalmer(db);
|
||||
private static RelatednessCalculator rc2 = new Resnik(db);
|
||||
private static RelatednessCalculator rc3 = new JiangConrath(db);
|
||||
private static RelatednessCalculator rc4 = new Lin(db);
|
||||
private static RelatednessCalculator rc5 = new LeacockChodorow(db);
|
||||
private static RelatednessCalculator rc6 = new Path(db);
|
||||
private static RelatednessCalculator rc7 = new Lesk(db);
|
||||
private static RelatednessCalculator rc8 = new HirstStOnge(db);
|
||||
|
||||
public static double SentenceMatcherSimilarityMatrix(String[] words1, String[] words2, double maxScore) {
|
||||
{
|
||||
double[][] s1 = getSimilarityMatrix(words1, words2, rc1);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s1[i][j] < maxScore && s1[i][j] > 0.0) {
|
||||
System.out.print(s1[i][j] + "\t");
|
||||
System.out.println("WuPalmer");
|
||||
maxScore = s1[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc2);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.println("Resnik");
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc3);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.println("JiangConrath");
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc4);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.println("Lin");
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc5);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
System.out.println("LeacockChodrow");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc6);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.println("Path");
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc7);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.println("Lesk");
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc8);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.println("HirstStOnge");
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return maxScore;
|
||||
}
|
||||
|
||||
public static double[][] getSimilarityMatrix(String[] words1, String[] words2, RelatednessCalculator rc) {
|
||||
double[][] result = new double[words1.length][words2.length];
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
double score = rc.calcRelatednessOfWords(words1[i], words2[j]);
|
||||
result[i][j] = score;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer.misc;
|
||||
|
||||
import edu.cmu.lti.lexical_db.ILexicalDatabase;
|
||||
import edu.cmu.lti.lexical_db.NictWordNet;
|
||||
import edu.cmu.lti.ws4j.RelatednessCalculator;
|
||||
import edu.cmu.lti.ws4j.impl.HirstStOnge;
|
||||
import edu.cmu.lti.ws4j.impl.JiangConrath;
|
||||
import edu.cmu.lti.ws4j.impl.LeacockChodorow;
|
||||
import edu.cmu.lti.ws4j.impl.Lesk;
|
||||
import edu.cmu.lti.ws4j.impl.Lin;
|
||||
import edu.cmu.lti.ws4j.impl.Path;
|
||||
import edu.cmu.lti.ws4j.impl.Resnik;
|
||||
import edu.cmu.lti.ws4j.impl.WuPalmer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
* http://ws4jdemo.appspot.com/?mode=s&s1=something+like+a+sentence&s2=should+not+be+like+the+first+one
|
||||
*/
|
||||
public class WordNetSimalarityTestObsolete {
|
||||
|
||||
private static ILexicalDatabase db = new NictWordNet();
|
||||
private static RelatednessCalculator rc1 = new WuPalmer(db);
|
||||
private static RelatednessCalculator rc2 = new Resnik(db);
|
||||
private static RelatednessCalculator rc3 = new JiangConrath(db);
|
||||
private static RelatednessCalculator rc4 = new Lin(db);
|
||||
private static RelatednessCalculator rc5 = new LeacockChodorow(db);
|
||||
private static RelatednessCalculator rc6 = new Path(db);
|
||||
private static RelatednessCalculator rc7 = new Lesk(db);
|
||||
private static RelatednessCalculator rc8 = new HirstStOnge(db);
|
||||
|
||||
public static double SentenceMatcherSimilarityMatrix(String[] words1, String[] words2, double maxScore) {
|
||||
boolean initial = maxScore == 0.0;
|
||||
List<RelatednessCalculator> RCList = new ArrayList();
|
||||
RCList.add(rc1);
|
||||
RCList.add(rc2);
|
||||
RCList.add(rc3);
|
||||
RCList.add(rc4);
|
||||
RCList.add(rc5);
|
||||
RCList.add(rc6);
|
||||
RCList.add(rc7);
|
||||
RCList.add(rc8);
|
||||
for (int h = 0; h < RCList.size(); h++) {
|
||||
double s1 = getSimilarityMatrix(words1, words2, RCList.get(h), maxScore, initial);
|
||||
System.out.println("s1: " + String.format("%.0f", s1) + " \nmaxScore: " + maxScore);
|
||||
if (s1 > 0.01 && (s1 < maxScore || initial)) {
|
||||
maxScore = s1;
|
||||
}
|
||||
}
|
||||
return maxScore;
|
||||
}
|
||||
|
||||
public static double getSimilarityMatrix(String[] words1, String[] words2, RelatednessCalculator rc, double maxScore, boolean initial) {
|
||||
double rtndouble = 0.01;
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (maxScore > rtndouble / words2.length || initial) {
|
||||
rtndouble += (rc.calcRelatednessOfWords(words1[i], words2[j]));
|
||||
//System.out.println("RelatednessCalculator: " + rc.toString());
|
||||
} else {
|
||||
return rtndouble;
|
||||
}
|
||||
}
|
||||
}
|
||||
return rtndouble;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer.misc;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class notes {
|
||||
/*
|
||||
|
||||
/*
|
||||
ILexicalDatabase db = new NictWordNet();
|
||||
RelatednessCalculator lesk = new Lesk(db);
|
||||
POS posWord1 = POS.n;
|
||||
POS posWord2 = POS.n;
|
||||
double maxScore = 0;
|
||||
WS4JConfiguration.getInstance().setMFS(true);
|
||||
List<Concept> synsets1 = (List<Concept>) db.getAllConcepts(strreturn, posWord1.name());
|
||||
for (int i = 0; i < allStringValuesPresent.size(); i++) {
|
||||
List<Concept> synsets2 = (List<Concept>) db.getAllConcepts(allStringValuesPresent.get(i), posWord2.name());
|
||||
for (Concept synset1 : synsets1) {
|
||||
for (Concept synset2 : synsets2) {
|
||||
Relatedness relatedness = lesk.calcRelatednessOfSynset(synset1, synset2);
|
||||
double score = relatedness.getScore();
|
||||
if (score > maxScore) {
|
||||
maxScore = score;
|
||||
index = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static RelatednessCalculator[] rcs;
|
||||
|
||||
static {
|
||||
WS4JConfiguration.getInstance().setMemoryDB(false);
|
||||
WS4JConfiguration.getInstance().setMFS(true);
|
||||
ILexicalDatabase db = new MITWordNet();
|
||||
rcs = new RelatednessCalculator[]{
|
||||
new HirstStOnge(db), new LeacockChodorow(db), new Lesk(db), new WuPalmer(db),
|
||||
new Resnik(db), new JiangConrath(db), new Lin(db), new Path(db)
|
||||
};
|
||||
}
|
||||
https://github.com/DonatoMeoli/WS4J
|
||||
https://www.programcreek.com/2014/01/calculate-words-similarity-using-wordnet-in-java/
|
||||
*/
|
||||
/*
|
||||
//available options of metrics
|
||||
private static RelatednessCalculator[] rcs = { new HirstStOnge(db),
|
||||
new LeacockChodorow(db), new Lesk(db), new WuPalmer(db),
|
||||
new Resnik(db), new JiangConrath(db), new Lin(db), new Path(db) };
|
||||
*/
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
|
||||
ps ax | grep EventNotfierDiscordBot-1.0
|
||||
kill $pid (number)
|
||||
|
||||
nohup screen -d -m -S nonRoot java -Xmx5048M -jar /home/Artificial_Autism/ArtificialAutism-1.0.jar
|
||||
nohup screen -d -m -S nonRoot java -Xmx4048M -jar /home/Artificial_Autism/ArtificialAutism-1.0.jar
|
||||
|
||||
nohup screen -d -m -S gameservers java -Xmx2450M -jar /home/gameservers/ArtificialAutism/ArtificialAutism-1.0.jar
|
||||
screen -ls (number1)
|
||||
screen -X -S (number1) quit
|
||||
*/
|
||||
//https://discordapp.com/developers/applications/
|
||||
//https://github.com/Javacord/Javacord
|
||||
package PresentationLayer;
|
||||
|
||||
import FunctionLayer.CustomError;
|
||||
import FunctionLayer.MYSQLDatahandler;
|
||||
import FunctionLayer.MessageResponseHandler;
|
||||
import FunctionLayer.StanfordParser.SentimentAnalyzerTest;
|
||||
import java.io.IOException;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import org.javacord.api.DiscordApi;
|
||||
import org.javacord.api.DiscordApiBuilder;
|
||||
import org.javacord.api.entity.user.User;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class DiscordHandler {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SentimentAnalyzerTest.shiftReduceParserInitiate();
|
||||
new Thread(() -> {
|
||||
try {
|
||||
MYSQLDatahandler.instance.initiateMYSQL();
|
||||
System.out.println("finished initiating MYSQL");
|
||||
} catch (SQLException | IOException ex) {
|
||||
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}).start();
|
||||
String token = "NTI5NzAxNTk5NjAyMjc4NDAx.Dw0vDg.7-aMjVWdQMYPl8qVNyvTCPS5F_A";
|
||||
DiscordApi api = new DiscordApiBuilder().setToken(token).login().join();
|
||||
api.addMessageCreateListener(event -> {
|
||||
if (!event.getMessage().getAuthor().isYourself()) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String strtest = event.getServerTextChannel().get().getCategory().toString();
|
||||
strtest = strtest.substring(9, strtest.length() - 1);
|
||||
boolean channelpermissionsDenied = false;
|
||||
switch (strtest) {
|
||||
case "Server Area": {
|
||||
if (!event.getServerTextChannel().get().toString().contains("chat-live")) {
|
||||
channelpermissionsDenied = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "Public Area": {
|
||||
break;
|
||||
}
|
||||
case "Information Area": {
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
channelpermissionsDenied = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!channelpermissionsDenied) {
|
||||
List<User> userlist = event.getMessage().getMentionedUsers();
|
||||
String strresult = event.getMessage().toString();
|
||||
if (userlist != null) {
|
||||
for (int i = 0; i < userlist.size(); i++) {
|
||||
strresult = strresult.replace(userlist.get(i).getIdAsString(), "");
|
||||
}
|
||||
}
|
||||
MessageResponseHandler.getMessage(strresult);
|
||||
try {
|
||||
MYSQLDatahandler.instance.checkIfUpdateStrings();
|
||||
MYSQLDatahandler.instance.checkIfUpdateMatrixes();
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
//contains to specify one channel where bot may always type
|
||||
if (event.getMessage().getMentionedUsers().contains(api.getYourself())
|
||||
|| event.getServerTextChannel().get().toString().contains("minor-test")) {
|
||||
String ResponseStr;
|
||||
try {
|
||||
ResponseStr = MessageResponseHandler.selectReponseMessage(event.getMessage().toString());
|
||||
if (!ResponseStr.isEmpty()) {
|
||||
System.out.print("\nResponseStr3: " + ResponseStr);
|
||||
event.getChannel().sendMessage(ResponseStr);
|
||||
}
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user