UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

replacing most arraylists with concurrentmaps due to them requirring less bytes per header since they should be stored bucketwise

Browse Source
This commit is contained in:
jenzur 2019-03-03 23:32:36 +01:00
parent f64ce5c5a0
commit aff2ddae5b
5 changed files with 200 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ArtificialAutism/src/main/java/DataLayer/DBCPDataSource.java
View File

@ -21,7 +21,6 @@ public class DBCPDataSource {
try { try {
ds.setDriver(new com.mysql.cj.jdbc.Driver()); ds.setDriver(new com.mysql.cj.jdbc.Driver());
ds.setUrl("jdbc:mysql://151.80.230.149:3306/ArtificialAutism?useLegacyDatetimeCode=false&serverTimezone=UTC"); ds.setUrl("jdbc:mysql://151.80.230.149:3306/ArtificialAutism?useLegacyDatetimeCode=false&serverTimezone=UTC");
//ds.setUrl("jdbc:mysql://localhost:3306/ArtificialAutism?useLegacyDatetimeCode=false&serverTimezone=UTC");
ds.setUsername("ArtificialAutism"); ds.setUsername("ArtificialAutism");
ds.setPassword("b423b54bwbfb1340438fn"); ds.setPassword("b423b54bwbfb1340438fn");
ds.setMaxTotal(-1); ds.setMaxTotal(-1);

60
ArtificialAutism/src/main/java/DataLayer/DataMapper.java
View File

@ -17,6 +17,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
@ -72,24 +73,22 @@ public class DataMapper {
return str; return str;
} }
public static void InsertMYSQLStrings(List<String> str) throws CustomError { public static void InsertMYSQLStrings(ConcurrentMap<Integer, String> str) throws CustomError {
Connection l_cCon = null; Connection l_cCon = null;
PreparedStatement l_pStatement = null; PreparedStatement l_pStatement = null;
ResultSet l_rsSearch = null; ResultSet l_rsSearch = null;
String l_sSQL = "INSERT IGNORE `Sentences` (`Strings`) VALUES (?)"; String l_sSQL = "INSERT IGNORE `Sentences` (`Strings`) VALUES (?)";
try { try {
if (str != null && str.size() > 0) { l_cCon = DBCPDataSource.getConnection();
l_cCon = DBCPDataSource.getConnection(); l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
java.sql.ResultSet.CONCUR_READ_ONLY); l_pStatement.setFetchSize(Integer.MIN_VALUE);
l_pStatement.setFetchSize(Integer.MIN_VALUE); for (String str1 : str.values()) {
for (String str1 : str) { System.out.println("adding str1: " + str1 + "\n");
System.out.println("adding str1: " + str1 + "\n"); l_pStatement.setString(1, str1);
l_pStatement.setString(1, str1); l_pStatement.addBatch();
l_pStatement.addBatch();
}
l_pStatement.executeBatch();
} }
l_pStatement.executeBatch();
} catch (SQLException ex) { } catch (SQLException ex) {
throw new CustomError("failed in DataMapper " + ex.getMessage()); throw new CustomError("failed in DataMapper " + ex.getMessage());
} finally { } finally {
@ -111,40 +110,7 @@ public class DataMapper {
return count; return count;
} }
public static List<SimilarityMatrix> getAllSementicMatrixes() throws CustomError { public static void insertSementicMatrixes(ConcurrentMap<Integer, SimilarityMatrix> WS4JListUpdate) throws CustomError {
//https://stackoverflow.com/questions/5157476/resultset-behavior-with-mysql-database-does-it-store-all-rows-in-memory/5159999#5159999
//https://stackoverflow.com/questions/3682614/how-to-read-all-rows-from-huge-table
int count = getSementicsDBRows();
int counter2 = 0;
int hardCapRetrieveCount = 500000;
List<SimilarityMatrix> WS4JList = new ArrayList(count + 1);
while (count > counter2) {
try (Connection l_cCon = DBCPDataSource.getConnection()) {
l_cCon.setAutoCommit(false);
String l_sSQL = "SELECT * FROM `WordMatrix` WHERE ID > " + counter2 + " AND ID < " + (counter2 + hardCapRetrieveCount);
try (PreparedStatement l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY)) {
l_pStatement.setFetchSize(Integer.MIN_VALUE);
try (ResultSet l_rsSearch = l_pStatement.executeQuery()) {
int i = 0;
while (l_rsSearch.next() && i < hardCapRetrieveCount) {
SimilarityMatrix ws4j = new SimilarityMatrix(l_rsSearch.getString(1), l_rsSearch.getString(2), l_rsSearch.getDouble(3));
//find something cheaper than arraylist probably
WS4JList.add(ws4j);
System.out.println("i: " + i + "\n" + "free memory: " + Runtime.getRuntime().freeMemory() + "\ncounter2: " + counter2 + "\n");
i++;
counter2++;
}
}
}
} catch (SQLException ex) {
Logger.getLogger(DataMapper.class.getName()).log(Level.SEVERE, null, ex);
}
}
return WS4JList;
}
public static void insertSementicMatrixes(List<SimilarityMatrix> WS4JListUpdate) throws CustomError {
Connection l_cCon = null; Connection l_cCon = null;
PreparedStatement l_pStatement = null; PreparedStatement l_pStatement = null;
ResultSet l_rsSearch = null; ResultSet l_rsSearch = null;
@ -155,7 +121,7 @@ public class DataMapper {
java.sql.ResultSet.CONCUR_READ_ONLY); java.sql.ResultSet.CONCUR_READ_ONLY);
l_pStatement.setFetchSize(Integer.MIN_VALUE); l_pStatement.setFetchSize(Integer.MIN_VALUE);
System.out.println("Matrix update size: " + WS4JListUpdate.size()); System.out.println("Matrix update size: " + WS4JListUpdate.size());
for (SimilarityMatrix ws4j : WS4JListUpdate) { for (SimilarityMatrix ws4j : WS4JListUpdate.values()) {
l_pStatement.setString(1, ws4j.getPrimaryString()); l_pStatement.setString(1, ws4j.getPrimaryString());
l_pStatement.setString(2, ws4j.getSecondaryString()); l_pStatement.setString(2, ws4j.getSecondaryString());
l_pStatement.setDouble(3, ws4j.getDistance()); l_pStatement.setDouble(3, ws4j.getDistance());

345
ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java
View File

@ -27,16 +27,13 @@ import java.io.IOException;
import java.io.StringReader; import java.io.StringReader;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Properties; import java.util.Properties;
import java.util.Random; import java.util.Random;
import java.util.Set;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
@ -44,7 +41,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
@ -167,141 +163,76 @@ public class MYSQLDatahandler {
public synchronized void checkIfUpdateMatrixes() { public synchronized void checkIfUpdateMatrixes() {
refreshMatrixFromDB = false; refreshMatrixFromDB = false;
int calculationBoundaries = 10; int calculationBoundaries = 6;
int updateBadgesInteger = 500; int updateBadgesInteger = 65;
if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) { while (lHMSMX.size() < (stringCache.values().size() * stringCache.values().size()) - stringCache.values().size()) {
refreshMatrixFromDB = true; if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) {
if (threadCounter == 0) { refreshMatrixFromDB = true;
lHMSMX = DataMapper.getAllRelationScores();
stopwatch1.reset();
}
}
if (stringCache.values().size() > 10 && !refreshMatrixFromDB) {
if (multiprocessCalculations.size() <= (calculationBoundaries * calculationBoundaries)) {
threadCounter++;
List<String> strList = new ArrayList(stringCache.values());
List<Integer> updateLocal = updatedRows;
int random = -1;
if (!updateLocal.contains(random)) {
updatedRows.add(random);
}
Collections.sort(updateLocal);
while (updateLocal.contains(random)) {
random = new Random().nextInt(strList.size() - 6);
int indexPrev = Collections.binarySearch(updateLocal, random);
int indexNext = Collections.binarySearch(updateLocal, random + 6);
//-1 will always be index 0
if (indexPrev > 0 && indexNext > 0) {
indexPrev = updateLocal.get(indexPrev);
indexNext = updateLocal.get(indexNext);
}
random = indexPrev < random - 5 && indexNext < random ? random : -1;
}
updatedRows.add(random);
semeticsUpdateCount = random;
int beginindex = semeticsUpdateCount;
semeticsUpdateCount += calculationBoundaries / 2;
int temp = semeticsUpdateCount;
System.out.println("beginindex: " + beginindex + "\ntemp: " + temp + "\n");
List<String> strIndexNavigator = new ArrayList();
strList.subList(beginindex, temp).forEach((str) -> {
strIndexNavigator.add(str);
multiprocessCalculations.add(str);
});
new Thread(() -> {
LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
List<String> strIndexNavigatorL = new ArrayList(strIndexNavigator);
List<String> strIndexAll = new ArrayList(strList);
List<String> randomIndexesToUpdate = new ArrayList();
int indexes = updateBadgesInteger;
if (indexes >= strIndexAll.size()) {
indexes = strIndexAll.size() - 1;
}
int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes);
strIndexAll.subList(beginindexes, beginindexes + indexes).forEach((str) -> {
randomIndexesToUpdate.add(str);
});
List<SimilarityMatrix> matrixUpdateList = new ArrayList();
List<Future<SimilarityMatrix>> futures = new ArrayList();
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
strIndexNavigatorL.forEach((str) -> {
randomIndexesToUpdate.stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> {
boolean present = false;
if (multiprocessCalculations.contains(str1)) {
present = true;
} else if (LHMSMXLocal.containsKey(str)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str);
if (orDefault.containsKey(str1)) {
present = true;
}
} else if (LHMSMXLocal.containsKey(str1)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str1);
if (orDefault.containsKey(str)) {
present = true;
}
}
if (!present) {
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX);
futures.add(executor.submit(worker));
}
});
});
executor.shutdown();
try {
System.out.println("finished worker assignment, futures size: " + futures.size() + "\n");
for (Future<SimilarityMatrix> future : futures) {
SimilarityMatrix SMX = future.get();
System.out.println("SMX primary: " + SMX.getPrimaryString() + "\nSMX Secondary: " + SMX.getSecondaryString()
+ "\nScore: " + SMX.getDistance() + "\n");
LinkedHashMap<String, Double> get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
if (get == null) {
get = new LinkedHashMap();
}
get.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), get);
matrixUpdateList.add(SMX);
}
} catch (InterruptedException | ExecutionException ex) {
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
}
new Thread(() -> {
try {
if (!matrixUpdateList.isEmpty()) {
DataMapper.insertSementicMatrixes(matrixUpdateList);
System.out.println("finished datamapper semetic insert");
}
threadCounter--;
System.out.println("\nthreadCounter: " + threadCounter + "\n");
} catch (CustomError ex) {
Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex);
}
}).start();
}).
start();
} else {
if (threadCounter == 0) { if (threadCounter == 0) {
lHMSMX = DataMapper.getAllRelationScores();
stopwatch1.reset();
}
}
if (stringCache.values().size() > 10 && !refreshMatrixFromDB) {
if (multiprocessCalculations.size() <= (calculationBoundaries * calculationBoundaries)) {
threadCounter++; threadCounter++;
ConcurrentMap<Integer, String> stringCachelocal = stringCache;
List<Integer> updateLocal = updatedRows;
int random = -1;
if (!updateLocal.contains(random)) {
updatedRows.add(random);
}
Collections.sort(updateLocal);
while (updateLocal.contains(random)) {
random = new Random().nextInt(stringCachelocal.values().size() - 6);
int indexPrev = Collections.binarySearch(updateLocal, random);
int indexNext = Collections.binarySearch(updateLocal, random + 6);
//-1 will always be index 0
if (indexPrev > 0 && indexNext > 0) {
indexPrev = updateLocal.get(indexPrev);
indexNext = updateLocal.get(indexNext);
}
random = indexPrev < random - 5 && indexNext < random ? random : -1;
}
updatedRows.add(random);
semeticsUpdateCount = random;
int beginindex = semeticsUpdateCount;
semeticsUpdateCount += calculationBoundaries / 2;
int temp = semeticsUpdateCount;
System.out.println("beginindex: " + beginindex + "\ntemp: " + temp + "\n");
ConcurrentMap<Integer, String> strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap();
int ij = 0;
while (beginindex + ij < temp) {
String get = stringCachelocal.get(beginindex + ij);
strIndexNavigator.put(ij, get);
multiprocessCalculations.add(get);
ij++;
}
new Thread(() -> { new Thread(() -> {
LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX; LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
List<String> strList = new ArrayList(stringCache.values()); ConcurrentMap<Integer, String> strIndexAll = stringCachelocal;
List<SimilarityMatrix> matrixUpdateList = new ArrayList(); ConcurrentMap<Integer, String> strIndexNavigatorL = strIndexNavigator;
List<String> randomStrList = new ArrayList(); ConcurrentMap<Integer, String> randomIndexesToUpdate = new MapMaker().concurrencyLevel(2).makeMap();
int indexes = updateBadgesInteger; int indexes = updateBadgesInteger;
if (indexes >= strList.size()) { if (indexes >= strIndexAll.size()) {
indexes = strList.size() - 1; indexes = strIndexAll.size() - 1;
} }
int beginindexes = new Random().nextInt((strList.size()) - indexes); int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes);
strList.subList(beginindexes, beginindexes + indexes).forEach((str) -> { int ij1 = 0;
randomStrList.add(str); while (beginindexes + ij1 < beginindexes + indexes) {
}); String get1 = strIndexAll.get(beginindexes + ij1);
List<Future<SimilarityMatrix>> futures = new ArrayList(); randomIndexesToUpdate.put(ij1, get1);
ij1++;
}
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
multiprocessCalculations.forEach((str) -> { strIndexNavigatorL.values().forEach((str) -> {
randomStrList.forEach((str1) -> { randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> {
boolean present = false; boolean present = false;
if (LHMSMXLocal.containsKey(str)) { if (multiprocessCalculations.contains(str1)) {
present = true;
} else if (LHMSMXLocal.containsKey(str)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str); LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str);
if (orDefault.containsKey(str1)) { if (orDefault.containsKey(str1)) {
present = true; present = true;
@ -315,13 +246,14 @@ public class MYSQLDatahandler {
if (!present) { if (!present) {
SimilarityMatrix SMX = new SimilarityMatrix(str, str1); SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX); Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX);
futures.add(executor.submit(worker)); futures.put(futures.size() + 1, executor.submit(worker));
} }
}); });
}); });
executor.shutdown(); executor.shutdown();
try { try {
for (Future<SimilarityMatrix> future : futures) { System.out.println("finished worker assignment, futures size: " + futures.size() + "\n");
for (Future<SimilarityMatrix> future : futures.values()) {
SimilarityMatrix SMX = future.get(); SimilarityMatrix SMX = future.get();
LinkedHashMap<String, Double> get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); LinkedHashMap<String, Double> get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
if (get == null) { if (get == null) {
@ -329,24 +261,97 @@ public class MYSQLDatahandler {
} }
get.put(SMX.getSecondaryString(), SMX.getDistance()); get.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), get); lHMSMX.put(SMX.getPrimaryString(), get);
matrixUpdateList.add(SMX); matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
} }
} catch (InterruptedException | ExecutionException ex) { } catch (InterruptedException | ExecutionException ex) {
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
} }
try { new Thread(() -> {
if (!matrixUpdateList.isEmpty()) { try {
DataMapper.insertSementicMatrixes(matrixUpdateList); if (!matrixUpdateList.isEmpty()) {
System.out.println("finished datamapper semetic insert"); DataMapper.insertSementicMatrixes(matrixUpdateList);
System.out.println("finished datamapper semetic insert");
}
threadCounter--;
System.out.println("\nthreadCounter: " + threadCounter + "\n");
} catch (CustomError ex) {
Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex);
} }
} catch (CustomError ex) { }).start();
Logger.getLogger(MYSQLDatahandler.class }).
.getName()).log(Level.SEVERE, null, ex); start();
} } else {
multiprocessCalculations = new ArrayList(); if (threadCounter == 0) {
updatedRows = new ArrayList(); threadCounter++;
threadCounter--; new Thread(() -> {
}).start(); LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
ConcurrentMap<Integer, String> strList = stringCache;
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> randomStrList = new MapMaker().concurrencyLevel(2).makeMap();
int indexes = updateBadgesInteger;
if (indexes >= strList.size()) {
indexes = strList.size() - 1;
}
int beginindexes = new Random().nextInt((strList.size()) - indexes);
int ij1 = 0;
while (beginindexes + ij1 < beginindexes + indexes) {
String get1 = strList.get(beginindexes + ij1);
randomStrList.put(ij1, get1);
ij1++;
}
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
multiprocessCalculations.forEach((str) -> {
randomStrList.values().forEach((str1) -> {
boolean present = false;
if (LHMSMXLocal.containsKey(str)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str);
if (orDefault.containsKey(str1)) {
present = true;
}
} else if (LHMSMXLocal.containsKey(str1)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str1);
if (orDefault.containsKey(str)) {
present = true;
}
}
if (!present) {
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX);
futures.put(futures.size() + 1, executor.submit(worker));
}
});
});
executor.shutdown();
try {
for (Future<SimilarityMatrix> future : futures.values()) {
SimilarityMatrix SMX = future.get();
LinkedHashMap<String, Double> get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
if (get == null) {
get = new LinkedHashMap();
}
get.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), get);
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
}
} catch (InterruptedException | ExecutionException ex) {
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
}
try {
if (!matrixUpdateList.isEmpty()) {
DataMapper.insertSementicMatrixes(matrixUpdateList);
System.out.println("finished datamapper semetic insert");
}
} catch (CustomError ex) {
Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex);
}
multiprocessCalculations = new ArrayList();
updatedRows = new ArrayList();
threadCounter--;
}).start();
}
} }
} }
} }
@ -355,22 +360,20 @@ public class MYSQLDatahandler {
public synchronized void checkIfUpdateStrings() throws CustomError { public synchronized void checkIfUpdateStrings() throws CustomError {
if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) { if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
new Thread(() -> { new Thread(() -> {
List<String> str = MessageResponseHandler.getStr(); ConcurrentMap<Integer, String> str = MessageResponseHandler.getStr();
str = cutContent(str); str = cutContent(str);
str = filterContent(str); str = filterContent(str);
str = removeSlacks(str); str = removeSlacks(str);
List<String> strUpdate = new ArrayList();
strUpdate.addAll(str);
try { try {
DataMapper.InsertMYSQLStrings(strUpdate); DataMapper.InsertMYSQLStrings(str);
} catch (CustomError ex) { } catch (CustomError ex) {
Logger.getLogger(MYSQLDatahandler.class Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex); .getName()).log(Level.SEVERE, null, ex);
} }
MessageResponseHandler.setStr(new ArrayList()); MessageResponseHandler.setStr(new MapMaker().concurrencyLevel(2).makeMap());
int j = stringCache.size() + 1; int j = stringCache.size() + 1;
for (String str1 : strUpdate) { for (String str1 : str.values()) {
stringCache.put(j, str1); stringCache.put(j, str1);
j++; j++;
} }
@ -394,7 +397,7 @@ public class MYSQLDatahandler {
SimilarityMatrix SMXreturn = new SimilarityMatrix("", ""); SimilarityMatrix SMXreturn = new SimilarityMatrix("", "");
System.out.println("pre mostSimilarSTR \n"); System.out.println("pre mostSimilarSTR \n");
String mostSimilarSTR = mostSimilar(str, strArrs); String mostSimilarSTR = mostSimilar(str, strArrs);
if (!mostSimilarSTR.isEmpty()) { if (mostSimilarSTR != null && !mostSimilarSTR.isEmpty()) {
System.out.println("mostSimilarSTR; " + mostSimilarSTR + "\n"); System.out.println("mostSimilarSTR; " + mostSimilarSTR + "\n");
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.getOrDefault(mostSimilarSTR, null); LinkedHashMap<String, Double> orDefault = LHMSMXLocal.getOrDefault(mostSimilarSTR, null);
if (orDefault != null) { if (orDefault != null) {
@ -471,11 +474,13 @@ public class MYSQLDatahandler {
for (Future<DistanceObject> future : futures) { for (Future<DistanceObject> future : futures) {
DistanceObject d = future.get(); DistanceObject d = future.get();
try { try {
int distance = d.getDistance(); if (d.getSentence() != null && d.getDistance() != null) {
System.out.println("distance: " + distance + "\n"); int distance = d.getDistance();
if (distance < minDistance) { System.out.println("distance: " + distance + "\n");
minDistance = distance; if (distance < minDistance) {
similar = d.getSentence(); minDistance = distance;
similar = d.getSentence();
}
} }
} catch (NullPointerException ex) { } catch (NullPointerException ex) {
System.out.println("failed future\n"); System.out.println("failed future\n");
@ -487,21 +492,21 @@ public class MYSQLDatahandler {
return similar; return similar;
} }
public static List<String> cutContent(List<String> str) { public static ConcurrentMap<Integer, String> cutContent(ConcurrentMap<Integer, String> str) {
List<String> returnlist = new ArrayList(); ConcurrentMap<Integer, String> returnlist = new MapMaker().concurrencyLevel(2).makeMap();
for (String str1 : str) { for (String str1 : str.values()) {
int iend = str1.indexOf("content: "); int iend = str1.indexOf("content: ");
if (iend != -1) { if (iend != -1) {
String trs = str1.substring(iend + 9); String trs = str1.substring(iend + 9);
returnlist.add(trs.substring(0, trs.length() - 1)); returnlist.put(returnlist.size() + 1, trs.substring(0, trs.length() - 1));
} }
} }
return returnlist; return returnlist;
} }
public static List<String> filterContent(List<String> str) { public static ConcurrentMap<Integer, String> filterContent(ConcurrentMap<Integer, String> str) {
List<String> strlistreturn = new ArrayList(); ConcurrentMap<Integer, String> strlistreturn = new MapMaker().concurrencyLevel(2).makeMap();
for (String str1 : str) { for (String str1 : str.values()) {
if (str1.isEmpty() || str1.length() < 3) { if (str1.isEmpty() || str1.length() < 3) {
continue; continue;
} }
@ -580,18 +585,18 @@ public class MYSQLDatahandler {
} }
str1 = str1.trim(); str1 = str1.trim();
if (str1.length() > 2 && (!str1.startsWith("!"))) { if (str1.length() > 2 && (!str1.startsWith("!"))) {
strlistreturn.add(str1); strlistreturn.put(strlistreturn.size() + 1, str1);
} }
} }
return strlistreturn; return strlistreturn;
} }
private List<String> removeSlacks(List<String> str) { private ConcurrentMap<Integer, String> removeSlacks(ConcurrentMap<Integer, String> str) {
ShiftReduceParser model = getModel(); ShiftReduceParser model = getModel();
MaxentTagger tagger = getTagger(); MaxentTagger tagger = getTagger();
List<TaggedWord> taggedWords; List<TaggedWord> taggedWords;
List<String> strreturn = new ArrayList(); ConcurrentMap<Integer, String> strreturn = new MapMaker().concurrencyLevel(2).makeMap();
for (String str1 : str) { for (String str1 : str.values()) {
int counter = 0; int counter = 0;
List<String> TGWList = new ArrayList(); List<String> TGWList = new ArrayList();
DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1)); DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1));
@ -638,7 +643,7 @@ public class MYSQLDatahandler {
} }
} }
if (!tooclosematch) { if (!tooclosematch) {
strreturn.add(str1); strreturn.put(strreturn.size() + 1, str1);
} }
} }
} }

14
ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java
View File

@ -5,8 +5,10 @@
*/ */
package FunctionLayer; package FunctionLayer;
import com.google.common.collect.MapMaker;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.ConcurrentMap;
/** /**
* *
@ -14,13 +16,13 @@ import java.util.List;
*/ */
public class MessageResponseHandler { public class MessageResponseHandler {
private static List<String> str = new ArrayList(); private static ConcurrentMap<Integer, String> str = new MapMaker().concurrencyLevel(2).makeMap();
public static List<String> getStr() { public static ConcurrentMap<Integer, String> getStr() {
return str; return str;
} }
public static void setStr(List<String> str) { public static void setStr(ConcurrentMap<Integer, String> str) {
MessageResponseHandler.str = str; MessageResponseHandler.str = str;
} }
@ -33,13 +35,13 @@ public class MessageResponseHandler {
if (message.startsWith("[ *")) { if (message.startsWith("[ *")) {
message = message.substring(message.indexOf("]")); message = message.substring(message.indexOf("]"));
} }
str.add(message); str.put(str.size() + 1, message);
} }
} }
public static String selectReponseMessage(String toString) throws CustomError { public static String selectReponseMessage(String toString) throws CustomError {
List<String> str1 = new ArrayList(); ConcurrentMap<Integer, String> str1 = new MapMaker().concurrencyLevel(2).makeMap();
str1.add(toString); str1.put(str1.size() +1 , toString);
str1 = MYSQLDatahandler.cutContent(str1); str1 = MYSQLDatahandler.cutContent(str1);
String strreturn = str1.get(0); String strreturn = str1.get(0);
String getResponseMsg = MYSQLDatahandler.instance.getResponseMsg(strreturn); String getResponseMsg = MYSQLDatahandler.instance.getResponseMsg(strreturn);

7
ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java
View File

@ -7,7 +7,7 @@
kill $pid (number) kill $pid (number)
nohup screen -d -m -S nonroot java -Xmx6048M -jar /home/javatests/ArtificialAutism-1.0.jar nohup screen -d -m -S nonroot java -Xmx6048M -jar /home/javatests/ArtificialAutism-1.0.jar
nohup screen -d -m -S nonroot java -Xmx4048M -jar /home/javatests/ArtificialAutism-1.0.jar nohup screen -d -m -S nonroot java -Xmx6800M -jar /home/javatests/ArtificialAutism-1.0.jar
screen -ls (number1) screen -ls (number1)
screen -X -S (number1) quit screen -X -S (number1) quit
@ -35,7 +35,6 @@ import org.javacord.api.entity.user.User;
public class DiscordHandler { public class DiscordHandler {
public static void main(String[] args) { public static void main(String[] args) {
MYSQLDatahandler.shiftReduceParserInitiate();
new Thread(() -> { new Thread(() -> {
try { try {
MYSQLDatahandler.instance.initiateMYSQL(); MYSQLDatahandler.instance.initiateMYSQL();
@ -44,6 +43,8 @@ public class DiscordHandler {
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex); Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
} }
}).start(); }).start();
MYSQLDatahandler.shiftReduceParserInitiate();
MYSQLDatahandler.instance.checkIfUpdateMatrixes();
String token = "NTI5NzAxNTk5NjAyMjc4NDAx.Dw0vDg.7-aMjVWdQMYPl8qVNyvTCPS5F_A"; String token = "NTI5NzAxNTk5NjAyMjc4NDAx.Dw0vDg.7-aMjVWdQMYPl8qVNyvTCPS5F_A";
DiscordApi api = new DiscordApiBuilder().setToken(token).login().join(); DiscordApi api = new DiscordApiBuilder().setToken(token).login().join();
api.addMessageCreateListener(event -> { api.addMessageCreateListener(event -> {
@ -81,7 +82,7 @@ public class DiscordHandler {
MessageResponseHandler.getMessage(strresult); MessageResponseHandler.getMessage(strresult);
try { try {
MYSQLDatahandler.instance.checkIfUpdateStrings(); MYSQLDatahandler.instance.checkIfUpdateStrings();
MYSQLDatahandler.instance.checkIfUpdateMatrixes(); //MYSQLDatahandler.instance.checkIfUpdateMatrixes();
} catch (CustomError ex) { } catch (CustomError ex) {
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex); Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
} }