UNLOZE/projects-jenz
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

replacing most arraylists with concurrentmaps due to them requirring less bytes per header since they should be stored bucketwise

Browse Source
This commit is contained in:
jenzur
2019-03-03 23:32:36 +01:00
parent f64ce5c5a0
commit aff2ddae5b
5 changed files with 200 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java
+175 -170
View File
@@ -27,16 +27,13 @@ import java.io.IOException;
import java.io.StringReader;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
@@ -44,7 +41,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -167,141 +163,76 @@ public class MYSQLDatahandler {
public synchronized void checkIfUpdateMatrixes() {
refreshMatrixFromDB = false;
int calculationBoundaries = 10;
int updateBadgesInteger = 500;
if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) {
refreshMatrixFromDB = true;
if (threadCounter == 0) {
lHMSMX = DataMapper.getAllRelationScores();
stopwatch1.reset();
}
}
if (stringCache.values().size() > 10 && !refreshMatrixFromDB) {
if (multiprocessCalculations.size() <= (calculationBoundaries * calculationBoundaries)) {
threadCounter++;
List<String> strList = new ArrayList(stringCache.values());
List<Integer> updateLocal = updatedRows;
int random = -1;
if (!updateLocal.contains(random)) {
updatedRows.add(random);
}
Collections.sort(updateLocal);
while (updateLocal.contains(random)) {
random = new Random().nextInt(strList.size() - 6);
int indexPrev = Collections.binarySearch(updateLocal, random);
int indexNext = Collections.binarySearch(updateLocal, random + 6);
//-1 will always be index 0
if (indexPrev > 0 && indexNext > 0) {
indexPrev = updateLocal.get(indexPrev);
indexNext = updateLocal.get(indexNext);
}
random = indexPrev < random - 5 && indexNext < random ? random : -1;
}
updatedRows.add(random);
semeticsUpdateCount = random;
int beginindex = semeticsUpdateCount;
semeticsUpdateCount += calculationBoundaries / 2;
int temp = semeticsUpdateCount;
System.out.println("beginindex: " + beginindex + "\ntemp: " + temp + "\n");
List<String> strIndexNavigator = new ArrayList();
strList.subList(beginindex, temp).forEach((str) -> {
strIndexNavigator.add(str);
multiprocessCalculations.add(str);
});
new Thread(() -> {
LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
List<String> strIndexNavigatorL = new ArrayList(strIndexNavigator);
List<String> strIndexAll = new ArrayList(strList);
List<String> randomIndexesToUpdate = new ArrayList();
int indexes = updateBadgesInteger;
if (indexes >= strIndexAll.size()) {
indexes = strIndexAll.size() - 1;
}
int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes);
strIndexAll.subList(beginindexes, beginindexes + indexes).forEach((str) -> {
randomIndexesToUpdate.add(str);
});
List<SimilarityMatrix> matrixUpdateList = new ArrayList();
List<Future<SimilarityMatrix>> futures = new ArrayList();
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
strIndexNavigatorL.forEach((str) -> {
randomIndexesToUpdate.stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> {
boolean present = false;
if (multiprocessCalculations.contains(str1)) {
present = true;
} else if (LHMSMXLocal.containsKey(str)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str);
if (orDefault.containsKey(str1)) {
present = true;
}
} else if (LHMSMXLocal.containsKey(str1)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str1);
if (orDefault.containsKey(str)) {
present = true;
}
}
if (!present) {
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX);
futures.add(executor.submit(worker));
}
});
});
executor.shutdown();
try {
System.out.println("finished worker assignment, futures size: " + futures.size() + "\n");
for (Future<SimilarityMatrix> future : futures) {
SimilarityMatrix SMX = future.get();
System.out.println("SMX primary: " + SMX.getPrimaryString() + "\nSMX Secondary: " + SMX.getSecondaryString()
+ "\nScore: " + SMX.getDistance() + "\n");
LinkedHashMap<String, Double> get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
if (get == null) {
get = new LinkedHashMap();
}
get.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), get);
matrixUpdateList.add(SMX);
}
} catch (InterruptedException | ExecutionException ex) {
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
}
new Thread(() -> {
try {
if (!matrixUpdateList.isEmpty()) {
DataMapper.insertSementicMatrixes(matrixUpdateList);
System.out.println("finished datamapper semetic insert");
}
threadCounter--;
System.out.println("\nthreadCounter: " + threadCounter + "\n");
} catch (CustomError ex) {
Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex);
}
}).start();
}).
start();
} else {
int calculationBoundaries = 6;
int updateBadgesInteger = 65;
while (lHMSMX.size() < (stringCache.values().size() * stringCache.values().size()) - stringCache.values().size()) {
if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) {
refreshMatrixFromDB = true;
if (threadCounter == 0) {
lHMSMX = DataMapper.getAllRelationScores();
stopwatch1.reset();
}
}
if (stringCache.values().size() > 10 && !refreshMatrixFromDB) {
if (multiprocessCalculations.size() <= (calculationBoundaries * calculationBoundaries)) {
threadCounter++;
ConcurrentMap<Integer, String> stringCachelocal = stringCache;
List<Integer> updateLocal = updatedRows;
int random = -1;
if (!updateLocal.contains(random)) {
updatedRows.add(random);
}
Collections.sort(updateLocal);
while (updateLocal.contains(random)) {
random = new Random().nextInt(stringCachelocal.values().size() - 6);
int indexPrev = Collections.binarySearch(updateLocal, random);
int indexNext = Collections.binarySearch(updateLocal, random + 6);
//-1 will always be index 0
if (indexPrev > 0 && indexNext > 0) {
indexPrev = updateLocal.get(indexPrev);
indexNext = updateLocal.get(indexNext);
}
random = indexPrev < random - 5 && indexNext < random ? random : -1;
}
updatedRows.add(random);
semeticsUpdateCount = random;
int beginindex = semeticsUpdateCount;
semeticsUpdateCount += calculationBoundaries / 2;
int temp = semeticsUpdateCount;
System.out.println("beginindex: " + beginindex + "\ntemp: " + temp + "\n");
ConcurrentMap<Integer, String> strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap();
int ij = 0;
while (beginindex + ij < temp) {
String get = stringCachelocal.get(beginindex + ij);
strIndexNavigator.put(ij, get);
multiprocessCalculations.add(get);
ij++;
}
new Thread(() -> {
LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
List<String> strList = new ArrayList(stringCache.values());
List<SimilarityMatrix> matrixUpdateList = new ArrayList();
List<String> randomStrList = new ArrayList();
ConcurrentMap<Integer, String> strIndexAll = stringCachelocal;
ConcurrentMap<Integer, String> strIndexNavigatorL = strIndexNavigator;
ConcurrentMap<Integer, String> randomIndexesToUpdate = new MapMaker().concurrencyLevel(2).makeMap();
int indexes = updateBadgesInteger;
if (indexes >= strList.size()) {
indexes = strList.size() - 1;
if (indexes >= strIndexAll.size()) {
indexes = strIndexAll.size() - 1;
}
int beginindexes = new Random().nextInt((strList.size()) - indexes);
strList.subList(beginindexes, beginindexes + indexes).forEach((str) -> {
randomStrList.add(str);
});
List<Future<SimilarityMatrix>> futures = new ArrayList();
int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes);
int ij1 = 0;
while (beginindexes + ij1 < beginindexes + indexes) {
String get1 = strIndexAll.get(beginindexes + ij1);
randomIndexesToUpdate.put(ij1, get1);
ij1++;
}
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
multiprocessCalculations.forEach((str) -> {
randomStrList.forEach((str1) -> {
strIndexNavigatorL.values().forEach((str) -> {
randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> {
boolean present = false;
if (LHMSMXLocal.containsKey(str)) {
if (multiprocessCalculations.contains(str1)) {
present = true;
} else if (LHMSMXLocal.containsKey(str)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str);
if (orDefault.containsKey(str1)) {
present = true;
@@ -315,13 +246,14 @@ public class MYSQLDatahandler {
if (!present) {
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX);
futures.add(executor.submit(worker));
futures.put(futures.size() + 1, executor.submit(worker));
}
});
});
executor.shutdown();
try {
for (Future<SimilarityMatrix> future : futures) {
System.out.println("finished worker assignment, futures size: " + futures.size() + "\n");
for (Future<SimilarityMatrix> future : futures.values()) {
SimilarityMatrix SMX = future.get();
LinkedHashMap<String, Double> get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
if (get == null) {
@@ -329,24 +261,97 @@ public class MYSQLDatahandler {
}
get.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), get);
matrixUpdateList.add(SMX);
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
}
} catch (InterruptedException | ExecutionException ex) {
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
}
try {
if (!matrixUpdateList.isEmpty()) {
DataMapper.insertSementicMatrixes(matrixUpdateList);
System.out.println("finished datamapper semetic insert");
new Thread(() -> {
try {
if (!matrixUpdateList.isEmpty()) {
DataMapper.insertSementicMatrixes(matrixUpdateList);
System.out.println("finished datamapper semetic insert");
}
threadCounter--;
System.out.println("\nthreadCounter: " + threadCounter + "\n");
} catch (CustomError ex) {
Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex);
}
} catch (CustomError ex) {
Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex);
}
multiprocessCalculations = new ArrayList();
updatedRows = new ArrayList();
threadCounter--;
}).start();
}).start();
}).
start();
} else {
if (threadCounter == 0) {
threadCounter++;
new Thread(() -> {
LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
ConcurrentMap<Integer, String> strList = stringCache;
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> randomStrList = new MapMaker().concurrencyLevel(2).makeMap();
int indexes = updateBadgesInteger;
if (indexes >= strList.size()) {
indexes = strList.size() - 1;
}
int beginindexes = new Random().nextInt((strList.size()) - indexes);
int ij1 = 0;
while (beginindexes + ij1 < beginindexes + indexes) {
String get1 = strList.get(beginindexes + ij1);
randomStrList.put(ij1, get1);
ij1++;
}
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
multiprocessCalculations.forEach((str) -> {
randomStrList.values().forEach((str1) -> {
boolean present = false;
if (LHMSMXLocal.containsKey(str)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str);
if (orDefault.containsKey(str1)) {
present = true;
}
} else if (LHMSMXLocal.containsKey(str1)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str1);
if (orDefault.containsKey(str)) {
present = true;
}
}
if (!present) {
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX);
futures.put(futures.size() + 1, executor.submit(worker));
}
});
});
executor.shutdown();
try {
for (Future<SimilarityMatrix> future : futures.values()) {
SimilarityMatrix SMX = future.get();
LinkedHashMap<String, Double> get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
if (get == null) {
get = new LinkedHashMap();
}
get.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), get);
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
}
} catch (InterruptedException | ExecutionException ex) {
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
}
try {
if (!matrixUpdateList.isEmpty()) {
DataMapper.insertSementicMatrixes(matrixUpdateList);
System.out.println("finished datamapper semetic insert");
}
} catch (CustomError ex) {
Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex);
}
multiprocessCalculations = new ArrayList();
updatedRows = new ArrayList();
threadCounter--;
}).start();
}
}
}
}
@@ -355,22 +360,20 @@ public class MYSQLDatahandler {
public synchronized void checkIfUpdateStrings() throws CustomError {
if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
new Thread(() -> {
List<String> str = MessageResponseHandler.getStr();
ConcurrentMap<Integer, String> str = MessageResponseHandler.getStr();
str = cutContent(str);
str = filterContent(str);
str = removeSlacks(str);
List<String> strUpdate = new ArrayList();
strUpdate.addAll(str);
try {
DataMapper.InsertMYSQLStrings(strUpdate);
DataMapper.InsertMYSQLStrings(str);
} catch (CustomError ex) {
Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex);
}
MessageResponseHandler.setStr(new ArrayList());
MessageResponseHandler.setStr(new MapMaker().concurrencyLevel(2).makeMap());
int j = stringCache.size() + 1;
for (String str1 : strUpdate) {
for (String str1 : str.values()) {
stringCache.put(j, str1);
j++;
}
@@ -394,7 +397,7 @@ public class MYSQLDatahandler {
SimilarityMatrix SMXreturn = new SimilarityMatrix("", "");
System.out.println("pre mostSimilarSTR \n");
String mostSimilarSTR = mostSimilar(str, strArrs);
if (!mostSimilarSTR.isEmpty()) {
if (mostSimilarSTR != null && !mostSimilarSTR.isEmpty()) {
System.out.println("mostSimilarSTR; " + mostSimilarSTR + "\n");
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.getOrDefault(mostSimilarSTR, null);
if (orDefault != null) {
@@ -471,11 +474,13 @@ public class MYSQLDatahandler {
for (Future<DistanceObject> future : futures) {
DistanceObject d = future.get();
try {
int distance = d.getDistance();
System.out.println("distance: " + distance + "\n");
if (distance < minDistance) {
minDistance = distance;
similar = d.getSentence();
if (d.getSentence() != null && d.getDistance() != null) {
int distance = d.getDistance();
System.out.println("distance: " + distance + "\n");
if (distance < minDistance) {
minDistance = distance;
similar = d.getSentence();
}
}
} catch (NullPointerException ex) {
System.out.println("failed future\n");
@@ -487,21 +492,21 @@ public class MYSQLDatahandler {
return similar;
}
public static List<String> cutContent(List<String> str) {
List<String> returnlist = new ArrayList();
for (String str1 : str) {
public static ConcurrentMap<Integer, String> cutContent(ConcurrentMap<Integer, String> str) {
ConcurrentMap<Integer, String> returnlist = new MapMaker().concurrencyLevel(2).makeMap();
for (String str1 : str.values()) {
int iend = str1.indexOf("content: ");
if (iend != -1) {
String trs = str1.substring(iend + 9);
returnlist.add(trs.substring(0, trs.length() - 1));
returnlist.put(returnlist.size() + 1, trs.substring(0, trs.length() - 1));
}
}
return returnlist;
}
public static List<String> filterContent(List<String> str) {
List<String> strlistreturn = new ArrayList();
for (String str1 : str) {
public static ConcurrentMap<Integer, String> filterContent(ConcurrentMap<Integer, String> str) {
ConcurrentMap<Integer, String> strlistreturn = new MapMaker().concurrencyLevel(2).makeMap();
for (String str1 : str.values()) {
if (str1.isEmpty() || str1.length() < 3) {
continue;
}
@@ -580,18 +585,18 @@ public class MYSQLDatahandler {
}
str1 = str1.trim();
if (str1.length() > 2 && (!str1.startsWith("!"))) {
strlistreturn.add(str1);
strlistreturn.put(strlistreturn.size() + 1, str1);
}
}
return strlistreturn;
}
private List<String> removeSlacks(List<String> str) {
private ConcurrentMap<Integer, String> removeSlacks(ConcurrentMap<Integer, String> str) {
ShiftReduceParser model = getModel();
MaxentTagger tagger = getTagger();
List<TaggedWord> taggedWords;
List<String> strreturn = new ArrayList();
for (String str1 : str) {
ConcurrentMap<Integer, String> strreturn = new MapMaker().concurrencyLevel(2).makeMap();
for (String str1 : str.values()) {
int counter = 0;
List<String> TGWList = new ArrayList();
DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1));
@@ -638,7 +643,7 @@ public class MYSQLDatahandler {
}
}
if (!tooclosematch) {
strreturn.add(str1);
strreturn.put(strreturn.size() + 1, str1);
}
}
}
ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java
+8 -6
View File
@@ -5,8 +5,10 @@
*/
package FunctionLayer;
import com.google.common.collect.MapMaker;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
/**
*
@@ -14,13 +16,13 @@ import java.util.List;
*/
public class MessageResponseHandler {
private static List<String> str = new ArrayList();
private static ConcurrentMap<Integer, String> str = new MapMaker().concurrencyLevel(2).makeMap();
public static List<String> getStr() {
public static ConcurrentMap<Integer, String> getStr() {
return str;
}
public static void setStr(List<String> str) {
public static void setStr(ConcurrentMap<Integer, String> str) {
MessageResponseHandler.str = str;
}
@@ -33,13 +35,13 @@ public class MessageResponseHandler {
if (message.startsWith("[ *")) {
message = message.substring(message.indexOf("]"));
}
str.add(message);
str.put(str.size() + 1, message);
}
}
public static String selectReponseMessage(String toString) throws CustomError {
List<String> str1 = new ArrayList();
str1.add(toString);
ConcurrentMap<Integer, String> str1 = new MapMaker().concurrencyLevel(2).makeMap();
str1.put(str1.size() +1 , toString);
str1 = MYSQLDatahandler.cutContent(str1);
String strreturn = str1.get(0);
String getResponseMsg = MYSQLDatahandler.instance.getResponseMsg(strreturn);