UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

simplifying matrix updating quite alot

Browse Source
This commit is contained in:
jenzur 2019-03-06 23:15:02 +01:00
parent 1fe4a268c4
commit 05683ce4be
2 changed files with 94 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

268
ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java
View File

@ -35,6 +35,7 @@ import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Properties; import java.util.Properties;
import java.util.Random; import java.util.Random;
import java.util.Set;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
@ -42,6 +43,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.ForkJoinPool; import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
@ -55,12 +57,9 @@ public class MYSQLDatahandler {
public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS); public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS);
public static MYSQLDatahandler instance = new MYSQLDatahandler(); public static MYSQLDatahandler instance = new MYSQLDatahandler();
public static int semeticsUpdateCount; public static int semeticsUpdateCount;
private static int counter = 0;
private volatile boolean refreshMatrixFromDB; private volatile boolean refreshMatrixFromDB;
private final ConcurrentMap<Integer, String> stringCache; private final ConcurrentMap<Integer, String> stringCache;
private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap(); private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap();
private ConcurrentMap<Integer, String> multiprocessCalculations;
private List<Integer> updatedRows = new ArrayList();
private final Stopwatch stopwatch; private final Stopwatch stopwatch;
private final Stopwatch stopwatch1; private final Stopwatch stopwatch1;
private static String modelPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz"; private static String modelPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz";
@ -99,7 +98,6 @@ public class MYSQLDatahandler {
this.stopwatch = Stopwatch.createUnstarted(); this.stopwatch = Stopwatch.createUnstarted();
this.stopwatch1 = Stopwatch.createStarted(); this.stopwatch1 = Stopwatch.createStarted();
this.stringCache = new MapMaker().concurrencyLevel(2).makeMap(); this.stringCache = new MapMaker().concurrencyLevel(2).makeMap();
this.multiprocessCalculations = new MapMaker().concurrencyLevel(2).makeMap();
} }
public static void shiftReduceParserInitiate() { public static void shiftReduceParserInitiate() {
@ -186,8 +184,8 @@ public class MYSQLDatahandler {
public synchronized void checkIfUpdateMatrixes() { public synchronized void checkIfUpdateMatrixes() {
refreshMatrixFromDB = false; refreshMatrixFromDB = false;
int calculationBoundaries = 5; int updateBadgesInteger = 900;
int updateBadgesInteger = 250; int counter = 0;
if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) { if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) {
refreshMatrixFromDB = true; refreshMatrixFromDB = true;
lHMSMX = DataMapper.getAllRelationScores(); lHMSMX = DataMapper.getAllRelationScores();
@ -195,162 +193,90 @@ public class MYSQLDatahandler {
} }
//requiring atleast 10 entries ensures no issues in case of empty stringcache //requiring atleast 10 entries ensures no issues in case of empty stringcache
if (stringCache.values().size() > 10 && !refreshMatrixFromDB) { if (stringCache.values().size() > 10 && !refreshMatrixFromDB) {
if (counter <= 5) { ConcurrentMap<Integer, String> stringCachelocal = stringCache;
int selectUpdate = -1;
LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
int ij2 = 0;
for (String str : stringCachelocal.values()) {
if (!LHMSMXLocal.keySet().contains(str)) {
selectUpdate = ij2;
LinkedHashMap<String, Double> newLHM = new LinkedHashMap();
lHMSMX.put(str, newLHM);
break;
}
ij2++;
}
if (selectUpdate == -1) {
ij2 = 0;
int valueSize = stringCachelocal.size();
for (LinkedHashMap<String, Double> values : LHMSMXLocal.values()) {
int lhmSize = values.size();
if (lhmSize < valueSize) {
selectUpdate = ij2;
}
ij2++;
}
}
semeticsUpdateCount = selectUpdate;
int beginindex = semeticsUpdateCount;
System.out.println("beginindex: " + beginindex + "\n");
ConcurrentMap<Integer, String> strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap();
String get = stringCachelocal.get(beginindex);
strIndexNavigator.put(strIndexNavigator.size() + 1, get);
ConcurrentMap<Integer, String> strIndexAll = stringCachelocal;
ConcurrentMap<Integer, String> randomIndexesToUpdate = new MapMaker().concurrencyLevel(2).makeMap();
int indexes = updateBadgesInteger;
if (indexes >= strIndexAll.size()) {
indexes = strIndexAll.size() - 1;
}
int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes);
int ij1 = 0;
while (beginindexes + ij1 < beginindexes + indexes) {
String get1 = strIndexAll.get(beginindexes + ij1);
randomIndexesToUpdate.put(ij1, get1);
ij1++;
}
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
strIndexNavigator.values().forEach((str) -> {
randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> {
boolean present = false;
if ((LHMSMXLocal.containsKey(str)) && LHMSMXLocal.get(str).containsKey(str1)
|| (LHMSMXLocal.containsKey(str1)) && LHMSMXLocal.get(str).containsKey(str)) {
present = true;
}
if (!present) {
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX);
futures.put(futures.size() + 1, executor.submit(worker));
}
});
});
System.out.println("finished worker assignment, futures size: " + futures.size() + "\n");
for (Future<SimilarityMatrix> future : futures.values()) {
System.out.println("counter: " + counter + "\n");
counter++; counter++;
ConcurrentMap<Integer, String> stringCachelocal = stringCache;
List<Integer> updateLocal = updatedRows;
int random = -1;
if (!updateLocal.contains(random)) {
updatedRows.add(random);
}
Collections.sort(updateLocal);
while (updateLocal.contains(random)) {
random = new Random().nextInt(stringCachelocal.values().size() - 6);
int indexPrev = Collections.binarySearch(updateLocal, random);
int indexNext = Collections.binarySearch(updateLocal, random + 6);
//-1 will always be index 0
if (indexPrev > 0 && indexNext > 0) {
indexPrev = updateLocal.get(indexPrev);
indexNext = updateLocal.get(indexNext);
}
random = indexPrev < random - 5 && indexNext < random ? random : -1;
}
updatedRows.add(random);
semeticsUpdateCount = random;
int beginindex = semeticsUpdateCount;
semeticsUpdateCount += calculationBoundaries;
int temp = semeticsUpdateCount;
System.out.println("beginindex: " + beginindex + "\ntemp: " + temp + "\n");
ConcurrentMap<Integer, String> strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap();
int ij = 0;
while (beginindex + ij < temp) {
String get = stringCachelocal.get(beginindex + ij);
strIndexNavigator.put(ij, get);
multiprocessCalculations.put(multiprocessCalculations.size() + 1, get);
ij++;
}
LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
ConcurrentMap<Integer, String> strIndexAll = stringCachelocal;
ConcurrentMap<Integer, String> strIndexNavigatorL = strIndexNavigator;
ConcurrentMap<Integer, String> randomIndexesToUpdate = new MapMaker().concurrencyLevel(2).makeMap();
int indexes = updateBadgesInteger;
if (indexes >= strIndexAll.size()) {
indexes = strIndexAll.size() - 1;
}
int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes);
int ij1 = 0;
while (beginindexes + ij1 < beginindexes + indexes) {
String get1 = strIndexAll.get(beginindexes + ij1);
randomIndexesToUpdate.put(ij1, get1);
ij1++;
}
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
strIndexNavigatorL.values().forEach((str) -> {
randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> {
boolean present = false;
if (multiprocessCalculations.values().contains(str1)) {
present = true;
} else if (LHMSMXLocal.containsKey(str)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str);
if (orDefault.containsKey(str1)) {
present = true;
}
} else if (LHMSMXLocal.containsKey(str1)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str1);
if (orDefault.containsKey(str)) {
present = true;
}
}
if (!present) {
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX);
futures.put(futures.size() + 1, executor.submit(worker));
}
});
});
try { try {
System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); SimilarityMatrix SMX = future.get(20, TimeUnit.SECONDS);
for (Future<SimilarityMatrix> future : futures.values()) { LinkedHashMap<String, Double> getFuture = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
SimilarityMatrix SMX = future.get(); if (getFuture == null) {
LinkedHashMap<String, Double> get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); getFuture = new LinkedHashMap();
if (get == null) {
get = new LinkedHashMap();
}
get.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), get);
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
} }
} catch (InterruptedException | ExecutionException ex) { getFuture.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), getFuture);
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
} }
new Thread(() -> { }
try { try {
if (!matrixUpdateList.isEmpty()) { if (!matrixUpdateList.isEmpty()) {
DataMapper.insertSementicMatrixes(matrixUpdateList); DataMapper.insertSementicMatrixes(matrixUpdateList);
System.out.println("finished datamapper semetic insert"); System.out.println("finished datamapper semetic insert");
}
} catch (CustomError ex) {
Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex);
}
}).start();
} else {
ConcurrentMap<Integer, String> stringCachelocal = stringCache;
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
Collection<String> values = multiprocessCalculations.values();
LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
values.forEach((str) -> {
stringCachelocal.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> {
boolean present = false;
if (LHMSMXLocal.containsKey(str)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str);
if (orDefault.containsKey(str1)) {
present = true;
}
} else if (LHMSMXLocal.containsKey(str1)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str1);
if (orDefault.containsKey(str)) {
present = true;
}
}
if (!present) {
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX);
futures.put(futures.size() + 1, executor.submit(worker));
}
});
});
try {
System.out.println("finished worker assignment, futures size: " + futures.size() + "\n");
for (Future<SimilarityMatrix> future : futures.values()) {
SimilarityMatrix SMX = future.get();
LinkedHashMap<String, Double> get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
if (get == null) {
get = new LinkedHashMap();
}
get.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), get);
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
}
} catch (InterruptedException | ExecutionException ex) {
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
} }
multiprocessCalculations = new MapMaker().concurrencyLevel(2).makeMap(); } catch (CustomError ex) {
updatedRows = new ArrayList(); Logger.getLogger(MYSQLDatahandler.class
counter = 0; .getName()).log(Level.SEVERE, null, ex);
new Thread(() -> {
try {
if (!matrixUpdateList.isEmpty()) {
DataMapper.insertSementicMatrixes(matrixUpdateList);
System.out.println("finished datamapper semetic insert");
}
} catch (CustomError ex) {
Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex);
}
}).start();
} }
} }
} }
@ -439,7 +365,7 @@ public class MYSQLDatahandler {
int index = 0; int index = 0;
for (Future<SimilarityMatrix> future : futureslocal.values()) { for (Future<SimilarityMatrix> future : futureslocal.values()) {
try { try {
SimilarityMatrix SMX = future.get(); SimilarityMatrix SMX = future.get((long) 20, TimeUnit.SECONDS);
double distance = SMX.getDistance(); double distance = SMX.getDistance();
System.out.println("index: " + index + "\nfutures size: " + futureslocal.values().size() + "\nScore: " + SMX.getDistance() + "\nSecondary: " System.out.println("index: " + index + "\nfutures size: " + futureslocal.values().size() + "\nScore: " + SMX.getDistance() + "\nSecondary: "
+ SMX.getSecondaryString() + "\nPrimary: " + SMX.getPrimaryString() + "\n"); + SMX.getSecondaryString() + "\nPrimary: " + SMX.getPrimaryString() + "\n");
@ -447,7 +373,7 @@ public class MYSQLDatahandler {
Score = distance; Score = distance;
SMXreturn = SMX; SMXreturn = SMX;
} }
} catch (InterruptedException | ExecutionException ex) { } catch (InterruptedException | ExecutionException | TimeoutException ex) {
System.out.println("ex: " + ex.getMessage() + "\n"); System.out.println("ex: " + ex.getMessage() + "\n");
} }
index++; index++;
@ -464,26 +390,18 @@ public class MYSQLDatahandler {
concurrentStrings.values().stream().map((str) -> new LevenshteinDistance(str, toBeCompared, new DistanceObject())).forEachOrdered((worker) -> { concurrentStrings.values().stream().map((str) -> new LevenshteinDistance(str, toBeCompared, new DistanceObject())).forEachOrdered((worker) -> {
futures.put(futures.size() + 1, executor.submit(worker)); futures.put(futures.size() + 1, executor.submit(worker));
}); });
try { for (Future<DistanceObject> future : futures.values()) {
for (Future<DistanceObject> future : futures.values()) { try {
DistanceObject d = future.get(); DistanceObject d = future.get();
try { int distance = d.getDistance();
if (d.getSentence() != null && d.getDistance() != null) { System.out.println("distance: " + distance + "\n");
int distance = d.getDistance(); if (distance < minDistance) {
System.out.println("distance: " + distance + "\n"); minDistance = distance;
if (distance < minDistance) { similar = d.getSentence();
minDistance = distance;
similar = d.getSentence();
}
}
} catch (NullPointerException ex) {
System.out.println("failed future\n");
} }
} catch (NullPointerException | InterruptedException | ExecutionException ex) {
System.out.println("failed future\n");
} }
} catch (InterruptedException | ExecutionException ex) {
Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex);
} }
return similar; return similar;
} }

2
ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java
View File

@ -45,7 +45,7 @@ public class DiscordHandler {
}).start(); }).start();
MYSQLDatahandler.shiftReduceParserInitiate(); MYSQLDatahandler.shiftReduceParserInitiate();
MYSQLDatahandler.instance.instantiateExecutor(); MYSQLDatahandler.instance.instantiateExecutor();
while (MYSQLDatahandler.instance.getlHMSMXSize() < (MYSQLDatahandler.instance.getstringCacheSize() while (MYSQLDatahandler.instance.getlHMSMXSize() / 2 < (MYSQLDatahandler.instance.getstringCacheSize()
* MYSQLDatahandler.instance.getstringCacheSize()) - MYSQLDatahandler.instance.getstringCacheSize()) { * MYSQLDatahandler.instance.getstringCacheSize()) - MYSQLDatahandler.instance.getstringCacheSize()) {
MYSQLDatahandler.instance.checkIfUpdateMatrixes(); MYSQLDatahandler.instance.checkIfUpdateMatrixes();
} }