UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

adopting strings correctly again, yikers

Browse Source
This commit is contained in:
jenzur 2019-03-10 19:19:33 +01:00
parent 7ab6c7ee0b
commit 43da2dd5d5
1 changed files with 51 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java
View File

@ -50,8 +50,8 @@ import java.util.logging.Logger;
* @author install1 * @author install1
*/ */
public class MYSQLDatahandler { public class MYSQLDatahandler {
public static final long EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(10, TimeUnit.MINUTES); public static final long EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(6, TimeUnit.MINUTES);
public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS); public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS);
public static MYSQLDatahandler instance = new MYSQLDatahandler(); public static MYSQLDatahandler instance = new MYSQLDatahandler();
private volatile boolean refreshMatrixFromDB; private volatile boolean refreshMatrixFromDB;
@ -77,27 +77,27 @@ public class MYSQLDatahandler {
private ExecutorService executor; private ExecutorService executor;
private static StanfordCoreNLP pipeline; private static StanfordCoreNLP pipeline;
private static StanfordCoreNLP pipelineSentiment; private static StanfordCoreNLP pipelineSentiment;
public static AbstractSequenceClassifier<CoreLabel> getClassifier() { public static AbstractSequenceClassifier<CoreLabel> getClassifier() {
return classifier; return classifier;
} }
public static void setClassifier(AbstractSequenceClassifier<CoreLabel> classifier) { public static void setClassifier(AbstractSequenceClassifier<CoreLabel> classifier) {
MYSQLDatahandler.classifier = classifier; MYSQLDatahandler.classifier = classifier;
} }
public void instantiateExecutor() { public void instantiateExecutor() {
this.executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), this.executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(),
ForkJoinPool.defaultForkJoinWorkerThreadFactory, ForkJoinPool.defaultForkJoinWorkerThreadFactory,
null, true); null, true);
} }
public MYSQLDatahandler() { public MYSQLDatahandler() {
this.stopwatch = Stopwatch.createUnstarted(); this.stopwatch = Stopwatch.createUnstarted();
this.stopwatch1 = Stopwatch.createStarted(); this.stopwatch1 = Stopwatch.createStarted();
this.stringCache = new MapMaker().concurrencyLevel(2).makeMap(); this.stringCache = new MapMaker().concurrencyLevel(2).makeMap();
} }
public static void shiftReduceParserInitiate() { public static void shiftReduceParserInitiate() {
try { try {
classifier = CRFClassifier.getClassifierNoExceptions(NERModel); classifier = CRFClassifier.getClassifierNoExceptions(NERModel);
@ -128,39 +128,39 @@ public class MYSQLDatahandler {
pipeline = new StanfordCoreNLP(props); pipeline = new StanfordCoreNLP(props);
pipelineSentiment = new StanfordCoreNLP(propsSentiment); pipelineSentiment = new StanfordCoreNLP(propsSentiment);
} }
public static GrammaticalStructureFactory getGsf() { public static GrammaticalStructureFactory getGsf() {
return gsf; return gsf;
} }
public static StanfordCoreNLP getPipeline() { public static StanfordCoreNLP getPipeline() {
return pipeline; return pipeline;
} }
public static StanfordCoreNLP getPipelineSentiment() { public static StanfordCoreNLP getPipelineSentiment() {
return pipelineSentiment; return pipelineSentiment;
} }
public static MaxentTagger getTagger() { public static MaxentTagger getTagger() {
return tagger; return tagger;
} }
public static ShiftReduceParser getModel() { public static ShiftReduceParser getModel() {
return model; return model;
} }
private Map<Integer, String> getCache() throws SQLException, IOException, CustomError { private Map<Integer, String> getCache() throws SQLException, IOException, CustomError {
return DataMapper.getAllStrings(); return DataMapper.getAllStrings();
} }
public int getlHMSMXSize() { public int getlHMSMXSize() {
return lHMSMX.size(); return lHMSMX.size();
} }
public int getstringCacheSize() { public int getstringCacheSize() {
return stringCache.size(); return stringCache.size();
} }
public void initiateMYSQL() throws SQLException, IOException { public void initiateMYSQL() throws SQLException, IOException {
try { try {
DataMapper.createTables(); DataMapper.createTables();
@ -171,7 +171,7 @@ public class MYSQLDatahandler {
.getName()).log(Level.SEVERE, null, ex); .getName()).log(Level.SEVERE, null, ex);
} }
} }
public synchronized void checkIfUpdateMatrixes() { public synchronized void checkIfUpdateMatrixes() {
refreshMatrixFromDB = false; refreshMatrixFromDB = false;
int counter = 0; int counter = 0;
@ -202,20 +202,21 @@ public class MYSQLDatahandler {
} }
if (selectUpdate == -1 || selectUpdate + 1 == stringCachelocal.size()) { if (selectUpdate == -1 || selectUpdate + 1 == stringCachelocal.size()) {
int valueSize = stringCachelocal.size(); int valueSize = stringCachelocal.size();
if (secondaryIterator >= valueSize) { if (secondaryIterator + 1 >= valueSize) {
secondaryIterator = 0; secondaryIterator = 0;
} }
selectUpdate = secondaryIterator; selectUpdate = secondaryIterator;
secondaryIterator++; secondaryIterator++;
} }
int beginindex = selectUpdate; int beginindex = selectUpdate;
System.out.println("beginindex: " + beginindex + "\n");
ConcurrentMap<Integer, String> strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, String> strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap();
String get = stringCachelocal.get(beginindex); String get = stringCachelocal.getOrDefault(beginindex, null);
if (get == null) {
get = stringCachelocal.get(new Random().nextInt(stringCachelocal.size() - 1));
}
strIndexNavigator.put(0, get); strIndexNavigator.put(0, get);
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, ConcurrentMap<String, String>> strMap = new MapMaker().concurrencyLevel(2).makeMap();
strIndexNavigator.values().forEach((str) -> { strIndexNavigator.values().forEach((str) -> {
stringCachelocal.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { stringCachelocal.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> {
boolean present = false; boolean present = false;
@ -237,27 +238,16 @@ public class MYSQLDatahandler {
} }
} }
} }
if (!present) {
for (ConcurrentMap<String, String> strconcuritr : strMap.values()) {
String orDefault1 = strconcuritr.getOrDefault(str, null);
if (orDefault1 != null && orDefault1.equals(str1)) {
present = true;
break;
}
orDefault1 = strconcuritr.getOrDefault(str1, null);
if (orDefault1 != null && orDefault1.equals(str)) {
present = true;
break;
}
}
}
if (!present) { if (!present) {
SimilarityMatrix SMX = new SimilarityMatrix(str, str1); SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX); Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX);
futures.put(futures.size() + 1, executor.submit(worker)); futures.put(futures.size() + 1, executor.submit(worker));
ConcurrentMap<String, String> strmapLocal = new MapMaker().concurrencyLevel(2).makeMap(); LinkedHashMap<String, Double> orDefault1 = LHMSMXLocal.getOrDefault(str, null);
strmapLocal.put(str, str1); if (orDefault1 == null) {
strMap.put(strMap.size() + 1, strmapLocal); orDefault1 = new LinkedHashMap<String, Double>();
}
orDefault1.put(str1, 0.0);
LHMSMXLocal.put(str, orDefault1);
} }
}); });
}); });
@ -265,18 +255,19 @@ public class MYSQLDatahandler {
for (Future<SimilarityMatrix> future : futures.values()) { for (Future<SimilarityMatrix> future : futures.values()) {
System.out.println("counter: " + counter + "\n"); System.out.println("counter: " + counter + "\n");
counter++; counter++;
SimilarityMatrix SMX = new SimilarityMatrix("", "");
try { try {
SimilarityMatrix SMX = future.get(20, TimeUnit.SECONDS); SMX = future.get(20, TimeUnit.SECONDS);
LinkedHashMap<String, Double> getFuture = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
if (getFuture == null) {
getFuture = new LinkedHashMap();
}
getFuture.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), getFuture);
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
} catch (InterruptedException | ExecutionException | TimeoutException ex) { } catch (InterruptedException | ExecutionException | TimeoutException ex) {
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
} }
LinkedHashMap<String, Double> getFuture = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
if (getFuture == null) {
getFuture = new LinkedHashMap<String, Double>();
}
getFuture.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), getFuture);
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
} }
try { try {
if (!matrixUpdateList.isEmpty()) { if (!matrixUpdateList.isEmpty()) {
@ -289,7 +280,7 @@ public class MYSQLDatahandler {
} }
} }
} }
public synchronized void checkIfUpdateStrings() throws CustomError { public synchronized void checkIfUpdateStrings() throws CustomError {
if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) { if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
ConcurrentMap<Integer, String> str = MessageResponseHandler.getStr(); ConcurrentMap<Integer, String> str = MessageResponseHandler.getStr();
@ -316,7 +307,7 @@ public class MYSQLDatahandler {
} }
} }
} }
public synchronized String getResponseMsg(String str) throws CustomError { public synchronized String getResponseMsg(String str) throws CustomError {
str = str.trim(); str = str.trim();
if (str.startsWith("<@")) { if (str.startsWith("<@")) {
@ -389,7 +380,7 @@ public class MYSQLDatahandler {
+ "\nScore: " + SMXreturn.getDistance()); + "\nScore: " + SMXreturn.getDistance());
return SMXreturn.getSecondaryString(); return SMXreturn.getSecondaryString();
} }
public String mostSimilar(String toBeCompared, ConcurrentMap<Integer, String> concurrentStrings) { public String mostSimilar(String toBeCompared, ConcurrentMap<Integer, String> concurrentStrings) {
int minDistance = 8; int minDistance = 8;
String similar = ""; String similar = "";
@ -412,7 +403,7 @@ public class MYSQLDatahandler {
} }
return similar; return similar;
} }
public static ConcurrentMap<Integer, String> cutContent(ConcurrentMap<Integer, String> str) { public static ConcurrentMap<Integer, String> cutContent(ConcurrentMap<Integer, String> str) {
ConcurrentMap<Integer, String> returnlist = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, String> returnlist = new MapMaker().concurrencyLevel(2).makeMap();
for (String str1 : str.values()) { for (String str1 : str.values()) {
@ -424,7 +415,7 @@ public class MYSQLDatahandler {
} }
return returnlist; return returnlist;
} }
public static ConcurrentMap<Integer, String> filterContent(ConcurrentMap<Integer, String> str) { public static ConcurrentMap<Integer, String> filterContent(ConcurrentMap<Integer, String> str) {
ConcurrentMap<Integer, String> strlistreturn = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, String> strlistreturn = new MapMaker().concurrencyLevel(2).makeMap();
for (String str1 : str.values()) { for (String str1 : str.values()) {
@ -532,7 +523,7 @@ public class MYSQLDatahandler {
} }
return strlistreturn; return strlistreturn;
} }
private ConcurrentMap<Integer, String> removeSlacks(ConcurrentMap<Integer, String> str) { private ConcurrentMap<Integer, String> removeSlacks(ConcurrentMap<Integer, String> str) {
ShiftReduceParser model = getModel(); ShiftReduceParser model = getModel();
MaxentTagger tagger = getTagger(); MaxentTagger tagger = getTagger();
@ -599,14 +590,11 @@ public class MYSQLDatahandler {
} }
return strreturn; return strreturn;
} }
private ConcurrentMap<Integer, String> verifyCalculationFitness(ConcurrentMap<Integer, String> strmap) { private ConcurrentMap<Integer, String> verifyCalculationFitness(ConcurrentMap<Integer, String> strmap) {
ConcurrentMap<Integer, String> returnmap = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, String> returnmap = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> allStrings = stringCache; ConcurrentMap<Integer, String> allStrings = stringCache;
if (allStrings.isEmpty()) { int intervalAprove = allStrings.values().size() / 10;
return strmap;
}
int intervalAprove = 450;
String str1 = "and to revise the questions and materials and such"; String str1 = "and to revise the questions and materials and such";
String str2 = "as William said earlier. Visiting your dentist once or twice a year is enough"; String str2 = "as William said earlier. Visiting your dentist once or twice a year is enough";
String str3 = "At least, you have more time to prepare then"; String str3 = "At least, you have more time to prepare then";
@ -619,14 +607,15 @@ public class MYSQLDatahandler {
worker = new SentimentAnalyzerTest(str, str3, new SimilarityMatrix(str, str3)); worker = new SentimentAnalyzerTest(str, str3, new SimilarityMatrix(str, str3));
futures.put(futures.size() + 1, executor.submit(worker)); futures.put(futures.size() + 1, executor.submit(worker));
int ij2 = 0; int ij2 = 0;
if (allStrings.isEmpty()) {
allStrings = strmap;
}
for (String strValues : allStrings.values()) { for (String strValues : allStrings.values()) {
if (ij2 > intervalAprove) { if (ij2 > intervalAprove) {
break; break;
} }
if (!strValues.equals(str1) && !strValues.equals(str2) && !strValues.equals(str3)) { worker = new SentimentAnalyzerTest(str, strValues, new SimilarityMatrix(str, strValues));
worker = new SentimentAnalyzerTest(str, strValues, new SimilarityMatrix(str, strValues)); futures.put(futures.size() + 1, executor.submit(worker));
futures.put(futures.size() + 1, executor.submit(worker));
}
ij2++; ij2++;
} }
int counter = 0; int counter = 0;
@ -635,7 +624,7 @@ public class MYSQLDatahandler {
try { try {
future.get(20, TimeUnit.SECONDS); future.get(20, TimeUnit.SECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException ex) { } catch (InterruptedException | ExecutionException | TimeoutException ex) {
//System.out.println("counter timeouts: " + counter + "\n"); System.out.println("counter timeouts: " + counter + "\n");
counter++; counter++;
if (counter >= 10) { if (counter >= 10) {
calculationIssues = true; calculationIssues = true;