UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

adopting strings correctly again, yikers

Browse Source
This commit is contained in:
jenzur 2019-03-10 19:19:33 +01:00
parent 7ab6c7ee0b
commit 43da2dd5d5
1 changed files with 51 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java
View File

@ -50,8 +50,8 @@ import java.util.logging.Logger;
* @author install1
*/
public class MYSQLDatahandler {
public static final long EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(10, TimeUnit.MINUTES);
public static final long EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(6, TimeUnit.MINUTES);
public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS);
public static MYSQLDatahandler instance = new MYSQLDatahandler();
private volatile boolean refreshMatrixFromDB;
@ -77,27 +77,27 @@ public class MYSQLDatahandler {
private ExecutorService executor;
private static StanfordCoreNLP pipeline;
private static StanfordCoreNLP pipelineSentiment;
public static AbstractSequenceClassifier<CoreLabel> getClassifier() {
return classifier;
}
public static void setClassifier(AbstractSequenceClassifier<CoreLabel> classifier) {
MYSQLDatahandler.classifier = classifier;
}
public void instantiateExecutor() {
this.executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(),
ForkJoinPool.defaultForkJoinWorkerThreadFactory,
null, true);
}
public MYSQLDatahandler() {
this.stopwatch = Stopwatch.createUnstarted();
this.stopwatch1 = Stopwatch.createStarted();
this.stringCache = new MapMaker().concurrencyLevel(2).makeMap();
}
public static void shiftReduceParserInitiate() {
try {
classifier = CRFClassifier.getClassifierNoExceptions(NERModel);
@ -128,39 +128,39 @@ public class MYSQLDatahandler {
pipeline = new StanfordCoreNLP(props);
pipelineSentiment = new StanfordCoreNLP(propsSentiment);
}
public static GrammaticalStructureFactory getGsf() {
return gsf;
}
public static StanfordCoreNLP getPipeline() {
return pipeline;
}
public static StanfordCoreNLP getPipelineSentiment() {
return pipelineSentiment;
}
public static MaxentTagger getTagger() {
return tagger;
}
public static ShiftReduceParser getModel() {
return model;
}
private Map<Integer, String> getCache() throws SQLException, IOException, CustomError {
return DataMapper.getAllStrings();
}
public int getlHMSMXSize() {
return lHMSMX.size();
}
public int getstringCacheSize() {
return stringCache.size();
}
public void initiateMYSQL() throws SQLException, IOException {
try {
DataMapper.createTables();
@ -171,7 +171,7 @@ public class MYSQLDatahandler {
.getName()).log(Level.SEVERE, null, ex);
}
}
public synchronized void checkIfUpdateMatrixes() {
refreshMatrixFromDB = false;
int counter = 0;
@ -202,20 +202,21 @@ public class MYSQLDatahandler {
}
if (selectUpdate == -1 || selectUpdate + 1 == stringCachelocal.size()) {
int valueSize = stringCachelocal.size();
if (secondaryIterator >= valueSize) {
if (secondaryIterator + 1 >= valueSize) {
secondaryIterator = 0;
}
selectUpdate = secondaryIterator;
secondaryIterator++;
}
int beginindex = selectUpdate;
System.out.println("beginindex: " + beginindex + "\n");
ConcurrentMap<Integer, String> strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap();
String get = stringCachelocal.get(beginindex);
String get = stringCachelocal.getOrDefault(beginindex, null);
if (get == null) {
get = stringCachelocal.get(new Random().nextInt(stringCachelocal.size() - 1));
}
strIndexNavigator.put(0, get);
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, ConcurrentMap<String, String>> strMap = new MapMaker().concurrencyLevel(2).makeMap();
strIndexNavigator.values().forEach((str) -> {
stringCachelocal.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> {
boolean present = false;
@ -237,27 +238,16 @@ public class MYSQLDatahandler {
}
}
}
if (!present) {
for (ConcurrentMap<String, String> strconcuritr : strMap.values()) {
String orDefault1 = strconcuritr.getOrDefault(str, null);
if (orDefault1 != null && orDefault1.equals(str1)) {
present = true;
break;
}
orDefault1 = strconcuritr.getOrDefault(str1, null);
if (orDefault1 != null && orDefault1.equals(str)) {
present = true;
break;
}
}
}
if (!present) {
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX);
futures.put(futures.size() + 1, executor.submit(worker));
ConcurrentMap<String, String> strmapLocal = new MapMaker().concurrencyLevel(2).makeMap();
strmapLocal.put(str, str1);
strMap.put(strMap.size() + 1, strmapLocal);
LinkedHashMap<String, Double> orDefault1 = LHMSMXLocal.getOrDefault(str, null);
if (orDefault1 == null) {
orDefault1 = new LinkedHashMap<String, Double>();
}
orDefault1.put(str1, 0.0);
LHMSMXLocal.put(str, orDefault1);
}
});
});
@ -265,18 +255,19 @@ public class MYSQLDatahandler {
for (Future<SimilarityMatrix> future : futures.values()) {
System.out.println("counter: " + counter + "\n");
counter++;
SimilarityMatrix SMX = new SimilarityMatrix("", "");
try {
SimilarityMatrix SMX = future.get(20, TimeUnit.SECONDS);
LinkedHashMap<String, Double> getFuture = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
if (getFuture == null) {
getFuture = new LinkedHashMap();
}
getFuture.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), getFuture);
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
SMX = future.get(20, TimeUnit.SECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
}
LinkedHashMap<String, Double> getFuture = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
if (getFuture == null) {
getFuture = new LinkedHashMap<String, Double>();
}
getFuture.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), getFuture);
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
}
try {
if (!matrixUpdateList.isEmpty()) {
@ -289,7 +280,7 @@ public class MYSQLDatahandler {
}
}
}
public synchronized void checkIfUpdateStrings() throws CustomError {
if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
ConcurrentMap<Integer, String> str = MessageResponseHandler.getStr();
@ -316,7 +307,7 @@ public class MYSQLDatahandler {
}
}
}
public synchronized String getResponseMsg(String str) throws CustomError {
str = str.trim();
if (str.startsWith("<@")) {
@ -389,7 +380,7 @@ public class MYSQLDatahandler {
+ "\nScore: " + SMXreturn.getDistance());
return SMXreturn.getSecondaryString();
}
public String mostSimilar(String toBeCompared, ConcurrentMap<Integer, String> concurrentStrings) {
int minDistance = 8;
String similar = "";
@ -412,7 +403,7 @@ public class MYSQLDatahandler {
}
return similar;
}
public static ConcurrentMap<Integer, String> cutContent(ConcurrentMap<Integer, String> str) {
ConcurrentMap<Integer, String> returnlist = new MapMaker().concurrencyLevel(2).makeMap();
for (String str1 : str.values()) {
@ -424,7 +415,7 @@ public class MYSQLDatahandler {
}
return returnlist;
}
public static ConcurrentMap<Integer, String> filterContent(ConcurrentMap<Integer, String> str) {
ConcurrentMap<Integer, String> strlistreturn = new MapMaker().concurrencyLevel(2).makeMap();
for (String str1 : str.values()) {
@ -532,7 +523,7 @@ public class MYSQLDatahandler {
}
return strlistreturn;
}
private ConcurrentMap<Integer, String> removeSlacks(ConcurrentMap<Integer, String> str) {
ShiftReduceParser model = getModel();
MaxentTagger tagger = getTagger();
@ -599,14 +590,11 @@ public class MYSQLDatahandler {
}
return strreturn;
}
private ConcurrentMap<Integer, String> verifyCalculationFitness(ConcurrentMap<Integer, String> strmap) {
ConcurrentMap<Integer, String> returnmap = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> allStrings = stringCache;
if (allStrings.isEmpty()) {
return strmap;
}
int intervalAprove = 450;
int intervalAprove = allStrings.values().size() / 10;
String str1 = "and to revise the questions and materials and such";
String str2 = "as William said earlier. Visiting your dentist once or twice a year is enough";
String str3 = "At least, you have more time to prepare then";
@ -619,14 +607,15 @@ public class MYSQLDatahandler {
worker = new SentimentAnalyzerTest(str, str3, new SimilarityMatrix(str, str3));
futures.put(futures.size() + 1, executor.submit(worker));
int ij2 = 0;
if (allStrings.isEmpty()) {
allStrings = strmap;
}
for (String strValues : allStrings.values()) {
if (ij2 > intervalAprove) {
break;
}
if (!strValues.equals(str1) && !strValues.equals(str2) && !strValues.equals(str3)) {
worker = new SentimentAnalyzerTest(str, strValues, new SimilarityMatrix(str, strValues));
futures.put(futures.size() + 1, executor.submit(worker));
}
worker = new SentimentAnalyzerTest(str, strValues, new SimilarityMatrix(str, strValues));
futures.put(futures.size() + 1, executor.submit(worker));
ij2++;
}
int counter = 0;
@ -635,7 +624,7 @@ public class MYSQLDatahandler {
try {
future.get(20, TimeUnit.SECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
//System.out.println("counter timeouts: " + counter + "\n");
System.out.println("counter timeouts: " + counter + "\n");
counter++;
if (counter >= 10) {
calculationIssues = true;