changed word list so hopefully no more undefined behaviour by list in coroutine environment

This commit is contained in:
christian 2021-12-01 23:45:36 +01:00
parent 59c255e615
commit 63be8078bc
5 changed files with 24 additions and 46 deletions

View File

@ -9,15 +9,16 @@ import java.sql.Connection;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
import org.apache.commons.dbcp2.BasicDataSource; import org.apache.commons.dbcp2.BasicDataSource;
import DataLayer.settings; import DataLayer.settings;
/** /**
*
* @author install1 * @author install1
*/ */
public class DBCPDataSource { public class DBCPDataSource {
private static BasicDataSource ds = new BasicDataSource(); private static BasicDataSource ds = new BasicDataSource();
static { static {
try { try {
ds.setDriver(new com.mysql.cj.jdbc.Driver()); ds.setDriver(new com.mysql.cj.jdbc.Driver());
@ -28,6 +29,7 @@ public class DBCPDataSource {
ds.setMinIdle(5); ds.setMinIdle(5);
ds.setMaxIdle(-1); ds.setMaxIdle(-1);
ds.setMaxOpenPreparedStatements(100); ds.setMaxOpenPreparedStatements(100);
System.out.println("called BasicDataSource ");
} catch (SQLException ex) { } catch (SQLException ex) {
Logger.getLogger(DBCPDataSource.class.getName()).log(Level.SEVERE, null, ex); Logger.getLogger(DBCPDataSource.class.getName()).log(Level.SEVERE, null, ex);
} }

View File

@ -39,7 +39,7 @@ public class DataMapper {
return arrayListStr; return arrayListStr;
} }
public static void InsertMYSQLStrings(ArrayList<String> str) throws SQLException { public static void InsertMYSQLStrings(String str) throws SQLException {
Connection l_cCon = null; Connection l_cCon = null;
PreparedStatement l_pStatement = null; PreparedStatement l_pStatement = null;
ResultSet l_rsSearch = null; ResultSet l_rsSearch = null;
@ -47,10 +47,8 @@ public class DataMapper {
try { try {
l_cCon = DBCPDataSource.getConnection(); l_cCon = DBCPDataSource.getConnection();
l_pStatement = l_cCon.prepareStatement(l_sSQL); l_pStatement = l_cCon.prepareStatement(l_sSQL);
for (String str1 : str) { l_pStatement.setString(1, str);
l_pStatement.setString(1, str1); l_pStatement.execute();
l_pStatement.execute();
}
} finally { } finally {
CloseConnections(l_pStatement, l_rsSearch, l_cCon); CloseConnections(l_pStatement, l_rsSearch, l_cCon);
} }
@ -88,7 +86,7 @@ public class DataMapper {
String CountSQL = "select count(*) from Sentences"; String CountSQL = "select count(*) from Sentences";
String l_sSQL = "delete from Sentences\n" + String l_sSQL = "delete from Sentences\n" +
" where DATE(last_used) < DATE_SUB(CURDATE(), INTERVAL 32 DAY)\n" + " where DATE(last_used) < DATE_SUB(CURDATE(), INTERVAL 32 DAY)\n" +
" order by last_used asc limit 5"; " order by last_used asc limit 2";
try { try {
l_cCon = DBCPDataSource.getConnection(); l_cCon = DBCPDataSource.getConnection();
l_pStatement = l_cCon.prepareStatement(CountSQL); l_pStatement = l_cCon.prepareStatement(CountSQL);

View File

@ -7,7 +7,6 @@ package FunctionLayer
import DataLayer.DataMapper import DataLayer.DataMapper
import FunctionLayer.StanfordParser.SentimentAnalyzerTest import FunctionLayer.StanfordParser.SentimentAnalyzerTest
import com.google.common.base.Stopwatch
import edu.mit.jmwe.data.IMWE import edu.mit.jmwe.data.IMWE
import edu.mit.jmwe.data.IToken import edu.mit.jmwe.data.IToken
import edu.stanford.nlp.ie.AbstractSequenceClassifier import edu.stanford.nlp.ie.AbstractSequenceClassifier
@ -36,8 +35,6 @@ import kotlin.collections.HashMap
* @author install1 * @author install1
*/ */
public class Datahandler { public class Datahandler {
private val stopwatch: Stopwatch
private val EXPIRE_TIME_IN_MINUTES = TimeUnit.MINUTES.convert(30, TimeUnit.MINUTES)
private var pipelineAnnotationCache: HashMap<String, Annotation> private var pipelineAnnotationCache: HashMap<String, Annotation>
private var pipelineSentimentAnnotationCache = HashMap<String, Annotation>() private var pipelineSentimentAnnotationCache = HashMap<String, Annotation>()
private var coreDocumentAnnotationCache: HashMap<String, CoreDocument> private var coreDocumentAnnotationCache: HashMap<String, CoreDocument>
@ -90,7 +87,6 @@ public class Datahandler {
private var PairCounterHashMap: HashMap<String, Int> = HashMap() private var PairCounterHashMap: HashMap<String, Int> = HashMap()
constructor() { constructor() {
stopwatch = Stopwatch.createUnstarted()
jmweAnnotationCache = HashMap<String, Annotation>() jmweAnnotationCache = HashMap<String, Annotation>()
pipelineAnnotationCache = HashMap<String, Annotation>() pipelineAnnotationCache = HashMap<String, Annotation>()
pipelineSentimentAnnotationCache = HashMap<String, Annotation>() pipelineSentimentAnnotationCache = HashMap<String, Annotation>()
@ -158,27 +154,12 @@ public class Datahandler {
return StanfordCoreNLP(propsSentiment) return StanfordCoreNLP(propsSentiment)
} }
@Synchronized
fun updateStringCache() { fun updateStringCache() {
if (stopwatch.elapsed(TimeUnit.MINUTES) >= EXPIRE_TIME_IN_MINUTES || !stopwatch.isRunning) { stringCache = ArrayList<String>();
if (!stopwatch.isRunning) { for (str: String in DataMapper.getAllStrings()) {
stopwatch.start() stringCache.add(str)
} else {
stopwatch.reset()
}
stringCache.sortWith(Comparator.comparingInt(String::length).reversed());
System.out.println("pre InsertMYSQLStrings")
val arrayList = java.util.ArrayList<String>(stringCache)
DataMapper.InsertMYSQLStrings(arrayList)
DataMapper.checkStringsToDelete();
for (str: String in DataMapper.getAllStrings()) {
if (str in stringCache) {
continue
}
stringCache.add(str)
}
System.out.println("post updateStringCache");
} }
stringCache.sortWith(Comparator.comparingInt(String::length).reversed());
} }
private fun trimString(str: String): String { private fun trimString(str: String): String {
@ -557,8 +538,11 @@ public class Datahandler {
} }
} }
val cacheRequirement = 6500; val cacheRequirement = 6500;
if (preRelationUserCounters > cacheRequirement && !stringCache.contains(strF) && filterContent(strF)) { if (preRelationUserCounters > cacheRequirement && !values_copy.contains(strF) && filterContent(strF)) {
DataMapper.InsertMYSQLStrings(strF)
DataMapper.checkStringsToDelete();
stringCache.add(strF) stringCache.add(strF)
stringCache.sortWith(Comparator.comparingInt(String::length).reversed());
} }
val randomLenghtPermit = strF.length * (Math.random() * Math.random() * Math.random() * (Math.random() * 10)) val randomLenghtPermit = strF.length * (Math.random() * Math.random() * Math.random() * (Math.random() * 10))
Collections.reverse(concurrentRelations) Collections.reverse(concurrentRelations)
@ -575,12 +559,7 @@ public class Datahandler {
if (SB.toString().isEmpty()) { if (SB.toString().isEmpty()) {
return "failure, preventing stuckness" return "failure, preventing stuckness"
} }
runBlocking { DataMapper.updateLastUsed(mysqlUpdateLastUsed);
CoroutineScope(launch(Dispatchers.IO) {
DataMapper.updateLastUsed(mysqlUpdateLastUsed);
yield()
})
}
return SB.toString() return SB.toString()
} }
@ -593,7 +572,7 @@ public class Datahandler {
stanfordCoreNLPSentiment: StanfordCoreNLP, ingameResponse: Boolean): String { stanfordCoreNLPSentiment: StanfordCoreNLP, ingameResponse: Boolean): String {
var responseFutures: String = "" var responseFutures: String = ""
runBlocking { runBlocking {
val launch1 = launch(Dispatchers.Default) { launch(Dispatchers.Default) {
var strF = trimString(str) var strF = trimString(str)
responseFutures = getResponseFutures(strF, stanfordCoreNLP, stanfordCoreNLPSentiment) responseFutures = getResponseFutures(strF, stanfordCoreNLP, stanfordCoreNLPSentiment)
if (!ingameResponse) { if (!ingameResponse) {
@ -601,8 +580,7 @@ public class Datahandler {
stanfordCoreNLPSentiment) stanfordCoreNLPSentiment)
} }
yield() yield()
} }.join()
launch1.join()
} }
return responseFutures return responseFutures
} }
@ -627,7 +605,6 @@ public class Datahandler {
if (emText != personName && !isMatched) { if (emText != personName && !isMatched) {
for (emLastMsg in pipelineCoreDcoumentLastMsg.entityMentions()) { for (emLastMsg in pipelineCoreDcoumentLastMsg.entityMentions()) {
if (emText != emLastMsg.text() && !Character.isDigit(emLastMsg.text().trim { it <= ' ' }[0])) { if (emText != emLastMsg.text() && !Character.isDigit(emLastMsg.text().trim { it <= ' ' }[0])) {
//System.out.println("emLastMsg.text(): " + emLastMsg.text());
str = (responseMsg.substring(0, responseMsg.indexOf(emText)) + " " str = (responseMsg.substring(0, responseMsg.indexOf(emText)) + " "
+ emLastMsg + " " + responseMsg.substring(responseMsg.indexOf(emText))) + emLastMsg + " " + responseMsg.substring(responseMsg.indexOf(emText)))
} }
@ -638,7 +615,7 @@ public class Datahandler {
} }
} }
} catch (e: Exception) { } catch (e: Exception) {
println("""SCUFFED JAYZ: ${e.localizedMessage}""".trimIndent()) println("""SCUFFED JAYZ: ${e.message}""".trimIndent())
} }
return responseMsg return responseMsg
} }

View File

@ -26,15 +26,14 @@ import reactor.core.publisher.Mono;
*/ */
public class DoStuff { public class DoStuff {
public synchronized static void doStuff(MessageCreateEvent event, String usernameBot, Datahandler datahandler, public static void doStuff(MessageCreateEvent event, String usernameBot, Datahandler datahandler,
StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) { StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) {
String username = ""; String username = "";
try { try {
username = event.getMessage().getAuthor().get().getUsername(); username = event.getMessage().getAuthor().get().getUsername();
} catch (java.util.NoSuchElementException e) { } catch (java.util.NoSuchElementException e) {
username = null; username = null;
} }
datahandler.updateStringCache();
if (username != null && !username.equals(usernameBot)) { if (username != null && !username.equals(usernameBot)) {
TextChannel block = event.getMessage().getChannel().cast(TextChannel.class).block(); TextChannel block = event.getMessage().getChannel().cast(TextChannel.class).block();
String name = block.getCategory().block().getName(); String name = block.getCategory().block().getName();
@ -73,6 +72,7 @@ public class DoStuff {
} }
if (mentionedBot || channelName.contains("general-autism")) { if (mentionedBot || channelName.contains("general-autism")) {
String ResponseStr; String ResponseStr;
System.out.println("reached datahandler.getResponseMsg");
ResponseStr = datahandler.getResponseMsg(content, username, stanfordCoreNLP, stanfordCoreNLPSentiment, ResponseStr = datahandler.getResponseMsg(content, username, stanfordCoreNLP, stanfordCoreNLPSentiment,
false); false);
if (!ResponseStr.isEmpty()) { if (!ResponseStr.isEmpty()) {

View File

@ -81,6 +81,7 @@ public class DiscordHandler {
public static void main(String[] args) throws IOException, SQLException { public static void main(String[] args) throws IOException, SQLException {
Datahandler datahandler = new Datahandler(); Datahandler datahandler = new Datahandler();
datahandler.updateStringCache(); datahandler.updateStringCache();
System.out.println("post updateStringCache");
PipelineJMWESingleton.getINSTANCE(); PipelineJMWESingleton.getINSTANCE();
StanfordCoreNLP stanfordCoreNLP = datahandler.pipeLineSetUp(); StanfordCoreNLP stanfordCoreNLP = datahandler.pipeLineSetUp();