reduced amount of strings and removed overuse of threads

This commit is contained in:
jenz 2023-02-21 21:35:05 +01:00
parent 138e887a80
commit 272024d9f4
5 changed files with 38 additions and 28 deletions

View File

@ -93,7 +93,7 @@ public class DataMapper {
ResultSet resultSet = l_pStatement.executeQuery(); ResultSet resultSet = l_pStatement.executeQuery();
if (resultSet.next()) { if (resultSet.next()) {
int count = resultSet.getInt(1); int count = resultSet.getInt(1);
if (count > 13000) { if (count > 8000) {
//System.out.println("cleaning strings: " + l_sSQL); //System.out.println("cleaning strings: " + l_sSQL);
l_pStatement = l_cCon.prepareStatement(l_sSQL); l_pStatement = l_cCon.prepareStatement(l_sSQL);
l_pStatement.executeUpdate(); l_pStatement.executeUpdate();

View File

@ -8,10 +8,17 @@ import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.*; import java.net.*;
import java.util.ArrayList;
import java.util.Properties; import java.util.Properties;
public class ThreadClient { public class ThreadClient {
public ThreadClient(int port, Datahandler datahandler, StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) { public ThreadClient(Datahandler datahandler, StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) {
ArrayList<Integer> ports = new ArrayList<Integer>();
ports.add(48475);
ports.add(48476);
ports.add(48477);
ports.add(48478);
Properties prop = new Properties(); Properties prop = new Properties();
String fileName = "app.config"; String fileName = "app.config";
try (FileInputStream fis = new FileInputStream(fileName)) { try (FileInputStream fis = new FileInputStream(fileName)) {
@ -21,33 +28,46 @@ public class ThreadClient {
} }
String hostIP = prop.getProperty("app.hostip"); String hostIP = prop.getProperty("app.hostip");
if (port == Integer.valueOf(prop.getProperty("app.hostport")) String hostIP2 = prop.getProperty("app.hostip2");
|| port == Integer.valueOf(prop.getProperty("app.hostport2"))) {
hostIP = prop.getProperty("app.hostip2");
}
try { try {
InetAddress ipAddress = InetAddress.getByName(hostIP);//used ip's InetAddress ipAddress = InetAddress.getByName(hostIP);//used ip's
try (DatagramSocket serverSocket = new DatagramSocket(port)) { InetAddress ipAddress2 = InetAddress.getByName(hostIP2);//used ip's
while (true) { try (DatagramSocket serverSocket = new DatagramSocket(ports.get(0))) {
receiveAndSendPacket(serverSocket, ipAddress, port, datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment); try (DatagramSocket serverSocket1 = new DatagramSocket(ports.get(1))) {
try (DatagramSocket serverSocket2 = new DatagramSocket(ports.get(2))) {
try (DatagramSocket serverSocket3 = new DatagramSocket(ports.get(3))) {
while (true) {
try {
receiveAndSendPacket(serverSocket, ipAddress, ports.get(0), datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment);
receiveAndSendPacket(serverSocket1, ipAddress, ports.get(1), datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment);
receiveAndSendPacket(serverSocket2, ipAddress2, ports.get(2), datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment);
receiveAndSendPacket(serverSocket3, ipAddress2, ports.get(3), datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
}
} }
} catch (SocketException e) { } catch (SocketException e) {
e.printStackTrace(); e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} }
} catch (UnknownHostException e) { } catch (UnknownHostException e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
private static void receiveAndSendPacket(DatagramSocket serverSocket, InetAddress ipAddress, int port, private static void receiveAndSendPacket(DatagramSocket serverSocket, InetAddress ipAddress, int port,
Datahandler datahandler, StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) throws Datahandler datahandler, StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) throws
IOException { IOException {
byte[] receiveData = new byte[4096]; byte[] receiveData = new byte[4096];
DatagramPacket receivePacket = new DatagramPacket(receiveData, receiveData.length); DatagramPacket receivePacket = new DatagramPacket(receiveData, receiveData.length);
try { try {
/*
Only one DatagramSocket can call receive at a time since its a blocking call. yet somehow
the other DatagramSockets still get their UDP packets from receive() even if the call is made
many minutes after the actual UDP packet was sent. Maybe Security manager context?
*/
serverSocket.receive(receivePacket); serverSocket.receive(receivePacket);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
@ -57,6 +77,7 @@ public class ThreadClient {
sentence = sentence.replace("clientmessage:", ""); sentence = sentence.replace("clientmessage:", "");
String ResponseMsg = datahandler.getResponseMsg(sentence, "", stanfordCoreNLP, stanfordCoreNLPSentiment, String ResponseMsg = datahandler.getResponseMsg(sentence, "", stanfordCoreNLP, stanfordCoreNLPSentiment,
true); true);
System.out.println("port: " + port + ". ResponseMsg ingame: " + ResponseMsg);
byte[] sendData = new byte[0]; byte[] sendData = new byte[0];
try { try {
sendData = ResponseMsg.getBytes("UTF-8"); sendData = ResponseMsg.getBytes("UTF-8");

View File

@ -27,7 +27,7 @@ import java.util.regex.Pattern;
public class Datahandler { public class Datahandler {
private ExecutorService pool = Executors.newFixedThreadPool(4); private ExecutorService pool = Executors.newFixedThreadPool(6);
private CompletionService completionService = new ExecutorCompletionService(pool); private CompletionService completionService = new ExecutorCompletionService(pool);
private HashMap<String, Annotation> pipelineAnnotationCache; private HashMap<String, Annotation> pipelineAnnotationCache;
private HashMap<String, Annotation> pipelineSentimentAnnotationCache; private HashMap<String, Annotation> pipelineSentimentAnnotationCache;
@ -139,7 +139,7 @@ public class Datahandler {
props.setProperty("parse.model", shiftReduceParserPath); props.setProperty("parse.model", shiftReduceParserPath);
props.setProperty("parse.maxlen", "90"); props.setProperty("parse.maxlen", "90");
props.setProperty("parse.binaryTrees", "true"); props.setProperty("parse.binaryTrees", "true");
props.setProperty("threads", "5"); props.setProperty("threads", "1");
props.setProperty("pos.maxlen", "90"); props.setProperty("pos.maxlen", "90");
props.setProperty("tokenize.maxlen", "90"); props.setProperty("tokenize.maxlen", "90");
props.setProperty("ssplit.maxlen", "90"); props.setProperty("ssplit.maxlen", "90");
@ -166,7 +166,7 @@ public class Datahandler {
propsSentiment.setProperty("parse.model", lexParserEnglishPCFG); propsSentiment.setProperty("parse.model", lexParserEnglishPCFG);
propsSentiment.setProperty("sentiment.model", sentimentModel); propsSentiment.setProperty("sentiment.model", sentimentModel);
propsSentiment.setProperty("parse.maxlen", "90"); propsSentiment.setProperty("parse.maxlen", "90");
propsSentiment.setProperty("threads", "5"); propsSentiment.setProperty("threads", "1");
propsSentiment.setProperty("pos.maxlen", "90"); propsSentiment.setProperty("pos.maxlen", "90");
propsSentiment.setProperty("tokenize.maxlen", "90"); propsSentiment.setProperty("tokenize.maxlen", "90");
propsSentiment.setProperty("ssplit.maxlen", "90"); propsSentiment.setProperty("ssplit.maxlen", "90");
@ -585,7 +585,6 @@ public class Datahandler {
for (String str1 : ues_copy) { for (String str1 : ues_copy) {
if (strF != str1) { if (strF != str1) {
//critical section //critical section
Future<SentimentAnalyzerTest> submit = completionService.submit(new get_res(strF, str1, stanfordCoreNLP, stanfordCoreNLPSentiment, Future<SentimentAnalyzerTest> submit = completionService.submit(new get_res(strF, str1, stanfordCoreNLP, stanfordCoreNLPSentiment,
coreMaps1, strAnno, strAnnoSentiment, coreDocument, tokenizeCountingF, taggedWordListF coreMaps1, strAnno, strAnnoSentiment, coreDocument, tokenizeCountingF, taggedWordListF

View File

@ -86,7 +86,7 @@ public class PipelineJMWESingleton {
propsJMWE = new Properties(); propsJMWE = new Properties();
propsJMWE.setProperty("annotators", "tokenize,ssplit,pos,lemma"); propsJMWE.setProperty("annotators", "tokenize,ssplit,pos,lemma");
propsJMWE.setProperty("tokenize.options", "untokenizable=firstKeep"); propsJMWE.setProperty("tokenize.options", "untokenizable=firstKeep");
propsJMWE.setProperty("threads", "5"); propsJMWE.setProperty("threads", "1");
propsJMWE.setProperty("pos.maxlen", "90"); propsJMWE.setProperty("pos.maxlen", "90");
propsJMWE.setProperty("tokenize.maxlen", "90"); propsJMWE.setProperty("tokenize.maxlen", "90");
propsJMWE.setProperty("ssplit.maxlen", "90"); propsJMWE.setProperty("ssplit.maxlen", "90");

View File

@ -66,17 +66,7 @@ public class DiscordHandler extends ListenerAdapter {
} catch (LoginException e) { } catch (LoginException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
new ThreadClient(datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment);
ArrayList<Integer> ports = new ArrayList<Integer>();
ports.add(48475);
ports.add(48476);
ports.add(48477);
ports.add(48478);
for (Integer port : ports) {
new Thread(() -> {
new ThreadClient(port, datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment);
}).start();
}
} }