reduced amount of strings and removed overuse of threads
This commit is contained in:
parent
a2309e7925
commit
e058788ad0
@ -93,7 +93,7 @@ public class DataMapper {
|
|||||||
ResultSet resultSet = l_pStatement.executeQuery();
|
ResultSet resultSet = l_pStatement.executeQuery();
|
||||||
if (resultSet.next()) {
|
if (resultSet.next()) {
|
||||||
int count = resultSet.getInt(1);
|
int count = resultSet.getInt(1);
|
||||||
if (count > 13000) {
|
if (count > 8000) {
|
||||||
//System.out.println("cleaning strings: " + l_sSQL);
|
//System.out.println("cleaning strings: " + l_sSQL);
|
||||||
l_pStatement = l_cCon.prepareStatement(l_sSQL);
|
l_pStatement = l_cCon.prepareStatement(l_sSQL);
|
||||||
l_pStatement.executeUpdate();
|
l_pStatement.executeUpdate();
|
||||||
|
@ -8,10 +8,17 @@ import java.io.FileNotFoundException;
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.UnsupportedEncodingException;
|
import java.io.UnsupportedEncodingException;
|
||||||
import java.net.*;
|
import java.net.*;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
|
|
||||||
public class ThreadClient {
|
public class ThreadClient {
|
||||||
public ThreadClient(int port, Datahandler datahandler, StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) {
|
public ThreadClient(Datahandler datahandler, StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) {
|
||||||
|
ArrayList<Integer> ports = new ArrayList<Integer>();
|
||||||
|
ports.add(48475);
|
||||||
|
ports.add(48476);
|
||||||
|
ports.add(48477);
|
||||||
|
ports.add(48478);
|
||||||
|
|
||||||
Properties prop = new Properties();
|
Properties prop = new Properties();
|
||||||
String fileName = "app.config";
|
String fileName = "app.config";
|
||||||
try (FileInputStream fis = new FileInputStream(fileName)) {
|
try (FileInputStream fis = new FileInputStream(fileName)) {
|
||||||
@ -21,33 +28,46 @@ public class ThreadClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
String hostIP = prop.getProperty("app.hostip");
|
String hostIP = prop.getProperty("app.hostip");
|
||||||
if (port == Integer.valueOf(prop.getProperty("app.hostport"))
|
String hostIP2 = prop.getProperty("app.hostip2");
|
||||||
|| port == Integer.valueOf(prop.getProperty("app.hostport2"))) {
|
|
||||||
hostIP = prop.getProperty("app.hostip2");
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
InetAddress ipAddress = InetAddress.getByName(hostIP);//used ip's
|
InetAddress ipAddress = InetAddress.getByName(hostIP);//used ip's
|
||||||
try (DatagramSocket serverSocket = new DatagramSocket(port)) {
|
InetAddress ipAddress2 = InetAddress.getByName(hostIP2);//used ip's
|
||||||
while (true) {
|
try (DatagramSocket serverSocket = new DatagramSocket(ports.get(0))) {
|
||||||
receiveAndSendPacket(serverSocket, ipAddress, port, datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment);
|
try (DatagramSocket serverSocket1 = new DatagramSocket(ports.get(1))) {
|
||||||
|
try (DatagramSocket serverSocket2 = new DatagramSocket(ports.get(2))) {
|
||||||
|
try (DatagramSocket serverSocket3 = new DatagramSocket(ports.get(3))) {
|
||||||
|
while (true) {
|
||||||
|
try {
|
||||||
|
receiveAndSendPacket(serverSocket, ipAddress, ports.get(0), datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment);
|
||||||
|
receiveAndSendPacket(serverSocket1, ipAddress, ports.get(1), datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment);
|
||||||
|
receiveAndSendPacket(serverSocket2, ipAddress2, ports.get(2), datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment);
|
||||||
|
receiveAndSendPacket(serverSocket3, ipAddress2, ports.get(3), datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} catch (SocketException e) {
|
} catch (SocketException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
} catch (IOException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
}
|
||||||
} catch (UnknownHostException e) {
|
} catch (UnknownHostException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private static void receiveAndSendPacket(DatagramSocket serverSocket, InetAddress ipAddress, int port,
|
private static void receiveAndSendPacket(DatagramSocket serverSocket, InetAddress ipAddress, int port,
|
||||||
Datahandler datahandler, StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) throws
|
Datahandler datahandler, StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) throws
|
||||||
IOException {
|
IOException {
|
||||||
byte[] receiveData = new byte[4096];
|
byte[] receiveData = new byte[4096];
|
||||||
DatagramPacket receivePacket = new DatagramPacket(receiveData, receiveData.length);
|
DatagramPacket receivePacket = new DatagramPacket(receiveData, receiveData.length);
|
||||||
try {
|
try {
|
||||||
|
/*
|
||||||
|
Only one DatagramSocket can call receive at a time since its a blocking call. yet somehow
|
||||||
|
the other DatagramSockets still get their UDP packets from receive() even if the call is made
|
||||||
|
many minutes after the actual UDP packet was sent. Maybe Security manager context?
|
||||||
|
*/
|
||||||
serverSocket.receive(receivePacket);
|
serverSocket.receive(receivePacket);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
@ -57,6 +77,7 @@ public class ThreadClient {
|
|||||||
sentence = sentence.replace("clientmessage:", "");
|
sentence = sentence.replace("clientmessage:", "");
|
||||||
String ResponseMsg = datahandler.getResponseMsg(sentence, "", stanfordCoreNLP, stanfordCoreNLPSentiment,
|
String ResponseMsg = datahandler.getResponseMsg(sentence, "", stanfordCoreNLP, stanfordCoreNLPSentiment,
|
||||||
true);
|
true);
|
||||||
|
System.out.println("port: " + port + ". ResponseMsg ingame: " + ResponseMsg);
|
||||||
byte[] sendData = new byte[0];
|
byte[] sendData = new byte[0];
|
||||||
try {
|
try {
|
||||||
sendData = ResponseMsg.getBytes("UTF-8");
|
sendData = ResponseMsg.getBytes("UTF-8");
|
||||||
|
@ -27,7 +27,7 @@ import java.util.regex.Pattern;
|
|||||||
|
|
||||||
public class Datahandler {
|
public class Datahandler {
|
||||||
|
|
||||||
private ExecutorService pool = Executors.newFixedThreadPool(4);
|
private ExecutorService pool = Executors.newFixedThreadPool(6);
|
||||||
private CompletionService completionService = new ExecutorCompletionService(pool);
|
private CompletionService completionService = new ExecutorCompletionService(pool);
|
||||||
private HashMap<String, Annotation> pipelineAnnotationCache;
|
private HashMap<String, Annotation> pipelineAnnotationCache;
|
||||||
private HashMap<String, Annotation> pipelineSentimentAnnotationCache;
|
private HashMap<String, Annotation> pipelineSentimentAnnotationCache;
|
||||||
@ -139,7 +139,7 @@ public class Datahandler {
|
|||||||
props.setProperty("parse.model", shiftReduceParserPath);
|
props.setProperty("parse.model", shiftReduceParserPath);
|
||||||
props.setProperty("parse.maxlen", "90");
|
props.setProperty("parse.maxlen", "90");
|
||||||
props.setProperty("parse.binaryTrees", "true");
|
props.setProperty("parse.binaryTrees", "true");
|
||||||
props.setProperty("threads", "5");
|
props.setProperty("threads", "1");
|
||||||
props.setProperty("pos.maxlen", "90");
|
props.setProperty("pos.maxlen", "90");
|
||||||
props.setProperty("tokenize.maxlen", "90");
|
props.setProperty("tokenize.maxlen", "90");
|
||||||
props.setProperty("ssplit.maxlen", "90");
|
props.setProperty("ssplit.maxlen", "90");
|
||||||
@ -166,7 +166,7 @@ public class Datahandler {
|
|||||||
propsSentiment.setProperty("parse.model", lexParserEnglishPCFG);
|
propsSentiment.setProperty("parse.model", lexParserEnglishPCFG);
|
||||||
propsSentiment.setProperty("sentiment.model", sentimentModel);
|
propsSentiment.setProperty("sentiment.model", sentimentModel);
|
||||||
propsSentiment.setProperty("parse.maxlen", "90");
|
propsSentiment.setProperty("parse.maxlen", "90");
|
||||||
propsSentiment.setProperty("threads", "5");
|
propsSentiment.setProperty("threads", "1");
|
||||||
propsSentiment.setProperty("pos.maxlen", "90");
|
propsSentiment.setProperty("pos.maxlen", "90");
|
||||||
propsSentiment.setProperty("tokenize.maxlen", "90");
|
propsSentiment.setProperty("tokenize.maxlen", "90");
|
||||||
propsSentiment.setProperty("ssplit.maxlen", "90");
|
propsSentiment.setProperty("ssplit.maxlen", "90");
|
||||||
@ -585,7 +585,6 @@ public class Datahandler {
|
|||||||
|
|
||||||
for (String str1 : ues_copy) {
|
for (String str1 : ues_copy) {
|
||||||
if (strF != str1) {
|
if (strF != str1) {
|
||||||
|
|
||||||
//critical section
|
//critical section
|
||||||
Future<SentimentAnalyzerTest> submit = completionService.submit(new get_res(strF, str1, stanfordCoreNLP, stanfordCoreNLPSentiment,
|
Future<SentimentAnalyzerTest> submit = completionService.submit(new get_res(strF, str1, stanfordCoreNLP, stanfordCoreNLPSentiment,
|
||||||
coreMaps1, strAnno, strAnnoSentiment, coreDocument, tokenizeCountingF, taggedWordListF
|
coreMaps1, strAnno, strAnnoSentiment, coreDocument, tokenizeCountingF, taggedWordListF
|
||||||
|
@ -86,7 +86,7 @@ public class PipelineJMWESingleton {
|
|||||||
propsJMWE = new Properties();
|
propsJMWE = new Properties();
|
||||||
propsJMWE.setProperty("annotators", "tokenize,ssplit,pos,lemma");
|
propsJMWE.setProperty("annotators", "tokenize,ssplit,pos,lemma");
|
||||||
propsJMWE.setProperty("tokenize.options", "untokenizable=firstKeep");
|
propsJMWE.setProperty("tokenize.options", "untokenizable=firstKeep");
|
||||||
propsJMWE.setProperty("threads", "5");
|
propsJMWE.setProperty("threads", "1");
|
||||||
propsJMWE.setProperty("pos.maxlen", "90");
|
propsJMWE.setProperty("pos.maxlen", "90");
|
||||||
propsJMWE.setProperty("tokenize.maxlen", "90");
|
propsJMWE.setProperty("tokenize.maxlen", "90");
|
||||||
propsJMWE.setProperty("ssplit.maxlen", "90");
|
propsJMWE.setProperty("ssplit.maxlen", "90");
|
||||||
|
@ -66,17 +66,7 @@ public class DiscordHandler extends ListenerAdapter {
|
|||||||
} catch (LoginException e) {
|
} catch (LoginException e) {
|
||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
|
new ThreadClient(datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment);
|
||||||
ArrayList<Integer> ports = new ArrayList<Integer>();
|
|
||||||
ports.add(48475);
|
|
||||||
ports.add(48476);
|
|
||||||
ports.add(48477);
|
|
||||||
ports.add(48478);
|
|
||||||
for (Integer port : ports) {
|
|
||||||
new Thread(() -> {
|
|
||||||
new ThreadClient(port, datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment);
|
|
||||||
}).start();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user