cleaning up code, added multi sentence support, build out caches, did other stuff i forgot
This commit is contained in:
jenzur
parent
f6da878ac0
commit
414806aa1f
@ -21,15 +21,20 @@ import edu.stanford.nlp.trees.GrammaticalStructureFactory;
|
||||
import edu.stanford.nlp.trees.TreebankLanguagePack;
|
||||
import java.io.IOException;
|
||||
import java.sql.SQLException;
|
||||
import java.util.AbstractMap;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import static java.util.Collections.reverseOrder;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import static java.util.Map.Entry.comparingByValue;
|
||||
import java.util.Properties;
|
||||
import java.util.Random;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
@ -40,6 +45,7 @@ import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import static java.util.stream.Collectors.toList;
|
||||
|
||||
/**
|
||||
*
|
||||
@ -55,6 +61,7 @@ public class Datahandler {
|
||||
private static Annotation strAnnoSentiment;
|
||||
private static Annotation strAnnoJMWE;
|
||||
private static CoreDocument coreDoc;
|
||||
private static ConcurrentMap<String, ConcurrentMap<String, Double>> sentenceRelationMap;
|
||||
private volatile boolean refreshMatrixFromDB;
|
||||
private static volatile int secondaryIterator = 0;
|
||||
private static volatile Double preRelationCounters = 0.0;
|
||||
@ -64,7 +71,8 @@ public class Datahandler {
|
||||
private static ConcurrentMap<String, Annotation> pipelineSentimentAnnotationCache;
|
||||
private static ConcurrentMap<String, Annotation> jmweAnnotationCache;
|
||||
private static ConcurrentMap<String, CoreDocument> coreDocumentAnnotationCache;
|
||||
private static ConcurrentMap<Integer, String> conversationMatchMap;
|
||||
private static ConcurrentMap<String, Integer> conversationMatchMap;
|
||||
private static ConcurrentMap<String, Integer> conversationUserMatchMap;
|
||||
private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap();
|
||||
private final Stopwatch stopwatch;
|
||||
private final Stopwatch stopwatch1;
|
||||
@ -99,6 +107,8 @@ public class Datahandler {
|
||||
this.pipelineSentimentAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
this.coreDocumentAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
this.conversationMatchMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
this.sentenceRelationMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
this.conversationUserMatchMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
}
|
||||
|
||||
public void shiftReduceParserInitiate() {
|
||||
@ -217,18 +227,15 @@ public class Datahandler {
|
||||
}
|
||||
|
||||
public void addHLstatsMessages() {
|
||||
ConcurrentMap<Integer, String> hlStatsMessages = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<String, Integer> hlStatsMessages = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strCacheLocal = stringCache;
|
||||
int hardcap = 10; //55000
|
||||
int ij = 0;
|
||||
for (String str : DataMapper.getHLstatsMessages().values()) {
|
||||
hlStatsMessages.put(ij, str);
|
||||
ij++;
|
||||
if (ij > hardcap) {
|
||||
break;
|
||||
Collection<String> strs = DataMapper.getHLstatsMessages().values();
|
||||
for (String str : strs) {
|
||||
if (hlStatsMessages.get(str) == null) {
|
||||
hlStatsMessages.put(str, hlStatsMessages.size());
|
||||
}
|
||||
}
|
||||
hlStatsMessages.values().parallelStream().forEach(str -> {
|
||||
hlStatsMessages.keySet().parallelStream().forEach(str -> {
|
||||
if (!str.startsWith("!")) {
|
||||
String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null);
|
||||
if (orElse == null) {
|
||||
@ -384,6 +391,13 @@ public class Datahandler {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* sentenceRelationMap only catches prior strF or already computed results
|
||||
* from same operation, so alot of times its null if msg from other channel
|
||||
*
|
||||
* @param strmap
|
||||
* @return
|
||||
*/
|
||||
public ConcurrentMap<Integer, String> removeNonSensicalStrings(ConcurrentMap<Integer, String> strmap) {
|
||||
ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
int relationCap = 20;
|
||||
@ -393,49 +407,72 @@ public class Datahandler {
|
||||
ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation = getMultiplePipelineSentimentAnnotation(strmap.values());
|
||||
ConcurrentMap<String, CoreDocument> localCoreDocumentMap = getMultipleCoreDocuments(strmap.values());
|
||||
for (String str : strmap.values()) {
|
||||
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, HashMap<String, String>> strsmaps = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (String str1 : strCacheLocal.values()) {
|
||||
HashMap HM1 = new HashMap();
|
||||
HM1.put(str, str1);
|
||||
if (!str.equals(str1) && !strsmaps.values().contains(HM1)) {
|
||||
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
|
||||
Callable<SimilarityMatrix> worker;
|
||||
if (stringCache.size() < 150) {
|
||||
worker = new SentimentAnalyzerTest(str, str1, SMX,
|
||||
localJMWEMap.get(str), localJMWEMap.get(str1), localPipelineAnnotation.get(str),
|
||||
localPipelineAnnotation.get(str1), localPipelineSentimentAnnotation.get(str),
|
||||
localPipelineSentimentAnnotation.get(str1), localCoreDocumentMap.get(str), localCoreDocumentMap.get(str1));
|
||||
} else {
|
||||
worker = new SentimentAnalyzerTest(str, str1, SMX,
|
||||
localJMWEMap.get(str), jmweAnnotationCache.get(str1), localPipelineAnnotation.get(str),
|
||||
pipelineAnnotationCache.get(str1), localPipelineSentimentAnnotation.get(str),
|
||||
pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1));
|
||||
}
|
||||
HashMap HM = new HashMap();
|
||||
HM.put(SMX.getPrimaryString(), SMX.getSecondaryString());
|
||||
strsmaps.put(strsmaps.size() + 1, HM);
|
||||
futures.put(futures.size() + 1, executor.submit(worker));
|
||||
}
|
||||
}
|
||||
int positiveRelationCounter = 0;
|
||||
for (Future<SimilarityMatrix> future : futures.values()) {
|
||||
try {
|
||||
SimilarityMatrix getSMX = future.get(5, TimeUnit.SECONDS);
|
||||
Double scoreRelationNewMsgToRecentMsg = getSMX.getDistance();
|
||||
int negativeRelationCounter = 0;
|
||||
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<String, Double> getMap = sentenceRelationMap.get(str);
|
||||
if (getMap == null) {
|
||||
ConcurrentMap<String, Double> mapUdate = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (String str1 : strCacheLocal.values()) {
|
||||
if (!str.equals(str1)) {
|
||||
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
|
||||
Callable<SimilarityMatrix> worker;
|
||||
if (stringCache.size() < 150) {
|
||||
worker = new SentimentAnalyzerTest(str, str1, SMX,
|
||||
localJMWEMap.get(str), localJMWEMap.get(str1), localPipelineAnnotation.get(str),
|
||||
localPipelineAnnotation.get(str1), localPipelineSentimentAnnotation.get(str),
|
||||
localPipelineSentimentAnnotation.get(str1), localCoreDocumentMap.get(str), localCoreDocumentMap.get(str1));
|
||||
futures.put(futures.size() + 1, executor.submit(worker));
|
||||
} else {
|
||||
worker = new SentimentAnalyzerTest(str, str1, SMX,
|
||||
localJMWEMap.get(str), jmweAnnotationCache.get(str1), localPipelineAnnotation.get(str),
|
||||
pipelineAnnotationCache.get(str1), localPipelineSentimentAnnotation.get(str),
|
||||
pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1));
|
||||
}
|
||||
futures.put(futures.size() + 1, executor.submit(worker));
|
||||
}
|
||||
}
|
||||
for (Future<SimilarityMatrix> future : futures.values()) {
|
||||
try {
|
||||
SimilarityMatrix getSMX = future.get(5, TimeUnit.SECONDS);
|
||||
Double scoreRelationNewMsgToRecentMsg = getSMX.getDistance();
|
||||
System.out.println("strmapreturn size: " + strmapreturn.size() + "\nscoreRelationNewMsgToRecentMsg: "
|
||||
+ scoreRelationNewMsgToRecentMsg + "\n");
|
||||
mapUdate.put(getSMX.getSecondaryString(), getSMX.getDistance());
|
||||
if (scoreRelationNewMsgToRecentMsg >= 5000.0) {
|
||||
positiveRelationCounter++;
|
||||
if (positiveRelationCounter > relationCap) {
|
||||
strmapreturn.put(strmapreturn.size() + 1, str);
|
||||
break;
|
||||
}
|
||||
} else if (scoreRelationNewMsgToRecentMsg <= -5000.0) {
|
||||
negativeRelationCounter++;
|
||||
if (negativeRelationCounter > relationCap * 2) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
|
||||
Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
sentenceRelationMap.put(str, mapUdate);
|
||||
} else {
|
||||
for (Entry<String, Double> mapValues : getMap.entrySet()) {
|
||||
Double scoreRelationNewMsgToRecentMsg = mapValues.getValue();
|
||||
System.out.println("strmapreturn size: " + strmapreturn.size() + "\nscoreRelationNewMsgToRecentMsg: "
|
||||
+ scoreRelationNewMsgToRecentMsg + "\n");
|
||||
if (scoreRelationNewMsgToRecentMsg >= 5000.0) {
|
||||
System.out.println("scoreRelationNewMsgToRecentMsg: " + scoreRelationNewMsgToRecentMsg + "\n");
|
||||
positiveRelationCounter++;
|
||||
if (positiveRelationCounter > relationCap) {
|
||||
strmapreturn.put(strmapreturn.size() + 1, str);
|
||||
break;
|
||||
}
|
||||
if (positiveRelationCounter > relationCap) {
|
||||
System.out.println("strmapreturn size: " + strmapreturn.size() + "\n");
|
||||
} else if (scoreRelationNewMsgToRecentMsg <= -5000.0) {
|
||||
negativeRelationCounter++;
|
||||
if (negativeRelationCounter > relationCap * 2) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
|
||||
Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -481,24 +518,37 @@ public class Datahandler {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param str is strF aka user message,
|
||||
* @param MostRecent String most recently responded with
|
||||
* @return
|
||||
* @throws CustomError
|
||||
*/
|
||||
public synchronized String getResponseMsg(String str, String MostRecent) throws CustomError {
|
||||
str = str.trim();
|
||||
if (str.startsWith("<@")) {
|
||||
str = str.substring(str.indexOf("> ") + 2);
|
||||
}
|
||||
SimilarityMatrix SMXreturn = new SimilarityMatrix("", "");
|
||||
ConcurrentMap<Integer, String> strCache = stringCache;
|
||||
ConcurrentMap<Integer, Future<SimilarityMatrix>> futureslocal = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, SimilarityMatrix> futurereturn = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, SimilarityMatrix> futureAndCacheCombineMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
String strF = str;
|
||||
getSingularAnnotation(strF);
|
||||
if (!conversationMatchMap.keySet().contains(MostRecent) && !MostRecent.isEmpty()) {
|
||||
conversationMatchMap.put(MostRecent, conversationMatchMap.size());
|
||||
}
|
||||
if (!conversationUserMatchMap.keySet().contains(strF)) {
|
||||
conversationUserMatchMap.put(strF, conversationUserMatchMap.size());
|
||||
}
|
||||
ConcurrentMap<String, Double> getPrimary = sentenceRelationMap.get(strF);
|
||||
strCache.values().parallelStream().forEach((str1) -> {
|
||||
if (!strF.equals(str1)) {
|
||||
boolean present = false;
|
||||
for (String strCons : conversationMatchMap.values()) {
|
||||
if (strCons.equals(str1)) {
|
||||
if (getPrimary != null) {
|
||||
Double getSecondary = getPrimary.get(str1);
|
||||
if (getSecondary != null) {
|
||||
present = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!present) {
|
||||
@ -511,44 +561,128 @@ public class Datahandler {
|
||||
}
|
||||
}
|
||||
});
|
||||
futureslocal.values().parallelStream().forEach((future) -> {
|
||||
futureslocal.values().forEach((future) -> {
|
||||
SimilarityMatrix SMX = new SimilarityMatrix("", "");
|
||||
try {
|
||||
SMX = future.get(5, TimeUnit.SECONDS);
|
||||
futurereturn.put(futurereturn.size() + 1, SMX);
|
||||
futureAndCacheCombineMap.put(futureAndCacheCombineMap.size(), SMX);
|
||||
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
|
||||
System.out.println("ex getResponsemsg: " + ex.getMessage() + "\n");
|
||||
}
|
||||
});
|
||||
preRelationCounters = 0.0;
|
||||
preRelationUserCounters = 0.0;
|
||||
conversationMatchMap.put(conversationMatchMap.size() + 1, MostRecent);
|
||||
Double scoreRelationNewMsgToRecentMsg = 0.0;
|
||||
for (String conversationStr : conversationMatchMap.values()) {
|
||||
scoreRelationNewMsgToRecentMsg += getScoreRelationNewMsgToRecentMsg(strF, conversationStr);
|
||||
if (getPrimary != null) {
|
||||
for (Entry<String, Double> cacheResults : getPrimary.entrySet()) {
|
||||
SimilarityMatrix SMX = new SimilarityMatrix(strF, cacheResults.getKey());
|
||||
SMX.setDistance(cacheResults.getValue());
|
||||
futureAndCacheCombineMap.put(futureAndCacheCombineMap.size(), SMX);
|
||||
}
|
||||
}
|
||||
futureAndCacheCombineMap.values().parallelStream().forEach((SMX) -> {
|
||||
if (sentenceRelationMap.get(strF) == null) {
|
||||
ConcurrentMap<String, Double> localMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
localMap.put(SMX.getSecondaryString(), SMX.getDistance());
|
||||
sentenceRelationMap.put(SMX.getPrimaryString(), localMap);
|
||||
} else {
|
||||
ConcurrentMap<String, Double> getPrimaryLocal = sentenceRelationMap.get(strF);
|
||||
Double doubleValue = getPrimaryLocal.get(SMX.getSecondaryString());
|
||||
if (doubleValue == null) {
|
||||
getPrimaryLocal.put(SMX.getSecondaryString(), SMX.getDistance());
|
||||
sentenceRelationMap.put(SMX.getPrimaryString(), getPrimaryLocal);
|
||||
}
|
||||
}
|
||||
});
|
||||
preRelationCounters = -100.0;
|
||||
preRelationUserCounters = -100.0;
|
||||
Double scoreRelationNewMsgToRecentMsg = 0.0;
|
||||
Double scoreRelationOldUserMsg = 0.0;
|
||||
ConcurrentMap<String, Double> getPrimaryLocal = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (String conversationStr : conversationMatchMap.keySet()) {
|
||||
getPrimaryLocal = sentenceRelationMap.get(strF);
|
||||
Double getSecondary = getPrimaryLocal.get(conversationStr);
|
||||
if (getSecondary == null) {
|
||||
getSecondary = getScoreRelationStrF(strF, conversationStr);
|
||||
getPrimaryLocal.put(conversationStr, getSecondary);
|
||||
sentenceRelationMap.put(strF, getPrimaryLocal);
|
||||
}
|
||||
scoreRelationNewMsgToRecentMsg += getSecondary;
|
||||
System.out.println("scoreRelationNewMsgToRecentMsg: " + scoreRelationNewMsgToRecentMsg + "\n");
|
||||
}
|
||||
for (String conversationUserStr : conversationUserMatchMap.keySet()) {
|
||||
if (!strF.equals(conversationUserStr)) {
|
||||
getPrimaryLocal = sentenceRelationMap.get(strF);
|
||||
Double getSecondary = getPrimaryLocal.get(conversationUserStr);
|
||||
if (getSecondary == null) {
|
||||
getSecondary = getScoreRelationStrF(strF, conversationUserStr);
|
||||
getPrimaryLocal.put(conversationUserStr, getSecondary);
|
||||
sentenceRelationMap.put(strF, getPrimaryLocal);
|
||||
}
|
||||
scoreRelationOldUserMsg += getSecondary;
|
||||
}
|
||||
}
|
||||
boolean userReponseRelated = scoreRelationOldUserMsg >= 250;
|
||||
boolean relatedReponse = scoreRelationNewMsgToRecentMsg >= 250;
|
||||
if (!userReponseRelated) {
|
||||
conversationUserMatchMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
}
|
||||
if (!relatedReponse) {
|
||||
conversationMatchMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
}
|
||||
for (SimilarityMatrix SMX : futurereturn.values()) {
|
||||
ConcurrentMap<Integer, Entry<Double, SimilarityMatrix>> concurrentRelationsMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Double> preRelationUserCountersMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (SimilarityMatrix SMX : futureAndCacheCombineMap.values()) {
|
||||
Double scoreRelation = 500.0;
|
||||
Double scoreRelationLastUserMsg = SMX.getDistance();
|
||||
if (relatedReponse) {
|
||||
for (String conversationStr : conversationMatchMap.values()) {
|
||||
scoreRelation += getScoreRelationNewMsgToRecentMsg(SMX.getSecondaryString(), conversationStr);
|
||||
for (String conversationStr : conversationMatchMap.keySet()) {
|
||||
Double relationNewMsg = 0.0;
|
||||
ConcurrentMap<String, Double> getMap = sentenceRelationMap.get(conversationStr);
|
||||
if (getMap != null) {
|
||||
Double getdoubleValue = getMap.get(conversationStr);
|
||||
if (getdoubleValue == null) {
|
||||
getdoubleValue = getScoreRelationNewMsgToRecentMsg(SMX.getSecondaryString(), conversationStr);
|
||||
getMap.put(conversationStr, getdoubleValue);
|
||||
sentenceRelationMap.put(SMX.getSecondaryString(), getMap);
|
||||
}
|
||||
relationNewMsg += getdoubleValue;
|
||||
} else {
|
||||
relationNewMsg = getScoreRelationNewMsgToRecentMsg(SMX.getSecondaryString(), conversationStr);
|
||||
ConcurrentMap<String, Double> localInnerMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
localInnerMap.put(conversationStr, relationNewMsg);
|
||||
sentenceRelationMap.put(SMX.getSecondaryString(), localInnerMap);
|
||||
}
|
||||
if (userReponseRelated) {
|
||||
relationNewMsg += scoreRelationOldUserMsg / conversationUserMatchMap.size();
|
||||
}
|
||||
scoreRelation += relationNewMsg;
|
||||
}
|
||||
}
|
||||
Double totalRelation = scoreRelation + scoreRelationLastUserMsg;
|
||||
if (totalRelation > preRelationCounters + preRelationUserCounters && scoreRelationLastUserMsg > preRelationUserCounters) {
|
||||
SMXreturn = SMX;
|
||||
if (totalRelation > preRelationCounters + preRelationUserCounters && scoreRelationLastUserMsg + (preRelationUserCounters / 10)
|
||||
> preRelationUserCounters) {
|
||||
Entry<Double, SimilarityMatrix> localEntry = new AbstractMap.SimpleEntry(totalRelation, SMX);
|
||||
concurrentRelationsMap.put(concurrentRelationsMap.size(), localEntry);
|
||||
preRelationUserCountersMap.put(preRelationUserCountersMap.size(), preRelationUserCounters);
|
||||
preRelationCounters = scoreRelation;
|
||||
preRelationUserCounters = scoreRelationLastUserMsg;
|
||||
}
|
||||
}
|
||||
System.out.println("Reached end: secondary: " + SMXreturn.getSecondaryString() + "\nPrimarY: " + SMXreturn.getPrimaryString()
|
||||
+ "\nScore: " + SMXreturn.getDistance());
|
||||
return SMXreturn.getSecondaryString();
|
||||
StringBuilder SB = new StringBuilder();
|
||||
int iterator = concurrentRelationsMap.size() - 1;
|
||||
System.out.println("iterator: " + iterator + "\n");
|
||||
while (iterator > -1) {
|
||||
Double preRelationUserCounterDouble = preRelationUserCountersMap.get(iterator);
|
||||
Entry<Double, SimilarityMatrix> getRelation = concurrentRelationsMap.get(iterator);
|
||||
Double result = preRelationUserCounterDouble + preRelationUserCounters;
|
||||
SB.append(getRelation.getValue().getSecondaryString()).append(" ");
|
||||
iterator--;
|
||||
System.out.println("result: " + result + "\ngetRelation.getKey(): " + getRelation.getKey() + "\npreRelationUserCounters: "
|
||||
+ preRelationUserCounters + "\npreRelationUserCounterDouble: " + preRelationUserCounterDouble + "\n");
|
||||
if (getRelation.getKey() * 2 < result) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
System.out.println("Reached end: SB: " + SB.toString() + "\n: ");
|
||||
return SB.toString();
|
||||
}
|
||||
|
||||
public void getSingularAnnotation(String str) {
|
||||
@ -621,6 +755,26 @@ public class Datahandler {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
private Double getScoreRelationStrF(String str, String mostRecentMsg) {
|
||||
SimilarityMatrix SMX = new SimilarityMatrix(str, mostRecentMsg);
|
||||
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, mostRecentMsg, SMX,
|
||||
strAnnoJMWE, jmweAnnotationCache.get(mostRecentMsg), strAnno,
|
||||
pipelineAnnotationCache.get(mostRecentMsg), strAnnoSentiment,
|
||||
pipelineSentimentAnnotationCache.get(mostRecentMsg), coreDoc, coreDocumentAnnotationCache.get(mostRecentMsg));
|
||||
SimilarityMatrix callSMX = null;
|
||||
try {
|
||||
callSMX = worker.call();
|
||||
} catch (Exception ex) {
|
||||
Logger.getLogger(Datahandler.class
|
||||
.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
if (callSMX != null) {
|
||||
double smxDistance = callSMX.getDistance();
|
||||
return smxDistance;
|
||||
}
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
public String mostSimilar(String toBeCompared, ConcurrentMap<Integer, String> concurrentStrings, String MostRecent) {
|
||||
similar = "";
|
||||
minDistance = 6.0;
|
||||
@ -825,6 +979,7 @@ public class Datahandler {
|
||||
if (stringCache.isEmpty()) {
|
||||
return str;
|
||||
}
|
||||
|
||||
Collection<String> values = stringCache.values();
|
||||
str.values().parallelStream().forEach(str1 -> {
|
||||
boolean tooclosematch = false;
|
||||
|
||||
@ -1,37 +0,0 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class DistanceObject {
|
||||
|
||||
private Integer distance;
|
||||
private String sentence;
|
||||
|
||||
public DistanceObject() {
|
||||
}
|
||||
|
||||
public Integer getDistance() {
|
||||
return distance;
|
||||
}
|
||||
|
||||
public void setDistance(Integer distance) {
|
||||
this.distance = distance;
|
||||
}
|
||||
|
||||
public String getSentence() {
|
||||
return sentence;
|
||||
}
|
||||
|
||||
|
||||
public DistanceObject(Integer distance, String sentence) {
|
||||
this.distance = distance;
|
||||
this.sentence = sentence;
|
||||
}
|
||||
}
|
||||
@ -1,733 +0,0 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer;
|
||||
|
||||
import DataLayer.DataMapper;
|
||||
import FunctionLayer.StanfordParser.SentimentAnalyzerTest;
|
||||
import com.google.common.base.Stopwatch;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import edu.stanford.nlp.ie.AbstractSequenceClassifier;
|
||||
import edu.stanford.nlp.ie.crf.CRFClassifier;
|
||||
import edu.stanford.nlp.ling.CoreLabel;
|
||||
import edu.stanford.nlp.ling.HasWord;
|
||||
import edu.stanford.nlp.ling.TaggedWord;
|
||||
import edu.stanford.nlp.ling.Word;
|
||||
import edu.stanford.nlp.parser.lexparser.LexicalizedParser;
|
||||
import edu.stanford.nlp.parser.shiftreduce.ShiftReduceParser;
|
||||
import edu.stanford.nlp.pipeline.Annotation;
|
||||
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
|
||||
import edu.stanford.nlp.process.DocumentPreprocessor;
|
||||
import edu.stanford.nlp.tagger.maxent.MaxentTagger;
|
||||
import edu.stanford.nlp.trees.GrammaticalStructureFactory;
|
||||
import edu.stanford.nlp.trees.Tree;
|
||||
import edu.stanford.nlp.trees.TreebankLanguagePack;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.sql.SQLException;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Properties;
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ForkJoinPool;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class MYSQLDatahandler {
|
||||
|
||||
public static final long EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(6, TimeUnit.MINUTES);
|
||||
public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS);
|
||||
public static MYSQLDatahandler instance = new MYSQLDatahandler();
|
||||
private volatile boolean refreshMatrixFromDB;
|
||||
private static int secondaryIterator = 0;
|
||||
private final ConcurrentMap<Integer, String> stringCache;
|
||||
private static ConcurrentMap<String, Annotation> pipelineAnnotationCache;
|
||||
private static ConcurrentMap<String, Annotation> pipelineSentimentAnnotationCache;
|
||||
private static ConcurrentMap<String, Annotation> jmweAnnotationCache;
|
||||
private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap();
|
||||
private final Stopwatch stopwatch;
|
||||
private final Stopwatch stopwatch1;
|
||||
private ForkJoinPool executor;
|
||||
private static String shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz";
|
||||
private static String sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz";
|
||||
private static String lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz";
|
||||
private static String taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger";
|
||||
private static String nerModel = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz";
|
||||
private static String jmweIndexData = "/home/javatests/lib/mweindex_wordnet3.0_semcor1.6.data"; // ./lib/mweindex_wordnet3.0_semcor1.6.data
|
||||
private static String jmweIndexDataLocalTest = "E:/java8/Projects/mweindex_wordnet3.0_semcor1.6.data";
|
||||
private static MaxentTagger tagger;
|
||||
private static ShiftReduceParser model;
|
||||
private static String[] options = {"-maxLength", "100"};
|
||||
private static Properties props = new Properties();
|
||||
private static Properties propsSentiment = new Properties();
|
||||
private static Properties propsJMWE = new Properties();
|
||||
private static GrammaticalStructureFactory gsf;
|
||||
private static LexicalizedParser lp;
|
||||
private static TreebankLanguagePack tlp;
|
||||
private static AbstractSequenceClassifier<CoreLabel> classifier;
|
||||
// set up Stanford CoreNLP pipeline
|
||||
private static StanfordCoreNLP pipeline;
|
||||
private static StanfordCoreNLP pipelineSentiment;
|
||||
private static StanfordCoreNLP pipelineJMWE;
|
||||
|
||||
public MYSQLDatahandler() {
|
||||
this.stopwatch = Stopwatch.createUnstarted();
|
||||
this.stopwatch1 = Stopwatch.createStarted();
|
||||
this.stringCache = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
//cant sadly just have one annotation for every pipelines, one pipeline per annotation is required
|
||||
this.jmweAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
this.pipelineAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
this.pipelineSentimentAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
}
|
||||
|
||||
public void shiftReduceParserInitiate() {
|
||||
try {
|
||||
classifier = CRFClassifier.getClassifierNoExceptions(nerModel);
|
||||
} catch (ClassCastException ex) {
|
||||
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
model = ShiftReduceParser.loadModel(shiftReduceParserPath, options);
|
||||
tagger = new MaxentTagger(taggerPath);
|
||||
lp = LexicalizedParser.loadModel(lexParserEnglishRNN, options);
|
||||
tlp = lp.getOp().langpack();
|
||||
gsf = tlp.grammaticalStructureFactory();
|
||||
// set up pipeline properties
|
||||
props.setProperty("parse.model", shiftReduceParserPath);
|
||||
props.setProperty("parse.maxlen", "100");
|
||||
props.setProperty("parse.binaryTrees", "true");
|
||||
props.setProperty("annotators", "tokenize,ssplit,pos,lemma,parse");
|
||||
propsSentiment.setProperty("parse.model", lexParserEnglishRNN);
|
||||
propsSentiment.setProperty("ner.model", nerModel);
|
||||
propsSentiment.setProperty("sentiment.model", sentimentModel);
|
||||
propsSentiment.setProperty("parse.maxlen", "100");
|
||||
propsSentiment.setProperty("annotators", "tokenize,ssplit,pos,parse,depparse,sentiment"); //coref too expensive memorywise
|
||||
propsJMWE.setProperty("customAnnotatorClass.jmwe", "edu.stanford.nlp.pipeline.JMWEAnnotator");
|
||||
propsJMWE.setProperty("customAnnotatorClass.jmwe.verbose", "false");
|
||||
propsJMWE.setProperty("customAnnotatorClass.jmwe.underscoreReplacement", "-");
|
||||
propsJMWE.setProperty("customAnnotatorClass.jmwe.indexData", jmweIndexData); //jmweIndexDataLocalTest jmweIndexData
|
||||
propsJMWE.setProperty("customAnnotatorClass.jmwe.detector", "Exhaustive");
|
||||
//"Consecutive", "Exhaustive", "ProperNouns", "Complex" and "CompositeConsecutiveProperNouns"
|
||||
propsJMWE.setProperty("annotators", "tokenize, ssplit, pos, lemma, jmwe");
|
||||
// set up pipeline
|
||||
pipeline = new StanfordCoreNLP(props);
|
||||
pipelineSentiment = new StanfordCoreNLP(propsSentiment);
|
||||
pipelineJMWE = new StanfordCoreNLP(propsJMWE);
|
||||
}
|
||||
|
||||
public static AbstractSequenceClassifier<CoreLabel> getClassifier() {
|
||||
return classifier;
|
||||
}
|
||||
|
||||
public static void setClassifier(AbstractSequenceClassifier<CoreLabel> classifier) {
|
||||
MYSQLDatahandler.classifier = classifier;
|
||||
}
|
||||
|
||||
public void updateStringCache() {
|
||||
try {
|
||||
checkIfUpdateStrings();
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
public void instantiateExecutor() {
|
||||
this.executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(),
|
||||
ForkJoinPool.defaultForkJoinWorkerThreadFactory,
|
||||
null, false); //true
|
||||
}
|
||||
|
||||
public static GrammaticalStructureFactory getGsf() {
|
||||
return gsf;
|
||||
}
|
||||
|
||||
public static StanfordCoreNLP getPipeline() {
|
||||
return pipeline;
|
||||
}
|
||||
|
||||
public static StanfordCoreNLP getPipelineSentiment() {
|
||||
return pipelineSentiment;
|
||||
}
|
||||
|
||||
public static MaxentTagger getTagger() {
|
||||
return tagger;
|
||||
}
|
||||
|
||||
public static ShiftReduceParser getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
private Map<Integer, String> getCache() throws SQLException, IOException, CustomError {
|
||||
return DataMapper.getAllStrings();
|
||||
}
|
||||
|
||||
public int getlHMSMXSize() {
|
||||
return lHMSMX.size();
|
||||
}
|
||||
|
||||
public int getstringCacheSize() {
|
||||
return stringCache.size();
|
||||
}
|
||||
|
||||
public void initiateMYSQL() throws SQLException, IOException {
|
||||
try {
|
||||
DataMapper.createTables();
|
||||
stringCache.putAll(getCache());
|
||||
addHLstatsMessages();
|
||||
lHMSMX = DataMapper.getAllRelationScores();
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(MYSQLDatahandler.class
|
||||
.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void addHLstatsMessages() {
|
||||
ConcurrentMap<Integer, String> hlStatsMessages = DataMapper.getHLstatsMessages();
|
||||
ConcurrentMap<Integer, String> strCacheLocal = stringCache;
|
||||
int hardcap = 300;
|
||||
int counter = 0;
|
||||
for (String str : hlStatsMessages.values()) {
|
||||
if (!str.startsWith("!")) {
|
||||
boolean present = false;
|
||||
for (String str1 : strCacheLocal.values()) {
|
||||
if (str.equals(str1)) {
|
||||
present = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!present) {
|
||||
MessageResponseHandler.getMessage(str);
|
||||
}
|
||||
}
|
||||
if (counter >= hardcap) {
|
||||
break;
|
||||
}
|
||||
counter++;
|
||||
}
|
||||
}
|
||||
|
||||
public void instantiateAnnotationMap() {
|
||||
for (String str : stringCache.values()) {
|
||||
System.out.println("str annotation jmwe: " + str + "\n");
|
||||
Annotation strAnno = new Annotation(str);
|
||||
pipelineJMWE.annotate(strAnno);
|
||||
jmweAnnotationCache.put(str, strAnno);
|
||||
}
|
||||
for (String str : stringCache.values()) {
|
||||
System.out.println("str annotation pipeline: " + str + "\n");
|
||||
Annotation strAnno = new Annotation(str);
|
||||
pipeline.annotate(strAnno);
|
||||
pipelineAnnotationCache.put(str, strAnno);
|
||||
}
|
||||
for (String str : stringCache.values()) {
|
||||
System.out.println("str annotation pipelinesentiment: " + str + "\n");
|
||||
Annotation strAnno = new Annotation(str);
|
||||
pipelineSentiment.annotate(strAnno);
|
||||
pipelineSentimentAnnotationCache.put(str, strAnno);
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void checkIfUpdateMatrixes() {
|
||||
refreshMatrixFromDB = false;
|
||||
int counter = 0;
|
||||
if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) {
|
||||
refreshMatrixFromDB = true;
|
||||
lHMSMX = DataMapper.getAllRelationScores();
|
||||
stopwatch1.reset();
|
||||
}
|
||||
//requiring atleast 10 entries ensures no issues in case of empty stringcache
|
||||
if (stringCache.values().size() > 10 && !refreshMatrixFromDB) {
|
||||
ConcurrentMap<Integer, String> stringCachelocal = stringCache;
|
||||
int selectUpdate = -1;
|
||||
LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
|
||||
int ij2 = 0;
|
||||
for (String str : stringCachelocal.values()) {
|
||||
boolean updatepresent = false;
|
||||
for (String strlocal : LHMSMXLocal.keySet()) {
|
||||
if (strlocal.equals(str)) {
|
||||
updatepresent = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!updatepresent) {
|
||||
selectUpdate = ij2;
|
||||
break;
|
||||
}
|
||||
ij2++;
|
||||
}
|
||||
if (selectUpdate == -1 || selectUpdate + 1 == stringCachelocal.size()) {
|
||||
int valueSize = stringCachelocal.size();
|
||||
if (secondaryIterator + 1 >= valueSize) {
|
||||
secondaryIterator = 0;
|
||||
}
|
||||
selectUpdate = secondaryIterator;
|
||||
secondaryIterator++;
|
||||
}
|
||||
ConcurrentMap<Integer, String> strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
String get = stringCachelocal.getOrDefault(selectUpdate, null);
|
||||
if (get == null) {
|
||||
get = stringCachelocal.get(new Random().nextInt(stringCachelocal.size() - 1));
|
||||
}
|
||||
strIndexNavigator.put(0, get);
|
||||
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
strIndexNavigator.values().forEach((str) -> {
|
||||
stringCachelocal.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> {
|
||||
boolean present = false;
|
||||
LinkedHashMap<String, Double> orDefault = lHMSMX.getOrDefault(str, null);
|
||||
if (orDefault != null) {
|
||||
for (String strkey : orDefault.keySet()) {
|
||||
if (strkey.equals(str1)) {
|
||||
present = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!present) {
|
||||
orDefault = lHMSMX.getOrDefault(str1, null);
|
||||
if (orDefault != null) {
|
||||
for (String strkey : orDefault.keySet()) {
|
||||
if (strkey.equals(str)) {
|
||||
present = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!present) {
|
||||
LinkedHashMap<String, Double> orDefault1 = lHMSMX.getOrDefault(str, null);
|
||||
if (orDefault1 == null) {
|
||||
orDefault1 = new LinkedHashMap<String, Double>();
|
||||
}
|
||||
orDefault1.put(str1, 0.0);
|
||||
lHMSMX.put(str, orDefault1);
|
||||
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
|
||||
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX, jmweAnnotationCache.get(str),
|
||||
jmweAnnotationCache.get(str1), pipelineAnnotationCache.get(str), pipelineAnnotationCache.get(str1),
|
||||
pipelineSentimentAnnotationCache.get(str), pipelineSentimentAnnotationCache.get(str1));
|
||||
futures.put(futures.size() + 1, executor.submit(worker));
|
||||
}
|
||||
});
|
||||
});
|
||||
System.out.println("finished worker assignment, futures size: " + futures.size() + "\n");
|
||||
for (Future<SimilarityMatrix> future : futures.values()) {
|
||||
SimilarityMatrix SMX = new SimilarityMatrix("", "");
|
||||
try {
|
||||
SMX = future.get(20, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
|
||||
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
SMX = null;
|
||||
}
|
||||
System.out.println("counter: " + counter + "\nSMX: " + SMX == null ? "SMX null\n" : "SMX not null\n");
|
||||
counter++;
|
||||
if (SMX != null) {
|
||||
LinkedHashMap<String, Double> getFuture = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
|
||||
getFuture.put(SMX.getSecondaryString(), SMX.getDistance());
|
||||
lHMSMX.put(SMX.getPrimaryString(), getFuture);
|
||||
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (!matrixUpdateList.isEmpty()) {
|
||||
DataMapper.insertSementicMatrixes(matrixUpdateList);
|
||||
System.out.println("finished datamapper semetic insert");
|
||||
}
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(MYSQLDatahandler.class
|
||||
.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void checkIfUpdateStrings() throws CustomError {
|
||||
if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
|
||||
ConcurrentMap<Integer, String> str = MessageResponseHandler.getStr();
|
||||
str = cutContent(str);
|
||||
str = filterContent(str);
|
||||
str = removeSlacks(str);
|
||||
str = verifyCalculationFitness(str);
|
||||
try {
|
||||
DataMapper.InsertMYSQLStrings(str);
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(MYSQLDatahandler.class
|
||||
.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
MessageResponseHandler.setStr(new MapMaker().concurrencyLevel(2).makeMap());
|
||||
int j = stringCache.size() + 1;
|
||||
for (String str1 : str.values()) {
|
||||
stringCache.put(j, str1);
|
||||
j++;
|
||||
Annotation strAnno = new Annotation(str1);
|
||||
pipelineJMWE.annotate(strAnno);
|
||||
jmweAnnotationCache.put(str1, strAnno);
|
||||
strAnno = new Annotation(str1);
|
||||
pipeline.annotate(strAnno);
|
||||
pipelineAnnotationCache.put(str1, strAnno);
|
||||
strAnno = new Annotation(str1);
|
||||
pipelineSentiment.annotate(strAnno);
|
||||
pipelineSentimentAnnotationCache.put(str1, strAnno);
|
||||
}
|
||||
if (!stopwatch.isRunning()) {
|
||||
stopwatch.start();
|
||||
} else {
|
||||
stopwatch.reset();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized String getResponseMsg(String str) throws CustomError {
|
||||
str = str.trim();
|
||||
if (str.startsWith("<@")) {
|
||||
str = str.substring(str.indexOf("> ") + 2);
|
||||
}
|
||||
final LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
|
||||
ConcurrentMap<Integer, String> strArrs = stringCache;
|
||||
double Score = -10000;
|
||||
SimilarityMatrix SMXreturn = new SimilarityMatrix("", "");
|
||||
System.out.println("pre mostSimilarSTR \n");
|
||||
String mostSimilarSTR = mostSimilar(str, strArrs);
|
||||
if (mostSimilarSTR != null) {
|
||||
System.out.println("mostSimilarSTR; " + mostSimilarSTR + "\n");
|
||||
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.getOrDefault(mostSimilarSTR, null);
|
||||
if (orDefault != null) {
|
||||
for (Entry<String, Double> entrySet : orDefault.entrySet()) {
|
||||
double smxDistance = entrySet.getValue();
|
||||
if (smxDistance > Score) {
|
||||
Score = smxDistance;
|
||||
SMXreturn = new SimilarityMatrix(mostSimilarSTR, entrySet.getKey(), smxDistance);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (Entry<String, LinkedHashMap<String, Double>> values1 : LHMSMXLocal.entrySet()) {
|
||||
LinkedHashMap<String, Double> value = values1.getValue();
|
||||
for (Entry<String, Double> keystr : value.entrySet()) {
|
||||
if (keystr.getKey().equals(mostSimilarSTR)) {
|
||||
double smxDistance = keystr.getValue();
|
||||
if (smxDistance > Score) {
|
||||
Score = smxDistance;
|
||||
SMXreturn = new SimilarityMatrix(values1.getKey(), keystr.getKey(), smxDistance);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!SMXreturn.getPrimaryString().isEmpty()) {
|
||||
if (SMXreturn.getPrimaryString().equals(mostSimilarSTR)) {
|
||||
return SMXreturn.getSecondaryString();
|
||||
} else {
|
||||
return SMXreturn.getPrimaryString();
|
||||
}
|
||||
}
|
||||
}
|
||||
System.out.println("none within 8 range");
|
||||
ConcurrentMap<Integer, String> strCache = stringCache;
|
||||
ConcurrentMap<Integer, Future<SimilarityMatrix>> futureslocal = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (String str1 : strCache.values()) {
|
||||
if (!str.equals(str1)) {
|
||||
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
|
||||
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX,
|
||||
jmweAnnotationCache.get(str), jmweAnnotationCache.get(str1), pipelineAnnotationCache.get(str),
|
||||
pipelineAnnotationCache.get(str1), pipelineSentimentAnnotationCache.get(str),
|
||||
pipelineSentimentAnnotationCache.get(str1));
|
||||
futureslocal.put(futureslocal.size() + 1, executor.submit(worker));
|
||||
}
|
||||
}
|
||||
int index = 0;
|
||||
for (Future<SimilarityMatrix> future : futureslocal.values()) {
|
||||
SimilarityMatrix SMX = new SimilarityMatrix("", "");
|
||||
try {
|
||||
SMX = future.get(20, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
|
||||
System.out.println("ex getResponsemsg: " + ex.getMessage() + "\n");
|
||||
SMX = null;
|
||||
}
|
||||
if (SMX != null) {
|
||||
double distance = SMX.getDistance();
|
||||
System.out.println("index: " + index + "\nfutures size: " + futureslocal.values().size() + "\nScore: " + SMX.getDistance() + "\nSecondary: "
|
||||
+ SMX.getSecondaryString() + "\nPrimary: " + SMX.getPrimaryString() + "\n");
|
||||
if (distance > Score) {
|
||||
Score = distance;
|
||||
SMXreturn = SMX;
|
||||
}
|
||||
}
|
||||
index++;
|
||||
}
|
||||
System.out.println("Reached end: secondary: " + SMXreturn.getSecondaryString() + "\nPrimarY: " + SMXreturn.getPrimaryString()
|
||||
+ "\nScore: " + SMXreturn.getDistance());
|
||||
return SMXreturn.getSecondaryString();
|
||||
}
|
||||
|
||||
public String mostSimilar(String toBeCompared, ConcurrentMap<Integer, String> concurrentStrings) {
|
||||
int minDistance = 8;
|
||||
String similar = "";
|
||||
ConcurrentMap<Integer, Future<Entry<String, Integer>>> futures = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (String str : concurrentStrings.values()) {
|
||||
Callable<Entry<String, Integer>> worker = new LevenshteinDistance(toBeCompared, str);
|
||||
futures.put(futures.size() + 1, executor.submit(worker));
|
||||
}
|
||||
for (Future<Entry<String, Integer>> future : futures.values()) {
|
||||
try {
|
||||
Entry<String, Integer> futureEntry = future.get();
|
||||
int distance = futureEntry.getValue();
|
||||
if (distance < minDistance) {
|
||||
System.out.println("distance: " + distance + "\n");
|
||||
minDistance = distance;
|
||||
similar = futureEntry.getKey();
|
||||
}
|
||||
} catch (NullPointerException | InterruptedException | ExecutionException ex) {
|
||||
System.out.println("failed future\n");
|
||||
}
|
||||
}
|
||||
return similar;
|
||||
}
|
||||
|
||||
public static ConcurrentMap<Integer, String> cutContent(ConcurrentMap<Integer, String> str) {
|
||||
ConcurrentMap<Integer, String> returnlist = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (String str1 : str.values()) {
|
||||
int iend = str1.indexOf("content: ");
|
||||
if (iend != -1) {
|
||||
String trs = str1.substring(iend + 9);
|
||||
returnlist.put(returnlist.size() + 1, trs.substring(0, trs.length() - 1));
|
||||
}
|
||||
}
|
||||
return returnlist;
|
||||
}
|
||||
|
||||
public static ConcurrentMap<Integer, String> filterContent(ConcurrentMap<Integer, String> str) {
|
||||
ConcurrentMap<Integer, String> strlistreturn = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (String str1 : str.values()) {
|
||||
if (str1.isEmpty() || str1.length() < 3) {
|
||||
continue;
|
||||
}
|
||||
str1 = str1.trim();
|
||||
if (str1.contains("PM*")) {
|
||||
str1 = str1.substring(str1.indexOf("PM*") + 5);
|
||||
}
|
||||
if (str1.contains("AM*")) {
|
||||
str1 = str1.substring(str1.indexOf("AM*") + 5);
|
||||
}
|
||||
for (Character c : str1.toCharArray()) {
|
||||
if (c == '?' || c == '°') {
|
||||
str1 = str1.replace("?", " <:wlenny:514861023002624001> ");
|
||||
str1 = str1.replace("°", " <:wlenny:514861023002624001> ");
|
||||
}
|
||||
}
|
||||
if (str1.contains("(Counter-Terrorist)")) {
|
||||
str1 = str1.replace("(Counter-Terrorist)", " ");
|
||||
}
|
||||
if (str1.contains("(Terrorist)")) {
|
||||
str1 = str1.replace("(Terrorist)", " ");
|
||||
}
|
||||
if (str1.contains("(Spectator)")) {
|
||||
str1 = str1.replace("(Spectator)", " ");
|
||||
}
|
||||
if (str1.contains("*DEAD*")) {
|
||||
str1 = str1.replace("*DEAD*", " ");
|
||||
}
|
||||
if (str1.contains("{red}")) {
|
||||
str1 = str1.replace("{red}", " ");
|
||||
}
|
||||
if (str1.contains("{orange}")) {
|
||||
str1 = str1.replace("{orange}", " ");
|
||||
}
|
||||
if (str1.contains("{yellow}")) {
|
||||
str1 = str1.replace("{yellow}", " ");
|
||||
}
|
||||
if (str1.contains("{green}")) {
|
||||
str1 = str1.replace("{green}", " ");
|
||||
}
|
||||
if (str1.contains("{lightblue}")) {
|
||||
str1 = str1.replace("{lightblue}", " ");
|
||||
}
|
||||
if (str1.contains("{blue}")) {
|
||||
str1 = str1.replace("{blue}", " ");
|
||||
}
|
||||
if (str1.contains("{purple}")) {
|
||||
str1 = str1.replace("{purple}", " ");
|
||||
}
|
||||
if (str1.contains("{white}")) {
|
||||
str1 = str1.replace("{white}", " ");
|
||||
}
|
||||
if (str1.contains("{fullblue}")) {
|
||||
str1 = str1.replace("{fullblue}", " ");
|
||||
}
|
||||
if (str1.contains("{cyan}")) {
|
||||
str1 = str1.replace("{cyan}", " ");
|
||||
}
|
||||
if (str1.contains("{lime}")) {
|
||||
str1 = str1.replace("{lime}", " ");
|
||||
}
|
||||
if (str1.contains("{deeppink}")) {
|
||||
str1 = str1.replace("{deeppink}", " ");
|
||||
}
|
||||
if (str1.contains("{slategray}")) {
|
||||
str1 = str1.replace("{slategray}", " ");
|
||||
}
|
||||
if (str1.contains("{dodgerblue}")) {
|
||||
str1 = str1.replace("{dodgerblue}", " ");
|
||||
}
|
||||
if (str1.contains("{black}")) {
|
||||
str1 = str1.replace("{black}", " ");
|
||||
}
|
||||
if (str1.contains("{orangered}")) {
|
||||
str1 = str1.replace("{orangered}", " ");
|
||||
}
|
||||
if (str1.contains("{darkorchid}")) {
|
||||
str1 = str1.replace("{darkorchid}", " ");
|
||||
}
|
||||
if (str1.contains("{pink}")) {
|
||||
str1 = str1.replace("{pink}", " ");
|
||||
}
|
||||
if (str1.contains("{lightyellow}")) {
|
||||
str1 = str1.replace("{lightyellow}", " ");
|
||||
}
|
||||
if (str1.contains("{chocolate}")) {
|
||||
str1 = str1.replace("{chocolate}", " ");
|
||||
}
|
||||
if (str1.contains("{beige}")) {
|
||||
str1 = str1.replace("{beige}", " ");
|
||||
}
|
||||
if (str1.contains("{azure}")) {
|
||||
str1 = str1.replace("{azure}", " ");
|
||||
}
|
||||
if (str1.contains("{yellowgreen}")) {
|
||||
str1 = str1.replace("{yellowgreen}", " ");
|
||||
}
|
||||
str1 = str1.trim();
|
||||
if (str1.length() > 2 && (!str1.startsWith("!"))) {
|
||||
strlistreturn.put(strlistreturn.size() + 1, str1);
|
||||
}
|
||||
}
|
||||
return strlistreturn;
|
||||
}
|
||||
|
||||
private ConcurrentMap<Integer, String> removeSlacks(ConcurrentMap<Integer, String> str) {
|
||||
ShiftReduceParser model = getModel();
|
||||
MaxentTagger tagger = getTagger();
|
||||
List<TaggedWord> taggedWords;
|
||||
ConcurrentMap<Integer, String> strreturn = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (String str1 : str.values()) {
|
||||
int counter = 0;
|
||||
ConcurrentMap<Integer, String> TGWList = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1));
|
||||
for (List<HasWord> sentence : tokenizer) {
|
||||
List<TaggedWord> tagged1 = tagger.tagSentence(sentence);
|
||||
Tree tree = model.apply(tagged1);
|
||||
taggedWords = tree.taggedYield();
|
||||
for (TaggedWord TGW : taggedWords) {
|
||||
if (!TGWList.values().contains(TGW.tag()) && !TGW.tag().equals(":") && !TGW.word().equals(TGW.tag())) {
|
||||
TGWList.put(TGWList.size() + 1, TGW.tag());
|
||||
counter++;
|
||||
}
|
||||
if (counter > 3) {
|
||||
int addCounter = 0;
|
||||
ConcurrentMap<Integer, Word> wordList = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (Word lab : tree.yieldWords()) {
|
||||
if (lab != null && lab.word() != null) {
|
||||
//System.out.println("lab: " + lab + " \n");
|
||||
if (!wordList.values().contains(lab) && lab.value() != null && !lab.value().equals(":")) {
|
||||
wordList.put(wordList.size() + 1, lab);
|
||||
addCounter++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (addCounter > 3) {
|
||||
addCounter = 0;
|
||||
ConcurrentMap<Integer, HasWord> HWlist = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (HasWord HW : tree.yieldHasWord()) {
|
||||
if (HW != null && HW.word() != null && !HWlist.values().contains(HW)) {
|
||||
addCounter++;
|
||||
HWlist.put(HWlist.size() + 1, HW);
|
||||
}
|
||||
}
|
||||
if (addCounter > 3) {
|
||||
boolean tooclosematch = false;
|
||||
Collection<String> values = stringCache.values();
|
||||
for (String strVals : values) {
|
||||
LevenshteinDistance leven = new LevenshteinDistance(strVals, str1);
|
||||
double Distance = leven.computeLevenshteinDistance();
|
||||
int maxpermittedDistance = 2;
|
||||
if (Distance < maxpermittedDistance) {
|
||||
tooclosematch = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!tooclosematch) {
|
||||
strreturn.put(strreturn.size() + 1, str1);
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (counter > 3) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return strreturn;
|
||||
}
|
||||
|
||||
private ConcurrentMap<Integer, String> verifyCalculationFitness(ConcurrentMap<Integer, String> strmap) {
|
||||
ConcurrentMap<Integer, String> returnmap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> allStrings = stringCache;
|
||||
int intervalAprove = allStrings.values().size() / 10;
|
||||
for (String str : strmap.values()) {
|
||||
int counter = 0;
|
||||
boolean calculationIssues = false;
|
||||
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
int ij2 = 0;
|
||||
if (allStrings.isEmpty()) {
|
||||
allStrings = strmap;
|
||||
}
|
||||
for (String strValues : allStrings.values()) {
|
||||
if (ij2 > intervalAprove) {
|
||||
break;
|
||||
}
|
||||
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, strValues, new SimilarityMatrix(str, strValues),
|
||||
jmweAnnotationCache.get(str), jmweAnnotationCache.get(strValues), pipelineAnnotationCache.get(str),
|
||||
pipelineAnnotationCache.get(strValues), pipelineSentimentAnnotationCache.get(str),
|
||||
pipelineSentimentAnnotationCache.get(strValues));
|
||||
futures.put(futures.size() + 1, executor.submit(worker));
|
||||
ij2++;
|
||||
}
|
||||
for (Future<SimilarityMatrix> future : futures.values()) {
|
||||
try {
|
||||
SimilarityMatrix get = future.get(20, TimeUnit.SECONDS);
|
||||
if (get == null) {
|
||||
System.out.println("counter timeouts: " + counter + "\n");
|
||||
counter++;
|
||||
if (counter >= 10) {
|
||||
calculationIssues = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
|
||||
System.out.println("counter timeouts: " + counter + "\n");
|
||||
counter++;
|
||||
if (counter >= 10) {
|
||||
calculationIssues = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!calculationIssues) {
|
||||
returnmap.put(returnmap.size() + 1, str);
|
||||
}
|
||||
}
|
||||
return returnmap;
|
||||
}
|
||||
}
|
||||
@ -52,20 +52,45 @@ public class MessageResponseHandler {
|
||||
}
|
||||
}
|
||||
String getResponseMsg = Datahandler.instance.getResponseMsg(strreturn, mostRecentMsg);
|
||||
getResponseMsg = checkPersonPresnetInSentence(personName, getResponseMsg);
|
||||
getResponseMsg = checkPersonPresentInSentence(personName, getResponseMsg, strreturn);
|
||||
return getResponseMsg;
|
||||
}
|
||||
|
||||
private static String checkPersonPresnetInSentence(String personName, String responseMsg) {
|
||||
private static String checkPersonPresentInSentence(String personName, String responseMsg, String userLastMessage) {
|
||||
//check if userlastmsg contains person as refference
|
||||
//check if first person is author or their person of mention
|
||||
String strreturn = responseMsg;
|
||||
CoreDocument pipelineCoreDcoument = new CoreDocument(responseMsg);
|
||||
CoreDocument pipelineCoreDcoumentLastMsg = new CoreDocument(userLastMessage);
|
||||
Datahandler.getPipeline().annotate(pipelineCoreDcoument);
|
||||
for (CoreEntityMention em : pipelineCoreDcoument.entityMentions()) {
|
||||
String entityType = em.entityType();
|
||||
if (entityType.equals("PERSON")) {
|
||||
String replace = strreturn.replaceFirst(em.text(), personName);
|
||||
return replace;
|
||||
Datahandler.getPipeline().annotate(pipelineCoreDcoumentLastMsg);
|
||||
try {
|
||||
for (CoreEntityMention em : pipelineCoreDcoument.entityMentions()) {
|
||||
String entityType = em.entityType();
|
||||
if (entityType.equals("PERSON")) {
|
||||
try {
|
||||
String str = strreturn;
|
||||
String emText = em.text();
|
||||
if (!emText.equals(personName)) {
|
||||
for (CoreEntityMention emLastMsg : pipelineCoreDcoumentLastMsg.entityMentions()) {
|
||||
if (!emText.equals(emLastMsg.text())) {
|
||||
str = strreturn.replaceFirst(emText, emLastMsg.text());
|
||||
}
|
||||
}
|
||||
try {
|
||||
str = str.replace(str.substring(str.lastIndexOf(emText), str.length()), personName);
|
||||
} catch (Exception e) {
|
||||
System.out.println("failed replacing: " + e.getLocalizedMessage() + "\n");
|
||||
}
|
||||
return str;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.out.println("failed person replacement: " + e.getLocalizedMessage() + "\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.out.println("SCUFFED JAYZ: " + e.getLocalizedMessage() + "\n");
|
||||
}
|
||||
return responseMsg;
|
||||
}
|
||||
|
||||
@ -1,114 +0,0 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer.StanfordParser;
|
||||
|
||||
import edu.cmu.lti.lexical_db.ILexicalDatabase;
|
||||
import edu.cmu.lti.lexical_db.NictWordNet;
|
||||
import edu.cmu.lti.ws4j.RelatednessCalculator;
|
||||
import edu.cmu.lti.ws4j.impl.HirstStOnge;
|
||||
import edu.cmu.lti.ws4j.impl.JiangConrath;
|
||||
import edu.cmu.lti.ws4j.impl.LeacockChodorow;
|
||||
import edu.cmu.lti.ws4j.impl.Lesk;
|
||||
import edu.cmu.lti.ws4j.impl.Lin;
|
||||
import edu.cmu.lti.ws4j.impl.Path;
|
||||
import edu.cmu.lti.ws4j.impl.Resnik;
|
||||
import edu.cmu.lti.ws4j.impl.WuPalmer;
|
||||
import edu.cmu.lti.ws4j.util.WS4JConfiguration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class SimilarityMatrix {
|
||||
|
||||
private String PrimaryString;
|
||||
private String SecondaryString;
|
||||
private double distance;
|
||||
|
||||
public double getDistance() {
|
||||
return distance;
|
||||
}
|
||||
|
||||
public void setDistance(double distance) {
|
||||
this.distance = distance;
|
||||
}
|
||||
|
||||
public SimilarityMatrix(String str1, String str2) {
|
||||
this.PrimaryString = str1;
|
||||
this.SecondaryString = str2;
|
||||
}
|
||||
|
||||
public SimilarityMatrix(String str1, String str2, double result) {
|
||||
this.PrimaryString = str1;
|
||||
this.SecondaryString = str2;
|
||||
this.distance = result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated
|
||||
* @return ws4j distance caluclation add infinitum absurdum
|
||||
*/
|
||||
public double getDistanceCalculations() {
|
||||
ILexicalDatabase db = new NictWordNet();
|
||||
WS4JConfiguration.getInstance().setMFS(true);
|
||||
RelatednessCalculator rc1 = new WuPalmer(db);
|
||||
RelatednessCalculator rc2 = new Resnik(db);
|
||||
RelatednessCalculator rc3 = new JiangConrath(db);
|
||||
RelatednessCalculator rc4 = new Lin(db);
|
||||
RelatednessCalculator rc5 = new LeacockChodorow(db);
|
||||
RelatednessCalculator rc6 = new Path(db);
|
||||
RelatednessCalculator rc7 = new Lesk(db);
|
||||
RelatednessCalculator rc8 = new HirstStOnge(db);
|
||||
double maxScore = -1D;
|
||||
List<RelatednessCalculator> RCList = new ArrayList();
|
||||
RCList.add(rc1);
|
||||
RCList.add(rc2);
|
||||
RCList.add(rc3);
|
||||
RCList.add(rc4);
|
||||
RCList.add(rc5);
|
||||
RCList.add(rc6);
|
||||
RCList.add(rc7);
|
||||
RCList.add(rc8);
|
||||
for (RelatednessCalculator rc : RCList) {
|
||||
double s = rc.calcRelatednessOfWords(PrimaryString, SecondaryString);
|
||||
s /= 1000;
|
||||
if (s > 0.000000) {
|
||||
System.out.println("s: " + s + "\n" + " PrimaryString: " + PrimaryString + "\n" + " SecondaryString: " + SecondaryString + "\n"
|
||||
+ " rc: " + rc.toString() + "\n");
|
||||
}
|
||||
String str = String.format("%.12f", s);
|
||||
if (str.contains(",")) {
|
||||
str = str.substring(0, str.indexOf(","));
|
||||
}
|
||||
int strend = str.length() > 6 ? 6 : str.length();
|
||||
str = str.substring(0, strend);
|
||||
double score = Double.valueOf(str);
|
||||
if (score > maxScore) {
|
||||
maxScore = score;
|
||||
}
|
||||
}
|
||||
return maxScore == -1D ? 0.00 : maxScore;
|
||||
}
|
||||
|
||||
public String getPrimaryString() {
|
||||
return PrimaryString;
|
||||
}
|
||||
|
||||
public void setPrimaryString(String PrimaryString) {
|
||||
this.PrimaryString = PrimaryString;
|
||||
}
|
||||
|
||||
public String getSecondaryString() {
|
||||
return SecondaryString;
|
||||
}
|
||||
|
||||
public void setSecondaryString(String SecondaryString) {
|
||||
this.SecondaryString = SecondaryString;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,73 +0,0 @@
|
||||
package FunctionLayer.misc;
|
||||
|
||||
import edu.stanford.nlp.ling.HasWord;
|
||||
import edu.stanford.nlp.ling.IndexedWord;
|
||||
import edu.stanford.nlp.ling.TaggedWord;
|
||||
import edu.stanford.nlp.ling.Word;
|
||||
import edu.stanford.nlp.parser.lexparser.LexicalizedParser;
|
||||
import edu.stanford.nlp.process.DocumentPreprocessor;
|
||||
import edu.stanford.nlp.process.Tokenizer;
|
||||
import edu.stanford.nlp.trees.GrammaticalStructure;
|
||||
import edu.stanford.nlp.trees.GrammaticalStructureFactory;
|
||||
import edu.stanford.nlp.trees.Tree;
|
||||
import edu.stanford.nlp.trees.TreebankLanguagePack;
|
||||
import edu.stanford.nlp.trees.TypedDependency;
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class SentimentAnalyzerTest {
|
||||
|
||||
public static SentimentAnalyzerTest instance = new SentimentAnalyzerTest();
|
||||
public static String grammar = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz";
|
||||
public static String[] options = {"-maxLength", "80", "-retainTmpSubcategories"};
|
||||
|
||||
public static LexicalizedParser initiateLexicalizedParser() {
|
||||
LexicalizedParser lp = LexicalizedParser.loadModel(grammar, options);
|
||||
return lp;
|
||||
}
|
||||
|
||||
public static TreebankLanguagePack initiateTreebankLanguagePack(LexicalizedParser lp) {
|
||||
TreebankLanguagePack tlp = lp.getOp().langpack();
|
||||
return tlp;
|
||||
}
|
||||
|
||||
public double sentimentanalyzing(String str, String str1, double sreturn, LexicalizedParser lp, TreebankLanguagePack tlp) {
|
||||
Iterable<List<? extends HasWord>> sentences;
|
||||
Tokenizer<? extends HasWord> toke
|
||||
= tlp.getTokenizerFactory().getTokenizer(new StringReader(str));
|
||||
List<? extends HasWord> sentence = toke.tokenize();
|
||||
String[] sent3 = {str1};
|
||||
String[] tag3 = {"PRP", "MD", "VB", "PRP", "."}; // Parser gets second "can" wrong without help
|
||||
List<TaggedWord> sentence2 = new ArrayList<>();
|
||||
for (int i = 0; i < sent3.length; i++) {
|
||||
sentence2.add(new TaggedWord(sent3[i], tag3[i]));
|
||||
}
|
||||
//parse.pennPrint();
|
||||
List<List<? extends HasWord>> tmp
|
||||
= new ArrayList<>();
|
||||
tmp.add(sentence2);
|
||||
tmp.add(sentence);
|
||||
sentences = tmp;
|
||||
for (List<? extends HasWord> sentence1 : sentences) {
|
||||
Tree parse1 = lp.parse(sentence1);
|
||||
GrammaticalStructureFactory gsf = tlp.grammaticalStructureFactory();
|
||||
GrammaticalStructure gs = gsf.newGrammaticalStructure(parse1);
|
||||
double score = parse1.score();
|
||||
if (score > sreturn) {
|
||||
//System.out.println("\n score : " + score + "\n");
|
||||
sreturn = score;
|
||||
}
|
||||
}
|
||||
return sreturn;
|
||||
}
|
||||
}
|
||||
@ -1,66 +0,0 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer.misc;
|
||||
|
||||
import DataLayer.DataMapper;
|
||||
import FunctionLayer.CustomError;
|
||||
import FunctionLayer.SimilarityMatrix;
|
||||
import static FunctionLayer.MYSQLDatahandler.EXPIRE_TIME_IN_SECONDS;
|
||||
import com.google.common.base.Stopwatch;
|
||||
import com.google.common.collect.ArrayListMultimap;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import com.google.common.collect.Multimap;
|
||||
import java.io.IOException;
|
||||
import java.sql.SQLException;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class SentimentSimilarityCacheObsolete {
|
||||
|
||||
public static final long EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(5, TimeUnit.HOURS);
|
||||
private static ConcurrentMap<String, List<SimilarityMatrix>> SimilarityMatrixCache;
|
||||
private static Stopwatch stopwatch;
|
||||
|
||||
public SentimentSimilarityCacheObsolete(ConcurrentMap<Integer, String> StringCache, Stopwatch stopwatch) {
|
||||
this.stopwatch = Stopwatch.createUnstarted();
|
||||
this.SimilarityMatrixCache = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
}
|
||||
|
||||
public void clearConCurrentMaps() {
|
||||
SimilarityMatrixCache.clear();
|
||||
}
|
||||
|
||||
private Multimap<String, SimilarityMatrix> getCache() throws SQLException, IOException, CustomError {
|
||||
List<SimilarityMatrix> matrixlist;
|
||||
matrixlist = DataMapper.getAllSementicMatrixes();
|
||||
Multimap<String, SimilarityMatrix> LHM = ArrayListMultimap.create();
|
||||
for (int i = 0; i < matrixlist.size(); i++) {
|
||||
LHM.put(matrixlist.get(i).getPrimaryString(), matrixlist.get(i));
|
||||
}
|
||||
return LHM;
|
||||
}
|
||||
|
||||
public static String getResponseStr(String strreturn) {
|
||||
String str = "";
|
||||
if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
|
||||
|
||||
if (!stopwatch.isRunning()) {
|
||||
stopwatch.start();
|
||||
} else {
|
||||
stopwatch.reset();
|
||||
}
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,152 +0,0 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
https://github.com/DonatoMeoli/WS4J
|
||||
*/
|
||||
package FunctionLayer.misc;
|
||||
|
||||
import edu.cmu.lti.jawjaw.pobj.POS;
|
||||
import edu.cmu.lti.lexical_db.ILexicalDatabase;
|
||||
import edu.cmu.lti.lexical_db.NictWordNet;
|
||||
import edu.cmu.lti.lexical_db.data.Concept;
|
||||
import edu.cmu.lti.ws4j.Relatedness;
|
||||
import edu.cmu.lti.ws4j.RelatednessCalculator;
|
||||
import edu.cmu.lti.ws4j.impl.HirstStOnge;
|
||||
import edu.cmu.lti.ws4j.impl.JiangConrath;
|
||||
import edu.cmu.lti.ws4j.impl.LeacockChodorow;
|
||||
import edu.cmu.lti.ws4j.impl.Lesk;
|
||||
import edu.cmu.lti.ws4j.impl.Lin;
|
||||
import edu.cmu.lti.ws4j.impl.Path;
|
||||
import edu.cmu.lti.ws4j.impl.Resnik;
|
||||
import edu.cmu.lti.ws4j.impl.WuPalmer;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
* https://www.programcreek.com/java-api-examples/?api=edu.cmu.lti.ws4j.RelatednessCalculator
|
||||
* https://stackoverflow.com/questions/36300485/how-to-resolve-the-difference-between-the-values-attained-in-the-web-api-and-the
|
||||
*/
|
||||
public class WordNetSimalarityObsolete {
|
||||
|
||||
private static ILexicalDatabase db = new NictWordNet();
|
||||
private static RelatednessCalculator rc1 = new WuPalmer(db);
|
||||
private static RelatednessCalculator rc2 = new Resnik(db);
|
||||
private static RelatednessCalculator rc3 = new JiangConrath(db);
|
||||
private static RelatednessCalculator rc4 = new Lin(db);
|
||||
private static RelatednessCalculator rc5 = new LeacockChodorow(db);
|
||||
private static RelatednessCalculator rc6 = new Path(db);
|
||||
private static RelatednessCalculator rc7 = new Lesk(db);
|
||||
private static RelatednessCalculator rc8 = new HirstStOnge(db);
|
||||
|
||||
public static double SentenceMatcherSimilarityMatrix(String[] words1, String[] words2, double maxScore) {
|
||||
{
|
||||
double[][] s1 = getSimilarityMatrix(words1, words2, rc1);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s1[i][j] < maxScore && s1[i][j] > 0.0) {
|
||||
System.out.print(s1[i][j] + "\t");
|
||||
System.out.println("WuPalmer");
|
||||
maxScore = s1[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc2);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.println("Resnik");
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc3);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.println("JiangConrath");
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc4);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.println("Lin");
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc5);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
System.out.println("LeacockChodrow");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc6);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.println("Path");
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc7);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.println("Lesk");
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
double[][] s2 = getSimilarityMatrix(words1, words2, rc8);
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (s2[i][j] < maxScore && s2[i][j] > 0.0) {
|
||||
System.out.println("HirstStOnge");
|
||||
System.out.print(s2[i][j] + "\t");
|
||||
maxScore = s2[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return maxScore;
|
||||
}
|
||||
|
||||
public static double[][] getSimilarityMatrix(String[] words1, String[] words2, RelatednessCalculator rc) {
|
||||
double[][] result = new double[words1.length][words2.length];
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
double score = rc.calcRelatednessOfWords(words1[i], words2[j]);
|
||||
result[i][j] = score;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@ -1,74 +0,0 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer.misc;
|
||||
|
||||
import edu.cmu.lti.lexical_db.ILexicalDatabase;
|
||||
import edu.cmu.lti.lexical_db.NictWordNet;
|
||||
import edu.cmu.lti.ws4j.RelatednessCalculator;
|
||||
import edu.cmu.lti.ws4j.impl.HirstStOnge;
|
||||
import edu.cmu.lti.ws4j.impl.JiangConrath;
|
||||
import edu.cmu.lti.ws4j.impl.LeacockChodorow;
|
||||
import edu.cmu.lti.ws4j.impl.Lesk;
|
||||
import edu.cmu.lti.ws4j.impl.Lin;
|
||||
import edu.cmu.lti.ws4j.impl.Path;
|
||||
import edu.cmu.lti.ws4j.impl.Resnik;
|
||||
import edu.cmu.lti.ws4j.impl.WuPalmer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
* http://ws4jdemo.appspot.com/?mode=s&s1=something+like+a+sentence&s2=should+not+be+like+the+first+one
|
||||
*/
|
||||
public class WordNetSimalarityTestObsolete {
|
||||
|
||||
private static ILexicalDatabase db = new NictWordNet();
|
||||
private static RelatednessCalculator rc1 = new WuPalmer(db);
|
||||
private static RelatednessCalculator rc2 = new Resnik(db);
|
||||
private static RelatednessCalculator rc3 = new JiangConrath(db);
|
||||
private static RelatednessCalculator rc4 = new Lin(db);
|
||||
private static RelatednessCalculator rc5 = new LeacockChodorow(db);
|
||||
private static RelatednessCalculator rc6 = new Path(db);
|
||||
private static RelatednessCalculator rc7 = new Lesk(db);
|
||||
private static RelatednessCalculator rc8 = new HirstStOnge(db);
|
||||
|
||||
public static double SentenceMatcherSimilarityMatrix(String[] words1, String[] words2, double maxScore) {
|
||||
boolean initial = maxScore == 0.0;
|
||||
List<RelatednessCalculator> RCList = new ArrayList();
|
||||
RCList.add(rc1);
|
||||
RCList.add(rc2);
|
||||
RCList.add(rc3);
|
||||
RCList.add(rc4);
|
||||
RCList.add(rc5);
|
||||
RCList.add(rc6);
|
||||
RCList.add(rc7);
|
||||
RCList.add(rc8);
|
||||
for (int h = 0; h < RCList.size(); h++) {
|
||||
double s1 = getSimilarityMatrix(words1, words2, RCList.get(h), maxScore, initial);
|
||||
System.out.println("s1: " + String.format("%.0f", s1) + " \nmaxScore: " + maxScore);
|
||||
if (s1 > 0.01 && (s1 < maxScore || initial)) {
|
||||
maxScore = s1;
|
||||
}
|
||||
}
|
||||
return maxScore;
|
||||
}
|
||||
|
||||
public static double getSimilarityMatrix(String[] words1, String[] words2, RelatednessCalculator rc, double maxScore, boolean initial) {
|
||||
double rtndouble = 0.01;
|
||||
for (int i = 0; i < words1.length; i++) {
|
||||
for (int j = 0; j < words2.length; j++) {
|
||||
if (maxScore > rtndouble / words2.length || initial) {
|
||||
rtndouble += (rc.calcRelatednessOfWords(words1[i], words2[j]));
|
||||
//System.out.println("RelatednessCalculator: " + rc.toString());
|
||||
} else {
|
||||
return rtndouble;
|
||||
}
|
||||
}
|
||||
}
|
||||
return rtndouble;
|
||||
}
|
||||
}
|
||||
@ -1,58 +0,0 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer.misc;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class notes {
|
||||
/*
|
||||
|
||||
/*
|
||||
ILexicalDatabase db = new NictWordNet();
|
||||
RelatednessCalculator lesk = new Lesk(db);
|
||||
POS posWord1 = POS.n;
|
||||
POS posWord2 = POS.n;
|
||||
double maxScore = 0;
|
||||
WS4JConfiguration.getInstance().setMFS(true);
|
||||
List<Concept> synsets1 = (List<Concept>) db.getAllConcepts(strreturn, posWord1.name());
|
||||
for (int i = 0; i < allStringValuesPresent.size(); i++) {
|
||||
List<Concept> synsets2 = (List<Concept>) db.getAllConcepts(allStringValuesPresent.get(i), posWord2.name());
|
||||
for (Concept synset1 : synsets1) {
|
||||
for (Concept synset2 : synsets2) {
|
||||
Relatedness relatedness = lesk.calcRelatednessOfSynset(synset1, synset2);
|
||||
double score = relatedness.getScore();
|
||||
if (score > maxScore) {
|
||||
maxScore = score;
|
||||
index = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static RelatednessCalculator[] rcs;
|
||||
|
||||
static {
|
||||
WS4JConfiguration.getInstance().setMemoryDB(false);
|
||||
WS4JConfiguration.getInstance().setMFS(true);
|
||||
ILexicalDatabase db = new MITWordNet();
|
||||
rcs = new RelatednessCalculator[]{
|
||||
new HirstStOnge(db), new LeacockChodorow(db), new Lesk(db), new WuPalmer(db),
|
||||
new Resnik(db), new JiangConrath(db), new Lin(db), new Path(db)
|
||||
};
|
||||
}
|
||||
https://github.com/DonatoMeoli/WS4J
|
||||
https://www.programcreek.com/2014/01/calculate-words-similarity-using-wordnet-in-java/
|
||||
*/
|
||||
/*
|
||||
//available options of metrics
|
||||
private static RelatednessCalculator[] rcs = { new HirstStOnge(db),
|
||||
new LeacockChodorow(db), new Lesk(db), new WuPalmer(db),
|
||||
new Resnik(db), new JiangConrath(db), new Lin(db), new Path(db) };
|
||||
*/
|
||||
}
|
||||
@ -33,7 +33,7 @@ import org.javacord.api.entity.user.User;
|
||||
*/
|
||||
public class DiscordHandler {
|
||||
|
||||
private static String MostRecentMsg = "how are you today bot";
|
||||
private static String MostRecentMsg = "";
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", "25");
|
||||
@ -49,8 +49,8 @@ public class DiscordHandler {
|
||||
Datahandler.instance.shiftReduceParserInitiate();
|
||||
Datahandler.instance.instantiateAnnotationMap();
|
||||
System.out.println("FINISHED ALL ANNOTATIONS");
|
||||
Datahandler.instance.addHLstatsMessages();
|
||||
Datahandler.instance.updateStringCache();
|
||||
//Datahandler.instance.addHLstatsMessages();
|
||||
//Datahandler.instance.updateStringCache();
|
||||
String token = "NTI5NzAxNTk5NjAyMjc4NDAx.Dw0vDg.7-aMjVWdQMYPl8qVNyvTCPS5F_A";
|
||||
DiscordApi api = new DiscordApiBuilder().setToken(token).login().join();
|
||||
api.addMessageCreateListener(event -> {
|
||||
@ -92,17 +92,17 @@ public class DiscordHandler {
|
||||
event.getChannel().sendMessage(ResponseStr);
|
||||
MostRecentMsg = ResponseStr;
|
||||
}
|
||||
new Thread(() -> {
|
||||
try {
|
||||
Datahandler.instance.checkIfUpdateStrings(false);
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}).start();
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
new Thread(() -> {
|
||||
try {
|
||||
Datahandler.instance.checkIfUpdateStrings(false);
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}).start();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Binary file not shown.
Loading…
Reference in New Issue
Block a user