UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

removed removeslack check as levensthein defunct now

Browse Source
This commit is contained in:
jenzuffer 2020-11-02 00:02:06 +01:00
parent e60ac109e8
commit 45e18c2a99
7 changed files with 7 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java
View File

@ -318,8 +318,6 @@ public class Datahandler {
SentimentValueCache sentimentCacheStr = sentimentCachingMap.getOrDefault(strCollection, null);
List<SimilarityMatrix> smxReturnList = new ArrayList();
List<String> randomIterationComparision = new ArrayList();
int iteratecap = strCacheLocal.size() > prefix_size ? strCacheLocal.size() - prefix_size : strCollection.size() - 1;
int iterator = ThreadLocalRandom.current().nextInt(0, iteratecap);
if (strCacheLocal.size() < prefix_size)
{
for (String colStr : strCollection)
@ -327,6 +325,9 @@ public class Datahandler {
strCacheLocal.put(strCacheLocal.size(), colStr);
}
}
int iteratecap = strCacheLocal.size() > prefix_size ? strCacheLocal.size() - prefix_size : strCacheLocal.size() - 1;
int iterator = ThreadLocalRandom.current().nextInt(0, iteratecap);
int iterated = 0;
for (String str1 : strCacheLocal.values()) {
if (iterated >= iterator && iterated < iterator + prefix_size) {
@ -404,7 +405,7 @@ public class Datahandler {
}
private ConcurrentMap<Integer, String> removeNonSensicalStrings(ConcurrentMap<Integer, String> strmap) {
final ConcurrentMap<Integer, String> strCacheLocal = stringCache.size() < 150 ? strmap : stringCache;
final ConcurrentMap<Integer, String> strCacheLocal = stringCache;
final ConcurrentMap<String, Annotation> localJMWEMap = getMultipleJMWEAnnotation(strmap.values());
final ConcurrentMap<String, Annotation> localPipelineAnnotation = getMultiplePipelineAnnotation(strmap.values());
final ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation = getMultiplePipelineSentimentAnnotation(strmap.values());
@ -416,12 +417,9 @@ public class Datahandler {
if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
ConcurrentMap<Integer, String> str = MessageResponseHandler.getStr();
System.out.println("str size: " + str.size());
str = cutContent(str, hlStatsMsg);
str = filterContent(str);
str = removeSlacks(str);
//System.out.println("finished removeSlacks \n" + str.size() + "\n");
str = removeNonSensicalStrings(str);
System.out.println("removeNonSensicalStrings str size POST: " + str.size() + "\n");
//System.out.println("removeNonSensicalStrings str size POST: " + str.size() + "\n");
str = annotationCacheUpdate(str);
System.out.println("annotationCacheUpdate str size POST: " + str.size() + "\n");
ConcurrentMap<Integer, String> strf = str;
@ -622,20 +620,6 @@ public class Datahandler {
return 0.0;
}
public static ConcurrentMap<Integer, String> cutContent(ConcurrentMap<Integer, String> str, boolean hlStatsMsg) {
ConcurrentMap<Integer, String> returnlist = new MapMaker().concurrencyLevel(2).makeMap();
str.values().forEach(str1 -> {
int iend = str1.indexOf("content: ");
if (iend != -1) {
String trs = str1.substring(iend + 9);
returnlist.put(returnlist.size() + 1, trs.substring(0, trs.length() - 1));
} else if (hlStatsMsg) {
returnlist.put(returnlist.size() + 1, str1);
}
});
return returnlist;
}
public static ConcurrentMap<Integer, String> filterContent(ConcurrentMap<Integer, String> str) {
ConcurrentMap<Integer, String> strlistreturn = new MapMaker().concurrencyLevel(2).makeMap();
str.values().forEach(str1 -> {
@ -739,38 +723,13 @@ public class Datahandler {
}
str1 = str1.trim();
if (str1.length() > 2 && (!str1.startsWith("!"))) {
strlistreturn.put(strlistreturn.size() + 1, str1);
strlistreturn.put(strlistreturn.size(), str1);
}
}
});
return strlistreturn;
}
private ConcurrentMap<Integer, String> removeSlacks(ConcurrentMap<Integer, String> str) {
ConcurrentMap<Integer, String> strreturn = new MapMaker().concurrencyLevel(2).makeMap();
if (stringCache.isEmpty()) {
return str;
}
Collection<String> values = stringCache.values();
str.values().forEach(str1 -> {
boolean tooclosematch = false;
for (String strVals : values) {
LevenshteinDistance leven = new LevenshteinDistance(strVals, str1);
double Distance = leven.computeLevenshteinDistance();
Double maxpermittedDistance = 2.5;
if (Distance < maxpermittedDistance) {
tooclosematch = true;
break;
}
}
if (!tooclosematch) {
strreturn.put(strreturn.size() + 1, str1);
//System.out.println("adding strreturn str1: " + str1 + "\n");
}
});
return strreturn;
}
private ConcurrentMap<Integer, String> annotationCacheUpdate(ConcurrentMap<Integer, String> strmap) {
ConcurrentMap<String, Annotation> jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strmap.values());
for (Entry<String, Annotation> jmweitr : jmweAnnotation.entrySet()) {

1
ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java
View File

@ -46,7 +46,6 @@ public class MessageResponseHandler {
public static String selectReponseMessage(String toString, String personName) throws CustomError {
ConcurrentMap<Integer, String> str1 = new MapMaker().concurrencyLevel(6).makeMap();
str1.put(str1.size() + 1, toString);
str1 = Datahandler.cutContent(str1, false);
String strreturn = "";
for (String str : str1.values()) {
if (!str.isEmpty()) {

BIN
ArtificialAutism/target/ArtificialAutism-1.0.jar
View File

Binary file not shown.

BIN
ArtificialAutism/target/classes/FunctionLayer/Datahandler$AnnotationCollector.class
View File

Binary file not shown.

BIN
ArtificialAutism/target/classes/FunctionLayer/Datahandler.class
View File

Binary file not shown.

BIN
ArtificialAutism/target/classes/FunctionLayer/MessageResponseHandler.class
View File

Binary file not shown.

2
ArtificialAutism/target/maven-archiver/pom.properties
View File

@ -1,5 +1,5 @@
#Generated by Maven
#Sun Nov 01 22:49:28 CET 2020
#Sun Nov 01 23:59:26 CET 2020
version=1.0
groupId=com.mycompany
artifactId=ArtificialAutism