diff --git a/ArtificialAutism/src/main/java/DataLayer/DataMapper.java b/ArtificialAutism/src/main/java/DataLayer/DataMapper.java index 4671ca99..46b0a258 100644 --- a/ArtificialAutism/src/main/java/DataLayer/DataMapper.java +++ b/ArtificialAutism/src/main/java/DataLayer/DataMapper.java @@ -5,6 +5,8 @@ */ package DataLayer; +import org.jetbrains.annotations.NotNull; + import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -78,4 +80,39 @@ public class DataMapper { } } } + + public static void checkStringsToDelete() { + Connection l_cCon = null; + PreparedStatement l_pStatement = null; + ResultSet l_rsSearch = null; + String l_sSQL = "delete from Sentences where last_used < NOW() - INTERVAL 4 WEEK LIMIT 25"; + try { + l_cCon = DBCPDataSource.getConnection(); + l_pStatement = l_cCon.prepareStatement(l_sSQL); + l_pStatement.execute(); + } catch (SQLException throwables) { + throwables.printStackTrace(); + } finally { + CloseConnections(l_pStatement, l_rsSearch, l_cCon); + } + } + + public static void updateLastUsed(@NotNull ArrayList mysqlUpdateLastUsed) { + Connection l_cCon = null; + PreparedStatement l_pStatement = null; + ResultSet l_rsSearch = null; + String l_sSQL = "update Sentences Set last_used = now() where Strings = (?)"; + try { + l_cCon = DBCPDataSource.getConnection(); + l_pStatement = l_cCon.prepareStatement(l_sSQL); + for (String str1 : mysqlUpdateLastUsed) { + l_pStatement.setString(1, str1); + l_pStatement.execute(); + } + } catch (SQLException throwables) { + throwables.printStackTrace(); + } finally { + CloseConnections(l_pStatement, l_rsSearch, l_cCon); + } + } } diff --git a/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.kt b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.kt index 7ca1bd14..44f2f10b 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.kt +++ b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.kt @@ -8,6 +8,8 @@ package FunctionLayer import DataLayer.DataMapper import FunctionLayer.StanfordParser.SentimentAnalyzerTest import com.google.common.base.Stopwatch +import edu.mit.jmwe.data.IMWE +import edu.mit.jmwe.data.IToken import edu.stanford.nlp.ie.AbstractSequenceClassifier import edu.stanford.nlp.ie.crf.CRFClassifier import edu.stanford.nlp.ling.CoreAnnotations @@ -18,13 +20,10 @@ import edu.stanford.nlp.pipeline.Annotation import edu.stanford.nlp.pipeline.CoreDocument import edu.stanford.nlp.pipeline.StanfordCoreNLP import edu.stanford.nlp.tagger.maxent.MaxentTagger -import edu.stanford.nlp.trees.GrammaticalStructureFactory -import edu.stanford.nlp.trees.Tree +import edu.stanford.nlp.trees.* import edu.stanford.nlp.util.CoreMap -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.yield +import kotlinx.coroutines.* +import org.ejml.simple.SimpleMatrix import java.util.* import java.util.concurrent.TimeUnit import java.util.regex.Pattern @@ -57,6 +56,36 @@ public class Datahandler { private var sentences1HashMap: HashMap> = HashMap() private var sentencesSentimentHashMap: HashMap> = HashMap() private var trees1HashMap: HashMap> = HashMap() + private var grammaticalStructureHashMap: HashMap> = + HashMap() + private var typedDependenciesHashMap: HashMap> = + HashMap() + private var rnnCoreAnnotationsPredictedHashMap: HashMap> = HashMap() + private var simpleMatricesHashMap: HashMap> = HashMap() + private var simpleMatricesNodevectorsHashMap: HashMap> = HashMap() + private var listHashMap: HashMap> = HashMap() + private var longestHashMap: HashMap = HashMap() + private var sentimentHashMap: HashMap = HashMap() + private var imwesHashMap: HashMap>> = HashMap() + private var InflectedCounterNegativeHashMap: HashMap = HashMap() + private var InflectedCounterPositiveHashMap: HashMap = HashMap() + private var tokenEntryHashMap: HashMap> = HashMap() + private var MarkedContinuousCounterHashMap: HashMap = HashMap() + private var UnmarkedPatternCounterHashMap: HashMap = HashMap() + private var strTokensIpartFormHashMap: HashMap> = HashMap() + private var tokenFormsHashMap: HashMap> = HashMap() + private var strTokenEntryGetPOSHashMap: HashMap> = HashMap() + private var intTokenEntyCountsHashMap: HashMap> = HashMap() + private var ITokenTagsHashMap: HashMap> = HashMap() + private var strTokenStemsHashMap: HashMap> = HashMap() + private var AnotatorcounterHashMap: HashMap = HashMap() + private var TokensCounterHashMap: HashMap = HashMap() + private var entityTokenTagsHashMap: HashMap> = HashMap() + private var nerEntitiesHashMap: HashMap> = HashMap() + private var nerEntitiesTypeHashMap: HashMap> = HashMap() + private var stopWordTokenHashMap: HashMap> = HashMap() + private var stopWordLemmaHashMap: HashMap> = HashMap() + private var PairCounterHashMap: HashMap = HashMap() constructor() { stopwatch = Stopwatch.createUnstarted() @@ -131,7 +160,9 @@ public class Datahandler { } stringCache.sortWith(Comparator.comparingInt(String::length).reversed()); System.out.println("pre InsertMYSQLStrings") - DataMapper.InsertMYSQLStrings(stringCache) + val arrayList = java.util.ArrayList(stringCache) + DataMapper.InsertMYSQLStrings(arrayList) + DataMapper.checkStringsToDelete(); } } @@ -191,8 +222,34 @@ public class Datahandler { var sentencesSentimentF: List? = null var coreMaps1: List = jmweAnnotationF.get(CoreAnnotations.SentencesAnnotation::class.java) var treesF: java.util.ArrayList? = null - - + var grammaticalStructuresF: ArrayList? = null + var typedDependenciesF: java.util.ArrayList? = null + var rnnCoreAnnotationsPredictedF: java.util.ArrayList? = null + var simpleMatricesF: java.util.ArrayList? = null + var simpleMatricesNodevectorsF: java.util.ArrayList? = null + var listF: MutableList? = null + var longestF: Int? = null + var sentimentLongestF: Int? = null + var imwesF: List>? = null + var InflectedCounterNegativeF: Int? = null + var InflectedCounterPositiveF: Int? = null + var tokenEntryF: ArrayList? = null + var MarkedContinuousCounterF: Int? = null + var UnmarkedPatternCounterF: Int? = null + var strTokensIpartFormF: ArrayList? = null + var tokenFormsF: java.util.ArrayList? = null + var strTokenEntryGetPOSF: ArrayList? = null + var intTokenEntyCountsF: java.util.ArrayList? = null + var ITokenTagsF: ArrayList? = null + var strTokenStemsF: java.util.ArrayList? = null + var AnotatorcounterF: Int? = null + var TokensCounterF: Int? = null + var entityTokenTagsF: java.util.ArrayList? = null + var nerEntitiesF: java.util.ArrayList? = null + var nerEntitiesTypeF: java.util.ArrayList? = null + var stopWordTokenF: java.util.ArrayList? = null + var stopWordLemmaF: java.util.ArrayList? = null + var PairCounterF: Int? = null for (str1 in values_copy) { if (strF != str1) { val annotation2 = pipelineSentimentAnnotationCache.getOrDefault(str1, null) @@ -219,63 +276,263 @@ public class Datahandler { val sentenceSentiment1: List? = sentencesSentimentHashMap.getOrDefault(str1, null) val trees1 = trees1HashMap.getOrDefault(str1, null) var coreMaps2: List = listOf() + val grammaticalStructures1 = grammaticalStructureHashMap.getOrDefault( + str1, null) if (jmweAnnotation != null) { coreMaps2 = jmweAnnotation.get(CoreAnnotations.SentencesAnnotation::class.java) } + val typedDependencies1 = typedDependenciesHashMap.getOrDefault(str1, null) + val rnnCoreAnnotationsPredicted1 = rnnCoreAnnotationsPredictedHashMap.getOrDefault(str1, null) + val simpleMatrices1 = simpleMatricesHashMap.getOrDefault(str1, null); + val simpleMatricesNodevectors1 = simpleMatricesNodevectorsHashMap.getOrDefault(str1, null); + val list1 = listHashMap.getOrDefault(str1, null); + val longest1 = longestHashMap.getOrDefault(str1, null); + val sentimentLongest1 = sentimentHashMap.getOrDefault(str1, null); + val imwes1 = imwesHashMap.getOrDefault(str1, null); + val InflectedCounterNegative1 = InflectedCounterNegativeHashMap.getOrDefault(str1, null); + val InflectedCounterPositive1 = InflectedCounterPositiveHashMap.getOrDefault(str1, null) + val tokenEntry1 = tokenEntryHashMap.getOrDefault(str1, null) + val MarkedContinuousCounter1 = MarkedContinuousCounterHashMap.getOrDefault(str1, null) + val UnmarkedPatternCounter1 = UnmarkedPatternCounterHashMap.getOrDefault(str1, null) + val strTokensIpartForm1 = strTokensIpartFormHashMap.getOrDefault(str1, null); + val tokenForms1 = tokenFormsHashMap.getOrDefault(str1, null); + val strTokenEntryGetPOS1 = strTokenEntryGetPOSHashMap.getOrDefault(str1, null) + val intTokenEntyCounts1 = intTokenEntyCountsHashMap.getOrDefault(str1, null); + val ITokenTags1 = ITokenTagsHashMap.getOrDefault(str1, null); + val strTokenStems1 = strTokenStemsHashMap.getOrDefault(str1, null); + val Anotatorcounter1 = AnotatorcounterHashMap.getOrDefault(str1, null); + val TokensCounter1 = TokensCounterHashMap.getOrDefault(str1, null); + val entityTokenTags1 = entityTokenTagsHashMap.getOrDefault(str1, null); + val nerEntities1 = nerEntitiesHashMap.getOrDefault(str1, null); + val nerEntitiesType1 = nerEntitiesTypeHashMap.getOrDefault(str1, null); + val stopWordToken1 = stopWordTokenHashMap.getOrDefault(str1, null); + val stopWordLemma1 = stopWordLemmaHashMap.getOrDefault(str1, null); + val PairCounter1 = PairCounterHashMap.getOrDefault(str1, null); + var SMX = SentimentAnalyzerTest(strF, str1, SimilarityMatrix(strF, str1), coreMaps1, coreMaps2, strAnno, pipelineAnnotationCache[str1], strAnnoSentiment, pipelineSentimentAnnotationCache[str1], coreDocument, coreDocumentAnnotationCache[str1], tagger, gsf, classifier, tokenizeCounting, tokenizeCountingF, taggedWordListF, taggedWordList1, retrieveTGWListF, retrieveTGWList1, - sentencesF, sentence1, sentencesSentimentF, sentenceSentiment1, treesF, trees1) + sentencesF, sentence1, sentencesSentimentF, sentenceSentiment1, treesF, trees1, + grammaticalStructuresF, grammaticalStructures1, typedDependenciesF, + typedDependencies1, rnnCoreAnnotationsPredictedF, rnnCoreAnnotationsPredicted1, + simpleMatricesF, simpleMatrices1, simpleMatricesNodevectorsF, simpleMatricesNodevectors1, + listF, list1, longestF, longest1, sentimentLongestF, sentimentLongest1, imwesF, + imwes1, InflectedCounterNegativeF, InflectedCounterNegative1, InflectedCounterPositiveF, + InflectedCounterPositive1, tokenEntryF, tokenEntry1, MarkedContinuousCounterF, + MarkedContinuousCounter1, UnmarkedPatternCounterF, UnmarkedPatternCounter1, + strTokensIpartFormF, strTokensIpartForm1, tokenFormsF, tokenForms1, + strTokenEntryGetPOSF, strTokenEntryGetPOS1, intTokenEntyCountsF, + intTokenEntyCounts1, ITokenTagsF, ITokenTags1, strTokenStemsF, strTokenStems1, + AnotatorcounterF, Anotatorcounter1, TokensCounterF, TokensCounter1, + entityTokenTagsF, entityTokenTags1, nerEntitiesF, nerEntities1, nerEntitiesTypeF, + nerEntitiesType1, stopWordTokenF, stopWordToken1, stopWordLemmaF, stopWordLemma1, + PairCounterF, PairCounter1) if (tokenizeCounting == null) { - val tokenizeCounting1 = SMX.getTokenizeCounting(); - tokenizeCountingHashMap.put(str1, tokenizeCounting1) + tokenizeCountingHashMap.put(str1, SMX.getTokenizeCounting()) } if (taggedWordList1 == null) { - val taggedWordList1Local = SMX.getTaggedWordList1(); - taggedWordListHashMap.put(str1, taggedWordList1Local) + taggedWordListHashMap.put(str1, SMX.getTaggedWordList1()) } if (tokenizeCountingF == null) { - val tokenizeCountingF1 = SMX.getTokenizeCountingF(); - tokenizeCountingF = tokenizeCountingF1; + tokenizeCountingF = SMX.getTokenizeCountingF(); } if (taggedWordListF == null) { - val taggedWordListF1 = SMX.getTaggedWordListF(); - taggedWordListF = taggedWordListF1; + taggedWordListF = SMX.getTaggedWordListF(); } if (retrieveTGWListF == null) { - val retrieveTGWListF1 = SMX.getRetrieveTGWListF(); - retrieveTGWListF = retrieveTGWListF1; + retrieveTGWListF = SMX.getRetrieveTGWListF(); } if (retrieveTGWList1 == null) { - val retrieveTGWList11 = SMX.getRetrieveTGWList1(); - retrieveTGWListHashMap.put(str1, retrieveTGWList11); + retrieveTGWListHashMap.put(str1, SMX.getRetrieveTGWList1()); } if (sentencesF == null) { - val sentencesF1 = SMX.getSentencesF(); - sentencesF = sentencesF1; + sentencesF = SMX.getSentencesF(); } if (sentence1 == null) { - val sentences1 = SMX.getSentences1(); - sentences1HashMap.put(str1, sentences1) + sentences1HashMap.put(str1, SMX.getSentences1()) } if (sentencesSentimentF == null) { - val sentencesSentimentF1 = SMX.getSentencesSentimentF(); - sentencesSentimentF = sentencesSentimentF1; + sentencesSentimentF = SMX.getSentencesSentimentF(); } if (sentenceSentiment1 == null) { - val sentencesSentiment1 = SMX.getSentencesSentiment1(); - sentencesSentimentHashMap.put(str1, sentencesSentiment1); + sentencesSentimentHashMap.put(str1, SMX.getSentencesSentiment1()); } if (treesF == null) { - val treesF1 = SMX.getTreesF(); - treesF = treesF1; + treesF = SMX.getTreesF(); } if (trees1 == null) { - val trees11 = SMX.getTrees1(); - trees1HashMap.put(str1, trees11) + trees1HashMap.put(str1, SMX.getTrees1()) + } + if (grammaticalStructuresF == null) { + grammaticalStructuresF = SMX.getGrammaticalStructuresF(); + } + if (grammaticalStructures1 == null) { + grammaticalStructureHashMap.put(str1, SMX.getGrammaticalStructures1()) + } + if (typedDependenciesF == null) { + typedDependenciesF = SMX.getTypedDependenciesF(); + } + if (typedDependencies1 == null) { + typedDependenciesHashMap.put(str1, SMX.getTypedDependencies1()) + } + if (rnnCoreAnnotationsPredictedF == null) { + rnnCoreAnnotationsPredictedF = SMX.getRnnCoreAnnotationsPredictedF() + } + if (rnnCoreAnnotationsPredicted1 == null) { + rnnCoreAnnotationsPredictedHashMap.put(str1, SMX.getRnnCoreAnnotationsPredicted1()) + } + if (simpleMatricesF == null) { + simpleMatricesF = SMX.getSimpleMatricesF(); + } + if (simpleMatrices1 == null) { + simpleMatricesHashMap.put(str1, SMX.getSimpleMatrices1()); + } + if (simpleMatricesNodevectorsF == null) { + simpleMatricesNodevectorsF = SMX.getSimpleMatricesNodevectorsF(); + } + if (simpleMatricesNodevectors1 == null) { + simpleMatricesNodevectorsHashMap.put(str1, SMX.getSimpleMatricesNodevectors1()); + } + if (listF == null) { + listF = SMX.getListF(); + } + if (list1 == null) { + listHashMap.put(str1, SMX.getList1()); + } + if (longestF == null) { + longestF = SMX.getLongestF(); + } + if (longest1 == null) { + longestHashMap.put(str1, SMX.getLongest1()); + } + if (sentimentLongestF == null) { + sentimentLongestF = SMX.getSentimentLongestF(); + } + if (sentimentLongest1 == null) { + sentimentHashMap.put(str1, SMX.getSentimentLongest1()); + } + if (imwesF == null) { + imwesF = SMX.getImwesF(); + } + if (imwes1 == null) { + imwesHashMap.put(str1, SMX.getImwes1()); + } + if (InflectedCounterNegativeF == null) { + InflectedCounterNegativeF = SMX.getInflectedCounterNegativeF(); + } + if (InflectedCounterNegative1 == null) { + InflectedCounterNegativeHashMap.put(str1, SMX.getInflectedCounterNegative1()); + } + if (InflectedCounterPositiveF == null) { + InflectedCounterPositiveF = SMX.getInflectedCounterPositiveF(); + } + if (InflectedCounterPositive1 == null) { + InflectedCounterPositiveHashMap.put(str1, SMX.getInflectedCounterPositive1()); + } + if (tokenEntryF == null) { + tokenEntryF = SMX.getTokenEntryF(); + } + if (tokenEntry1 == null) { + tokenEntryHashMap.put(str1, SMX.getTokenEntry1()) + } + if (MarkedContinuousCounterF == null) { + MarkedContinuousCounterF = SMX.getMarkedContinuousCounterF(); + } + if (MarkedContinuousCounter1 == null) { + MarkedContinuousCounterHashMap.put(str1, SMX.getMarkedContinuousCounter1()); + } + if (UnmarkedPatternCounterF == null) { + UnmarkedPatternCounterF = SMX.getUnmarkedPatternCounterF(); + } + if (UnmarkedPatternCounter1 == null) { + UnmarkedPatternCounterHashMap.put(str1, SMX.getUnmarkedPatternCounter1()); + } + if (strTokensIpartFormF == null) { + strTokensIpartFormF = SMX.getStrTokensIpartFormF(); + } + if (strTokensIpartForm1 == null) { + strTokensIpartFormHashMap.put(str1, SMX.getStrTokensIpartForm1()); + } + if (tokenFormsF == null) { + tokenFormsF = SMX.getTokenFormsF(); + } + if (tokenForms1 == null) { + tokenFormsHashMap.put(str1, SMX.getTokenForms1()); + } + if (strTokenEntryGetPOSF == null) { + strTokenEntryGetPOSF = SMX.getStrTokenEntryGetPOSF(); + } + if (strTokenEntryGetPOS1 == null) { + strTokenEntryGetPOSHashMap.put(str1, SMX.getStrTokenEntryGetPOS1()) + } + if (intTokenEntyCountsF == null) { + intTokenEntyCountsF = SMX.getIntTokenEntyCountsF(); + } + if (intTokenEntyCounts1 == null) { + intTokenEntyCountsHashMap.put(str1, SMX.getIntTokenEntyCounts1()); + } + if (ITokenTagsF == null) { + ITokenTagsF = SMX.getITokenTagsF(); + } + if (ITokenTags1 == null) { + ITokenTagsHashMap.put(str1, SMX.getITokenTags1()); + } + if (strTokenStemsF == null) { + strTokenStemsF = SMX.getStrTokenStemsF(); + } + if (strTokenStems1 == null) { + strTokenStemsHashMap.put(str1, SMX.getStrTokenStems1()); + } + if (AnotatorcounterF == null) { + AnotatorcounterF = SMX.getAnotatorcounterF(); + } + if (Anotatorcounter1 == null) { + AnotatorcounterHashMap.put(str1, SMX.getAnotatorcounter1()); + } + if (TokensCounterF == null) { + TokensCounterF = SMX.getTokensCounterF(); + } + if (TokensCounter1 == null) { + TokensCounterHashMap.put(str1, SMX.getTokensCounter1()); + } + if (entityTokenTagsF == null) { + entityTokenTagsF = SMX.getEntityTokenTagsF(); + } + if (entityTokenTags1 == null) { + entityTokenTagsHashMap.put(str1, SMX.getEntityTokenTags1()); + } + if (nerEntitiesF == null) { + nerEntitiesF = SMX.getNerEntitiesF(); + } + if (nerEntities1 == null) { + nerEntitiesHashMap.put(str1, SMX.getNerEntities1()); + } + if (nerEntitiesTypeF == null) { + nerEntitiesTypeF = SMX.getNerEntitiesTypeF(); + } + if (nerEntitiesType1 == null) { + nerEntitiesTypeHashMap.put(str1, SMX.getNerEntitiesType1()); + } + if (stopWordTokenF == null) { + stopWordTokenF = SMX.getStopWordTokenF(); + } + if (stopWordToken1 == null) { + stopWordTokenHashMap.put(str1, SMX.getStopWordToken1()); + } + if (stopWordLemmaF == null) { + stopWordLemmaF = SMX.getStopWordLemmaF(); + } + if (stopWordLemma1 == null) { + stopWordLemmaHashMap.put(str1, SMX.getStopWordLemma1()); + } + if (PairCounterF == null) { + PairCounterF = SMX.getPairCounterF(); + } + if (PairCounter1 == null) { + PairCounterHashMap.put(str1, SMX.getPairCounter1()); } var getSMX: SimilarityMatrix = SMX.callSMX() @@ -292,17 +549,25 @@ public class Datahandler { } val randomLenghtPermit = strF.length * (Math.random() * Math.random() * Math.random() * (Math.random() * 10)) Collections.reverse(concurrentRelations) + val mysqlUpdateLastUsed: ArrayList = ArrayList() if (!concurrentRelations.isEmpty()) { for (secondaryRelation in concurrentRelations) { if (SB.toString().length > randomLenghtPermit && !SB.toString().isEmpty()) { break } SB.append(secondaryRelation).append(" ") + mysqlUpdateLastUsed.add(secondaryRelation) } } if (SB.toString().isEmpty()) { return "failure, preventing stuckness" } + runBlocking { + CoroutineScope(launch(Dispatchers.IO) { + DataMapper.updateLastUsed(mysqlUpdateLastUsed); + yield() + }) + } return SB.toString() } diff --git a/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java b/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java index 3fe79975..d2c62a6c 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java @@ -115,6 +115,231 @@ public class SentimentAnalyzerTest { return trees1; } + + public ArrayList getGrammaticalStructuresF() { + return grammaticalStructuresF; + } + + public ArrayList getGrammaticalStructures1() { + return grammaticalStructures1; + } + + public ArrayList getTypedDependenciesF() { + return typedDependenciesF; + } + + public ArrayList getTypedDependencies1() { + return typedDependencies1; + } + + public ArrayList getRnnCoreAnnotationsPredictedF() { + return rnnCoreAnnotationsPredictedF; + } + + public ArrayList getRnnCoreAnnotationsPredicted1() { + return rnnCoreAnnotationsPredicted1; + } + + public ArrayList getSimpleMatricesF() { + return simpleMatricesF; + } + + public ArrayList getSimpleMatrices1() { + return simpleMatrices1; + } + + public ArrayList getSimpleMatricesNodevectorsF() { + return simpleMatricesNodevectorsF; + } + + public List getListF() { + return listF; + } + + public List getList1() { + return list1; + } + + public ArrayList getSimpleMatricesNodevectors1() { + return simpleMatricesNodevectors1; + } + + public Integer getLongestF() { + return longestF; + } + + public Integer getSentimentLongestF() { + return sentimentLongestF; + } + + public Integer getSentimentLongest1() { + return sentimentLongest1; + } + + public List> getImwesF() { + return imwesF; + } + + public List> getImwes1() { + return imwes1; + } + + public Integer getLongest1() { + return longest1; + } + + public Integer getInflectedCounterNegativeF() { + return InflectedCounterNegativeF; + } + + public Integer getInflectedCounterPositiveF() { + return InflectedCounterPositiveF; + } + + public Integer getInflectedCounterPositive1() { + return InflectedCounterPositive1; + } + + public Integer getInflectedCounterNegative1() { + return InflectedCounterNegative1; + } + + public ArrayList getTokenEntryF() { + return tokenEntryF; + } + + public ArrayList getTokenEntry1() { + return tokenEntry1; + } + + public Integer getMarkedContinuousCounterF() { + return MarkedContinuousCounterF; + } + + public Integer getMarkedContinuousCounter1() { + return MarkedContinuousCounter1; + } + + public Integer getUnmarkedPatternCounterF() { + return UnmarkedPatternCounterF; + } + + public Integer getUnmarkedPatternCounter1() { + return UnmarkedPatternCounter1; + } + + public ArrayList getStrTokensIpartFormF() { + return strTokensIpartFormF; + } + + public ArrayList getStrTokensIpartForm1() { + return strTokensIpartForm1; + } + + public ArrayList getTokenFormsF() { + return tokenFormsF; + } + + public ArrayList getTokenForms1() { + return tokenForms1; + } + + public ArrayList getStrTokenEntryGetPOSF() { + return strTokenEntryGetPOSF; + } + + public ArrayList getStrTokenEntryGetPOS1() { + return strTokenEntryGetPOS1; + } + + public ArrayList getIntTokenEntyCountsF() { + return intTokenEntyCountsF; + } + + public ArrayList getIntTokenEntyCounts1() { + return intTokenEntyCounts1; + } + + public ArrayList getITokenTagsF() { + return ITokenTagsF; + } + + public ArrayList getITokenTags1() { + return ITokenTags1; + } + + public ArrayList getStrTokenStemsF() { + return strTokenStemsF; + } + + public ArrayList getStrTokenStems1() { + return strTokenStems1; + } + + public Integer getAnotatorcounterF() { + return AnotatorcounterF; + } + + public Integer getAnotatorcounter1() { + return Anotatorcounter1; + } + + public Integer getTokensCounterF() { + return TokensCounterF; + } + + public Integer getTokensCounter1() { + return TokensCounter1; + } + + public ArrayList getEntityTokenTagsF() { + return entityTokenTagsF; + } + + public ArrayList getEntityTokenTags1() { + return entityTokenTags1; + } + + public ArrayList getNerEntitiesF() { + return nerEntitiesF; + } + + public ArrayList getNerEntities1() { + return nerEntities1; + } + + public ArrayList getNerEntitiesTypeF() { + return nerEntitiesTypeF; + } + + public ArrayList getNerEntitiesType1() { + return nerEntitiesType1; + } + + public ArrayList getStopWordTokenF() { + return stopWordTokenF; + } + + public ArrayList getStopWordToken1() { + return stopWordToken1; + } + + public ArrayList getStopWordLemmaF() { + return stopWordLemmaF; + } + + public ArrayList getStopWordLemma1() { + return stopWordLemma1; + } + + public Integer getPairCounterF() { + return PairCounterF; + } + + public Integer getPairCounter1() { + return PairCounter1; + } + //caches private Integer tokenizeCounting; private Integer tokenizeCountingF; @@ -128,7 +353,62 @@ public class SentimentAnalyzerTest { private List sentencesSentiment1; private ArrayList treesF; private ArrayList trees1; - + private ArrayList grammaticalStructuresF; + private ArrayList grammaticalStructures1; + private ArrayList typedDependenciesF; + private ArrayList typedDependencies1; + private ArrayList rnnCoreAnnotationsPredictedF; + private ArrayList rnnCoreAnnotationsPredicted1; + private ArrayList simpleMatricesF; + private ArrayList simpleMatrices1; + private ArrayList simpleMatricesNodevectorsF; + private ArrayList simpleMatricesNodevectors1; + private List listF; + private List list1; + private Integer longestF; + private Integer longest1; + private Integer sentimentLongestF; + private Integer sentimentLongest1; + private List> imwesF; + private List> imwes1; + private Integer InflectedCounterNegativeF; + private Integer InflectedCounterNegative1; + private Integer InflectedCounterPositiveF; + private Integer InflectedCounterPositive1; + private ArrayList tokenEntryF; + private ArrayList tokenEntry1; + private Integer MarkedContinuousCounterF; + private Integer MarkedContinuousCounter1; + private Integer UnmarkedPatternCounterF; + private Integer UnmarkedPatternCounter1; + private ArrayList strTokensIpartFormF; + private ArrayList strTokensIpartForm1; + private ArrayList tokenFormsF; + private ArrayList tokenForms1; + private ArrayList strTokenEntryGetPOSF; + private ArrayList strTokenEntryGetPOS1; + private ArrayList intTokenEntyCountsF; + private ArrayList intTokenEntyCounts1; + private ArrayList ITokenTagsF; + private ArrayList ITokenTags1; + private ArrayList strTokenStemsF; + private ArrayList strTokenStems1; + private Integer AnotatorcounterF; + private Integer Anotatorcounter1; + private Integer TokensCounterF; + private Integer TokensCounter1; + private ArrayList entityTokenTagsF; + private ArrayList entityTokenTags1; + private ArrayList nerEntitiesF; + private ArrayList nerEntities1; + private ArrayList nerEntitiesTypeF; + private ArrayList nerEntitiesType1; + private ArrayList stopWordTokenF; + private ArrayList stopWordToken1; + private ArrayList stopWordLemmaF; + private ArrayList stopWordLemma1; + private Integer PairCounterF; + private Integer PairCounter1; public SentimentAnalyzerTest(String str, String str1, SimilarityMatrix smxParam, List coreMaps1, List coreMaps2, Annotation strPipeline1, Annotation strPipeline2, Annotation strPipeSentiment1, Annotation strPipeSentiment2, @@ -140,7 +420,40 @@ public class SentimentAnalyzerTest { retrieveTGWListF, ArrayList retrieveTGWList1, List sentencesF, List sentences1, List sentencesSentimentF, List sentencesSentiment1, - ArrayList treesF, ArrayList trees1) { + ArrayList treesF, ArrayList trees1, + ArrayList grammaticalStructuresF, + ArrayList grammaticalStructures1, + ArrayList typedDependenciesF, + ArrayList typedDependencies1, + ArrayList rnnCoreAnnotationsPredictedF, + ArrayList rnnCoreAnnotationsPredicted1, + ArrayList simpleMatricesF, + ArrayList simpleMatrices1, + ArrayList simpleMatricesNodevectorsF, + ArrayList simpleMatricesNodevectors1, + List listF, List list1, Integer longestF, Integer longest1, + Integer sentimentLongestF, Integer sentimentLongest1, + List> imwesF, List> imwes1, + Integer InflectedCounterNegativeF, + Integer InflectedCounterNegative1, Integer InflectedCounterPositiveF, + Integer InflectedCounterPositive1, ArrayList tokenEntryF, + ArrayList tokenEntry1, Integer MarkedContinuousCounterF, + Integer MarkedContinuousCounter1, Integer UnmarkedPatternCounterF, + Integer UnmarkedPatternCounter1, ArrayList strTokensIpartFormF, + ArrayList strTokensIpartForm1, ArrayList tokenFormsF, + ArrayList tokenForms1, ArrayList strTokenEntryGetPOSF, + ArrayList strTokenEntryGetPOS1, ArrayList intTokenEntyCountsF, + ArrayList intTokenEntyCounts1, ArrayList ITokenTagsF, + ArrayList ITokenTags1, ArrayList strTokenStemsF, + ArrayList strTokenStems1, Integer AnotatorcounterF, + Integer Anotatorcounter1, Integer TokensCounterF, + Integer TokensCounter1, ArrayList entityTokenTagsF, + ArrayList entityTokenTags1, ArrayList nerEntitiesF, + ArrayList nerEntities1, ArrayList nerEntitiesTypeF, + ArrayList nerEntitiesType1, ArrayList stopWordTokenF, + ArrayList stopWordToken1, ArrayList stopWordLemmaF, + ArrayList stopWordLemma1, Integer PairCounterF, + Integer PairCounter1) { this.str = str; this.str1 = str1; this.smxParam = smxParam; @@ -167,6 +480,62 @@ public class SentimentAnalyzerTest { this.sentencesSentiment1 = sentencesSentiment1; this.treesF = treesF; this.trees1 = trees1; + this.grammaticalStructuresF = grammaticalStructuresF; + this.grammaticalStructures1 = grammaticalStructures1; + this.typedDependenciesF = typedDependenciesF; + this.typedDependencies1 = typedDependencies1; + this.rnnCoreAnnotationsPredictedF = rnnCoreAnnotationsPredictedF; + this.rnnCoreAnnotationsPredicted1 = rnnCoreAnnotationsPredicted1; + this.simpleMatricesF = simpleMatricesF; + this.simpleMatrices1 = simpleMatrices1; + this.simpleMatricesNodevectorsF = simpleMatricesNodevectorsF; + this.simpleMatricesNodevectors1 = simpleMatricesNodevectors1; + this.listF = listF; + this.list1 = list1; + this.longestF = longestF; + this.longest1 = longest1; + this.sentimentLongestF = sentimentLongestF; + this.sentimentLongest1 = sentimentLongest1; + this.imwesF = imwesF; + this.imwes1 = imwes1; + this.InflectedCounterNegativeF = InflectedCounterNegativeF; + this.InflectedCounterNegative1 = InflectedCounterNegative1; + this.InflectedCounterPositiveF = InflectedCounterPositiveF; + this.InflectedCounterPositive1 = InflectedCounterPositive1; + this.tokenEntryF = tokenEntryF; + this.tokenEntry1 = tokenEntry1; + this.MarkedContinuousCounterF = MarkedContinuousCounterF; + this.MarkedContinuousCounter1 = MarkedContinuousCounter1; + this.UnmarkedPatternCounterF = UnmarkedPatternCounterF; + this.UnmarkedPatternCounter1 = UnmarkedPatternCounter1; + this.strTokensIpartFormF = strTokensIpartFormF; + this.strTokensIpartForm1 = strTokensIpartForm1; + this.tokenFormsF = tokenFormsF; + this.tokenForms1 = tokenForms1; + this.strTokenEntryGetPOSF = strTokenEntryGetPOSF; + this.strTokenEntryGetPOS1 = strTokenEntryGetPOS1; + this.intTokenEntyCountsF = intTokenEntyCountsF; + this.intTokenEntyCounts1 = intTokenEntyCounts1; + this.ITokenTagsF = ITokenTagsF; + this.ITokenTags1 = ITokenTags1; + this.strTokenStemsF = strTokenStemsF; + this.strTokenStems1 = strTokenStems1; + this.AnotatorcounterF = AnotatorcounterF; + this.Anotatorcounter1 = Anotatorcounter1; + this.TokensCounterF = TokensCounterF; + this.TokensCounter1 = TokensCounter1; + this.entityTokenTagsF = entityTokenTagsF; + this.entityTokenTags1 = entityTokenTags1; + this.nerEntitiesF = nerEntitiesF; + this.nerEntities1 = nerEntities1; + this.nerEntitiesTypeF = nerEntitiesTypeF; + this.nerEntitiesType1 = nerEntitiesType1; + this.stopWordTokenF = stopWordTokenF; + this.stopWordToken1 = stopWordToken1; + this.stopWordLemmaF = stopWordLemmaF; + this.stopWordLemma1 = stopWordLemma1; + this.PairCounterF = PairCounterF; + this.PairCounter1 = PairCounter1; } private List> getTaggedWordList(String message) { @@ -210,83 +579,97 @@ public class SentimentAnalyzerTest { int constituencySize = sentenceConstituencyParseList1.size() + sentenceConstituencyParseList2.size(); for (final Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2) { int constiRelationsize = 0; - for (final Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1) { - Set constinuent1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse2); - Set constinuent2 = Tdiff.markDiff(sentenceConstituencyParse2, sentenceConstituencyParse1); - ArrayList constiLabels = new ArrayList(); - for (final Constituent consti : constinuent1) { - for (final Constituent consti1 : constinuent2) { - if (consti.value().equals(consti1.value()) && !constiLabels.contains(consti.value())) { - constiLabels.add(consti.value()); - constiRelationsize++; + try { + if (sentenceConstituencyParse2 != null && !sentenceConstituencyParse2.isEmpty()) { + for (final Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1) { + try { + if (sentenceConstituencyParse1 != null && !sentenceConstituencyParse1.isEmpty()) { + Set constinuent1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse2); + Set constinuent2 = Tdiff.markDiff(sentenceConstituencyParse2, sentenceConstituencyParse1); + ArrayList constiLabels = new ArrayList(); + for (final Constituent consti : constinuent1) { + for (final Constituent consti1 : constinuent2) { + if (consti.value().equals(consti1.value()) && !constiLabels.contains(consti.value())) { + constiLabels.add(consti.value()); + constiRelationsize++; + } + } + } + int constituents1 = constinuent1.size() - constiRelationsize; + int constituents2 = constinuent2.size() - constiRelationsize; + constituentsMap.add(constituents1); + constituentsMap.add(constituents2); + constituentsMap.add(constiRelationsize); + if (constituentsMap.size() < 4) { + if ((constituents1 * 5 < constituents2 || constituents2 * 5 < constituents1) && constituents1 > 0 && constituents2 > 0) { + score -= (constituents1 + constituents2) * 200; + } else if ((constituents1 == 0 || constituents2 == 0) && (constituents1 >= constituents2 + 4 || constituents2 >= constituents1 + 4)) { + score -= constituents1 > constituents2 ? constituents1 * 500 : constituents2 * 500; + } else if (constiRelationsize >= constituents1 + constituents2 && (constituents1 > 2 && constituents2 > 2)) { + score += (constiRelationsize + constituents1 + constituents2) * 350; + } else if (constituents1 >= 2 && constituents2 >= 2 && constituents1 * 1.5 > constituents2 && constituents2 * 1.5 > constituents1) { + if (constituents1 == constituents2 && constiRelationsize - constituents1 == 1) { + score += (constiRelationsize + constituents1 + constituents2) * 1550; + } else if (constiRelationsize >= constituents1 && constituents1 == constituents2) { + score -= (constiRelationsize + constituents1 + constituents2) * 550; + } else if (constiRelationsize < constituents1 && constiRelationsize < constituents2) { + score += 800; + } else if ((constiRelationsize == constituents1 || constiRelationsize == constituents2) && constituents1 * 1.5 > constituents2 + && constituents2 * 1.5 > constituents1) { + score += (constiRelationsize + constituents1 + constituents2) * 350; + } + } else if (constiRelationsize > constituents1 + constituents2) { + score += 2500; + } else if (constiRelationsize * 5 < constituents1 || constiRelationsize * 5 < constituents2) { + score -= (constituents1 + constituents2) * 400; + } + } else { + score = preConstituentsScore; + int n1 = constituentsMap.get(0); + int n2 = constituentsMap.get(1); + int n3 = constituentsMap.get(2); + int cap = 0; + if (n1 > n2 && n1 > n3) { + cap = n1; + } else if (n2 > n3 && n2 > n1) { + cap = n2; + } else { + cap = n3; + } + int overheat = 0; + for (int iterator = 3; iterator < constituentsMap.size(); iterator++) { + Integer getConstituent = constituentsMap.get(iterator); + if (getConstituent > cap) { + overheat++; + } + } + if (overheat > 1) { + score -= overheat * 800; + } else { + score += 1300; + } + } + } + } catch (NoSuchElementException e) { } } + if (constituencySize > 10) { + score -= constituencySize * 400; + } } - int constituents1 = constinuent1.size() - constiRelationsize; - int constituents2 = constinuent2.size() - constiRelationsize; - constituentsMap.add(constituents1); - constituentsMap.add(constituents2); - constituentsMap.add(constiRelationsize); - if (constituentsMap.size() < 4) { - if ((constituents1 * 5 < constituents2 || constituents2 * 5 < constituents1) && constituents1 > 0 && constituents2 > 0) { - score -= (constituents1 + constituents2) * 200; - } else if ((constituents1 == 0 || constituents2 == 0) && (constituents1 >= constituents2 + 4 || constituents2 >= constituents1 + 4)) { - score -= constituents1 > constituents2 ? constituents1 * 500 : constituents2 * 500; - } else if (constiRelationsize >= constituents1 + constituents2 && (constituents1 > 2 && constituents2 > 2)) { - score += (constiRelationsize + constituents1 + constituents2) * 350; - } else if (constituents1 >= 2 && constituents2 >= 2 && constituents1 * 1.5 > constituents2 && constituents2 * 1.5 > constituents1) { - if (constituents1 == constituents2 && constiRelationsize - constituents1 == 1) { - score += (constiRelationsize + constituents1 + constituents2) * 1550; - } else if (constiRelationsize >= constituents1 && constituents1 == constituents2) { - score -= (constiRelationsize + constituents1 + constituents2) * 550; - } else if (constiRelationsize < constituents1 && constiRelationsize < constituents2) { - score += 800; - } else if ((constiRelationsize == constituents1 || constiRelationsize == constituents2) && constituents1 * 1.5 > constituents2 - && constituents2 * 1.5 > constituents1) { - score += (constiRelationsize + constituents1 + constituents2) * 350; - } - } else if (constiRelationsize > constituents1 + constituents2) { - score += 2500; - } else if (constiRelationsize * 5 < constituents1 || constiRelationsize * 5 < constituents2) { - score -= (constituents1 + constituents2) * 400; - } - } else { - score = preConstituentsScore; - int n1 = constituentsMap.get(0); - int n2 = constituentsMap.get(1); - int n3 = constituentsMap.get(2); - int cap = 0; - if (n1 > n2 && n1 > n3) { - cap = n1; - } else if (n2 > n3 && n2 > n1) { - cap = n2; - } else { - cap = n3; - } - int overheat = 0; - for (int iterator = 3; iterator < constituentsMap.size(); iterator++) { - Integer getConstituent = constituentsMap.get(iterator); - if (getConstituent > cap) { - overheat++; - } - } - if (overheat > 1) { - score -= overheat * 800; - } else { - score += 1300; - } - } + + } catch (NoSuchElementException e) { + } } - if (constituencySize > 10) { - score -= constituencySize * 400; - } return score; } - private Double typeDependenciesGrammaticalRelation(Collection allTypedDependencies1, Collection allTypedDependencies2, - Double score, ArrayList grammaticalMap1, ArrayList grammaticalMap2, - ArrayList sentenceConstituencyParseList1, ArrayList sentenceConstituencyParseList2) { + private Double typeDependenciesGrammaticalRelation + (Collection allTypedDependencies1, Collection allTypedDependencies2, + Double score, ArrayList grammaticalMap1, + ArrayList grammaticalMap2, + ArrayList sentenceConstituencyParseList1, ArrayList sentenceConstituencyParseList2) { ArrayList alltypeDepsSize1 = new ArrayList(); ArrayList summationList = new ArrayList(); int relationApplicable1 = 0; @@ -301,16 +684,22 @@ public class SentimentAnalyzerTest { for (GrammaticalStructure gs : grammaticalMap1) { GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep); for (Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2) { - if (grammaticalRelation.isApplicable(sentenceConstituencyParse2) && !treeCollectionGramatical.contains(sentenceConstituencyParse2)) { - score += 700; - grammaticalRelation1++; - treeCollectionGramatical.add(sentenceConstituencyParse2); - } - GrammaticalRelation reln = TDY1.reln(); - if (reln.isApplicable(sentenceConstituencyParse2) && !treeCollectionReln.contains(sentenceConstituencyParse2)) { - score += 525; - relationApplicable1++; - treeCollectionReln.add(sentenceConstituencyParse2); + try { + if (sentenceConstituencyParse2 != null && !sentenceConstituencyParse2.isEmpty()) { + if (grammaticalRelation.isApplicable(sentenceConstituencyParse2) && !treeCollectionGramatical.contains(sentenceConstituencyParse2)) { + score += 700; + grammaticalRelation1++; + treeCollectionGramatical.add(sentenceConstituencyParse2); + } + GrammaticalRelation reln = TDY1.reln(); + if (reln.isApplicable(sentenceConstituencyParse2) && !treeCollectionReln.contains(sentenceConstituencyParse2)) { + score += 525; + relationApplicable1++; + treeCollectionReln.add(sentenceConstituencyParse2); + } + } + } catch (NoSuchElementException e) { + } } } @@ -323,17 +712,23 @@ public class SentimentAnalyzerTest { for (GrammaticalStructure gs : grammaticalMap2) { GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep); for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1) { - if (grammaticalRelation.isApplicable(sentenceConstituencyParse1) && !treeCollectionGramatical.contains(sentenceConstituencyParse1)) { - score += 700; - grammaticalRelation2++; - treeCollectionGramatical.add(sentenceConstituencyParse1); - } - GrammaticalRelation reln = TDY.reln(); - //sentenceConstituencyParse1 - if (reln.isApplicable(sentenceConstituencyParse1) && !treeCollectionReln.contains(sentenceConstituencyParse1)) { - score += 525; - relationApplicable2++; - treeCollectionReln.add(sentenceConstituencyParse1); + try { + if (sentenceConstituencyParse1 != null && !sentenceConstituencyParse1.isEmpty()) { + if (grammaticalRelation.isApplicable(sentenceConstituencyParse1) && !treeCollectionGramatical.contains(sentenceConstituencyParse1)) { + score += 700; + grammaticalRelation2++; + treeCollectionGramatical.add(sentenceConstituencyParse1); + } + GrammaticalRelation reln = TDY.reln(); + //sentenceConstituencyParse1 + if (reln.isApplicable(sentenceConstituencyParse1) && !treeCollectionReln.contains(sentenceConstituencyParse1)) { + score += 525; + relationApplicable2++; + treeCollectionReln.add(sentenceConstituencyParse1); + } + } + } catch (NoSuchElementException r) { + } } } @@ -397,24 +792,36 @@ public class SentimentAnalyzerTest { ArrayList filerTreeContent = new ArrayList(); int runCount1 = 0; for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1) { - for (Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2) { - for (CoreLabel LBW : sentenceConstituencyParse1.taggedLabeledYield()) { - for (CoreLabel LBW1 : sentenceConstituencyParse2.taggedLabeledYield()) { - if (LBW.lemma().equals(LBW1.lemma())) { - boolean found = false; - for (String str : filerTreeContent) { - if (str.equals(LBW.lemma())) { - found = true; - break; + try { + if (sentenceConstituencyParse1 != null && !sentenceConstituencyParse1.isEmpty()) { + for (Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2) { + try { + if (sentenceConstituencyParse2 != null && !sentenceConstituencyParse2.isEmpty()) { + for (CoreLabel LBW : sentenceConstituencyParse1.taggedLabeledYield()) { + for (CoreLabel LBW1 : sentenceConstituencyParse2.taggedLabeledYield()) { + if (LBW.lemma().equals(LBW1.lemma())) { + boolean found = false; + for (String str : filerTreeContent) { + if (str.equals(LBW.lemma())) { + found = true; + break; + } + } + if (!found) { + filerTreeContent.add(LBW.lemma()); + runCount1++; + } + } + } } } - if (!found) { - filerTreeContent.add(LBW.lemma()); - runCount1++; - } + } catch (NoSuchElementException e) { + } } } + } catch (NoSuchElementException e) { + } } score += runCount1 * 250; @@ -608,7 +1015,8 @@ public class SentimentAnalyzerTest { return score; } - private Double simpleRNNMaxtrixVectors(Double score, ArrayList simpleSMXlistVector1, ArrayList simpleSMXlistVector2) { + private Double simpleRNNMaxtrixVectors(Double + score, ArrayList simpleSMXlistVector1, ArrayList simpleSMXlistVector2) { ArrayList elementSumCounter = new ArrayList<>(); ArrayList dotMap = new ArrayList<>(); ArrayList elementSumMap = new ArrayList<>(); @@ -705,7 +1113,8 @@ public class SentimentAnalyzerTest { return score; } - private Double elementsAndDotsRelation(Double score, ArrayList dotMap, ArrayList elementSumCounter) { + private Double elementsAndDotsRelation(Double + score, ArrayList dotMap, ArrayList elementSumCounter) { OptionalDouble minvalueDots = dotMap.stream().mapToDouble(Double::doubleValue).min(); OptionalDouble maxvalueDots = dotMap.stream().mapToDouble(Double::doubleValue).max(); boolean permitted = false; @@ -751,7 +1160,8 @@ public class SentimentAnalyzerTest { return score; } - private Double sentimentMatrixVariances(Double score, int longest1, int longest2, int mainSentiment1, int mainSentiment2) { + private Double sentimentMatrixVariances(Double score, int longest1, int longest2, int mainSentiment1, + int mainSentiment2) { if (longest1 != longest2) { long deffLongest = longest1 > longest2 ? longest1 : longest2; long deffshorter = longest1 < longest2 ? longest1 : longest2; @@ -788,15 +1198,13 @@ public class SentimentAnalyzerTest { } private int classifyRawEvaluation() { - DocumentReaderAndWriter readerAndWriter = classifier.makePlainTextReaderAndWriter(); - List list1 = classifier.classifyRaw(str, readerAndWriter); - List list2 = classifier.classifyRaw(str1, readerAndWriter); - final List classifyRaw1 = list1; - final List classifyRaw2 = list2; + final List classifyRaw1 = this.listF; + final List classifyRaw2 = this.list1; return (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200; } - private Double entryCountsRelation(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + private Double entryCountsRelation(Double + score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { int entry1 = cacheSentimentLocal1.size(); int entry2 = cacheSentimentLocal2.size(); if (entry1 > 0 && entry2 > 0) { @@ -828,8 +1236,14 @@ public class SentimentAnalyzerTest { private ArrayList grammaticalStructureSetup(ArrayList trees) { ArrayList grammaticalStructures = new ArrayList(); for (Tree tree : trees) { - GrammaticalStructure gs = gsf.newGrammaticalStructure(tree); - grammaticalStructures.add(gs); + try { + if (!tree.isEmpty()) { + GrammaticalStructure gs = gsf.newGrammaticalStructure(tree); + grammaticalStructures.add(gs); + } + } catch (NoSuchElementException e) { + + } } return grammaticalStructures; } @@ -909,7 +1323,8 @@ public class SentimentAnalyzerTest { return longest; } - private Double entryCountsScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + private Double entryCountsScoring(Double + score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { ArrayList countsMap = new ArrayList(); int totalsize = cacheSentimentLocal1.size() + cacheSentimentLocal2.size(); for (int counts : cacheSentimentLocal1) { @@ -927,7 +1342,8 @@ public class SentimentAnalyzerTest { return score; } - private Double tokenEntryPosScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + private Double tokenEntryPosScoring(Double + score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { if (cacheSentimentLocal1.size() > 1 && cacheSentimentLocal2.size() > 1) { for (String strTokenPos1 : cacheSentimentLocal1) { for (String strTokenPos2 : cacheSentimentLocal2) { @@ -945,7 +1361,8 @@ public class SentimentAnalyzerTest { return score; } - private Double unmarkedPatternCounterScoring(Double score, int UnmarkedPatternCounter1, int UnmarkedPatternCounter2) { + private Double unmarkedPatternCounterScoring(Double score, int UnmarkedPatternCounter1, + int UnmarkedPatternCounter2) { if (UnmarkedPatternCounter1 > 0 && UnmarkedPatternCounter2 > 0) { if (UnmarkedPatternCounter1 < 100 && UnmarkedPatternCounter2 < 100) { if (UnmarkedPatternCounter1 * 2 > UnmarkedPatternCounter2 && UnmarkedPatternCounter2 * 2 > UnmarkedPatternCounter1) { @@ -960,7 +1377,8 @@ public class SentimentAnalyzerTest { return score; } - private Double markedContiniousCounterScoring(Double score, int MarkedContinuousCounter1, int MarkedContinuousCounter2) { + private Double markedContiniousCounterScoring(Double score, int MarkedContinuousCounter1, + int MarkedContinuousCounter2) { int MarkedContiniousCounter1Entries = MarkedContinuousCounter1; int MarkedContiniousCounter2Entries = MarkedContinuousCounter2; if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) { @@ -990,7 +1408,8 @@ public class SentimentAnalyzerTest { return score; } - private Double strTokensMapScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + private Double strTokensMapScoring(Double + score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { ArrayList strtokensMap = new ArrayList(); for (String strTokeniPart1 : cacheSentimentLocal1) { for (String strTokeniPart2 : cacheSentimentLocal2) { @@ -1012,7 +1431,8 @@ public class SentimentAnalyzerTest { return score; } - private Double strTokenEntryScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + private Double strTokenEntryScoring(Double + score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { int tokenEntry1 = cacheSentimentLocal1.size(); int tokenEntry2 = cacheSentimentLocal2.size(); boolean boundaryLeaks = false; @@ -1039,7 +1459,8 @@ public class SentimentAnalyzerTest { return score; } - private Double strTokenMapTagsScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + private Double strTokenMapTagsScoring(Double + score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { ArrayList iTokenMapTagsMap = new ArrayList(); for (String strmapTag : cacheSentimentLocal1) { for (String strmapTag1 : cacheSentimentLocal2) { @@ -1063,7 +1484,8 @@ public class SentimentAnalyzerTest { return score; } - private Double tokenformSizeScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + private Double tokenformSizeScoring(Double + score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { int tokenform1size = cacheSentimentLocal1.size(); int tokenform2size = cacheSentimentLocal2.size(); if (tokenform1size > 0 || tokenform2size > 0) { @@ -1088,7 +1510,8 @@ public class SentimentAnalyzerTest { return score; } - private Double tokenStemmingMapScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + private Double tokenStemmingMapScoring(Double + score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { ArrayList tokenStemmingMap = new ArrayList(); for (String strTokenStem : cacheSentimentLocal1) { for (String strTokenStem1 : cacheSentimentLocal2) { @@ -1101,7 +1524,8 @@ public class SentimentAnalyzerTest { return score; } - private Double inflectedCounterScoring(Double score, int inflectedCounterPositive1, int inflectedCounterPositive2, + private Double inflectedCounterScoring(Double score, int inflectedCounterPositive1, + int inflectedCounterPositive2, int inflectedCounterNegative1, int inflectedCounterNegative2) { int inflectedCounterNegative = inflectedCounterNegative1 > inflectedCounterNegative1 ? inflectedCounterNegative1 - inflectedCounterNegative2 @@ -1262,7 +1686,236 @@ public class SentimentAnalyzerTest { return score; } + private List> getIMWES(List coreMaps) { + List> tokenList = new ArrayList<>(); + for (CoreMap sentence : coreMaps) { + List> imwes = sentence.get(JMWEAnnotation.class); + tokenList.addAll(imwes); + } + return tokenList; + } + + private int getInflictedCounterPositive(List> imwesFLocal) { + int InflectedCounterPositive = 0; + for (IMWE token : imwesFLocal) { + if (token.isInflected()) { + InflectedCounterPositive++; + } + } + return InflectedCounterPositive; + } + + private int getUnmarkedPatterns(List> imwesFLocal) { + int unmarked = 0; + for (IMWE token : imwesFLocal) { + IMWEDesc entry = token.getEntry(); + unmarked += entry.getUnmarkedPattern(); + } + return unmarked; + } + + private ArrayList gettokenForms(List> imwesFLocal) { + ArrayList arrs = new ArrayList<>(); + for (IMWE token : imwesFLocal) { + String form = token.getForm(); + arrs.add(form); + } + return arrs; + } + + private ArrayList getStrtokenEntryPos(List> imwesFLocal) { + ArrayList arrs = new ArrayList<>(); + for (IMWE token : imwesFLocal) { + IMWEDesc entry = token.getEntry(); + for (String strPostPrefix : entry.getPOS().getPrefixes()) { + arrs.add(strPostPrefix); + } + } + return arrs; + } + + private ArrayList getintTokenEntyCounts(List> imwesFLocal) { + ArrayList arrs = new ArrayList<>(); + for (IMWE token : imwesFLocal) { + IMWEDesc entry = token.getEntry(); + for (int counts : entry.getCounts()) { + arrs.add(counts); + } + } + return arrs; + } + + private ArrayList getITokenTags(List> imwesFLocal) { + ArrayList arrs = new ArrayList<>(); + for (IMWE token : imwesFLocal) { + for (IToken tokens : token.getTokens()) { + arrs.add(tokens.getTag()); + } + } + return arrs; + } + + private ArrayList getstrTokenStems(List> imwesFLocal) { + ArrayList arrs = new ArrayList<>(); + for (IMWE token : imwesFLocal) { + for (IToken tokens : token.getTokens()) { + for (String strtoken : tokens.getStems()) { + arrs.add(strtoken); + } + } + } + return arrs; + } + + private Integer getAnotatorcounter(List> imwesFLocal) { + return imwesFLocal.size(); + } + + private ArrayList getnerEntities(CoreDocument coreDocument) { + ArrayList arrs = new ArrayList<>(); + for (CoreEntityMention em : coreDocument.entityMentions()) { + if (!arrs.contains(em.text())) { + arrs.add(em.text()); + } + } + return arrs; + } + + private ArrayList getnerEntitiesType(CoreDocument coreDocument) { + ArrayList arrs = new ArrayList<>(); + for (CoreEntityMention em : coreDocument.entityMentions()) { + if (!arrs.contains(em.entityType())) { + arrs.add(em.entityType()); + } + } + return arrs; + } + + private Integer getPairCounter(Annotation pipelineAnnotationSentiment) { + int counter = 0; + List tokensSentiment = pipelineAnnotationSentiment. + get(CoreAnnotations.TokensAnnotation.class); + for (CoreLabel token : tokensSentiment) { + Pair stopword = token.get(StopwordAnnotator.class); + if (stopword.first() && stopword.second()) { + counter++; + } + } + return counter; + } + + private ArrayList getstopWordLemma(Annotation pipelineAnnotationSentiment) { + ArrayList arrs = new ArrayList<>(); + List tokensSentiment = pipelineAnnotationSentiment. + get(CoreAnnotations.TokensAnnotation.class); + String customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for," + + "if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these," + + "they,this,to,was,will,with"; + for (CoreLabel token : tokensSentiment) { + Set stopWords = StopAnalyzer.ENGLISH_STOP_WORDS_SET; + Set stopWordsCustom = StopwordAnnotator.getStopWordList(customStopWordList, true); + String lemma = token.lemma().toLowerCase(); + if (stopWords.contains(lemma) || stopWordsCustom.contains(lemma)) { + arrs.add(lemma); + } + } + return arrs; + } + + private ArrayList getstopWordToken(Annotation pipelineAnnotationSentiment) { + ArrayList arrs = new ArrayList<>(); + List tokensSentiment = pipelineAnnotationSentiment. + get(CoreAnnotations.TokensAnnotation.class); + String customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for," + + "if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these," + + "they,this,to,was,will,with"; + for (CoreLabel token : tokensSentiment) { + String word = token.word().toLowerCase(); + Set stopWords = StopAnalyzer.ENGLISH_STOP_WORDS_SET; + Set stopWordsCustom = StopwordAnnotator.getStopWordList(customStopWordList, true); + if (stopWords.contains(word) || stopWordsCustom.contains(word)) { + arrs.add(word); + } + } + return arrs; + } + + private ArrayList getentityTokenTags(CoreDocument coreDocument) { + ArrayList arrs = new ArrayList<>(); + for (CoreEntityMention em : coreDocument.entityMentions()) { + List tokens = em.tokens(); + String entityType = em.entityType(); + Double EntityConfidences = 0.0; + Set> entrySet = em.entityTypeConfidences().entrySet(); + for (Map.Entry entries : entrySet) { + if (EntityConfidences < entries.getValue()) { + EntityConfidences = entries.getValue(); + } + } + for (CoreLabel token : tokens) { + if (token != null) { + if (!arrs.contains(token.tag())) { + if (entityType.equals("PERSON") && EntityConfidences > 0.80) { + arrs.add(token.tag()); + } + } + } + } + } + return arrs; + } + + private ArrayList getstrTokensIpartForm(List> imwesFLocal) { + ArrayList arrs = new ArrayList<>(); + for (IMWE token : imwesFLocal) { + Collection values = token.getPartMap().values(); + for (IMWEDesc.IPart iPart : values) { + String iPartForm = iPart.getForm(); + arrs.add(iPartForm); + } + } + return arrs; + } + + private int getMarkedCounter(List> imwesFLocal) { + int marked = 0; + for (IMWE token : imwesFLocal) { + IMWEDesc entry = token.getEntry(); + marked += entry.getMarkedContinuous(); + for (IToken tokens : token.getTokens()) { + marked += tokens.getStems().size(); + } + } + return marked; + } + + public int getInflictedCounterNegative(List> imwesFLocal) { + int InflectedCounterNegative = 0; + Collection> tokeninflectionMap = new ArrayList(); + for (IMWE token : imwesFLocal) { + if (!token.isInflected() && !tokeninflectionMap.contains(token)) { + InflectedCounterNegative++; + tokeninflectionMap.add(token); + } + } + return InflectedCounterNegative; + } + + public ArrayList getTokenEntries(List> imwesFLocal) { + ArrayList tokenStrList = new ArrayList<>(); + for (IMWE token : imwesFLocal) { + final String substring = token.getEntry().toString().substring(token.getEntry() + .toString().length() - 1); + tokenStrList.add(substring); + } + return tokenStrList; + } + + public void validateStringCaches() { + Class sentimentAnnotatedTreeClass = + SentimentCoreAnnotations.SentimentAnnotatedTree.class; + if (this.tokenizeCountingF == null) { this.tokenizeCountingF = tokenizeCounting(getTaggedWordList(str1)); } @@ -1299,6 +1952,178 @@ public class SentimentAnalyzerTest { if (this.trees1 == null) { this.trees1 = retrieveTrees(this.sentences1); } + if (this.grammaticalStructuresF == null) { + this.grammaticalStructuresF = grammaticalStructureSetup(this.treesF); + } + if (this.grammaticalStructures1 == null) { + this.grammaticalStructures1 = grammaticalStructureSetup(this.trees1); + } + if (this.typedDependenciesF == null) { + this.typedDependenciesF = grammaticalStructureAllTypedDependencies(this.grammaticalStructuresF); + } + if (this.typedDependencies1 == null) { + this.typedDependencies1 = grammaticalStructureAllTypedDependencies(this.grammaticalStructures1); + } + if (this.rnnCoreAnnotationsPredictedF == null) { + this.rnnCoreAnnotationsPredictedF = sentimentRNNCoreAnnotations(this.sentencesSentimentF, + sentimentAnnotatedTreeClass); + } + if (this.rnnCoreAnnotationsPredicted1 == null) { + this.rnnCoreAnnotationsPredicted1 = sentimentRNNCoreAnnotations(this.sentencesSentiment1, + sentimentAnnotatedTreeClass); + } + if (this.simpleMatricesF == null) { + this.simpleMatricesF = sentimentRNNCorePredicted(this.sentencesSentimentF, sentimentAnnotatedTreeClass); + } + if (this.simpleMatrices1 == null) { + this.simpleMatrices1 = sentimentRNNCorePredicted(this.sentencesSentiment1, sentimentAnnotatedTreeClass); + } + if (this.simpleMatricesNodevectorsF == null) { + this.simpleMatricesNodevectorsF = sentimentRNNCoreNodevectors(this.sentencesSentimentF, sentimentAnnotatedTreeClass); + } + if (this.simpleMatricesNodevectors1 == null) { + this.simpleMatricesNodevectors1 = sentimentRNNCoreNodevectors(this.sentencesSentiment1, sentimentAnnotatedTreeClass); + } + if (this.listF == null) { + DocumentReaderAndWriter readerAndWriter = classifier.makePlainTextReaderAndWriter(); + this.listF = classifier.classifyRaw(str, readerAndWriter); + } + if (this.list1 == null) { + DocumentReaderAndWriter readerAndWriter = classifier.makePlainTextReaderAndWriter(); + this.list1 = classifier.classifyRaw(str1, readerAndWriter); + } + if (this.longestF == null) { + this.longestF = setupMainLongest(this.sentencesSentimentF); + } + if (this.longest1 == null) { + this.longest1 = setupMainLongest(this.sentencesSentiment1); + } + if (this.sentimentLongestF == null) { + this.sentimentLongestF = setupMainSentiment(this.sentencesSentimentF, sentimentAnnotatedTreeClass); + } + if (this.sentimentLongest1 == null) { + this.sentimentLongest1 = setupMainSentiment(this.sentencesSentiment1, sentimentAnnotatedTreeClass); + } + if (this.imwesF == null) { + this.imwesF = getIMWES(this.coreMaps1); + } + if (this.imwes1 == null) { + this.imwes1 = getIMWES(this.coreMaps2); + } + if (this.InflectedCounterNegativeF == null) { + this.InflectedCounterNegativeF = getInflictedCounterNegative(this.imwesF); + } + if (this.InflectedCounterNegative1 == null) { + this.InflectedCounterNegative1 = getInflictedCounterNegative(this.imwes1); + } + if (this.InflectedCounterPositiveF == null) { + this.InflectedCounterPositiveF = getInflictedCounterPositive(this.imwesF); + } + if (this.InflectedCounterPositive1 == null) { + this.InflectedCounterPositive1 = getInflictedCounterPositive(this.imwes1); + } + if (this.tokenEntryF == null) { + this.tokenEntryF = getTokenEntries(this.imwesF); + } + if (this.tokenEntry1 == null) { + this.tokenEntry1 = getTokenEntries(this.imwes1); + } + if (this.MarkedContinuousCounterF == null) { + this.MarkedContinuousCounterF = getMarkedCounter(this.imwesF); + } + if (this.MarkedContinuousCounter1 == null) { + this.MarkedContinuousCounter1 = getMarkedCounter(this.imwes1); + } + if (this.UnmarkedPatternCounterF == null) { + this.UnmarkedPatternCounterF = getUnmarkedPatterns(this.imwesF); + } + if (this.UnmarkedPatternCounter1 == null) { + this.UnmarkedPatternCounter1 = getUnmarkedPatterns(this.imwes1); + } + if (this.strTokensIpartFormF == null) { + this.strTokensIpartFormF = getstrTokensIpartForm(this.imwesF); + } + if (this.strTokensIpartForm1 == null) { + this.strTokensIpartForm1 = getstrTokensIpartForm(this.imwes1); + } + if (this.tokenFormsF == null) { + this.tokenFormsF = gettokenForms(this.imwesF); + } + if (this.tokenForms1 == null) { + this.tokenForms1 = gettokenForms(this.imwes1); + } + if (this.strTokenEntryGetPOSF == null) { + this.strTokenEntryGetPOSF = getStrtokenEntryPos(this.imwesF); + } + if (this.strTokenEntryGetPOS1 == null) { + this.strTokenEntryGetPOS1 = getStrtokenEntryPos(this.imwes1); + } + if (this.intTokenEntyCountsF == null) { + this.intTokenEntyCountsF = getintTokenEntyCounts(this.imwesF); + } + if (this.intTokenEntyCounts1 == null) { + this.intTokenEntyCounts1 = getintTokenEntyCounts(this.imwes1); + } + if (this.ITokenTagsF == null) { + this.ITokenTagsF = getITokenTags(this.imwesF); + } + if (this.ITokenTags1 == null) { + this.ITokenTags1 = getITokenTags(this.imwes1); + } + if (this.strTokenStemsF == null) { + this.strTokenStemsF = getstrTokenStems(this.imwesF); + } + if (this.strTokenStems1 == null) { + this.strTokenStems1 = getstrTokenStems(this.imwes1); + } + if (this.AnotatorcounterF == null) { + this.AnotatorcounterF = getAnotatorcounter(this.imwesF); + } + if (this.Anotatorcounter1 == null) { + this.Anotatorcounter1 = getAnotatorcounter(this.imwes1); + } + if (this.TokensCounterF == null) { + this.TokensCounterF = getAnotatorcounter(this.imwesF); + } + if (this.TokensCounter1 == null) { + this.TokensCounter1 = getAnotatorcounter(this.imwes1); + } + if (this.entityTokenTagsF == null) { + this.entityTokenTagsF = getentityTokenTags(this.pipelineCoreDcoument1); + } + if (this.entityTokenTags1 == null) { + this.entityTokenTags1 = getentityTokenTags(this.pipelineCoreDcoument2); + } + if (this.nerEntitiesF == null) { + this.nerEntitiesF = getnerEntities(this.pipelineCoreDcoument1); + } + if (this.nerEntities1 == null) { + this.nerEntities1 = getnerEntities(this.pipelineCoreDcoument2); + } + if (this.nerEntitiesTypeF == null) { + this.nerEntitiesTypeF = getnerEntitiesType(this.pipelineCoreDcoument1); + } + if (this.nerEntitiesType1 == null) { + this.nerEntitiesType1 = getnerEntitiesType(this.pipelineCoreDcoument2); + } + if (this.stopWordTokenF == null) { + this.stopWordTokenF = getstopWordToken(this.pipelineAnnotation1Sentiment); + } + if (this.stopWordToken1 == null) { + this.stopWordToken1 = getstopWordToken(this.pipelineAnnotation2Sentiment); + } + if (this.stopWordLemmaF == null) { + this.stopWordLemmaF = getstopWordLemma(this.pipelineAnnotation1Sentiment); + } + if (this.stopWordLemma1 == null) { + this.stopWordLemma1 = getstopWordLemma(this.pipelineAnnotation2Sentiment); + } + if (this.PairCounterF == null) { + this.PairCounterF = getPairCounter(this.pipelineAnnotation1Sentiment); + } + if (this.PairCounter1 == null) { + this.PairCounter1 = getPairCounter(this.pipelineAnnotation2Sentiment); + } } @@ -1315,281 +2140,47 @@ public class SentimentAnalyzerTest { Class sentimentAnnotatedTreeClass = SentimentCoreAnnotations.SentimentAnnotatedTree.class; - ArrayList grammaticalStructures1 = grammaticalStructureSetup(this.treesF); - ArrayList grammaticalStructures2 = grammaticalStructureSetup(this.trees1); - ArrayList typedDependencies1 = grammaticalStructureAllTypedDependencies(grammaticalStructures1); - ArrayList typedDependencies2 = grammaticalStructureAllTypedDependencies(grammaticalStructures2); score = iterateTrees(this.treesF, this.trees1, score); - score = typeDependenciesGrammaticalRelation(typedDependencies1, typedDependencies2, score, grammaticalStructures1, grammaticalStructures2, + score = typeDependenciesGrammaticalRelation(this.typedDependenciesF, typedDependencies1, score, this.grammaticalStructuresF, this.grammaticalStructures1, this.treesF, this.trees1); - ArrayList rnnCoreAnnotationsPredicted1 = sentimentRNNCoreAnnotations(this.sentencesSentimentF, sentimentAnnotatedTreeClass); - ArrayList rnnCoreAnnotationsPredicted2 = sentimentRNNCoreAnnotations(this.sentencesSentiment1, sentimentAnnotatedTreeClass); - ArrayList simpleMatrices1 = sentimentRNNCorePredicted(this.sentencesSentimentF, sentimentAnnotatedTreeClass); - ArrayList simpleMatrices2 = sentimentRNNCorePredicted(this.sentencesSentiment1, sentimentAnnotatedTreeClass); - ArrayList simpleMatricesNodevectors1 = sentimentRNNCoreNodevectors(this.sentencesSentimentF, sentimentAnnotatedTreeClass); - ArrayList simpleMatricesNodevectors2 = sentimentRNNCoreNodevectors(this.sentencesSentiment1, sentimentAnnotatedTreeClass); - score = simpleRNNMatrixCalculations(score, simpleMatrices1, simpleMatrices2); - score = simpleRNNMaxtrixVectors(score, simpleMatricesNodevectors1, simpleMatricesNodevectors2); - Integer sentiment1 = rnnCoreAnnotationsPredicted1.size(); - Integer sentiment2 = rnnCoreAnnotationsPredicted2.size(); + + score = simpleRNNMatrixCalculations(score, this.simpleMatricesF, this.simpleMatrices1); + score = simpleRNNMaxtrixVectors(score, this.simpleMatricesNodevectorsF, this.simpleMatricesNodevectors1); + Integer sentiment1 = this.rnnCoreAnnotationsPredictedF.size(); + Integer sentiment2 = this.rnnCoreAnnotationsPredicted1.size(); score -= (sentiment1 > sentiment2 ? sentiment2 - sentiment1 : sentiment1 - sentiment2) * 500; score -= classifyRawEvaluation(); - int longest1 = setupMainLongest(this.sentencesSentimentF); - int longest2 = setupMainLongest(this.sentencesSentiment1); - int sentimentLongest1 = setupMainSentiment(this.sentencesSentimentF, sentimentAnnotatedTreeClass); - int sentimentLongest2 = setupMainSentiment(this.sentencesSentiment1, sentimentAnnotatedTreeClass); - score = sentimentMatrixVariances(score, longest1, longest2, sentimentLongest1, sentimentLongest2); + score = sentimentMatrixVariances(score, this.longestF, this.longest1, this.sentimentLongestF, this.sentimentLongest1); - int InflectedCounterNegative1 = 0; - int InflectedCounterNegative2 = 0; - int InflectedCounterPositive1 = 0; - int InflectedCounterPositive2 = 0; + score = entryCountsRelation(score, this.intTokenEntyCountsF, this.intTokenEntyCounts1); + score = entryCountsScoring(score, this.intTokenEntyCountsF, this.intTokenEntyCounts1); + score = tokenEntryPosScoring(score, this.strTokenEntryGetPOSF, this.strTokenEntryGetPOS1); + score = unmarkedPatternCounterScoring(score, this.UnmarkedPatternCounterF, + this.UnmarkedPatternCounter1); + score = markedContiniousCounterScoring(score, this.MarkedContinuousCounterF, + this.MarkedContinuousCounter1); + score = strTokensMapScoring(score, this.strTokensIpartFormF, this.strTokensIpartForm1); + score = strTokenEntryScoring(score, this.tokenEntryF, this.tokenEntry1); + score = strTokenMapTagsScoring(score, this.ITokenTagsF, this.ITokenTags1); + score = tokenformSizeScoring(score, this.tokenFormsF, this.tokenForms1); + score = tokenStemmingMapScoring(score, this.strTokenStemsF, this.strTokenStems1); - ArrayList> coreMapsList = new ArrayList<>(); - coreMapsList.add(this.coreMaps1); - coreMapsList.add(this.coreMaps2); - - int MarkedContinuousCounter1 = 0; - int MarkedContinuousCounter2 = 0; - int UnmarkedPatternCounter1 = 0; - int UnmarkedPatternCounter2 = 0; - ArrayList strTokensIpartForm1 = new ArrayList<>(); - ArrayList strTokensIpartForm2 = new ArrayList<>(); - ArrayList tokenEntry1 = new ArrayList<>(); - ArrayList tokenEntry2 = new ArrayList<>(); - ArrayList tokenForms1 = new ArrayList<>(); - ArrayList tokenForms2 = new ArrayList<>(); - ArrayList strTokenEntryGetPOS1 = new ArrayList<>(); - ArrayList strTokenEntryGetPOS2 = new ArrayList<>(); - ArrayList intTokenEntyCounts1 = new ArrayList(); - ArrayList intTokenEntyCounts2 = new ArrayList(); - ArrayList ITokenTags1 = new ArrayList<>(); - ArrayList ITokenTags2 = new ArrayList<>(); - ArrayList strTokenStems1 = new ArrayList<>(); - ArrayList strTokenStems2 = new ArrayList<>(); - int Anotatorcounter1 = 0; - int Anotatorcounter2 = 0; - int TokensCounter1 = 0; - int TokensCounter2 = 0; - boolean firstAnnotation = true; - - for (List coreMaps : coreMapsList) { - int tokenadder = 0; - Collection> tokeninflectionMap = new ArrayList(); - for (CoreMap sentence : coreMaps) { - List> imwes = sentence.get(JMWEAnnotation.class); - for (IMWE token : imwes) { - Collection values = token.getPartMap().values(); - IMWEDesc entry = token.getEntry(); - if (!token.isInflected() && !tokeninflectionMap.contains(token)) { - if (firstAnnotation) { - InflectedCounterNegative1++; - } else { - InflectedCounterNegative2++; - } - tokeninflectionMap.add(token); - } else if (token.isInflected()) { - if (firstAnnotation) { - InflectedCounterPositive1++; - } else { - InflectedCounterPositive2++; - } - } - - String form = token.getForm(); - final String substring = token.getEntry().toString().substring(token.getEntry().toString().length() - 1); - if (firstAnnotation) { - tokenEntry1.add(substring); - tokenForms1.add(form); - } else { - tokenEntry2.add(substring); - tokenForms2.add(form); - } - if (firstAnnotation) { - MarkedContinuousCounter1 += entry.getMarkedContinuous(); - } else { - MarkedContinuousCounter2 += entry.getMarkedContinuous(); - } - if (firstAnnotation) { - UnmarkedPatternCounter1 += entry.getUnmarkedPattern(); - } else { - UnmarkedPatternCounter2 += entry.getUnmarkedPattern(); - } - for (IMWEDesc.IPart iPart : values) { - String iPartForm = iPart.getForm(); - if (firstAnnotation) { - strTokensIpartForm1.add(iPartForm); - } else { - strTokensIpartForm2.add(iPartForm); - } - } - for (String strPostPrefix : entry.getPOS().getPrefixes()) { - if (firstAnnotation) { - strTokenEntryGetPOS1.add(strPostPrefix); - } else { - strTokenEntryGetPOS2.add(strPostPrefix); - } - } - for (int counts : entry.getCounts()) { - if (firstAnnotation) { - intTokenEntyCounts1.add(counts); - } else { - intTokenEntyCounts2.add(counts); - } - } - for (IToken tokens : token.getTokens()) { - if (firstAnnotation) { - ITokenTags1.add(tokens.getTag()); - } else { - ITokenTags2.add(tokens.getTag()); - } - for (String strtoken : tokens.getStems()) { - if (firstAnnotation) { - strTokenStems1.add(strtoken); - } else { - strTokenStems2.add(strtoken); - } - if (firstAnnotation) { - MarkedContinuousCounter1++; - } else { - MarkedContinuousCounter2++; - } - } - } - tokenadder += 1; - } - if (firstAnnotation) { - Anotatorcounter1++; - } else { - Anotatorcounter2++; - } - } - firstAnnotation = false; - if (firstAnnotation) { - TokensCounter1 = tokenadder; - } else { - TokensCounter2 = tokenadder; - } - } - - score = entryCountsRelation(score, intTokenEntyCounts1, intTokenEntyCounts2); - score = entryCountsScoring(score, intTokenEntyCounts1, intTokenEntyCounts2); - score = tokenEntryPosScoring(score, strTokenEntryGetPOS1, strTokenEntryGetPOS2); - score = unmarkedPatternCounterScoring(score, UnmarkedPatternCounter1, UnmarkedPatternCounter2); - score = markedContiniousCounterScoring(score, MarkedContinuousCounter1, MarkedContinuousCounter2); - score = strTokensMapScoring(score, strTokensIpartForm1, strTokensIpartForm2); - score = strTokenEntryScoring(score, tokenEntry1, tokenEntry2); - score = strTokenMapTagsScoring(score, ITokenTags1, ITokenTags2); - score = tokenformSizeScoring(score, tokenForms1, tokenForms1); - score = tokenStemmingMapScoring(score, strTokenStems1, strTokenStems2); - - score = inflectedCounterScoring(score, InflectedCounterPositive1, InflectedCounterPositive2, - InflectedCounterNegative1, InflectedCounterNegative2); - score = annotatorCountScoring(score, Anotatorcounter1, Anotatorcounter2); - score = tokensCounterScoring(score, TokensCounter1, TokensCounter2); + score = inflectedCounterScoring(score, this.InflectedCounterPositiveF, this.InflectedCounterPositive1, + this.InflectedCounterNegativeF, this.InflectedCounterNegative1); + score = annotatorCountScoring(score, this.AnotatorcounterF, this.Anotatorcounter1); + score = tokensCounterScoring(score, this.TokensCounterF, this.TokensCounter1); LevenshteinDistance leven = new LevenshteinDistance(str, str1); double SentenceScoreDiff = leven.computeLevenshteinDistance(); SentenceScoreDiff *= 15; score -= SentenceScoreDiff; + score = nerEntitiesAndTokenScoring(score, this.entityTokenTagsF, this.entityTokenTags1, + this.nerEntitiesF, this.nerEntities1); - ArrayList entityTokenTags1 = new ArrayList<>(); - ArrayList entityTokenTags2 = new ArrayList<>(); - ArrayList nerEntities1 = new ArrayList<>(); - ArrayList nerEntities2 = new ArrayList<>(); - ArrayList nerEntitiesType = new ArrayList<>(); - ArrayList pipelineCoreDocuments = new ArrayList<>(); - pipelineCoreDocuments.add(pipelineCoreDcoument1); - pipelineCoreDocuments.add(pipelineCoreDcoument2); - firstAnnotation = true; - for (CoreDocument coreDocument : pipelineCoreDocuments) { - for (CoreEntityMention em : coreDocument.entityMentions()) { - Set> entrySet = em.entityTypeConfidences().entrySet(); - String entityType = em.entityType(); - Double EntityConfidences = 0.0; - for (Map.Entry entries : entrySet) { - EntityConfidences = entries.getValue(); - } - List tokens = em.tokens(); - for (CoreLabel token : tokens) { - if (token != null) { - if (firstAnnotation) { - if (!entityTokenTags1.contains(token.tag())) { - if (entityType.equals("PERSON") && EntityConfidences > 0.80) { - entityTokenTags1.add(token.tag()); - } - } - } else { - if (!entityTokenTags2.contains(token.tag())) { - if (entityType.equals("PERSON") && EntityConfidences > 0.80) { - entityTokenTags2.add(token.tag()); - } - } - } - } - } - if (firstAnnotation) { - if (!nerEntities1.contains(em.text())) { - nerEntities1.add(em.text()); - nerEntitiesType.add(em.entityType()); - } - } else { - if (!nerEntities2.contains(em.text())) { - nerEntities2.add(em.text()); - nerEntitiesType.add(em.entityType()); - } - } - } - firstAnnotation = false; - } - score = nerEntitiesAndTokenScoring(score, entityTokenTags1, entityTokenTags2, nerEntities1, nerEntities2); - ArrayList pipelineAnnotationSentiments = new ArrayList<>(); - ArrayList stopWordToken1 = new ArrayList<>(); - ArrayList stopWordToken2 = new ArrayList<>(); - ArrayList stopWordLemma1 = new ArrayList<>(); - ArrayList stopWordLemma2 = new ArrayList<>(); - int PairCounter1 = 0; - int PairCounter2 = 0; - pipelineAnnotationSentiments.add(pipelineAnnotation1Sentiment); - pipelineAnnotationSentiments.add(pipelineAnnotation2Sentiment); - - firstAnnotation = true; - for (Annotation pipelineAnnotationSentiment : pipelineAnnotationSentiments) { - String customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in," + - "into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with"; - List tokensSentiment = pipelineAnnotationSentiment.get(CoreAnnotations.TokensAnnotation.class); - Set stopWords = StopAnalyzer.ENGLISH_STOP_WORDS_SET; - Set stopWordsCustom = StopwordAnnotator.getStopWordList(customStopWordList, true); - for (CoreLabel token : tokensSentiment) { - Pair stopword = token.get(StopwordAnnotator.class); - String word = token.word().toLowerCase(); - if (stopWords.contains(word) || stopWordsCustom.contains(word)) { - if (firstAnnotation) { - stopWordToken1.add(word); - } else { - stopWordToken2.add(word); - } - } - String lemma = token.lemma().toLowerCase(); - if (stopWords.contains(lemma) || stopWordsCustom.contains(lemma)) { - if (firstAnnotation) { - stopWordLemma1.add(lemma); - } else { - stopWordLemma2.add(lemma); - } - } - if (stopword.first() && stopword.second()) { - if (firstAnnotation) { - PairCounter1++; - } else { - PairCounter2++; - } - } - } - firstAnnotation = false; - } - score = stopWordTokenLemmaScoring(score, stopWordToken1, stopWordToken2, stopWordLemma1, stopWordLemma2); - score = stopwordTokenPairCounterScoring(score, stopWordToken1, stopWordToken2, PairCounter1, PairCounter2); + score = stopWordTokenLemmaScoring(score, this.stopWordTokenF, this.stopWordToken1, + this.stopWordLemmaF, this.stopWordLemma1); + score = stopwordTokenPairCounterScoring(score, this.stopWordTokenF, this.stopWordToken1, + this.PairCounterF, this.PairCounter1); smxParam.setDistance(score); return smxParam; } diff --git a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java index a00c4c89..953c7429 100644 --- a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java +++ b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java @@ -1,17 +1,3 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - - ps ax | grep EventNotfierDiscordBot-1.0 - kill $pid (number) - -nohup screen -d -m -S nonroot java -Xmx6048M -jar /home/javatests/ArtificialAutism-1.0.jar -nohup screen -d -m -S nonroot java -Xmx6800M -jar /home/javatests/ArtificialAutism-1.0.jar - -screen -ls (number1) -screen -X -S (number1) quit - */ package PresentationLayer; import DataLayer.settings; @@ -117,5 +103,5 @@ public class DiscordHandler { FunctionLayer.DoStuff.doStuff(event, usernameBot, datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment); }); gateway.onDisconnect().block(); - } + } //3.1.1 discord4j version } diff --git a/suggestionsbot/discord_suggestions.py b/suggestionsbot/discord_suggestions.py index a9ab86ff..7a048fb0 100644 --- a/suggestionsbot/discord_suggestions.py +++ b/suggestionsbot/discord_suggestions.py @@ -1,11 +1,13 @@ import discord from discord.ext import commands from discord.ext.tasks import loop +from discord import HTTPException from settings import token +import time import datetime client = discord.Client() -ignore_list = [] +ignore_list = [846756271910158378] #shit api. got switched like 1 year ago so like every single discusison about it is outdated. def get_suggestion_type(msg): @@ -35,7 +37,7 @@ async def on_message(message): if message.author.bot: return if message.author.id in ignore_list: - msg = f'{message.author.name} Your suggestion was ignored because you made too many shitty suggestions already. Do not post in this channel anymore.' + msg = f'{message.author.name} Your suggestion was ignored because you made too many shitty suggestions already. Do not post in this channel anymore because your retarded.' await message.channel.send() return if message.channel.name == 'suggestion-box': #suggestion-box @@ -44,12 +46,13 @@ async def on_message(message): suggestion_type, message_content = get_suggestion_type(message_content) now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") send_msg = f"""```DISCORD USER ID: {message.author.id}\nUSERNAME: {message.author}\nTYPE: {suggestion_type}\nTIME: {now}\nSTATUS: No admin interested yet (mark with an emote to prevent suggestion deletion within 48 hours)\nSUGGESTION: {message_content}```""" - send_channel_msg = f"""{message.author} your suggestion was sent to admins as type: {suggestion_type}. Specify Suggestion types by appending your message with one of the following commands: !ze !zr !mg !forum !discord !meta""" + send_channel_msg = f"""{message.author} your suggestion was sent to admins as type: {suggestion_type}. Specify Suggestion types by appending your message with one of the following commands: !ze !zr !mg !forum !discord !meta. \nSpamming the suggestion channel with useless crap will lead to ban.""" await message.channel.send(send_channel_msg) await channel.send(send_msg) @loop(seconds = 10) async def check_suggestions_to_delete(): + global client for channel in client.get_all_channels(): if channel.name == 'suggestion-admin': msgs_history = channel.history() @@ -79,7 +82,13 @@ async def check_suggestions_to_delete(): if user.name not in user_reactions: user_reactions.append(user.name) final_msg = f'{first_part}STATUS: Admins interested in topic:{user_reactions}\n{suggestion}{last_part}' - await msg.edit(content=final_msg) + try: + await msg.edit(content=final_msg) + except HTTPException: + time.sleep(300) + client = discord.Client() + check_suggestions_to_delete.start() + client.run(token) #the entire syntax that is visisble from python3 terminal is not very well displayed in the docs of discord.py and nobody has the current syntax in any suggestions anywhere on the internet. almost every single thing is out of date reee if __name__ == '__main__':